Open yonicd opened 5 years ago
I think this is the equiv
cl <- parallel::makeCluster(Sys.getenv()["SLURM_NTASKS"], type = "MPI")
future::plan(future::cluster, workers = cl)
furrr::future_map(1:4,.f=function(i){
paste(i,Sys.info()["nodename"], sep = ":")
})
parallel::stopCluster(cl)
not sure how to mimic Rmpi::mpi.comm.rank()
I have this version of multiprocess/sge/mpi working, which somewhat answers my initial query. Is there any way to float up the cl <- parallel::makeCluster(2, type = "MPI")
into the plan
step?
the sge template is one that is referenced in issue https://github.com/HenrikBengtsson/future.batchtools/issues/39#issue-414824200
library(future)
library(batchtools)
library(future.batchtools)
sge <- future::tweak(
future.batchtools::batchtools_sge,
label = 'test',
template = 'batchtools.sge-mrg.tmpl',
resources = list(slots = 4)
)
future::plan(list(future::multiprocess,sge))
f <- future::future({
future.apply::future_lapply(1:4,FUN=function(i){
hello_world <- function() {
## Print the hostname and MPI worker rank.
paste(i,Sys.info()["nodename"],Rmpi::mpi.comm.rank(), sep = ":")
}
cl <- parallel::makeCluster(2, type = "MPI")
on.exit(parallel::stopCluster(cl),add = TRUE)
parallel::clusterCall(cl, hello_world)
})
})
future::value(f)
# 2 slaves are spawned successfully. 0 failed.
# 2 slaves are spawned successfully. 0 failed.
# 2 slaves are spawned successfully. 0 failed.
# 2 slaves are spawned successfully. 0 failed.
# Loading required namespace: Rmpi
# Loading required namespace: Rmpi
# Loading required namespace: Rmpi
# Loading required namespace: Rmpi
# [[1]]
# [[1]][[1]]
# [1] "1:ip-172-16-2-22:1"
#
# [[1]][[2]]
# [1] "1:ip-172-16-2-22:2"
#
#
# [[2]]
# [[2]][[1]]
# [1] "2:ip-172-16-2-129:1"
#
# [[2]][[2]]
# [1] "2:ip-172-16-2-129:2"
#
#
# [[3]]
# [[3]][[1]]
# [1] "3:ip-172-16-2-39:1"
#
# [[3]][[2]]
# [1] "3:ip-172-16-2-39:2"
#
#
# [[4]]
# [[4]][[1]]
# [1] "4:ip-172-16-2-22:1"
#
# [[4]][[2]]
# [1] "4:ip-172-16-2-22:2"
is there an analogous setting in future::plan for rmpi?
where the plan is set on the workers and the call to slurm/grid is done via bash template.