setting strategy to mimic rmpi/dompi

[yonicd]

is there an analogous setting in future::plan for rmpi?

library(parallel) 
hello_world <- function() { 
## Print the hostname and MPI worker rank. 
paste(Sys.info()["nodename"],Rmpi::mpi.comm.rank(), sep = ":") 
} 

cl <- makeCluster(Sys.getenv()["SLURM_NTASKS"], type = "MPI") 
clusterCall(cl, hello_world) 
stopCluster(cl)

where the plan is set on the workers and the call to slurm/grid is done via bash template.

[yonicd]

I think this is the equiv

cl <- parallel::makeCluster(Sys.getenv()["SLURM_NTASKS"], type = "MPI") 

future::plan(future::cluster, workers = cl)

furrr::future_map(1:4,.f=function(i){
paste(i,Sys.info()["nodename"], sep = ":") 
})

parallel::stopCluster(cl)

not sure how to mimic Rmpi::mpi.comm.rank()

[yonicd]

I have this version of multiprocess/sge/mpi working, which somewhat answers my initial query. Is there any way to float up the cl <- parallel::makeCluster(2, type = "MPI") into the plan step?

the sge template is one that is referenced in issue https://github.com/HenrikBengtsson/future.batchtools/issues/39#issue-414824200

library(future)
library(batchtools)
library(future.batchtools)

sge <- future::tweak(
  future.batchtools::batchtools_sge,
  label = 'test',
  template = 'batchtools.sge-mrg.tmpl',
  resources = list(slots = 4)
)

future::plan(list(future::multiprocess,sge))

f <- future::future({
  future.apply::future_lapply(1:4,FUN=function(i){

    hello_world <- function() {
      ## Print the hostname and MPI worker rank.
      paste(i,Sys.info()["nodename"],Rmpi::mpi.comm.rank(), sep = ":")
    }

    cl <- parallel::makeCluster(2, type = "MPI")
    on.exit(parallel::stopCluster(cl),add = TRUE)

    parallel::clusterCall(cl, hello_world)

  })
})

future::value(f)

# 2 slaves are spawned successfully. 0 failed.
# 2 slaves are spawned successfully. 0 failed.
# 2 slaves are spawned successfully. 0 failed.
# 2 slaves are spawned successfully. 0 failed.
# Loading required namespace: Rmpi
# Loading required namespace: Rmpi
# Loading required namespace: Rmpi
# Loading required namespace: Rmpi
# [[1]]
# [[1]][[1]]
# [1] "1:ip-172-16-2-22:1"
# 
# [[1]][[2]]
# [1] "1:ip-172-16-2-22:2"
# 
# 
# [[2]]
# [[2]][[1]]
# [1] "2:ip-172-16-2-129:1"
# 
# [[2]][[2]]
# [1] "2:ip-172-16-2-129:2"
# 
# 
# [[3]]
# [[3]][[1]]
# [1] "3:ip-172-16-2-39:1"
# 
# [[3]][[2]]
# [1] "3:ip-172-16-2-39:2"
# 
# 
# [[4]]
# [[4]][[1]]
# [1] "4:ip-172-16-2-22:1"
# 
# [[4]][[2]]
# [1] "4:ip-172-16-2-22:2"