c-blake / cligen

Nim library to infer/generate command-line-interfaces / option / argument parsing; Docs at
https://c-blake.github.io/cligen/
ISC License
496 stars 23 forks source link

`procpool` helpers for flat `seq[T]` types #211

Closed Vindaar closed 2 years ago

Vindaar commented 2 years ago

This adds two helpers to more easily use seq[T] types together with procpool.

  1. wrLenSeq similar to wrLenBuf. Writes the seq[T] (as long as T is flat, use at your own risk) with an integer length prefix using writev to a given file descriptor.
  2. toSeq for MSlice. Copies over the data in the given slice to a sequence of seq[T] (again, flat T).

Combined they can be used to easily write data from a kid worker back to the parent and fill a resulting sequence.

Until we have an example using this in the repository, here's a snippet of the main parts of my own code for which I just wrote this (it's a MC code where I parallelize over multiple processes to draw more toys).

type
  ProcData = object
    id: int
    nmc: int

proc computeParallelLimits(ctx: Context, limitKind: LimitKind, nmc: int): seq[(float, int)] =
  var pp = initProcPool(
    (proc(r, w: cint) =
       let i = open(r)
       var p: ProcData
       while i.uRd(p):
         echo "Starting work for ", p
         var rnd = wrap(initMersenneTwister(p.id.uint32))
         var results = newSeq[(float, int)]()
         for i in 0 ..< p.nmc:
           echo "MC index ", i
           ctx.mcIdx = i
           let cands = ctx.drawCandidates(rnd)
           let limit = ctx.computeLimit(rnd, cands, limitKind)
           let cInSens = candsInSens(ctx, cands)
           results.add (limit, cInSens)
         echo "Bytes: ", w.wrLenSeq results),
    framesLenPfx, # use `framesLenPfx` as `wrLenSeq` writes with an int length prefix of the data
    jobs = countProcessors()
  )

  var work = newSeq[ProcData]()
  for i in 0 ..< countProcessors():
    work.add ProcData(id: i, nmc: nmc)

  var limits = newSeq[tuple[limit: float, cInSens: int]]()
  var getRes = proc(s: MSlice) =
    var res: seq[tuple[limit: float, cInSens: int]]
    # use `toSeq` to copy over the slice data back into `res`
    s.toSeq(res)
    limits.add res
  pp.evalOb work, getRes
  result = limits

The code as it is of course doesn't compile, but it should get the idea across as a reference for now.