High memory usage when running in parallel

[cedric-cordenier]

Describe the bug

I'm exploring various wasm runtimes to see which are best from a performance perspective. One issue I'm running into is that when testing with wasmer, memory usage is quite high (on the order of tens of GBs) for the duration of the test. This is with the minimal example below:

package main

import (
    "fmt"
    "log"
    "os"
    "runtime"
    "runtime/pprof"
    "time"

    "github.com/wasmerio/wasmer-go/wasmer"
)

func NewModule() (*Module, error) {
    wasm := []byte(`
    (module
      (import "testmodule" "hello" (func $hello))
      (func (export "_start")
        (call $hello)
      )
    )
    `)

    engine := wasmer.NewEngine()
    store := wasmer.NewStore(engine)

    module, err := wasmer.NewModule(store, wasm)
    if err != nil {
        return nil, err
    }

    imports := wasmer.NewImportObject()
    imports.Register("testmodule", map[string]wasmer.IntoExtern{
        "hello": wasmer.NewFunction(
            store,
            wasmer.NewFunctionType(
                wasmer.NewValueTypes(),
                wasmer.NewValueTypes(),
            ),
            func(args []wasmer.Value) ([]wasmer.Value, error) {
                return []wasmer.Value{}, nil
            },
        ),
    })

    instance, err := wasmer.NewInstance(module, imports)
    if err != nil {
        return nil, err
    }

    return &Module{module: module, engine: engine, instance: instance, store: store}, nil
}

func (m *Module) Start() error {
    start, err := m.instance.Exports.GetFunction("_start")
    start()
    return err
}

func (m *Module) Close() error {
    m.store.Close()
    m.instance.Close()
    m.module.Close()
    return nil
}

type Module struct {
    module   *wasmer.Module
    engine   *wasmer.Engine
    instance *wasmer.Instance
    store    *wasmer.Store
}

func work() {
    start := time.Now()
    m, err := NewModule()
    if err != nil {
        log.Fatal(err)
    }

    _ = m.Start()
    fmt.Printf("d: %s\n", time.Now().Sub(start))

    m.Close()
    return
}

func main() {
    sem := make(chan bool, 10)
    after := time.After(30 * time.Second)
    for {
        select {
        case <-after:
            f, err := os.Create("mem.out")
            if err != nil {
                log.Fatal("could not create memory profile: ", err)
            }
            defer f.Close() // error handling omitted for example
            runtime.GC()    // get up-to-date statistics
            if err := pprof.WriteHeapProfile(f); err != nil {
                log.Fatal("could not write memory profile: ", err)
            }
            f, err = os.Create("cpu.out")
            if err != nil {
                log.Fatal("could not create CPU profile: ", err)
            }
            defer f.Close() // error handling omitted for example
            if err := pprof.StartCPUProfile(f); err != nil {
                log.Fatal("could not start CPU profile: ", err)
            }
            defer pprof.StopCPUProfile()
            return
        case sem <- true:
            go func() {
                defer func() { <-sem }()
                work()
                time.Sleep(1 * time.Second)
                return
            }()
        }
    }
}

Is this expected? Am I misusing the API somehow? Thanks for your help :)

Steps to reproduce

1. Run the script above
2. Open up htop/activity monitor and check memory usage

Expected behavior

I'd expect less memory to be used.

Actual behavior

A lot of memory is being used.

Additional context

[syrusakbary]

Thanks for opening the issue. I think the issue is more related to wasmer-go than to wasmer (since it's bit a bit unmaintained). We have to update our go bindings and revisit this issue