kriszyp / lmdb-js

Simple, efficient, ultra-fast, scalable data store wrapper for LMDB
Other
505 stars 42 forks source link

Creating and dropping multiple environments causes memory to balloon in process #26

Closed kylebernhardy closed 3 years ago

kylebernhardy commented 3 years ago

This is an odd one I have a script where I am creating and deleting many environments in a single process. The memory balloons progressively the more iterations the code executes. You will see from the code the environments are being closed & the files deleted, I have also verified the process has no references to deleted files. This came up as we are seeing this on a smaller scale where users are creating/dropping environments without restarting our product.

Thank you for your help.


const fs = require('fs');
const path = require('path');
const lmdb = require('lmdb-store');
const {promisify} = require('util');
const p_sleep = promisify(setTimeout);

function createEnv(name){
    let environment_path = path.join(__dirname, 'data', name);
    try {
        fs.rmdirSync(environment_path, {recursive:true})
        fs.mkdirSync(environment_path);
    }catch(e){
        console.error(e);
    }
    let env_init = {
        "path": environment_path,
        "mapSize": 10*1024*1024,
        "maxReaders": 1000
    };
    let env = lmdb.open(env_init);

    return env;
}

async function run(seed){
    const prefix = seed + 'env';
    await p_sleep(1000);
let es = [];
    for(let x = 0; x < 1000; x++){
        es.push(createEnv(prefix + x));
    }
    console.log('added');
    await p_sleep(1000);

    console.log('checked');
    await p_sleep(1000);

    for(let x = 0; x < 1000; x++){
        es[x].close();
        let environment_path = path.join(__dirname, 'data', prefix + x);
        fs.rmdirSync(environment_path, {recursive:true})
    }
    console.log('deleted');
    await p_sleep(1000);

    es = [];
}

(async ()=>{
    await p_sleep(5000);
    await run(1);
    await run(2);
    await run(3);
    await run(4);
    await run(5);

    setInterval(()=>{}, 1000);
})();
kylebernhardy commented 3 years ago

Apologies I meant to post this on the node-lmdb repo