Bug: Out of memory if reloading the page excessively

[Complexicon]

Hi, i noticed after using this awesome library, that if you reload a webpage containing this library over and over again it will consume increasingly more RAM. I'd guess that there are some resources that aren't freed on page reload, i can reliably reproduce this behaviour on the official demo site (https://sql.js.org/examples/GUI/) here is a screenshot of the chrome performance monitor

as you can see, each time i reload the page some memory is allocated in the js heap that isn't being freed afterwards. i am not really sure if this is more of a chrome type bug or a bug from webassembly. it's also possible i overread something in the documentation how to properly handle deallocation of each wasm instance...

i've hit tab out of memory exceptions multiple times while developing a page with this library, but i dont think this is a particular high priority issue as it only happens if you EXCESSIVELY reload a webpage. i'd still though i should report this

[lovasoa]

It's the browser who is responsible to free memory on page reload, whatever the webpage itself does...

[lovasoa]

Just to be clear, does this happen when you reload the page, or when whatever framework you are using hot reloads modules ?

[Complexicon]

it happens if i full reload the page, i've even tried the ol' shift+f5 for a full reload. i tested a bit more and if i load a rather sizeable sqlite database it makes it like 10x worse. i'd also think the browser is responsible to free it but i could not reproduce this with other wasm libraries sooo maybe some weird edge case?

[nenge123]

how about share woker?

//A page
let p = await (new WorkerSQL({
            src:this.assets+'vendor/sql.js/sqlWorker.js',
            type:'share',
            data:false,
            tables:{
        data:{
            gameID:'int primary key',
            title:'char',
            type:'char',
            region:'char',
            binary:'char',
            language:'char',
            genre:'char',
            titleScreenImage:'char',
        },
        tag:{
            name:'char',
            num:'int'
        }
    }
        }).ready);
await p.getResult({
    result:['data',{
        "gameID": "1",
        "title": "GB热血！沙滩排球 国雄君(繁)(AXI)(JP)(8Mb)",
        "type": "gb",
        "binary": "37aceb63eca3157e2cf58b9f61110ecdba31ab510946b0515ad0065ce8f2ff6a.gb.zip",
        "region": "jp",
        "language": "zh-Hant",
        "titleScreenImage": "eaa9ddc2ee09d627cdbcc81e52c4b42814b181afbaa66115eead86d63261c646.webp"
    }],method:'insert'
});

//B page
let p = await (new WorkerSQL({src:this.assets+'vendor/sql.js/sqlWorker.js',type:'share'});
await p.all('select * from `data`')

client.js

class WorkerMessage{ constructor(port){ if('postMessage' in port){ this.port = port; } } feedback = {}; getRandom(){ return crypto?crypto.randomUUID():btoa(performance.now()+Math.random()); } async getMessage(result,method){ return await this.getResult({result,method}); } async getResult(result){ return new Promise((back,error)=>{ const workerId = this.getRandom(); this.feedback[workerId] = function (data){ const workerId = data.workerId; if(workerId&&this[workerId] instanceof Function){ delete this[workerId]; } if(data.throw)return error(data.throw); back(data.result); }; result.workerId = workerId; this.postMessage(result); }); } postMessage(result){ this.port.postMessage(result); } } class WorkerSQL extends WorkerMessage{ constructor(ARG){ let worker; if(ARG.type=='share'){ let share = new SharedWorker(ARG.src,'SQLite'); super(share.port); this.ready = this.getReady(share.port,ARG); share.port.start(); }else{ worker = new Worker(ARG.src); super(worker); this.ready = this.getReady(worker,ARG); } } getReady(worker,ARG){ return new Promise((back,error)=>{ worker.addEventListener('message',async e=>{ if(e.data=='complete'||e.data&&e.data.result=='complete'){ worker.addEventListener('message',x=>{ let xx = x.data; if (xx && xx.constructor === Object) { let workerId = xx.workerId; if(this.feedback[workerId] instanceof Function){ return this.feedback[workerId](xx); } } }); if(!e.data.database){ await this.install(ARG.data,ARG.tables,back); }else{ await this.setMethod(); back(this); } } },{once:true}); }); } async getMethod(){ return await this.getResult({method:'getMethod'}); } async setMethod(method){ if(!method) method = await this.getMethod(); method.forEach(v=>{ if(v=='constructor')return; this[v] = new Function('...result','return this.getResult({result,method:"'+v+'"})'); }); } async install(data,tablelist,fn){ let method = await this.getResult({result:[data,tablelist],method:'install'}); this.setMethod(method); if(fn instanceof Function)fn(this); return method; } run(query,param){ return this.getResult({ result:[query,param], method:'run' }) } exec(query,param){ return this.getResult({ result:[query,param], method:'exec' }) } closeworker(){ return this.getResult({method:'closeworker'}); } save2exit(){ return this.getResult({method:'save2exit'}); } }

work.js

new class { ports = []; constructor(name, table) { const S = this; self.S = S; const isLocal = /(127\.0\.0\.1|localhost|local\.nenge\.net)/.test(location.host); const root = self.location.href.split('/').slice(0, -1).join('/') + '/'; this.name = name || 'sql-lite'; this.table = table || 'files'; this.jsfile = isLocal ? root + 'sql-wasm.js' : 'https: this.wasmfile = isLocal ? root + 'sql-wasm.wasm' : 'https: this.ready = new Promise(async (ok, error) => { await this.loadDB().catch(e => { error(e); throw e; }); if (self.postMessage) { self.addEventListener('message', this.baseMessage); self.postMessage({ result: 'complete', database: self.database instanceof self.SQLite }); } ok(true); }); self.addEventListener("rejectionhandled", (event) => { console.log(`Promise rejected; reason: ${event.reason}`); }); self.addEventListener("unhandledrejection", (event) => { console.log(`Promise unhandled rejected; reason: ${event.reason}`); }); if (self.SharedWorkerGlobalScope && self instanceof self.SharedWorkerGlobalScope) { console.log(self); self.onconnect = async function (e) { console.log(e); const source = e.source; await self.S.ready; source.onmessage = self.S.baseMessage; source.postMessage({ result: 'complete', database: self.database instanceof self.SQLite }); } } } async loadImport() { importScripts(this.jsfile); this.onRuntimeInitialized(await initSqlJsPromise); return true; } error(e) { postMessage({ throw: e }); throw e; } async loadDB() { let script = await this.getItem('sql-wasm.js'); if (!script) { let response = await fetch(this.jsfile).catch(e => this.error(e)); if (response && response.status == 200) { script = await response.blob(); this.setItem('sql-wasm.js', script); } else { throw 'file error'; } } let initSqlJs = (new Function(await script.text() + ';return initSqlJs;'))(); let wasmBinary = await this.getItem('sql-wasm.wasm'); if (!wasmBinary) { let response = await fetch(this.wasmfile).catch(e => this.error(e)); if (response && response.status == 200) { wasmBinary = new Uint8Array(await response.arrayBuffer()); this.setItem('sql-wasm.wasm', wasmBinary); } else { throw 'file error'; } } this.onRuntimeInitialized(await initSqlJs({ wasmBinary })); return true; } methods = { install(result) { self.database = new self.SQLite(...result); return self.S.methods.getMethod(); }, export(result) { return self.database.export(); }, async reload(result) { if (self.database) self.database.close(); if (result instanceof Boolean) result = await self.S.getItem('data.sqlite3'); if (!(result instanceof Uint8Array)) throw 'data not uint8'; self.database = new self.SQLite(result); return true; }, getMethod() { return Reflect.ownKeys(self.database.constructor.prototype); } } onRuntimeInitialized(SQL) { const S = this; delete SQL.wasmBinary; self.SQLite = class extends SQL.Database { constructor(data, tablelist) { super(data instanceof Uint8Array ? data : undefined); if (tablelist && !(data instanceof Uint8Array)) { this.createtable(tablelist); } } all(sql, params, limit) { let result = this.exec(sql, params); if (result[0]) { let data = []; for (let value of result[0].values) { data.push(Object.fromEntries(value.map((v, k) => [result[0].columns[k], v]))); if (limit) break; } return data; } } one(sql, params) { let result = this.all(sql, params, 1); if (result && result[0]) { return result[0]; } } columns(index, sql, params) { let result = Object.values(this.one(sql, params) || []); if (result.length) { return result[index || 0]; } } result(sql, params) { return this.columns(0, sql, params); } insert(table, data, where) { if (where) { let whereStr = Array.from(Object.keys(where), e => '`' + e + '` = ? ').join(' AND '); this.run('DELETE FROM `' + table + '` WHERE ' + whereStr + ';', Object.values(where)); } return this.run('INSERT INTO `' + table + '` (' + Array.from(Object.keys(data), e => '`' + e + '`').join(',') + ') VALUES (' + Object.keys(data).fill('?').join(',') + ');', Object.values(data)); } update(table, data, where) { let setStr = Array.from(Object.keys(data), e => '`' + e + '` = ?').join(','); let whereStr = Array.from(Object.keys(where), e => '`' + e + '` = ? ').join(' AND '); let param = Object.values(data); param.push(...Object.values(where)); return this.run('UPDATE `' + table + '` SET ' + setStr + ' WHERE ' + whereStr + ' ;', param); } createtable(tablelist) { Array.from(Object.entries(tablelist) || [], entry => { let keys = Array.from(Object.entries(entry[1]), sub => { return '`' + sub[0] + '` ' + sub[1]; }).join(','); this.run(`CREATE TABLE \`${entry[0]}\` (${keys});`); }); } async savedata() { return await S.setItem('data.sqlite3', this.export()); } } } async open(version) { if (this.idb instanceof Promise) return await this.idb; if (!this.idb) { this.idb = new Promise(resolve => { let req = indexedDB.open(this.name, version); req.addEventListener("upgradeneeded", e => { const db = req.result; if (!db.objectStoreNames.contains(this.table)) { const store = db.createObjectStore(this.table); store.createIndex('timestamp', 'timestamp', { "unique": false }); } }, { once: true }); req.addEventListener('success', async e => { const db = req.result; if (!db.objectStoreNames.contains(this.table)) { let version = db.version += 1; db.close(); return resolve(await this.open(version)); } return resolve(db); }, { once: true }); }); } return this.idb; } async ObjectStore(ReadMode) { const db = await this.open(); const transaction = db.transaction([this.table], ReadMode ? undefined : "readwrite"); return transaction.objectStore(this.table); } readOnly() { return this.ObjectStore(!0); } readWrite() { return this.ObjectStore(); } async getItem(name) { let request = (await this.readOnly()).get(name) return new Promise((resolve) => request.addEventListener('success', (e) => { resolve(e.target.result && e.target.result.contents || e.target.result); })); } async setItem(name, contents) { let request = (await this.readWrite()).put({ contents, timestamp: new Date }, name) return new Promise((resolve) => request.addEventListener('success', (e) => { resolve(e.target.result); })); } async baseMessage(e) { const data = e.data; if (data && data.constructor === Object) { const method = data.method; const workerId = data.workerId; const result = data.result; let newResult = { workerId }; if (self.database && self.database[method] instanceof Function) { newResult.result = self.database[method](...result); } else if (self.S.methods[method] instanceof Function) { newResult.result = self.S.methods[method](result, data) } else { switch (method) { case 'closeworker': this.postMessage({ workerId, result: true }); return self.close(); break; case 'save2exit': await self.database.savedata(); this.postMessage({ workerId, result: true }); return self.close(); break; default: newResult.throw = 'undefine method'; break; } } if (newResult.result instanceof Promise) { newResult.result = await newResult.result.catch(e => { newResult.throw = e; }); if (newResult.throw) delete newResult.result; } this.postMessage(newResult); } } }