ironSource / parquetjs

fully asynchronous, pure JavaScript implementation of the Parquet file format
MIT License
348 stars 175 forks source link

Cannot use 'in' operator to search error #138

Closed namoona1 closed 1 year ago

namoona1 commented 1 year ago

I am getting the following error when trying to append a row in parquet format, it works fine when I pass static values. I tried a number of options with no luck, any help will be appreciated

/mnt/volume_nyc3_01/data/node_modules/brotli/build/encode.js:3 1<process.argv.length?process.argv[1].replace(/\/g,"/"):"unknown-program");b.arguments=process.argv.slice(2);"undefined"!==typeof module&&(module.exports=b);process.on("uncaughtException",function(a){if(!(a instanceof y))throw a;});b.inspect=function(){return"[Emscripten Module object]"}}else if(x)b.print||(b.print=print),"undefined"!=typeof printErr&&(b.printErr=printErr),b.read="undefined"!=typeof read?read:function(){throw"no read() available (jsc?)";},b.readBinary=function(a){if("function"=== ^

TypeError: Cannot use 'in' operator to search for 'tx_hash' in {"0x2fd4ead5f5525ecb62e73ce0a2e1df4ec69580a4e760374e8309918150c3b949",block:15903683} at shredRecordInternal (/mnt/volume_nyc3_01/data/node_modules/parquetjs/lib/shred.js:82:30) at exports.shredRecord (/mnt/volume_nyc3_01/data/node_modules/parquetjs/lib/shred.js:40:3) at ParquetWriter.appendRow (/mnt/volume_nyc3_01/data/node_modules/parquetjs/lib/writer.js:94:22) at file:///mnt/volume_nyc3_01/data/test_parquet.js:70:16 at process.processTicksAndRejections (node:internal/process/task_queues:95:5) Thrown at: at shredRecordInternal (/mnt/volume_nyc3_01/data/node_modules/parquetjs/lib/shred.js:82:30) at exports.shredRecord (/mnt/volume_nyc3_01/data/node_modules/parquetjs/lib/shred.js:40:3) at appendRow (/mnt/volume_nyc3_01/data/node_modules/parquetjs/lib/writer.js:94:22) at file:///mnt/volume_nyc3_01/data/test_parquet.js:70:16 at processTicksAndRejections (node:internal/process/task_queues:95:5)

Code:

var schema = new pkg.ParquetSchema({ tx_hash: { type: 'UTF8' , encoding: 'PLAIN'}, block: { type: 'INT64' } // optional fromaddr: { type: 'UTF8' }, // optional toaddr: { type: 'UTF8' }, //optional value: { type: 'DOUBLE' } });

let writer = await pkg.ParquetWriter.openFile(schema, '/mnt/volume_nyc3_01/data/transactions.parquet');

for (let j=0; j < 1000; j++) {

    const cblock = latestBlock - j;
    const block = await alchemy.core.getBlockWithTransactions(cblock);

    const blockStr = block.number + "," + block.miner.toLowerCase() + "\n";

    const len = block.transactions.length;

    for (let i=0 ; i < len - 1; i++) {
            const tx = block.transactions[i];
            const fromaddr = tx.from?.toLowerCase() || 'x';
            const toaddr = tx.to?.toLowerCase() || 'x';

            const transStr = '{tx_hash:"' + tx.hash.toLowerCase() + '",block:' + cblock + '}';
            console.log(transStr);

            await writer.appendRow(transStr);

} } await writer.close();

punowo commented 1 year ago

Sorry, how did you solve this @namoona1 ?