vertica / vertica-nodejs

Official native node.js client for the Vertica Analytics Database.
https://www.vertica.com/
Apache License 2.0
12 stars 14 forks source link

Cannot use stream to bulk insert #105

Closed negreanucalin closed 1 year ago

negreanucalin commented 1 year ago

Hello, I am trying to insert multiple rows and am trying to do this using copy streams. I tried running a modified version of this integration test but there seem to be issues:

P.S: Connection works (read/write)

Code:

[...]
pool.connect(function (err: any, client: any, done: any) {
    if (err) throw err

    var stream = client.query(copyFrom('COPY pt.employee(name,lastname) FROM STDIN'))
    stream.on('end', function () {
        setTimeout(() => {
            done()
            console.log('End');
            client.query('COMMIT;', (err: any, result: any) => {
                if (err) throw err;
                console.log('Commited');
                pool.end()
            })
        }, 50)
    })
    for (var i = 1; i <= 5; i++) {
        var line = ['test', '|', 'user']
        stream.write(line.join(''))
        console.log('Write: ' + line.join(''));
    }
    // stream.write('\\.');   <--- Found this in the [docs](https://www.vertica.com/docs/10.0.x/HTML/Content/Authoring/DataLoad/LoadingUTF-8FormatData.htm)
    stream.end();
})

Output

root@c39d544f53f6:/usr/src/app# ts-node src/test_vertica.ts
Write: test|user
Write: test|user
Write: test|user
Write: test|user
Write: test|user
End
Commited
root@c39d544f53f6:/usr/src/app# 

Versions: Vertica: Vertica Analytic Database v11.1.1-0 Node: v16.18.1

negreanucalin commented 1 year ago

Figured it out, here is my working code:

pool.connect(function (err: any, client: any, done: any) {
    if (err) throw err

    var stream = client.query(copyFrom('COPY pt.employee(name,lastname) FROM STDIN'))

    for (var i = 1; i <= 5; i++) {
        var line = ['test', '|', 'user', '\n']     // <- Used the default column delimiter ( | ) and new row delimiter ( \n )
        stream.write(line.join(''))
        console.log('Write: ' + line.join(''));
    }
    stream.end(); // Cleanup here because stream.on('end', ... ); does not get called
    done();
})

P.S: with pg-copy-streams at 6.0.4