mariadb-corporation / mariadb-connector-nodejs

MariaDB Connector/Node.js is used to connect applications developed on Node.js to MariaDB and MySQL databases. MariaDB Connector/Node.js is LGPL licensed.
GNU Lesser General Public License v2.1
369 stars 91 forks source link

Large Insert never completes with .batch() and node 3.2.3 #272

Closed markddrake closed 5 months ago

markddrake commented 8 months ago

Running the following code

"use strict"
import mariadb from 'mariadb';
import fs from 'fs';

async function main() {

    const connectionDetails = {
            host      : "yadamu-db1"
           ,user      : "root"
           ,password  : "oracle"
           ,port      : 3307
           ,database  : "mysql"
           ,multipleStatements: true
           ,typeCast:true
           ,bigNumberStrings:true
           ,dateStrings:true
           ,rowsAsArray:true}

  let results;

  try {

    const pool = mariadb.createPool(connectionDetails);
    const conn = await pool.getConnection();
    results = await conn.query(`SET AUTOCOMMIT = 0, TIME_ZONE = '+00:00',SESSION INTERACTIVE_TIMEOUT = 600000, WAIT_TIMEOUT = 600000, SQL_MODE='ANSI_QUOTES,PAD_CHAR_TO_FULL_LENGTH', GROUP_CONCAT_MAX_LEN = 1024000, GLOBAL LOCAL_INFILE = 'ON';`);
    results = await conn.query(`drop schema if exists "YADAMU_TEST";`);    
    results = await conn.query(`create schema "YADAMU_TEST";`);    

    results = await conn.query(`USE "YADAMU_TEST"`);    
    results = await conn.query(`create table  if not exists "YADAMU_TEST"."character_types"(
  "character_col" char(1) ,"character_varying_col" longtext ,"character_max_col" mediumtext ,"character_varying_max_col" mediumtext ,"character_4000_col" varchar(4000) ,"character_varying_4000_col" varchar(4000) ,"character_64k_col" text ,"character_varying_64k_col" text ,"text_col" longtext)`);

    const testData = fs.readFileSync('TEST_DATA.json');
    const data = JSON.parse(testData)
    console.log('ROWS',data.length)

    const sql = `insert into "YADAMU_TEST"."character_types" ("character_col","character_varying_col","character_max_col","character_varying_max_col","character_4000_col","character_varying_4000_col","character_64k_col","character_varying_64k_col","text_col") values  (?,?,?,?,?,?,?,?,?)`;

    try {

      results = await conn.beginTransaction()
      results = await conn.query(`SAVEPOINT YADAMU_INSERT; `)
      console.log(1)
      results = await conn.batch(sql,data);
      console.log(2)
      results = await conn.commit()
    } catch (e) {
      console.log(3)
      console.log(e)
    } 

   conn.end()
   pool.end();
 } catch(e) {
   console.log(e);
 }
}

main().then(() => {console.log("Success")}).catch((e) => {console.log(e)})

Results in

C:\Development\YADAMU\src\scratch\mariadb>time
The current time is:  0:35:02.89
Enter the new time:

C:\Development\YADAMU\src\scratch\mariadb>node insertIssue.js
ROWS 4
1
^C
C:\Development\YADAMU\src\scratch\mariadb>time
The current time is:  0:36:30.80
Enter the new time:

C:\Development\YADAMU\src\scratch\mariadb>

As you can see, after a unreasonably long pause the code had not returned from the batch operation.

markddrake commented 8 months ago

Test Data file TEST_DATA.zip

markddrake commented 8 months ago

C:\Development\YADAMU\src\scratch\mariadb>node -v v20.7.0

C:\Development\YADAMU\src\scratch\mariadb>npm ls yadamu@1.0.0 C:\Development\YADAMU\src +-- @aws-sdk/client-s3@3.507.0 +-- @aws-sdk/lib-storage@3.507.0 +-- @azure/storage-blob@12.17.0 +-- @electron/remote@2.1.2 +-- bootstrap-icons@1.11.3 +-- bootstrap@5.3.2 +-- cookie-parser@1.4.6 +-- csv-parser@3.0.0 +-- electron-packager@17.1.2 +-- electron@27.3.1 +-- express-session@1.18.0 +-- express@4.18.2 +-- font-awesome@4.7.0 +-- ibm_db_electron@npm:ibm_db@3.2.3 +-- ibm_db@3.2.3 +-- install@0.13.0 +-- jquery@3.7.1 +-- mariadb@3.2.3 +-- mime-types@2.1.35 +-- mongodb@6.3.0 +-- mssql@10.0.2 +-- mysql@2.18.1 +-- npm@10.4.0 +-- oracledb@6.3.0 +-- pg-copy-streams@6.0.6 +-- pg-query-stream@4.5.3 +-- pg@8.11.3 +-- readable-stream@4.5.2 +-- snowflake-sdk@1.9.3 +-- uuid@9.0.1 `-- wkx@0.5.0

rusher commented 5 months ago

closing since this must have been corrected in 3.3.0. If that's not the case, feel free to create new issue related to this one