search

alibaba / DataX

DataX是阿里云DataWorks数据集成的开源版本。
Other
15.99k stars 5.45k forks source link

DataX3:Mylsql8.0.27ToStarRocks3.2.8,StarRocksWriter:Code:[DBUtilErrorCode-05], Description:[往您配置的写入表中写入数据时失败.]. - java.lang.RuntimeException: Writing records to StarRocks failed. #2195

Open gearsir opened 3 months ago

gearsir commented 3 months ago

其他驱动成功传输： DataX数据传输测试：均成功，测试数据：120万条。

SQLServer12 To SQLServer12：OK SQLServer12 To Mysql8.0.27：OK SQLServer12 To StarRocks3.2.8（Writer使用Mysql驱动）：OK

Mysql8.0.27 To Mysql8.0.27：OK Mysql8.0.27 To SQLServer12：OK Mysql8.0.27 To StarRocks3.2.8（Writer使用Mysql驱动）：OK

StarRocks3.2.8 To Mysql8.0.27：OK StarRocks3.2.8 To StarRocks3.2.8：OK StarRocks3.2.8 To SQLServer12：OK

Hyper-V上CentOS-Stream9-server ping192.168.1.3:8040是通的，但一直报127.0.0.1:8040连接失败（是不是DxtaX在执行job的时候访问的是CentOS9本机的127.0.0.1:8040？） 但使用StarRocksWriter驱动修改job数据库连接方式后，就报错误了，脚本： {

“job”: {

"setting": {

"speed": {

   "channel": 1

},

"errorLimit": {

  "record": 0,

  "percentage": 0

}

},

"content": [

{

"reader": {

  "name": "mysqlreader",

  "parameter": {

    "connection": [

      {

        "jdbcUrl":["jdbc:mysql://192.168.1.3:3307/regulationdb?characterEncoding=UTF-8&serverTimezone=Asia/Shanghai"],

        "table": ["datamonitoring"]

      }

    ],

    "username": "root",

    "password": "123456",

    "column": ["id","MarketID"

               ,"MarketIName"

               ,"TerminalId"

               ,"TerminalName"

               ,"ShopingCartID"

               ,"SequenceNumberInShopingCart"

               ,"NameID"

               ,"NameIDSku"

               ,"MarketDefinedProductCode"

               ,"SellingCount"

               ,"Price"

               ,"SellingTime"

               ,"Valuation"

               ,"UploadingTime"]

  }

},

"writer": {

    "name":"starrockswriter",

   "parameter": {

    "username":"root",

    "password":"123456",

    "database":"regulationdb",

    "table":"datamonitoring_copy3",

    "column": ["id","MarketID"

               ,"MarketIName"

               ,"TerminalId"

               ,"TerminalName"

               ,"ShopingCartID"

               ,"SequenceNumberInShopingCart"

               ,"NameID"

               ,"NameIDSku"

               ,"MarketDefinedProductCode"

               ,"SellingCount"

               ,"Price"

               ,"SellingTime"

               ,"Valuation"

               ,"UploadingTime"],

    "field": [

      {  

       "name": "id",  

       "type": "int"  

       },

       {  

       "name": "MarketID",  

       "type": "int"  

       },

       {  

       "name": "MarketIName",  

       "type": "varchar(255)"  

       },

       {  

       "name": "TerminalId",  

       "type": "int"  

       },

       {  

       "name": "TerminalName",  

       "type": "varchar(255)"  

       },

       {  

       "name": "ShopingCartID",  

       "type": "varchar(255)"  

       },

       {  

         "name": "SequenceNumberInShopingCart",  

         "type": "varchar(255)"  

       },

       {  

       "name": "NameID",  

       "type": "int"  

       },

       {  

       "name": "NameIDSku",  

       "type": "varchar(255)"  

       },

       {  

       "name": "MarketDefinedProductCode",  

       "type": "varchar(255)"  

       },

       {  

       "name": "SellingCount",  

       "type": "decimal(6, 3)"  

       },

       {  

       "name": "Price",  

       "type": "decimal(8, 2)"  

       },

       {  

       "name": "SellingTime",  

       "type": "datetime"  

       },

       {  

       "name": "Valuation",  

       "type": "varchar(255)"  

       },

       {  

       "name": "UploadingTime",  

       "type": "datetime "  

       }

   ],

    "writeMode":"insert",

    "jdbcUrl":"jdbc:mysql://192.168.1.3:9030/",

    "loadUrl":["192.168.1.3:8030"],

    "preSql": [],

    "postSql": [],

    "loadProps": {

      "format": "json",

      "strip_outer_array": true

    }

}

}

}

]

}

}

图片 图片 图片941×502 24.4 KB

图片 图片1691×594 28.6 KB

2024-08-28 15:10:14 **** 2024-08-28 15:10:14 ____ ___ ____ ____ ____ ____ ____ _ __ 2024-08-28 15:10:14 |\ | \ |\ |\ |\ |\ |\ |\ \ |\ \ 2024-08-28 15:10:14 * \ \ || \ \ \ |\ \ \ |\ \ \ |\ \ \ |\ \ \ |\ \ / /|\ \ | 2024-08-28 15:10:14 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ___ \ 2024-08-28 15:10:14 ||\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ||\ \ 2024-08-28 15:10:14 * \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ 2024-08-28 15:10:14 * |\ || |||||||||||||||||| || 2024-08-28 15:10:14 || || 2024-08-28 15:10:14 2024-08-28 15:10:14 * ____ ____ 2024-08-28 15:10:14 * |\ |\ \ |\ \ |\ |\ \ / 2024-08-28 15:10:14 \ \ |\ \ \ \ \ \ \ \ \ \ \ \ \ |/|\ 2024-08-28 15:10:14 \ \ \ \ \ \ \ \ \ \ \ \ \ \ |/ \ \ 2024-08-28 15:10:14 \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ 2024-08-28 15:10:14 \ \ \ \ \ \ \ \ \ 2024-08-28 15:10:14 |||||||||||| || || 2024-08-28 15:10:14 2024-08-28 15:10:14 **** 2024-08-28 15:10:14 2024-08-28 15:10:14 StarRocks [(Blazing Fast)]> _ 2024-08-28 15:10:14 2024-08-28 15:10:14 2024-08-28 15:10:15 2024-08-28 07:10:15,213 INFO Set uid to user 0 succeeded 2024-08-28 15:10:15 2024-08-28 07:10:15,516 INFO RPC interface ‘supervisor’ initialized 2024-08-28 15:10:15 2024-08-28 07:10:15,516 CRIT Server ‘unix_http_server’ running without any HTTP authentication checking 2024-08-28 15:10:15 2024-08-28 07:10:15,516 INFO supervisord started with pid 7 2024-08-28 15:10:16 2024-08-28 07:10:16,518 INFO spawned: ‘beservice’ with pid 555 2024-08-28 15:10:16 2024-08-28 07:10:16,519 INFO spawned: ‘broker’ with pid 556 2024-08-28 15:10:16 2024-08-28 07:10:16,521 INFO spawned: ‘director’ with pid 557 2024-08-28 15:10:16 2024-08-28 07:10:16,522 INFO spawned: ‘feproxy’ with pid 558 2024-08-28 15:10:16 2024-08-28 07:10:16,523 INFO spawned: ‘feservice’ with pid 560 2024-08-28 15:10:16 2024-08-28 07:10:16+00:00 INFO checking if need to perform auto registring Backend and Broker … 2024-08-28 15:10:16 2024-08-28 07:10:16,583 INFO exited: broker (exit status 1; not expected) 2024-08-28 15:10:16 2024-08-28 07:10:16+00:00 INFO checking if FE service query port:9030 alive or not … 2024-08-28 15:10:16 2024-08-28 07:10:16+00:00 WARN FE service query port:9030 is NOT alive yet! 2024-08-28 15:10:17 2024-08-28 07:10:17,790 INFO spawned: ‘broker’ with pid 3024 2024-08-28 15:10:18 2024-08-28 07:10:18+00:00 WARN FE service query port:9030 is NOT alive yet! 2024-08-28 15:10:20 2024-08-28 07:10:20+00:00 WARN FE service query port:9030 is NOT alive yet! 2024-08-28 15:10:21 2024-08-28 07:10:21,741 INFO success: beservice entered RUNNING state, process has stayed up for > than 5 seconds (startsecs) 2024-08-28 15:10:21 2024-08-28 07:10:21,741 INFO success: director entered RUNNING state, process has stayed up for > than 5 seconds (startsecs) 2024-08-28 15:10:21 2024-08-28 07:10:21,741 INFO success: feproxy entered RUNNING state, process has stayed up for > than 5 seconds (startsecs) 2024-08-28 15:10:21 2024-08-28 07:10:21,741 INFO success: feservice entered RUNNING state, process has stayed up for > than 5 seconds (startsecs) 2024-08-28 15:10:22 2024-08-28 07:10:22+00:00 INFO FE service query port:9030 is alive! 2024-08-28 15:10:22 2024-08-28 07:10:22+00:00 INFO generate my.cnf file … 2024-08-28 15:10:22 2024-08-28 07:10:22+00:00 INFO check if need to add BE into FE service … 2024-08-28 15:10:22 2024-08-28 07:10:22,929 INFO success: broker entered RUNNING state, process has stayed up for > than 5 seconds (startsecs) 2024-08-28 15:10:26 2024-08-28 07:10:26+00:00 INFO BE service already added into FE service … 2024-08-28 15:10:26 2024-08-28 07:10:26+00:00 INFO check if need to add BROKER into FE service … 2024-08-28 15:10:26 2024-08-28 07:10:26+00:00 INFO broker service already added into FE service … 2024-08-28 15:10:15 Unlinking stale socket /data/deploy/starrocks/supervisor/supervisor.sock 2024-08-28 15:10:26 2024-08-28 07:10:26+00:00 INFO cluster initialization DONE! 2024-08-28 15:10:26 2024-08-28 07:10:26+00:00 INFO wait a few seconds for BE and Broker’s heartbeat … 2024-08-28 15:10:36 2024-08-28 07:10:36+00:00 INFO StarRocks Cluster information details: 2024-08-28 15:10:36 1. row 2024-08-28 15:10:36 Name: 127.0.0.1_9010_1724058294712 2024-08-28 15:10:36 IP: 127.0.0.1 2024-08-28 15:10:36 EditLogPort: 9010 2024-08-28 15:10:36 HttpPort: 8030 2024-08-28 15:10:36 QueryPort: 9030 2024-08-28 15:10:36 RpcPort: 9020 2024-08-28 15:10:36 Role: LEADER 2024-08-28 15:10:36 ClusterId: 1869614770 2024-08-28 15:10:36 Join: true 2024-08-28 15:10:36 Alive: true 2024-08-28 15:10:36 ReplayedJournalId: 147393 2024-08-28 15:10:36 LastHeartbeat: 2024-08-28 07:10:34 2024-08-28 15:10:36 IsHelper: true 2024-08-28 15:10:36 ErrMsg: 2024-08-28 15:10:36 StartTime: 2024-08-28 07:10:21 2024-08-28 15:10:36 Version: 3.2.8-759cc78 2024-08-28 15:10:36 1. row 2024-08-28 15:10:36 BackendId: 10004 2024-08-28 15:10:36 IP: 127.0.0.1 2024-08-28 15:10:36 HeartbeatPort: 9050 2024-08-28 15:10:36 BePort: 9060 2024-08-28 15:10:36 HttpPort: 8040 2024-08-28 15:10:36 BrpcPort: 8060 2024-08-28 15:10:36 LastStartTime: 2024-08-28 07:10:19 2024-08-28 15:10:36 LastHeartbeat: 2024-08-28 07:10:34 2024-08-28 15:10:36 Alive: true 2024-08-28 15:10:36 SystemDecommissioned: false 2024-08-28 15:10:36 ClusterDecommissioned: false 2024-08-28 15:10:36 TabletNum: 62 2024-08-28 15:10:36 DataUsedCapacity: 0.000 B 2024-08-28 15:10:36 AvailCapacity: 91.113 GB 2024-08-28 15:10:36 TotalCapacity: 165.451 GB 2024-08-28 15:10:36 UsedPct: 44.93 % 2024-08-28 15:10:36 MaxDiskUsedPct: 44.93 % 2024-08-28 15:10:36 ErrMsg: 2024-08-28 15:10:36 Version: 3.2.8-759cc78 2024-08-28 15:10:36 Status: {“lastSuccessReportTabletsTime”:“N/A”} 2024-08-28 15:10:36 DataTotalCapacity: 91.113 GB 2024-08-28 15:10:36 DataUsedPct: 0.00 % 2024-08-28 15:10:36 CpuCores: 16 2024-08-28 15:10:36 NumRunningQueries: 0 2024-08-28 15:10:36 MemUsedPct: 0.80 % 2024-08-28 15:10:36 CpuUsedPct: 0.0 % 2024-08-28 15:10:36 Location: 2024-08-28 15:10:36 1. row 2024-08-28 15:10:36 Name: allin1broker 2024-08-28 15:10:36 IP: 127.0.0.1 2024-08-28 15:10:36 Port: 8000 2024-08-28 15:10:36 Alive: true 2024-08-28 15:10:36 LastStartTime: 2024-08-19 09:05:05 2024-08-28 15:10:36 LastUpdateTime: 2024-08-28 07:10:34 2024-08-28 15:10:36 ErrMsg: 2024-08-28 15:10:36 2024-08-28 07:10:36+00:00 INFO 2024-08-28 15:10:36 2024-08-28 07:10:36+00:00 INFO 2024-08-28 15:10:36 2024-08-28 07:10:36+00:00 INFO IMPORTANT NOTICE! 2024-08-28 15:10:36 2024-08-28 15:10:36 If FE/BE state needs to be persisted, please be sure the following directories are mounted: 2024-08-28 15:10:36 FE service meta: /data/deploy/starrocks/fe/meta 2024-08-28 15:10:36 * BE service storage: /data/deploy/starrocks/be/storage 2024-08-28 15:10:36 2024-08-28 15:10:36 2024-08-28 07:10:36+00:00 INFO 2024-08-28 15:10:36 2024-08-28 07:10:36+00:00 INFO FE mysql query port: 9030 2024-08-28 15:10:36 2024-08-28 07:10:36+00:00 INFO FE http service port: 8080 2024-08-28 15:10:36 2024-08-28 07:10:36+00:00 INFO 2024-08-28 15:10:36 2024-08-28 07:10:36+00:00 INFO Enjoy the journal to StarRocks blazing-fast lake-house engine!

gearsir commented 2 months ago

f20047856fd2a845bfd4f7710b046d11a5559da5