apache / seatunnel

SeaTunnel is a next-generation super high-performance, distributed, massive data integration tool.
https://seatunnel.apache.org/
Apache License 2.0
7.82k stars 1.76k forks source link

[Bug] [mysql cdc] can't read data from mysql cdc source #5831

Open wilsonfu88 opened 10 months ago

wilsonfu88 commented 10 months ago

Search before asking

What happened

mysqlcdc2mysql.config file content env { job.name="SeaTunnel" execution.parallelism = 1 job.mode = "STREAMING" checkpoint.interval = 10000
}

source { MySQL-CDC { result_table_name = "book_1110"

database-names = ["test_db2"]

#server-id = 1
username = "root"
password = "root"
#driver="com.mysql.cj.jdbc.Driver"
table-names = ["test_db2.book"]
base-url = "jdbc:mysql://127.0.0.1:3306/test_db2?useSSL=false&rewriteBatchedStatements=true"

} }

sink { Console{ } }

SeaTunnel Version

seatunnel 2.3.3

SeaTunnel Config

2.3.3

Running Command

bin/seatunnel.sh -e local -c config/mysqlcdc2mysql.config

Error Exception

2023-11-11 12:51:08,198 ERROR org.apache.seatunnel.core.starter.SeaTunnel - Fatal Error,

2023-11-11 12:51:08,198 ERROR org.apache.seatunnel.core.starter.SeaTunnel - Please submit bug report in https://github.com/apache/seatunnel/issues

2023-11-11 12:51:08,198 ERROR org.apache.seatunnel.core.starter.SeaTunnel - Reason:SeaTunnel job executed failed

2023-11-11 12:51:08,202 ERROR org.apache.seatunnel.core.starter.SeaTunnel - Exception StackTrace:org.apache.seatunnel.core.starter.exception.CommandExecuteException: SeaTunnel job executed failed
        at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:191)
        at org.apache.seatunnel.core.starter.SeaTunnel.run(SeaTunnel.java:40)
        at org.apache.seatunnel.core.starter.seatunnel.SeaTunnelClient.main(SeaTunnelClient.java:34)
Caused by: org.apache.seatunnel.engine.common.exception.SeaTunnelEngineException: java.util.concurrent.CompletionException: java.lang.ClassCastException: cannot assign instance of io.debezium.relational.TableId to field org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit.tableId of type io.debezium.relational.TableId in instance of org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit
        at com.hazelcast.spi.impl.AbstractInvocationFuture.returnOrThrowWithJoinConventions(AbstractInvocationFuture.java:819)
        at com.hazelcast.spi.impl.AbstractInvocationFuture.resolveAndThrowWithJoinConvention(AbstractInvocationFuture.java:835)
        at com.hazelcast.spi.impl.AbstractInvocationFuture.join(AbstractInvocationFuture.java:553)
        at org.apache.seatunnel.engine.server.task.context.SeaTunnelSplitEnumeratorContext.assignSplit(SeaTunnelSplitEnumeratorContext.java:82)
        at org.apache.seatunnel.api.source.SourceSplitEnumerator$Context.assignSplit(SourceSplitEnumerator.java:101)
        at org.apache.seatunnel.connectors.cdc.base.source.enumerator.IncrementalSourceEnumerator.assignSplits(IncrementalSourceEnumerator.java:164)
        at org.apache.seatunnel.connectors.cdc.base.source.enumerator.IncrementalSourceEnumerator.run(IncrementalSourceEnumerator.java:69)
        at org.apache.seatunnel.engine.server.task.SourceSplitEnumeratorTask.stateProcess(SourceSplitEnumeratorTask.java:303)
        at org.apache.seatunnel.engine.server.task.SourceSplitEnumeratorTask.call(SourceSplitEnumeratorTask.java:134)
        at org.apache.seatunnel.engine.server.TaskExecutionService$BlockingWorker.run(TaskExecutionService.java:613)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassCastException: cannot assign instance of io.debezium.relational.TableId to field org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit.tableId of type io.debezium.relational.TableId in instance of org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit
        at java.io.ObjectStreamClass$FieldReflector.setObjFieldValues(ObjectStreamClass.java:2287)
        at java.io.ObjectStreamClass.setObjFieldValues(ObjectStreamClass.java:1417)
        at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2293)
        at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2211)
        at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2069)
        at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
        at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431)
        at org.apache.seatunnel.common.utils.SerializationUtils.deserialize(SerializationUtils.java:74)
        at org.apache.seatunnel.api.serialization.DefaultSerializer.deserialize(DefaultSerializer.java:41)
        at org.apache.seatunnel.api.serialization.DefaultSerializer.deserialize(DefaultSerializer.java:25)
        at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.lambda$run$0(AssignSplitOperation.java:67)
        at org.apache.seatunnel.common.utils.RetryUtils.retryWithException(RetryUtils.java:48)
        at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.run(AssignSplitOperation.java:54)
        at com.hazelcast.spi.impl.operationservice.Operation.call(Operation.java:189)
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:273)
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:248)
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:213)
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:175)
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:139)
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.executeRun(OperationThread.java:123)
        at com.hazelcast.internal.util.executor.HazelcastManagedThread.run(HazelcastManagedThread.java:102)

        at org.apache.seatunnel.engine.client.job.ClientJobProxy.waitForJobComplete(ClientJobProxy.java:122)
        at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:184)
        ... 2 more

2023-11-11 12:51:08,202 ERROR org.apache.seatunnel.core.starter.SeaTunnel -
===============================================================================

Exception in thread "main" org.apache.seatunnel.core.starter.exception.CommandExecuteException: SeaTunnel job executed failed
        at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:191)
        at org.apache.seatunnel.core.starter.SeaTunnel.run(SeaTunnel.java:40)
        at org.apache.seatunnel.core.starter.seatunnel.SeaTunnelClient.main(SeaTunnelClient.java:34)
Caused by: org.apache.seatunnel.engine.common.exception.SeaTunnelEngineException: java.util.concurrent.CompletionException: java.lang.ClassCastException: cannot assign instance of io.debezium.relational.TableId to field org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit.tableId of type io.debezium.relational.TableId in instance of org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit
        at com.hazelcast.spi.impl.AbstractInvocationFuture.returnOrThrowWithJoinConventions(AbstractInvocationFuture.java:819)
        at com.hazelcast.spi.impl.AbstractInvocationFuture.resolveAndThrowWithJoinConvention(AbstractInvocationFuture.java:835)
        at com.hazelcast.spi.impl.AbstractInvocationFuture.join(AbstractInvocationFuture.java:553)
        at org.apache.seatunnel.engine.server.task.context.SeaTunnelSplitEnumeratorContext.assignSplit(SeaTunnelSplitEnumeratorContext.java:82)
        at org.apache.seatunnel.api.source.SourceSplitEnumerator$Context.assignSplit(SourceSplitEnumerator.java:101)
        at org.apache.seatunnel.connectors.cdc.base.source.enumerator.IncrementalSourceEnumerator.assignSplits(IncrementalSourceEnumerator.java:164)
        at org.apache.seatunnel.connectors.cdc.base.source.enumerator.IncrementalSourceEnumerator.run(IncrementalSourceEnumerator.java:69)
        at org.apache.seatunnel.engine.server.task.SourceSplitEnumeratorTask.stateProcess(SourceSplitEnumeratorTask.java:303)
        at org.apache.seatunnel.engine.server.task.SourceSplitEnumeratorTask.call(SourceSplitEnumeratorTask.java:134)
        at org.apache.seatunnel.engine.server.TaskExecutionService$BlockingWorker.run(TaskExecutionService.java:613)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassCastException: cannot assign instance of io.debezium.relational.TableId to field org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit.tableId of type io.debezium.relational.TableId in instance of org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit
        at java.io.ObjectStreamClass$FieldReflector.setObjFieldValues(ObjectStreamClass.java:2287)
        at java.io.ObjectStreamClass.setObjFieldValues(ObjectStreamClass.java:1417)
        at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2293)
        at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2211)
        at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2069)
        at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573)
        at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431)
        at org.apache.seatunnel.common.utils.SerializationUtils.deserialize(SerializationUtils.java:74)
        at org.apache.seatunnel.api.serialization.DefaultSerializer.deserialize(DefaultSerializer.java:41)
        at org.apache.seatunnel.api.serialization.DefaultSerializer.deserialize(DefaultSerializer.java:25)
        at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.lambda$run$0(AssignSplitOperation.java:67)
        at org.apache.seatunnel.common.utils.RetryUtils.retryWithException(RetryUtils.java:48)
        at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.run(AssignSplitOperation.java:54)
        at com.hazelcast.spi.impl.operationservice.Operation.call(Operation.java:189)
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:273)
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:248)
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:213)
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:175)
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:139)
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.executeRun(OperationThread.java:123)
        at com.hazelcast.internal.util.executor.HazelcastManagedThread.run(HazelcastManagedThread.java:102)

        at org.apache.seatunnel.engine.client.job.ClientJobProxy.waitForJobComplete(ClientJobProxy.java:122)
        at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:184)
        ... 2 more
2023-11-11 12:51:08,206 INFO  org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand - run shutdown hook because get close signal

Zeta or Flink or Spark Version

seatunnel 2.3.3

Java or Scala Version

jdk1.8

Screenshots

image

Are you willing to submit PR?

Code of Conduct

zhilinli123 commented 10 months ago

You should add more details, such as mysql DDL information Mysql data volume

wilsonfu88 commented 10 months ago

use seatunnel 2.3.1 can run success,it's a bug in 2.3.3? DDL: -- test_db2.book definition

CREATE TABLE book ( id int NOT NULL AUTO_INCREMENT, name varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL, author varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL, publishDate datetime DEFAULT CURRENT_TIMESTAMP, sex bit(1) DEFAULT NULL, total_page int DEFAULT NULL, PRIMARY KEY (id) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=158 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC;

tableData:

INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('JAVA 入门指南测试3','wilson.fu','2019-09-06 22:33:02',1,1), ('VUE入门','wilson.fu01',NULL,0,2), ('VUE入门','wilson.fu01',NULL,0,3), ('VUE入门','wilson.fu01','2019-09-06 22:52:26',0,4), ('VUE入门','wilson.fu01','2019-09-06 22:52:27',0,5), ('VUE入门','wilson.fu01','2019-09-06 23:01:46',0,6), ('VUE入门','wilson.fu01','2019-09-06 23:01:47',0,7), ('VUE入门','wilson.fu01','2019-09-06 23:01:52',0,8), ('VUE入门','wilson.fu01','2019-09-06 23:01:53',0,9), ('VUE入门','wilson.fu01','2019-09-06 23:01:54',0,10); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('VUE入门','wilson.fu01','2019-09-06 23:01:54',0,11), ('VUE入门','wilson.fu01','2019-09-06 23:01:54',0,12), ('VUE入门','wilson.fu01','2019-09-06 23:01:54',0,13), ('VUE入门','wilson.fu01','2019-09-06 23:18:07',0,14), ('VUE入门','wilson.fu01','2019-09-06 23:18:09',0,15), ('VUE入门','wilson.fu01','2019-09-06 23:18:10',0,16), ('VUE入门','wilson.fu01','2019-09-06 23:18:10',0,17), ('VUE入门','wilson.fu01','2019-09-06 23:18:10',0,18), ('VUE入门','wilson.fu01','2019-09-06 23:18:10',0,19), ('VUE入门','wilson.fu01','2019-09-06 23:18:10',0,20); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('VUE入门','wilson.fu01','2019-09-06 23:18:16',0,21), ('VUE入门','wilson.fu01','2019-09-07 14:25:43',0,22), ('VUE入门','wilson.fu01','2019-09-07 14:25:49',0,23), ('VUE入门','wilson.fu01','2019-09-07 14:26:13',0,24), ('VUE入门','wilson.fu01','2019-09-07 14:30:31',0,25), ('VUE入门','wilson.fu01','2019-09-07 14:30:38',0,26), ('VUE入门','wilson.fu01','2019-09-07 14:30:42',0,27), ('VUE入门','wilson.fu01','2019-09-07 14:30:42',0,28), ('VUE入门','wilson.fu01','2019-09-07 14:30:43',0,29), ('VUE入门','wilson.fu01','2019-09-07 14:30:43',0,30); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('VUE入门','wilson.fu01','2019-09-07 14:30:43',0,31), ('C#入门。','wilson.fu88','2019-09-07 15:16:09',0,32), ('C#入门。','wilson.fu88','2019-09-07 15:16:12',0,33), ('C#入门。','wilson.fu88','2019-09-07 15:20:23',0,34), (NULL,NULL,'2019-09-07 22:51:26',0,35), ('c#','wilson.fu','2019-09-07 22:57:50',0,36), ('c#','wilson.fu','2019-09-07 22:57:59',0,37), ('c#','wilson.fu','2019-09-07 22:58:01',0,38), ('c#','wilson.fu','2019-09-07 22:58:06',0,39), ('c#','wilson.fu','2019-09-07 22:58:11',0,40); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:07:51',0,41), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:07:54',0,42), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:07:56',0,43), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:08:42',0,44), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:08:44',0,45), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:08:45',0,46), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:08:52',0,47), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:09:36',0,48), ('c#','wilson.fu','2019-09-07 23:12:39',0,49), ('Spring Boot 入门。','wilson.fu1982',NULL,0,50); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('Spring Boot 入门。','wilson.fu1982',NULL,0,51), ('Spring Boot 入门。','wilson.fu1982',NULL,0,52), ('Spring Boot 入门。','wilson.fu1982',NULL,0,53), ('Spring Boot 入门。','wilson.fu1982',NULL,0,54), ('Spring Boot 入门。','wilson.fu1982',NULL,0,55), ('Spring Boot 入门。','wilson.fu1982',NULL,0,56), ('Spring Boot 入门。','wilson.fu1982',NULL,0,57), ('Spring Boot 入门。','wilson.fu1982',NULL,0,58), ('Spring Boot 入门。','wilson.fu1982',NULL,0,59), ('Spring Boot 入门。','wilson.fu1982',NULL,0,60); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('Spring Boot 入门。','wilson.fu1982',NULL,0,61), ('Spring Boot 入门。','wilson.fu1982',NULL,0,62), ('Spring Boot 入门。','wilson.fu1982',NULL,0,63), ('Spring Boot 入门。','wilson.fu1982',NULL,0,64), ('Spring Boot 入门。','wilson.fu1982',NULL,0,65), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:12',0,66), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:13',0,67), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:13',0,68), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:13',0,69), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:14',0,70); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:14',0,71), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:14',0,72), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:14',0,73), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:14',0,74), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:15',0,75), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:15',0,76), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:15',0,77), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:15:15',0,78), ('C#入门。','wilson.fu88','2019-09-07 23:43:13',0,79), ('Spring Boot 入门。','wilson.fu1982','2019-09-07 23:43:20',0,80); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('JAVA0078dc1e-d93f-4ac3-9993-6160264d63a7','WILSON.FU0908Sun Sep 08 19:00:28 CST 2019','2019-09-08 19:00:28',0,81), ('JAVA1d651323-45ec-448b-b590-591f48f66a75','WILSON.FU0908Sun Sep 08 19:02:44 CST 2019','2019-09-08 19:02:44',0,82), ('JAVA2078c661-1515-4a0a-8323-ae1eeb8691ff','WILSON.FU0908Sun Sep 08 19:02:48 CST 2019','2019-09-08 19:02:49',0,83), ('JAVA3b988be7-c6c7-4560-8efb-b248bc1b9d30','WILSON.FU0908Sun Sep 08 19:02:49 CST 2019','2019-09-08 19:02:50',0,84), ('JAVA9d6fe599-fef9-4ce9-90ca-3daa42561a89','WILSON.FU0908Sun Sep 08 19:02:51 CST 2019','2019-09-08 19:02:51',0,85), ('JAVAe3acc578-2242-4263-bbd0-8289760881ec','WILSON.FU0908Sun Sep 08 19:02:52 CST 2019','2019-09-08 19:02:52',0,86), ('JAVA3ed87c2a-c5e9-4db3-840c-209883bc24b3','WILSON.FU0908Sun Sep 08 19:02:53 CST 2019','2019-09-08 19:02:53',0,87), ('JAVA45ba371f-ce41-4915-9172-77899188831b','WILSON.FU0908Sun Sep 08 19:02:53 CST 2019','2019-09-08 19:02:54',0,88), ('JAVAf20d6940-10ad-491c-83cd-38fd873f651f','WILSON.FU0908Sun Sep 08 19:02:54 CST 2019','2019-09-08 19:02:55',0,89), ('JAVA53af9317-9890-4436-840a-a620669dba0b','WILSON.FU0908Sun Sep 08 19:02:54 CST 2019','2019-09-08 19:02:55',0,90); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('JAVAe9b37003-303b-47eb-b3a6-509d1d3f9636','WILSON.FU0908Sun Sep 08 19:02:56 CST 2019','2019-09-08 19:02:57',0,91), ('JAVA0a392db1-5357-448c-ad11-4cbf4c44c224','WILSON.FU0908Sun Sep 08 19:03:30 CST 2019','2019-09-08 19:03:30',0,92), ('JAVAf961febe-4362-44c6-bbb3-871b12b23eb2','WILSON.FU0908Sun Sep 08 19:03:31 CST 2019','2019-09-08 19:03:31',0,93), ('JAVA7e19fa3a-9c95-44da-bf77-95a4e4b04b43','WILSON.FU0908Sun Sep 08 19:03:34 CST 2019','2019-09-08 19:03:34',0,94), ('JAVA6fa6384c-1a2c-4032-b456-bd7d992a22b0','WILSON.FU0908Sun Sep 08 19:03:37 CST 2019','2019-09-08 19:03:37',0,95), ('JAVA48ca6979-749b-488d-bd5d-f99402997ef4','WILSON.FU0908Sun Sep 08 19:03:38 CST 2019','2019-09-08 19:03:39',0,96), ('JAVA2b1b53f3-d752-4dd2-b890-24fe84071c13','WILSON.FU0908Sun Sep 08 19:03:39 CST 2019','2019-09-08 19:03:39',0,97), ('JAVA3a1b55e1-db10-4fcf-b180-498d21c01635','WILSON.FU0908Sun Sep 08 19:03:39 CST 2019','2019-09-08 19:03:40',0,98), ('JAVA5dbdcc40-81cd-405e-9cd1-31d449a25632','WILSON.FU0908Sun Sep 08 19:03:40 CST 2019','2019-09-08 19:03:40',0,99), ('JAVA690b4b18-ccc4-4111-96fa-e1f568883569','WILSON.FU0908Sun Sep 08 19:03:40 CST 2019','2019-09-08 19:03:40',0,100); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('JAVAdcfcd67d-dd2c-4439-ab89-1f7128b58ce2','WILSON.FU0908Sun Sep 08 19:03:40 CST 2019','2019-09-08 19:03:41',0,101), ('JAVAf73622bd-fea6-412e-8f96-9070dc9e8b6a','WILSON.FU0908Sun Sep 08 19:03:41 CST 2019','2019-09-08 19:03:41',0,102), ('JAVAdbafce77-3391-4a48-8670-b7f819565c9a','WILSON.FU0908Sun Sep 08 19:03:45 CST 2019','2019-09-08 19:03:45',0,103), ('JAVAdd013c04-1dc2-4e16-b2a2-07dc2a969046','WILSON.FU0908Sun Sep 08 19:04:22 CST 2019','2019-09-08 19:04:23',0,104), ('JAVA694c760a-f4ba-44cf-aa7f-a144994924a5','WILSON.FU0908Sun Sep 08 19:04:22 CST 2019','2019-09-08 19:04:23',0,105), ('JAVA2e8d4c80-a003-4940-8c8d-0e810f5b3bfe','WILSON.FU0908Sun Sep 08 19:04:22 CST 2019','2019-09-08 19:04:23',0,106), ('JAVAb952405c-144a-40f2-a13b-29a8f562f651','WILSON.FU0908Sun Sep 08 19:04:23 CST 2019','2019-09-08 19:04:23',0,107), ('JAVAbd0480f9-0ffb-429b-b4ff-2c4444e60a33','WILSON.FU0908Sun Sep 08 19:04:23 CST 2019','2019-09-08 19:04:23',0,108), ('JAVA202a6000-1311-4df6-b129-8a915fdbdb53','WILSON.FU0908Sun Sep 08 19:04:23 CST 2019','2019-09-08 19:04:24',0,109), ('JAVA1d684bde-1936-4701-a48d-85c9f5b54ec8','WILSON.FU0908Sun Sep 08 19:04:23 CST 2019','2019-09-08 19:04:24',0,110); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('JAVAc6823417-958d-4812-a928-2f9e1ddaadfb','WILSON.FU0908Sun Sep 08 19:04:23 CST 2019','2019-09-08 19:04:24',0,111), ('JAVAc16e615e-86bb-4ec6-85ef-9fc4f8bb3bab','WILSON.FU0908Sun Sep 08 19:04:24 CST 2019','2019-09-08 19:04:24',0,112), ('JAVA1e48f2bd-5362-48d6-b19c-de2017bfc155','WILSON.FU0908Sun Sep 08 19:04:24 CST 2019','2019-09-08 19:04:25',0,113), ('JAVAb664730c-3468-4d29-9bfc-914a400bbbe1','WILSON.FU0908Sun Sep 08 19:04:25 CST 2019','2019-09-08 19:04:26',0,114), ('JAVA0ae4a82f-b56f-4cb4-ae5a-56d8e9900fa4','WILSON.FU0908Sun Sep 08 19:04:26 CST 2019','2019-09-08 19:04:26',0,115), ('JAVA8a2dedfb-2abb-4376-845b-40be69c3cab4','WILSON.FU0908Sun Sep 08 19:04:54 CST 2019','2019-09-08 19:04:54',0,116), ('JAVA72d0eb10-7654-4ff2-9355-bfe0a3311fa0','WILSON.FU0908Sun Sep 08 19:04:54 CST 2019','2019-09-08 19:04:55',0,117), ('JAVAa58c96fd-cc74-404d-ad8d-31f30957b01f','WILSON.FU0908Sun Sep 08 19:04:55 CST 2019','2019-09-08 19:04:55',0,118), ('JAVA070f1629-e068-41ea-8303-a3675537f210','WILSON.FU0908Sun Sep 08 19:04:55 CST 2019','2019-09-08 19:04:55',0,119), ('JAVA5b3fe961-e8a2-4a3b-bea8-b05eeac8689c','WILSON.FU0908Sun Sep 08 19:04:55 CST 2019','2019-09-08 19:04:56',0,120); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('JAVAa0e19952-ca87-4d21-8a80-3d34f2c9226e','WILSON.FU0908Sun Sep 08 19:04:55 CST 2019','2019-09-08 19:04:56',0,121), ('JAVAf799ecfc-c211-49a8-85ba-8c676b14492f','WILSON.FU0908Sun Sep 08 19:04:56 CST 2019','2019-09-08 19:04:56',0,122), ('JAVA8fbf80d8-ae57-4831-9585-e920a922a52e','WILSON.FU0908Sun Sep 08 19:04:56 CST 2019','2019-09-08 19:04:56',0,123), ('JAVAdd1c45fd-32cc-4c25-8146-e106c2c0d8a1','WILSON.FU0908Sun Sep 08 19:05:34 CST 2019','2019-09-08 19:05:35',0,124), ('JAVA21fe0d2c-fb78-40b4-aa66-c51758255823','WILSON.FU0908Sun Sep 08 19:05:36 CST 2019','2019-09-08 19:05:36',0,125), ('JAVAfa18b183-25e0-4d1c-aada-049ea210b794','WILSON.FU0908Sun Sep 08 19:05:37 CST 2019','2019-09-08 19:05:37',0,126), ('JAVAd13e517f-8db4-40df-8005-a4997845b4c9','WILSON.FU0908Sun Sep 08 19:05:37 CST 2019','2019-09-08 19:05:38',0,127), ('JAVA52893b13-6aa7-46fa-ae9f-759bc6d9f5b7','WILSON.FU0908Sun Sep 08 19:05:38 CST 2019','2019-09-08 19:05:38',0,128), ('JAVA063f9927-8734-4fad-96f3-38fecd02e01c','wilson.fu-0908Sun Sep 08 19:24:35 CST 2019','2019-09-08 19:24:36',0,129), ('JAVA48e6d5ba-cfdb-4fb5-9a22-1267b905a929','wilson.fu-0908Sun Sep 08 19:24:38 CST 2019','2019-09-08 19:24:38',0,130); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('JAVA2cc46624-b490-4adb-a655-cd10ee3db092','wilson.fu-0908Sun Sep 08 19:24:38 CST 2019','2019-09-08 19:24:39',0,131), ('JAVA65a84c9e-b65d-41a9-a03a-9e855299d58e','wilson.fu-0908Sun Sep 08 19:24:39 CST 2019','2019-09-08 19:24:39',0,132), ('JAVA6e92bb16-875f-4ebf-907a-7af0d5ec7fb3','wilson.fu-0908Sun Sep 08 19:24:40 CST 2019','2019-09-08 19:24:40',0,133), ('JAVA94037d6f-6d4e-4fdb-8c3d-826a99100df2','wilson.fu-0908Sun Sep 08 19:24:40 CST 2019','2019-09-08 19:24:41',0,134), ('JAVA8d72c2d9-bfe5-49e7-899d-7b6d26db2b84','wilson.fu-0908Sun Sep 08 19:24:41 CST 2019','2019-09-08 19:24:41',0,135), ('JAVAa4ab0c4d-03d4-4f9a-af9f-285b716b0178','wilson.fu-0908Sun Sep 08 19:24:41 CST 2019','2019-09-08 19:24:42',0,136), ('JAVAc3d351b3-66b6-4cae-9003-bb8761ea8557','wilson.fu-0908Sun Sep 08 19:24:42 CST 2019','2019-09-08 19:24:42',0,137), ('JAVA7313d7d5-22bf-47fb-bba6-39bd482a9178','wilson.fu-0908Sun Sep 08 19:24:42 CST 2019','2019-09-08 19:24:43',0,138), ('JAVA307f40e1-13e7-4307-a8a2-d881bd959508','wilson.fu-0908Sun Sep 08 19:24:43 CST 2019','2019-09-08 19:24:43',0,139), ('JAVAe9c30ee0-4459-4582-8b99-f0362d5a4e1b','wilson.fu-0908Sun Sep 08 19:24:44 CST 2019','2019-09-08 19:24:44',0,140); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('JAVA954eb90f-ce8f-465c-bae6-231deea428b4','wilson.fu-0908Sun Sep 08 22:27:18 CST 2019','2019-09-08 22:27:18',0,141), ('JAVA8bfdd54b-7b68-40b7-a46e-789d3968b4c6','wilson.fu-0908Sun Sep 08 22:27:20 CST 2019','2019-09-08 22:27:20',0,142), ('JAVA1a947afb-a4ae-45e6-bdd1-d89c04c47f9f','wilson.fu-0908Sun Sep 08 22:27:20 CST 2019','2019-09-08 22:27:21',0,143), ('JAVAd329187f-6bd9-4d5b-aa54-c1a550c74764','wilson.fu-0908Sun Sep 08 22:27:48 CST 2019','2019-09-08 22:27:48',0,144), ('JAVAd4e51f99-83d1-434c-a43c-62b118370c60','WILSON.FU0908Sun Sep 08 22:27:50 CST 2019','2019-09-08 22:27:51',0,145), ('JAVAc4c8797b-3dbd-46ab-a240-83f09c7809c4','WILSON.FU0908Sun Sep 08 22:34:47 CST 2019','2019-09-08 22:34:47',0,146), ('JAVAc549b6e2-5006-4e50-bee7-29c210179640','WILSON.FU0908Sun Sep 08 22:34:47 CST 2019','2019-09-08 22:34:47',0,147), ('JAVA3d174778-490a-4c33-951a-224e1df53920','WILSON.FU0908Sun Sep 08 22:34:48 CST 2019','2019-09-08 22:34:49',0,148), ('JAVAeeef825f-ea4d-4763-aa12-dd3a65fed837','WILSON.FU0908Sun Sep 08 22:34:49 CST 2019','2019-09-08 22:34:49',0,149), ('JAVA08e12602-a833-41ca-ae24-38948ccf95c2','WILSON.FU0908Sun Sep 08 22:34:49 CST 2019','2019-09-08 22:34:50',0,150); INSERT INTO test_db2.book (name,author,publishDate,sex,total_page) VALUES ('JAVAaaeec543-f0ea-4d6e-826f-e6f15e7de5f6','WILSON.FU0908Sun Sep 08 22:34:50 CST 2019','2019-09-08 22:34:50',0,151), ('JAVA45bb7dda-7572-49b3-af7f-bac63b241017','WILSON.FU0908Sun Sep 08 22:34:50 CST 2019','2019-09-08 22:34:51',0,152), ('JAVAf2b24fd0-0b14-443d-ba08-70c9ec31c72e','WILSON.FU0908Sun Sep 08 22:34:51 CST 2019','2019-09-08 22:34:51',0,153), ('JAVAe6722dd2-c2f3-4121-8be9-19e34acbb1d0','WILSON.FU0908Sun Sep 08 22:34:51 CST 2019','2019-09-08 22:34:52',0,154), ('JAVA9394f35b-28ef-4721-acbc-2d11f5278be8','WILSON.FU0908Sun Sep 08 22:35:08 CST 2019','2019-09-08 22:35:09',0,155), ('JAVAfa5f33f7-8288-47e0-8451-16176dc82b41','wilson.fu-0908Sun Sep 08 23:45:01 CST 2019','2019-09-08 23:45:02',0,156), ('JAVAb3d88d55-4802-4ae3-9847-69b6056941d3','WILSON.FU0908Sun Sep 08 23:45:13 CST 2019','2019-09-08 23:45:13',0,157);

zhilinli123 commented 9 months ago

I'll get around to looking at that soon @wilsonfu88

fengcheche commented 9 months ago

I also encountered the same problem, the mysql-cdc created by seatunnel-web was synchronized in real time, the task was executed, and the log reported errors

Below is the task configuration json, and below is an error message

{
    "env" : {
        "job.mode" : "STREAMING",
        "job.name" : "SeaTunnel_Job"
    },
    "source" : [
        {
            "inverse-sampling.rate" : 1000,
            "catalog" : {
                "factory" : "Mysql"
            },
            "parallelism" : 1,
            "table-names" : [
                "seatunnel.user",
                "seatunnel.user_login_log"
            ],
            "chunk-key.even-distribution.factor.lower-bound" : 0.05,
            "database-names" : [
                "seatunnel"
            ],
            "password" : "&YGV9ijn*UHB",
            "sample-sharding.threshold" : 1000,
            "incremental.parallelism" : 1,
            "snapshot.fetch.size" : 1024,
            "connect.max-retries" : 3,
            "base-url" : "jdbc:mysql://192.168.3.29:3306/seatunnel",
            "startup.mode" : "INITIAL",
            "format" : "DEFAULT",
            "result_table_name" : "Table11707699295552",
            "server-time-zone" : "UTC",
            "plugin_name" : "MySQL-CDC",
            "exactly_once" : "true",
            "connection.pool.size" : 20,
            "snapshot.split.size" : 8096,
            "stop.mode" : "NEVER",
            "chunk-key.even-distribution.factor.upper-bound" : 100,
            "connect.timeout.ms" : 30000,
            "dag-parsing.mode" : "MULTIPLEX",
            "username" : "root"
        }
    ],
    "transform" : [],
    "sink" : [
        {
            "batch_size" : 1000,
            "primary_keys" : [
                "id",
                "id"
            ],
            "max_retries" : "3",
            "catalog" : {
                "factory" : "MySQL",
                "username" : "root",
                "password" : "&YGV9ijn*UHB",
                "base-url" : "jdbc:mysql://192.168.3.29:3306/seatunnel"
            },
            "source_table_name" : "Table11707699295552",
            "max_commit_attempts" : 3,
            "auto_commit" : "true",
            "plugin_name" : "Jdbc",
            "url" : "jdbc:mysql://192.168.3.29:3306/seatunnel",
            "is_exactly_once" : "false",
            "database" : "seatunnel-synctest",
            "password" : "&YGV9ijn*UHB",
            "transaction_timeout_sec" : -1,
            "driver" : "com.mysql.cj.jdbc.Driver",
            "support_upsert_by_query_primary_key_exist" : "false",
            "connection_check_timeout_sec" : 30,
            "generate_sink_sql" : true,
            "user" : "root"
        }
    ]
}
fengcheche commented 9 months ago
java.lang.ClassCastException: cannot assign instance of io.debezium.relational.TableId to field org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit.tableId of type io.debezium.relational.TableId in instance of org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit
        at java.io.ObjectStreamClass$FieldReflector.setObjFieldValues(ObjectStreamClass.java:2133) ~[?:1.8.0_65]
        at java.io.ObjectStreamClass.setObjFieldValues(ObjectStreamClass.java:1305) ~[?:1.8.0_65]
        at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2006) ~[?:1.8.0_65]
        at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1924) ~[?:1.8.0_65]
        at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) ~[?:1.8.0_65]
        at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) ~[?:1.8.0_65]
        at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371) ~[?:1.8.0_65]
        at org.apache.seatunnel.common.utils.SerializationUtils.deserialize(SerializationUtils.java:74) ~[connector-kudu-2.3.3.jar:2.3.3]
        at org.apache.seatunnel.api.serialization.DefaultSerializer.deserialize(DefaultSerializer.java:41) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.api.serialization.DefaultSerializer.deserialize(DefaultSerializer.java:25) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.lambda$run$0(AssignSplitOperation.java:67) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.common.utils.RetryUtils.retryWithException(RetryUtils.java:48) ~[connector-kudu-2.3.3.jar:2.3.3]
        at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.run(AssignSplitOperation.java:54) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationservice.Operation.call(Operation.java:189) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:273) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:248) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:213) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:175) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:139) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.executeRun(OperationThread.java:123) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.internal.util.executor.HazelcastManagedThread.run(HazelcastManagedThread.java:102) ~[seatunnel-starter.jar:2.3.3]
2023-11-25 15:31:55,706 WARN  org.apache.seatunnel.engine.server.TaskExecutionService - [localhost]:5801 [seatunnel] [5.1] Exception in org.apache.seatunnel.engine.server.task.SourceSplitEnumeratorTask@6ffc9f2f
java.util.concurrent.CompletionException: java.lang.ClassCastException: cannot assign instance of io.debezium.relational.TableId to field org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit.tableId of type io.debezium.relational.TableId in instance of org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit
        at com.hazelcast.spi.impl.AbstractInvocationFuture.returnOrThrowWithJoinConventions(AbstractInvocationFuture.java:819) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.AbstractInvocationFuture.resolveAndThrowWithJoinConvention(AbstractInvocationFuture.java:835) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.AbstractInvocationFuture.join(AbstractInvocationFuture.java:553) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.engine.server.task.context.SeaTunnelSplitEnumeratorContext.assignSplit(SeaTunnelSplitEnumeratorContext.java:82) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.api.source.SourceSplitEnumerator$Context.assignSplit(SourceSplitEnumerator.java:101) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.connectors.cdc.base.source.enumerator.IncrementalSourceEnumerator.assignSplits(IncrementalSourceEnumerator.java:164) ~[connector-cdc-sqlserver-2.3.3.jar:2.3.3]
        at org.apache.seatunnel.connectors.cdc.base.source.enumerator.IncrementalSourceEnumerator.run(IncrementalSourceEnumerator.java:69) ~[connector-cdc-sqlserver-2.3.3.jar:2.3.3]
        at org.apache.seatunnel.engine.server.task.SourceSplitEnumeratorTask.stateProcess(SourceSplitEnumeratorTask.java:303) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.engine.server.task.SourceSplitEnumeratorTask.call(SourceSplitEnumeratorTask.java:134) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.engine.server.TaskExecutionService$BlockingWorker.run(TaskExecutionService.java:613) ~[seatunnel-starter.jar:2.3.3]
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_65]
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_65]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_65]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_65]
        at java.lang.Thread.run(Thread.java:745) [?:1.8.0_65]
Caused by: java.lang.ClassCastException: cannot assign instance of io.debezium.relational.TableId to field org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit.tableId of type io.debezium.relational.TableId in instance of org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit
        at java.io.ObjectStreamClass$FieldReflector.setObjFieldValues(ObjectStreamClass.java:2133) ~[?:1.8.0_65]
        at java.io.ObjectStreamClass.setObjFieldValues(ObjectStreamClass.java:1305) ~[?:1.8.0_65]
        at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2006) ~[?:1.8.0_65]
        at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1924) ~[?:1.8.0_65]
        at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801) ~[?:1.8.0_65]
        at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351) ~[?:1.8.0_65]
        at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371) ~[?:1.8.0_65]
        at org.apache.seatunnel.common.utils.SerializationUtils.deserialize(SerializationUtils.java:74) ~[connector-kudu-2.3.3.jar:2.3.3]
        at org.apache.seatunnel.api.serialization.DefaultSerializer.deserialize(DefaultSerializer.java:41) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.api.serialization.DefaultSerializer.deserialize(DefaultSerializer.java:25) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.lambda$run$0(AssignSplitOperation.java:67) ~[seatunnel-starter.jar:2.3.3]
        at org.apache.seatunnel.common.utils.RetryUtils.retryWithException(RetryUtils.java:48) ~[connector-kudu-2.3.3.jar:2.3.3]
        at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.run(AssignSplitOperation.java:54) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationservice.Operation.call(Operation.java:189) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:273) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:248) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:213) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:175) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:139) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.executeRun(OperationThread.java:123) ~[seatunnel-starter.jar:2.3.3]
        at com.hazelcast.internal.util.executor.HazelcastManagedThread.run(HazelcastManagedThread.java:102) ~[seatunnel-starter.jar:2.3.3]
fengcheche commented 9 months ago

不用web配置,手动输入切换成下面这样去启动还是报一样的错误

Without web configuration, manually enter to switch to the following to start still report the same error

env {
  execution.parallelism = 2
  job.mode = "STREAMING"
  checkpoint.interval = 10000
}

source {
  MySQL-CDC {
    username = "root"
    password = "&YGV9ijn*UHB"
    table-names = ["seatunnel.user"]
    base-url = "jdbc:mysql://192.168.3.29:3306/seatunnel"
  }
}

sink {
  jdbc {
    url = "jdbc:mysql://192.168.3.29:3306/seatunnel-synctest"
    driver = "com.mysql.cj.jdbc.Driver"
    username = "root"
    password = "&YGV9ijn*UHB"
    generate_sink_sql = true
    database = "seatunnel-synctest"
    table = "user"
    primary_keys = ["id"]
  }

}
fengcheche commented 9 months ago

image

zhilinli123 commented 9 months ago

image @fengcheche cc @Carl-Zhou-CN close issues

Carl-Zhou-CN commented 9 months ago

@zhilinli123 Thank you. Good check,We wait for a reply for some time

wilsonfu88 commented 9 months ago

I still can't find a solution,wait for seatunnel v2.3.4 release ?

Carl-Zhou-CN commented 9 months ago

I still can't find a solution,wait for seatunnel v2.3.4 release ?

@wilsonfu88 Can you help to test whether the dev branch still has this problem?

wilsonfu88 commented 9 months ago

I still can't find a solution,wait for seatunnel v2.3.4 release ?

@wilsonfu88 Can you help to test whether the dev branch still has this problem? Use dev branch code run this bug disappear,but happen others error!

The detail is below 2023-12-10 09:08:05,904 INFO com.hazelcast.client.impl.statistics.ClientStatisticsService - Client statistics is enabled with period 5 seconds. 2023-12-10 09:08:05,952 INFO com.hazelcast.core.LifecycleService - hz.client_1 [seatunnel-197763] [5.1] HazelcastClient 5.1 (20220228 - 21f20e7) is SHUTTING_DOWN 2023-12-10 09:08:05,955 INFO com.hazelcast.internal.server.tcp.TcpServerConnection - [localhost]:5801 [seatunnel-197763] [5.1] Connection[id=1, /127.0.0.1:5801->/127.0.0.1:60751, qualifier=null, endpoint=[127.0.0.1]:60751, remoteUuid=785df542-0839-4f62-a375-12178e82f758, alive=false, connectionType=JVM, planeIndex=-1] closed. Reason: Connection closed by the other side 2023-12-10 09:08:05,955 INFO com.hazelcast.client.impl.connection.ClientConnectionManager - hz.client_1 [seatunnel-197763] [5.1] Removed connection to endpoint: [localhost]:5801:a1c596b5-15de-4ec1-a3b5-335fd2ff1934, connection: ClientConnection{alive=false, connectionId=1, channel=NioChannel{/127.0.0.1:60751->localhost/127.0.0.1:5801}, remoteAddress=[localhost]:5801, lastReadTime=2023-12-10 09:08:05.916, lastWriteTime=2023-12-10 09:08:05.914, closedTime=2023-12-10 09:08:05.953, connected server version=5.1} 2023-12-10 09:08:05,955 INFO com.hazelcast.core.LifecycleService - hz.client_1 [seatunnel-197763] [5.1] HazelcastClient 5.1 (20220228 - 21f20e7) is CLIENT_DISCONNECTED 2023-12-10 09:08:05,959 INFO com.hazelcast.client.impl.ClientEndpointManager - [localhost]:5801 [seatunnel-197763] [5.1] Destroying ClientEndpoint{connection=Connection[id=1, /127.0.0.1:5801->/127.0.0.1:60751, qualifier=null, endpoint=[127.0.0.1]:60751, remoteUuid=785df542-0839-4f62-a375-12178e82f758, alive=false, connectionType=JVM, planeIndex=-1], clientUuid=785df542-0839-4f62-a375-12178e82f758, clientName=hz.client_1, authenticated=true, clientVersion=5.1, creationTime=1702170485865, latest clientAttributes=lastStatisticsCollectionTime=1702170485904,enterprise=false,clientType=JVM,clientVersion=5.1,clusterConnectionTimestamp=1702170485854,clientAddress=127.0.0.1,clientName=hz.client_1,credentials.principal=null,os.committedVirtualMemorySize=7049457664,os.freePhysicalMemorySize=228626432,os.freeSwapSpaceSize=5254148096,os.maxFileDescriptorCount=4096,os.openFileDescriptorCount=52,os.processCpuTime=7160000000,os.systemLoadAverage=0.35,os.totalPhysicalMemorySize=33566445568,os.totalSwapSpaceSize=5368705024,runtime.availableProcessors=8,runtime.freeMemory=967933400,runtime.maxMemory=1029177344,runtime.totalMemory=1029177344,runtime.uptime=2952,runtime.usedMemory=61243944, labels=[]} 2023-12-10 09:08:05,960 INFO com.hazelcast.core.LifecycleService - hz.client_1 [seatunnel-197763] [5.1] HazelcastClient 5.1 (20220228 - 21f20e7) is SHUTDOWN 2023-12-10 09:08:05,961 INFO org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand - Closed SeaTunnel client...... 2023-12-10 09:08:05,961 INFO com.hazelcast.core.LifecycleService - [localhost]:5801 [seatunnel-197763] [5.1] [localhost]:5801 is SHUTTING_DOWN 2023-12-10 09:08:05,966 INFO com.hazelcast.instance.impl.Node - [localhost]:5801 [seatunnel-197763] [5.1] Shutting down connection manager... 2023-12-10 09:08:05,969 INFO com.hazelcast.instance.impl.Node - [localhost]:5801 [seatunnel-197763] [5.1] Shutting down node engine... 2023-12-10 09:08:05,977 WARN com.hazelcast.spi.impl.operationservice.impl.Invocation - [localhost]:5801 [seatunnel-197763] [5.1] Future.complete(Object) on completed future. Request: Invocation{op=com.hazelcast.map.impl.query.QueryOperation{serviceName='hz:impl:mapService', identityHash=1921544358, partitionId=-1, replicaIndex=0, callId=-2, invocationTime=1702170485961 (2023-12-10 09:08:05.961), waitTimeout=-1, callTimeout=60000, tenantControl=com.hazelcast.spi.impl.tenantcontrol.NoopTenantControl@0, name=engine_runningJobInfo}, tryCount=20, tryPauseMillis=500, invokeCount=1, callTimeoutMillis=60000, firstInvocationTimeMs=1702170485961, firstInvocationTime='2023-12-10 09:08:05.961', lastHeartbeatMillis=0, lastHeartbeatTime='1970-01-01 08:00:00.000', target=[localhost]:5801, pendingResponse={VOID}, backupsAcksExpected=-1, backupsAcksReceived=0, connection=null}, current value: ExceptionalResult{cause=com.hazelcast.core.HazelcastInstanceNotActiveException: Hazelcast instance is not active!}, offered value: com.hazelcast.map.impl.query.QueryResult@4a01f35 java.lang.Exception: null at com.hazelcast.spi.impl.AbstractInvocationFuture.warnIfSuspiciousDoubleCompletion(AbstractInvocationFuture.java:1253) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.AbstractInvocationFuture.complete0(AbstractInvocationFuture.java:1230) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.AbstractInvocationFuture.complete(AbstractInvocationFuture.java:1219) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.impl.Invocation.complete(Invocation.java:673) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.impl.Invocation.sendResponse(Invocation.java:236) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.Operation.sendResponse(Operation.java:483) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:283) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:248) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:213) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationexecutor.impl.OperationExecutorImpl.run(OperationExecutorImpl.java:411) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationexecutor.impl.OperationExecutorImpl.runOrExecute(OperationExecutorImpl.java:438) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.impl.Invocation.doInvokeLocal(Invocation.java:601) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.impl.Invocation.doInvoke(Invocation.java:580) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.impl.Invocation.invoke0(Invocation.java:541) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.impl.Invocation.invoke(Invocation.java:241) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.spi.impl.operationservice.impl.OperationServiceImpl.invokeOnTarget(OperationServiceImpl.java:365) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.query.QueryEngineImpl.dispatchFullQueryOnAllMembersOnQueryThread(QueryEngineImpl.java:315) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.query.QueryEngineImpl.dispatchFullQueryOnQueryThread(QueryEngineImpl.java:284) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.query.QueryEngineImpl.dispatchOnQueryThreads(QueryEngineImpl.java:171) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.query.QueryEngineImpl.doRunOnQueryThreads(QueryEngineImpl.java:164) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.query.QueryEngineImpl.runOnGivenPartitions(QueryEngineImpl.java:141) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.query.QueryEngineImpl.execute(QueryEngineImpl.java:109) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.proxy.MapProxySupport.executeQueryInternal(MapProxySupport.java:1415) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.proxy.MapProxySupport.executeQueryInternal(MapProxySupport.java:1383) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.proxy.MapProxyImpl.executePredicate(MapProxyImpl.java:796) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.proxy.MapProxyImpl.entrySet(MapProxyImpl.java:745) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at com.hazelcast.map.impl.proxy.MapProxyImpl.entrySet(MapProxyImpl.java:739) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at org.apache.seatunnel.engine.server.CoordinatorService.initCoordinatorService(CoordinatorService.java:236) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at org.apache.seatunnel.engine.server.CoordinatorService.checkNewActiveMaster(CoordinatorService.java:335) ~[seatunnel-starter.jar:2.3.4-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_202] at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [?:1.8.0_202] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_202] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [?:1.8.0_202] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_202] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_202] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_202] 2023-12-10 09:08:06,011 ERROR org.apache.seatunnel.engine.server.CoordinatorService - [localhost]:5801 [seatunnel-197763] [5.1] com.hazelcast.core.HazelcastInstanceNotActiveException: Hazelcast instance is not active! at com.hazelcast.spi.impl.operationservice.impl.InvocationRegistry.register(InvocationRegistry.java:140) at com.hazelcast.spi.impl.operationservice.impl.Invocation.doInvoke(Invocation.java:570) at com.hazelcast.spi.impl.operationservice.impl.Invocation.invoke0(Invocation.java:541) at com.hazelcast.spi.impl.operationservice.impl.Invocation.invoke(Invocation.java:241) at com.hazelcast.spi.impl.operationservice.impl.OperationServiceImpl.invokeOnPartition(OperationServiceImpl.java:330) at com.hazelcast.map.impl.query.QueryEngineImpl.dispatchPartitionScanQueryOnOwnerMemberOnPartitionThread(QueryEngineImpl.java:341) at com.hazelcast.map.impl.query.QueryEngineImpl.lambda$dispatchPartitionScanQueryOnOwnerMemberOnPartitionThread$1(QueryEngineImpl.java:333) at java.util.PrimitiveIterator$OfInt.forEachRemaining(PrimitiveIterator.java:115) at com.hazelcast.map.impl.query.QueryEngineImpl.dispatchPartitionScanQueryOnOwnerMemberOnPartitionThread(QueryEngineImpl.java:332) at com.hazelcast.map.impl.query.QueryEngineImpl.doRunOnPartitionThreads(QueryEngineImpl.java:198) at com.hazelcast.map.impl.query.QueryEngineImpl.runOnGivenPartitions(QueryEngineImpl.java:144) at com.hazelcast.map.impl.query.QueryEngineImpl.execute(QueryEngineImpl.java:109) at com.hazelcast.map.impl.proxy.MapProxySupport.executeQueryInternal(MapProxySupport.java:1415) at com.hazelcast.map.impl.proxy.MapProxySupport.executeQueryInternal(MapProxySupport.java:1383) at com.hazelcast.map.impl.proxy.MapProxyImpl.executePredicate(MapProxyImpl.java:796) at com.hazelcast.map.impl.proxy.MapProxyImpl.entrySet(MapProxyImpl.java:745) at com.hazelcast.map.impl.proxy.MapProxyImpl.entrySet(MapProxyImpl.java:739) at org.apache.seatunnel.engine.server.CoordinatorService.initCoordinatorService(CoordinatorService.java:236) at org.apache.seatunnel.engine.server.CoordinatorService.checkNewActiveMaster(CoordinatorService.java:335) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) at ------ submitted from ------.() at com.hazelcast.internal.util.ExceptionUtil.cloneExceptionWithFixedAsyncStackTrace(ExceptionUtil.java:336) at com.hazelcast.spi.impl.operationservice.impl.InvocationFuture.returnOrThrowWithGetConventions(InvocationFuture.java:112) at com.hazelcast.spi.impl.operationservice.impl.InvocationFuture.resolveAndThrowIfException(InvocationFuture.java:100) at com.hazelcast.spi.impl.AbstractInvocationFuture.get(AbstractInvocationFuture.java:609) at com.hazelcast.map.impl.query.QueryEngineImpl.addResultsOfPredicate(QueryEngineImpl.java:214) at com.hazelcast.map.impl.query.QueryEngineImpl.doRunOnPartitionThreads(QueryEngineImpl.java:200) at com.hazelcast.map.impl.query.QueryEngineImpl.runOnGivenPartitions(QueryEngineImpl.java:144) at com.hazelcast.map.impl.query.QueryEngineImpl.execute(QueryEngineImpl.java:109) at com.hazelcast.map.impl.proxy.MapProxySupport.executeQueryInternal(MapProxySupport.java:1415) at com.hazelcast.map.impl.proxy.MapProxySupport.executeQueryInternal(MapProxySupport.java:1383) at com.hazelcast.map.impl.proxy.MapProxyImpl.executePredicate(MapProxyImpl.java:796) at com.hazelcast.map.impl.proxy.MapProxyImpl.entrySet(MapProxyImpl.java:745) at com.hazelcast.map.impl.proxy.MapProxyImpl.entrySet(MapProxyImpl.java:739) at org.apache.seatunnel.engine.server.CoordinatorService.initCoordinatorService(CoordinatorService.java:236) at org.apache.seatunnel.engine.server.CoordinatorService.checkNewActiveMaster(CoordinatorService.java:335) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)

2023-12-10 09:08:08,986 INFO com.hazelcast.instance.impl.NodeExtension - [localhost]:5801 [seatunnel-197763] [5.1] Destroying node NodeExtension. 2023-12-10 09:08:08,991 INFO com.hazelcast.instance.impl.Node - [localhost]:5801 [seatunnel-197763] [5.1] Hazelcast Shutdown is completed in 3026 ms. 2023-12-10 09:08:08,992 INFO com.hazelcast.core.LifecycleService - [localhost]:5801 [seatunnel-197763] [5.1] [localhost]:5801 is SHUTDOWN 2023-12-10 09:08:08,992 INFO org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand - Closed HazelcastInstance ...... 2023-12-10 09:08:08,992 ERROR org.apache.seatunnel.core.starter.SeaTunnel -

===============================================================================

2023-12-10 09:08:08,992 ERROR org.apache.seatunnel.core.starter.SeaTunnel - Fatal Error,

2023-12-10 09:08:08,992 ERROR org.apache.seatunnel.core.starter.SeaTunnel - Please submit bug report in https://github.com/apache/seatunnel/issues

2023-12-10 09:08:08,994 ERROR org.apache.seatunnel.core.starter.SeaTunnel - Reason:SeaTunnel job executed failed

2023-12-10 09:08:08,996 ERROR org.apache.seatunnel.core.starter.SeaTunnel - Exception StackTrace:org.apache.seatunnel.core.starter.exception.CommandExecuteException: SeaTunnel job executed failed at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:192) at org.apache.seatunnel.core.starter.SeaTunnel.run(SeaTunnel.java:40) at org.apache.seatunnel.core.starter.seatunnel.SeaTunnelClient.main(SeaTunnelClient.java:34) Caused by: java.lang.NullPointerException at sun.nio.fs.UnixPath.normalizeAndCheck(UnixPath.java:77) at sun.nio.fs.UnixPath.(UnixPath.java:71) at sun.nio.fs.UnixFileSystem.getPath(UnixFileSystem.java:281) at java.nio.file.Paths.get(Paths.java:84) at org.apache.seatunnel.core.starter.utils.FileUtils.getConfigPath(FileUtils.java:50) at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:123) ... 2 more

2023-12-10 09:08:08,996 ERROR org.apache.seatunnel.core.starter.SeaTunnel -

Exception in thread "main" org.apache.seatunnel.core.starter.exception.CommandExecuteException: SeaTunnel job executed failed at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:192) at org.apache.seatunnel.core.starter.SeaTunnel.run(SeaTunnel.java:40) at org.apache.seatunnel.core.starter.seatunnel.SeaTunnelClient.main(SeaTunnelClient.java:34) Caused by: java.lang.NullPointerException at sun.nio.fs.UnixPath.normalizeAndCheck(UnixPath.java:77) at sun.nio.fs.UnixPath.(UnixPath.java:71) at sun.nio.fs.UnixFileSystem.getPath(UnixFileSystem.java:281) at java.nio.file.Paths.get(Paths.java:84) at org.apache.seatunnel.core.starter.utils.FileUtils.getConfigPath(FileUtils.java:50) at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:123) ... 2 more

avatarTaier commented 6 months ago

@zhilinli123 @fengcheche cc @Carl-Zhou-CN v2.3.3 can't read data from mysql cdc source :

Caused by: java.lang.ClassCastException: cannot assign instance of io.debezium.relational.TableId to field org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit.tableId of type io.debezium.relational.TableId in instance of org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit at java.base/java.io.ObjectStreamClass$FieldReflector.setObjFieldValues(ObjectStreamClass.java:2087) at java.base/java.io.ObjectStreamClass$FieldReflector.checkObjectFieldValueTypes(ObjectStreamClass.java:2050) at java.base/java.io.ObjectStreamClass.checkObjFieldValueTypes(ObjectStreamClass.java:1304) at java.base/java.io.ObjectInputStream.defaultCheckFieldValues(ObjectInputStream.java:2534) at java.base/java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2441) at java.base/java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2250) at java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1709) at java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:500) at java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:458) at org.apache.seatunnel.common.utils.SerializationUtils.deserialize(SerializationUtils.java:74) at org.apache.seatunnel.api.serialization.DefaultSerializer.deserialize(DefaultSerializer.java:41) at org.apache.seatunnel.api.serialization.DefaultSerializer.deserialize(DefaultSerializer.java:25) at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.lambda$run$0(AssignSplitOperation.java:67) at org.apache.seatunnel.common.utils.RetryUtils.retryWithException(RetryUtils.java:48) at org.apache.seatunnel.engine.server.task.operation.source.AssignSplitOperation.run(AssignSplitOperation.java:54) at com.hazelcast.spi.impl.operationservice.Operation.call(Operation.java:189) at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:273) at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:248) at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:213) at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:175) at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:139) at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.executeRun(OperationThread.java:123) at com.hazelcast.internal.util.executor.HazelcastManagedThread.run(HazelcastManagedThread.java:102) at org.apache.seatunnel.engine.client.job.ClientJobProxy.waitForJobComplete(ClientJobProxy.java:122) at org.apache.seatunnel.core.starter.seatunnel.command.ClientExecuteCommand.execute(ClientExecuteCommand.java:184) ... 2 more

avatarTaier commented 6 months ago

env { execution.parallelism = 2 job.mode = "STREAMING" checkpoint.interval = 10000 }

source { MySQL-CDC { username = "root" password = "123456" table-names = ["seatunnel.user"] base-url = "jdbc:mysql://localhost:3306/seatunnel" } }

sink { jdbc { url = "jdbc:mysql://localhost:3306/test" driver = "com.mysql.cj.jdbc.Driver" user = "root" password = "123456" generate_sink_sql = true database = "test" table = "user" primary_keys = ["id"] } }

Yuuurj commented 1 month ago

@avatarTaier @wilsonfu88 How did you solve this problem