search

actiontech / dtle

Distributed Data Transfer Service for MySQL
https://actiontech.github.io/dtle-docs-cn
Mozilla Public License 2.0
549 stars 132 forks source link

mysql fault - kill or stop src MySQL at incr repl stage cause panic #917

Closed asiroliu closed 2 years ago

asiroliu commented 2 years ago

Description

mysql fault - kill or stop src MySQL at incr repl stage cause panic

Steps to reproduce the issue

  1. insert data on src mysql 
    shell> sysbench /usr/share/sysbench/oltp_common.lua --mysql-host=172.100.9.1 --mysql-port=3306 --mysql-user=test --mysql-password=test --create_secondary=off --report-interval=10 --time=0 --mysql-db=action_db_1 --tables=1 --table_size=5000 prepare
  2. create mysql-mysql job 
    {
"job_id": "kill_scr_mysql_incr_repl",
"is_password_encrypted": false,
"task_step_name": "all",
"failover": true,
"retry": 2,
"src_task": {
"task_name": "src",
"node_id": "61c04b14-4db3-485e-5b9e-e30ef72f1475",
"mysql_src_task_config": {
  "gtid": "",
  "binlog_relay": false
},
"drop_table_if_exists": true,
"skip_create_db_table": false,
"repl_chan_buffer_size": 120,
"chunk_size": 2000,
"group_max_size": 1,
"group_timeout": 100,
"connection_config": {
  "database_type": "MySQL",
  "host": "172.100.9.2",
  "port": 3306,
  "user": "test_src",
  "password": "test_src"
},
"replicate_do_db": [
  {
    "table_schema": "action_db_1",
    "tables": [
      {
        "table_name": "sbtest1"
      }
    ]
  }
]
},
"dest_task": {
"task_name": "dest",
"node_id": "b4647efe-47fa-c017-372d-9a98e2c02407",
"mysql_dest_task_config": {
  "use_my_sql_dependency": false,
  "dependency_history_size": 2500,
  "parallel_workers": 32
},
"connection_config": {
  "database_type": "MySQL",
  "host": "172.100.9.1",
  "port": 3306,
  "user": "test_dest",
  "password": "test_dest"
}
}
}
  3. wait for full repl finish
  4. insert data on src mysql 
    shell > sysbench /usr/share/sysbench/oltp_write_only.lua --mysql-host=172.100.9.2 --mysql-port=3306 --mysql-user=test --mysql-password=test --create_secondary=off --report-interval=10 --time=0 --mysql-db=action_db_1 --tables=1 --table_size=5000 --events=5000 run
  5. kill or stop src MySQL when incr repl is running
  6. check src dtle log 
    2022-03-21T15:50:35.826+0800 [ERROR] client.driver_mgr.dtle: error GetEvent.: driver=dtle @module=reader err="dial tcp 172.100.9.2:3306: connect: connection refused" job=kill_scr_mysql_incr_repl-migration timestamp=2022-03-21T15:50:35.826+0800
2022-03-21T15:50:35.826+0800 [ERROR] client.driver_mgr.dtle: onError: driver=dtle @module=dtle.extractor err="StreamEvents encountered unexpected error: dial tcp 172.100.9.2:3306: connect: connection refused
github.com/pingcap/errors.AddStack
    /universe/src/github.com/actiontech/dtle/vendor/github.com/pingcap/errors/errors.go:174
github.com/pingcap/errors.Trace
    /universe/src/github.com/actiontech/dtle/vendor/github.com/pingcap/errors/juju_adaptor.go:15
github.com/go-mysql-org/go-mysql/client.Connect
    /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/client/conn.go:57
github.com/go-mysql-org/go-mysql/replication.(*BinlogSyncer).newConnection
    /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go:893
github.com/go-mysql-org/go-mysql/replication.(*BinlogSyncer).registerSlave
    /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go:236
github.com/go-mysql-org/go-mysql/replication.(*BinlogSyncer).prepare
    /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go:354
github.com/go-mysql-org/go-mysql/replication.(*BinlogSyncer).prepareSyncGTID
    /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go:643
github.com/go-mysql-org/go-mysql/replication.(*BinlogSyncer).retrySync
    /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go:606
github.com/go-mysql-org/go-mysql/replication.(*BinlogSyncer).onStream
    /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go:701
runtime.goexit
    /opt/go/src/runtime/asm_amd64.s:1371" job=kill_scr_mysql_incr_repl-migration timestamp=2022-03-21T15:50:35.826+0800
2022-03-21T15:50:35.826+0800 [DEBUG] client.driver_mgr.dtle: extractor shutdown: driver=dtle job=kill_scr_mysql_incr_repl-migration @module=dtle.extractor timestamp=2022-03-21T15:50:35.826+0800
2022-03-21T15:50:35.826+0800 [INFO]  client.driver_mgr.dtle: extractor shutdown: driver=dtle @module=dtle.extractor job=kill_scr_mysql_incr_repl-migration timestamp=2022-03-21T15:50:35.826+0800
2022-03-21T15:50:35.826+0800 [DEBUG] client.driver_mgr.dtle: BinlogReader.Close: driver=dtle @module=reader job=kill_scr_mysql_incr_repl-migration timestamp=2022-03-21T15:50:35.826+0800
2022-03-21T15:50:35.828+0800 [INFO]  client.driver_mgr.dtle: DestroyTask: driver=dtle @module=dtle id=17f646b9-4729-23e7-7659-aaa5600be72c/src/481c6c5a timestamp=2022-03-21T15:50:35.828+0800
2022-03-21T15:50:35.828+0800 [DEBUG] client.driver_mgr.dtle: extractor shutdown: driver=dtle job=kill_scr_mysql_incr_repl-migration @module=dtle.extractor timestamp=2022-03-21T15:50:35.828+0800
2022-03-21T15:50:35.881+0800 [DEBUG] client.driver_mgr.dtle: StreamEvents goroutine exited: driver=dtle job=kill_scr_mysql_incr_repl-migration @module=dtle.extractor timestamp=2022-03-21T15:50:35.881+0800
2022-03-21T15:50:35.881+0800 [INFO]  client.driver_mgr.dtle: Shutting down: driver=dtle @module=dtle.extractor job=kill_scr_mysql_incr_repl-migration timestamp=2022-03-21T15:50:35.881+0800
2022-03-21T15:50:35.882+0800 [INFO]  client.alloc_runner.task_runner: restarting task: alloc_id=17f646b9-4729-23e7-7659-aaa5600be72c task=src reason="Restart within policy" delay=17.839111074s
  7. the src task restart, job keep running

Output of ./dtle version:**

9.9.9.9-master-03e7307
asiroliu commented 2 years ago

/debug/pprof/goroutine?debug=1

goroutine profile: total 88
8 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0x1799136 0x474801
#   0x4709ed    sync.runtime_notifyListWait+0xcd            /opt/go/src/runtime/sema.go:513
#   0x48c838    sync.(*Cond).Wait+0x98                  /opt/go/src/sync/cond.go:56
#   0x1799135   github.com/nats-io/nats%2ego.(*Conn).waitForMsgs+0x635  /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2626

4 @ 0x43d0c5 0x40854f 0x4081cb 0x179a62f 0x474801
#   0x179a62e   github.com/nats-io/nats%2ego.(*Conn).flusher+0xee   /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2978

4 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x1508c2c 0x170e5c5 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54                     /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44                     /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4e8734    internal/poll.(*pollDesc).waitRead+0x1d4                    /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4e8716    internal/poll.(*FD).Read+0x1b6                          /opt/go/src/internal/poll/fd_unix.go:166
#   0x651eae    net.(*netFD).Read+0x4e                              /opt/go/src/net/fd_posix.go:55
#   0x666c30    net.(*conn).Read+0x90                               /opt/go/src/net/net.go:183
#   0x1508c2b   github.com/nats-io/nats-server/v2/server.(*client).readLoop+0x110b      /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/client.go:1187
#   0x170e5c4   github.com/nats-io/nats-server/v2/server.(*Server).createClient.func1+0x44  /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:2568

4 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x1793ef8 0x17989a8 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54         /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44         /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4e8734    internal/poll.(*pollDesc).waitRead+0x1d4        /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4e8716    internal/poll.(*FD).Read+0x1b6              /opt/go/src/internal/poll/fd_unix.go:166
#   0x651eae    net.(*netFD).Read+0x4e                  /opt/go/src/net/fd_posix.go:55
#   0x666c30    net.(*conn).Read+0x90                   /opt/go/src/net/net.go:183
#   0x1793ef7   github.com/nats-io/nats%2ego.(*natsReader).Read+0xb7    /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:1649
#   0x17989a7   github.com/nats-io/nats%2ego.(*Conn).readLoop+0xc7  /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2584

4 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0x150765e 0x170e60a 0x474801
#   0x4709ed    sync.runtime_notifyListWait+0xcd                        /opt/go/src/runtime/sema.go:513
#   0x48c838    sync.(*Cond).Wait+0x98                              /opt/go/src/sync/cond.go:56
#   0x150765d   github.com/nats-io/nats-server/v2/server.(*client).writeLoop+0x23d      /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/client.go:1061
#   0x170e609   github.com/nats-io/nats-server/v2/server.(*Server).createClient.func2+0x29  /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:2571

4 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0x1798615 0x474801
#   0x4709ed    sync.runtime_notifyListWait+0xcd                        /opt/go/src/runtime/sema.go:513
#   0x48c838    sync.(*Cond).Wait+0x98                              /opt/go/src/sync/cond.go:56
#   0x1798614   github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher+0x34    /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2509

4 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0xff1ef3 0x474801
#   0x4709ed    sync.runtime_notifyListWait+0xcd            /opt/go/src/runtime/sema.go:513
#   0x48c838    sync.(*Cond).Wait+0x98                  /opt/go/src/sync/cond.go:56
#   0xff1ef2    github.com/nats-io/go-nats.(*Conn).waitForMsgs+0x452    /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:2014

3 @ 0x43d0c5 0x44e2d7 0x820875 0x474801
#   0x820874    database/sql.(*DB).connectionOpener+0xb4    /opt/go/src/database/sql/sql.go:1133

2 @ 0x43d0c5 0x40854f 0x4081cb 0x147695d 0x13f4ff6 0xbd1278 0xbd2948 0xbe0a0b 0x474801
#   0x147695c   github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).TaskStats+0x19c    /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:251
#   0x13f4ff5   github.com/hashicorp/nomad/plugins/drivers/proto._Driver_TaskStats_Handler+0x115    /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4509
#   0xbd1277    google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329
#   0xbd2947    google.golang.org/grpc.(*Server).handleStream+0xc67                 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409
#   0xbe0a0a    google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746

2 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x4f1cb7 0x4f1c87 0x545382 0x1371ac5 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54     /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44     /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4e8734    internal/poll.(*pollDesc).waitRead+0x1d4    /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4e8716    internal/poll.(*FD).Read+0x1b6          /opt/go/src/internal/poll/fd_unix.go:166
#   0x4f1cb6    os.(*File).read+0x76                /opt/go/src/os/file_posix.go:31
#   0x4f1c86    os.(*File).Read+0x46                /opt/go/src/os/file.go:117
#   0x545381    bufio.(*Reader).Read+0x221          /opt/go/src/bufio/bufio.go:227
#   0x1371ac4   github.com/hashicorp/go-plugin.copyChan+0xc4    /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_stdio.go:181

2 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x794917 0x544ce8 0x544e6f 0x795608 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54     /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44     /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4e8734    internal/poll.(*pollDesc).waitRead+0x1d4    /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4e8716    internal/poll.(*FD).Read+0x1b6          /opt/go/src/internal/poll/fd_unix.go:166
#   0x651eae    net.(*netFD).Read+0x4e              /opt/go/src/net/fd_posix.go:55
#   0x666c30    net.(*conn).Read+0x90               /opt/go/src/net/net.go:183
#   0x794916    net/http.(*persistConn).Read+0x76       /opt/go/src/net/http/transport.go:1922
#   0x544ce7    bufio.(*Reader).fill+0x107          /opt/go/src/bufio/bufio.go:101
#   0x544e6e    bufio.(*Reader).Peek+0x4e           /opt/go/src/bufio/bufio.go:139
#   0x795607    net/http.(*persistConn).readLoop+0x1a7      /opt/go/src/net/http/transport.go:2083

2 @ 0x43d0c5 0x44e2d7 0x7971d7 0x474801
#   0x7971d6    net/http.(*persistConn).writeLoop+0xf6  /opt/go/src/net/http/transport.go:2382

1 @ 0x40e414 0x470f85 0x7c73e5 0x474801
#   0x470f84    os/signal.signal_recv+0xa4  /opt/go/src/runtime/sigqueue.go:168
#   0x7c73e4    os/signal.loop+0x24     /opt/go/src/os/signal/signal_unix.go:23

1 @ 0x43d0c5 0x40854f 0x40818b 0x1377889 0x474801
#   0x1377888   github.com/hashicorp/go-plugin.Serve.func3+0x88 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/server.go:433

1 @ 0x43d0c5 0x40854f 0x40818b 0x1477a38 0x13f51f6 0xbd1278 0xbd2948 0xbe0a0b 0x474801
#   0x1477a37   github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).TaskEvents+0x97    /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:358
#   0x13f51f5   github.com/hashicorp/nomad/plugins/drivers/proto._Driver_TaskEvents_Handler+0x115   /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4530
#   0xbd1277    google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329
#   0xbd2947    google.golang.org/grpc.(*Server).handleStream+0xc67                 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409
#   0xbe0a0a    google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746

1 @ 0x43d0c5 0x40854f 0x40818b 0x7d03c9 0x474801
#   0x7d03c8    github.com/actiontech/dtle/g.DumpLoop+0xa8  /universe/src/github.com/actiontech/dtle/g/dump.go:21

1 @ 0x43d0c5 0x40854f 0x40818b 0x7d122f 0x474801
#   0x7d122e    github.com/actiontech/dtle/g.FreeMemoryWorker+0x4e  /universe/src/github.com/actiontech/dtle/g/g.go:92

1 @ 0x43d0c5 0x40854f 0x40818b 0x7d166b 0x474801
#   0x7d166a    github.com/actiontech/dtle/g.MemoryMonitor+0x6a /universe/src/github.com/actiontech/dtle/g/g.go:141

1 @ 0x43d0c5 0x40854f 0x4081cb 0xff2a8a 0x474801
#   0xff2a89    github.com/nats-io/go-nats.(*Conn).flusher+0xe9 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:2219

1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x545382 0x4bb1a7 0x848af1 0x848ab0 0x848689 0x85798d 0x857962 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54                     /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44                     /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4e8734    internal/poll.(*pollDesc).waitRead+0x1d4                    /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4e8716    internal/poll.(*FD).Read+0x1b6                          /opt/go/src/internal/poll/fd_unix.go:166
#   0x651eae    net.(*netFD).Read+0x4e                              /opt/go/src/net/fd_posix.go:55
#   0x666c30    net.(*conn).Read+0x90                               /opt/go/src/net/net.go:183
#   0x545381    bufio.(*Reader).Read+0x221                          /opt/go/src/bufio/bufio.go:227
#   0x4bb1a6    io.ReadAtLeast+0x86                             /opt/go/src/io/io.go:328
#   0x848af0    io.ReadFull+0x70                                /opt/go/src/io/io.go:347
#   0x848aaf    github.com/go-mysql-org/go-mysql/packet.(*Conn).ReadPacketTo+0x2f       /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/packet/conn.go:132
#   0x848688    github.com/go-mysql-org/go-mysql/packet.(*Conn).ReadPacketReuseMem+0xa8     /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/packet/conn.go:98
#   0x85798c    github.com/go-mysql-org/go-mysql/packet.(*Conn).ReadPacket+0x30c        /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/packet/conn.go:90
#   0x857961    github.com/go-mysql-org/go-mysql/replication.(*BinlogSyncer).onStream+0x2e1 /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go:670

1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x545382 0x4bb1a7 0xb51989 0xb51922 0xb52205 0xb8b85b 0xbccea2 0xbe091f 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54                     /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44                     /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4e8734    internal/poll.(*pollDesc).waitRead+0x1d4                    /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4e8716    internal/poll.(*FD).Read+0x1b6                          /opt/go/src/internal/poll/fd_unix.go:166
#   0x651eae    net.(*netFD).Read+0x4e                              /opt/go/src/net/fd_posix.go:55
#   0x666c30    net.(*conn).Read+0x90                               /opt/go/src/net/net.go:183
#   0x545381    bufio.(*Reader).Read+0x221                          /opt/go/src/bufio/bufio.go:227
#   0x4bb1a6    io.ReadAtLeast+0x86                             /opt/go/src/io/io.go:328
#   0xb51988    io.ReadFull+0x88                                /opt/go/src/io/io.go:347
#   0xb51921    golang.org/x/net/http2.readFrameHeader+0x21                 /universe/src/github.com/actiontech/dtle/vendor/golang.org/x/net/http2/frame.go:237
#   0xb52204    golang.org/x/net/http2.(*Framer).ReadFrame+0xa4                 /universe/src/github.com/actiontech/dtle/vendor/golang.org/x/net/http2/frame.go:498
#   0xb8b85a    google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams+0x9a /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453
#   0xbccea1    google.golang.org/grpc.(*Server).serveStreams+0xe1              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:742
#   0xbe091e    google.golang.org/grpc.(*Server).handleRawConn.func1+0x3e           /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:703

1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x773298 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54     /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44     /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4e8734    internal/poll.(*pollDesc).waitRead+0x1d4    /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4e8716    internal/poll.(*FD).Read+0x1b6          /opt/go/src/internal/poll/fd_unix.go:166
#   0x651eae    net.(*netFD).Read+0x4e              /opt/go/src/net/fd_posix.go:55
#   0x666c30    net.(*conn).Read+0x90               /opt/go/src/net/net.go:183
#   0x773297    net/http.(*connReader).backgroundRead+0x57  /opt/go/src/net/http/server.go:672

1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0xff17ca 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54         /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44         /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4e8734    internal/poll.(*pollDesc).waitRead+0x1d4        /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4e8716    internal/poll.(*FD).Read+0x1b6              /opt/go/src/internal/poll/fd_unix.go:166
#   0x651eae    net.(*netFD).Read+0x4e                  /opt/go/src/net/fd_posix.go:55
#   0x666c30    net.(*conn).Read+0x90                   /opt/go/src/net/net.go:183
#   0xff17c9    github.com/nats-io/go-nats.(*Conn).readLoop+0x189   /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:1978

1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4ea4b2 0x4ea494 0x6534e5 0x673172 0x671dc5 0x25d57ef 0x77ea85 0x25d33b3 0x29164f5 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54                 /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44                 /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4ea4b1    internal/poll.(*pollDesc).waitRead+0x211                /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4ea493    internal/poll.(*FD).Accept+0x1f3                    /opt/go/src/internal/poll/fd_unix.go:401
#   0x6534e4    net.(*netFD).accept+0x44                        /opt/go/src/net/fd_unix.go:172
#   0x673171    net.(*TCPListener).accept+0x31                      /opt/go/src/net/tcpsock_posix.go:139
#   0x671dc4    net.(*TCPListener).AcceptTCP+0x64                   /opt/go/src/net/tcpsock.go:248
#   0x25d57ee   github.com/labstack/echo/v4.tcpKeepAliveListener.Accept+0x2e        /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:963
#   0x77ea84    net/http.(*Server).Serve+0x284                      /opt/go/src/net/http/server.go:2961
#   0x25d33b2   github.com/labstack/echo/v4.(*Echo).Start+0xf2              /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:671
#   0x29164f4   github.com/actiontech/dtle/drivers/api.SetupApiServer.func3+0x74    /universe/src/github.com/actiontech/dtle/drivers/api/route.go:179

1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4ea4b2 0x4ea494 0x6534e5 0x673172 0x671f65 0x16bad02 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54                     /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44                     /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4ea4b1    internal/poll.(*pollDesc).waitRead+0x211                    /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4ea493    internal/poll.(*FD).Accept+0x1f3                        /opt/go/src/internal/poll/fd_unix.go:401
#   0x6534e4    net.(*netFD).accept+0x44                            /opt/go/src/net/fd_unix.go:172
#   0x673171    net.(*TCPListener).accept+0x31                          /opt/go/src/net/tcpsock_posix.go:139
#   0x671f64    net.(*TCPListener).Accept+0x64                          /opt/go/src/net/tcpsock.go:261
#   0x16bad01   github.com/nats-io/nats-server/v2/server.(*Server).acceptConnections+0x41   /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:2088

1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4ea4b2 0x4ea494 0x6534e5 0x6797f2 0x677a45 0xbcbc9f 0x13714ee 0x474801
#   0x46e774    internal/poll.runtime_pollWait+0x54         /opt/go/src/runtime/netpoll.go:227
#   0x4e7644    internal/poll.(*pollDesc).wait+0x44         /opt/go/src/internal/poll/fd_poll_runtime.go:87
#   0x4ea4b1    internal/poll.(*pollDesc).waitRead+0x211        /opt/go/src/internal/poll/fd_poll_runtime.go:92
#   0x4ea493    internal/poll.(*FD).Accept+0x1f3            /opt/go/src/internal/poll/fd_unix.go:401
#   0x6534e4    net.(*netFD).accept+0x44                /opt/go/src/net/fd_unix.go:172
#   0x6797f1    net.(*UnixListener).accept+0x31             /opt/go/src/net/unixsock_posix.go:162
#   0x677a44    net.(*UnixListener).Accept+0x64             /opt/go/src/net/unixsock.go:260
#   0xbcbc9e    google.golang.org/grpc.(*Server).Serve+0x27e        /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:621
#   0x13714ed   github.com/hashicorp/go-plugin.(*GRPCServer).Serve+0x6d /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_server.go:138

1 @ 0x43d0c5 0x44e2d7 0x11a3869 0x266bed1 0x474801
#   0x11a3868   github.com/actiontech/dtle/drivers/mysql/common.RegularlyUpdateJobStatus+0xe8           /universe/src/github.com/actiontech/dtle/drivers/mysql/common/common.go:170
#   0x266bed0   github.com/actiontech/dtle/drivers/mysql/mysql.(*Extractor).initiateStreaming.func2+0x90    /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:691

1 @ 0x43d0c5 0x44e2d7 0x11b7014 0x474801
#   0x11b7013   github.com/actiontech/dtle/drivers/mysql/common.(*StoreManager).SrcWatchNats.func1+0x93 /universe/src/github.com/actiontech/dtle/drivers/mysql/common/store.go:219

1 @ 0x43d0c5 0x44e2d7 0x136f1e6 0x136ff7a 0x474801
#   0x136f1e5   github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv+0x85    /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:121
#   0x136ff79   github.com/hashicorp/go-plugin.(*GRPCBroker).Run+0x79       /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:411

1 @ 0x43d0c5 0x44e2d7 0x13718c5 0x135ca53 0xbd1278 0xbd2948 0xbe0a0b 0x474801
#   0x13718c4   github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio+0x144         /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_stdio.go:58
#   0x135ca52   github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler+0x112 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/internal/plugin/grpc_stdio.pb.go:185
#   0xbd1277    google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329
#   0xbd2947    google.golang.org/grpc.(*Server).handleStream+0xc67                 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409
#   0xbe0a0a    google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746

1 @ 0x43d0c5 0x44e2d7 0x1375534 0x1474e7f 0x29177a8 0x291763e 0x2917a97 0x43cc96 0x474801
#   0x1375533   github.com/hashicorp/go-plugin.Serve+0x953      /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/server.go:468
#   0x1474e7e   github.com/hashicorp/nomad/plugins/drivers.Serve+0x23e  /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/plugin.go:53
#   0x29177a7   github.com/hashicorp/nomad/plugins.serve+0x147      /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/serve.go:47
#   0x291763d   github.com/hashicorp/nomad/plugins.Serve+0xbd       /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/serve.go:26
#   0x2917a96   main.main+0x2d6                     /universe/src/github.com/actiontech/dtle/cmd/nomad-plugin/main.go:30
#   0x43cc95    runtime.main+0x255                  /opt/go/src/runtime/proc.go:225

1 @ 0x43d0c5 0x44e2d7 0x137732f 0x474801
#   0x137732e   github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1+0x10e  /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:70

1 @ 0x43d0c5 0x44e2d7 0x14756c5 0x13f3e33 0xbd1278 0xbd2948 0xbe0a0b 0x474801
#   0x14756c4   github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).Fingerprint+0x124  /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:75
#   0x13f3e32   github.com/hashicorp/nomad/plugins/drivers/proto._Driver_Fingerprint_Handler+0x112  /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4380
#   0xbd1277    google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329
#   0xbd2947    google.golang.org/grpc.(*Server).handleStream+0xc67                 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409
#   0xbe0a0a    google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746

1 @ 0x43d0c5 0x44e2d7 0x14760a5 0x13f4677 0xbce86b 0xbd29af 0xbe0a0b 0x474801
#   0x14760a4   github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).WaitTask+0x124 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:148
#   0x13f4676   github.com/hashicorp/nomad/plugins/drivers/proto._Driver_WaitTask_Handler+0x216 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4438
#   0xbce86a    google.golang.org/grpc.(*Server).processUnaryRPC+0x52a              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1082
#   0xbd29ae    google.golang.org/grpc.(*Server).handleStream+0xcce             /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1405
#   0xbe0a0a    google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa          /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746

1 @ 0x43d0c5 0x44e2d7 0x147f4ad 0x474801
#   0x147f4ac   github.com/hashicorp/nomad/drivers/shared/eventer.(*Eventer).eventLoop+0xec /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/drivers/shared/eventer/eventer.go:66

1 @ 0x43d0c5 0x44e2d7 0x153e237 0x474801
#   0x153e236   github.com/nats-io/nats-server/v2/server.(*Server).internalSendLoop+0x116   /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/events.go:319

1 @ 0x43d0c5 0x44e2d7 0x16aea25 0x474801
#   0x16aea24   github.com/nats-io/nats-server/v2/server.(*sendq).internalLoop+0x464    /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/sendq.go:54

1 @ 0x43d0c5 0x44e2d7 0x16c6e9c 0x474801
#   0x16c6e9b   github.com/nats-io/nats-server/v2/server.(*srvIPQueueLogger).run+0x11b  /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:3645

1 @ 0x43d0c5 0x44e2d7 0x16ff425 0x474801
#   0x16ff424   github.com/nats-io/nats-server/v2/server.(*Server).startGWReplyMapExpiration.func1+0x124    /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/gateway.go:3143

1 @ 0x43d0c5 0x44e2d7 0x170eee9 0x474801
#   0x170eee8   github.com/nats-io/nats-server/v2/server.(*Server).handleSignals.func1+0xa8 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/signal.go:48

1 @ 0x43d0c5 0x44e2d7 0x184454f 0x474801
#   0x184454e   github.com/nats-io/nats-streaming-server/server.(*StanServer).ioLoop+0x74e  /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-streaming-server/server/server.go:4379

1 @ 0x43d0c5 0x44e2d7 0x184aae5 0x474801
#   0x184aae4   github.com/nats-io/nats-streaming-server/server.(*StanServer).processSubscriptionsStart+0xe4    /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-streaming-server/server/server.go:5393

1 @ 0x43d0c5 0x44e2d7 0x2425ced 0x474801
#   0x2425cec   go.opencensus.io/stats/view.(*worker).start+0xcc    /universe/src/github.com/actiontech/dtle/vendor/go.opencensus.io/stats/view/worker.go:276

1 @ 0x43d0c5 0x44e2d7 0x265d7af 0x474801
#   0x265d7ae   github.com/actiontech/dtle/drivers/mysql/mysql.(*TimestampContext).Handle+0xce  /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:952

1 @ 0x43d0c5 0x44e2d7 0x266c825 0x474801
#   0x266c824   github.com/actiontech/dtle/drivers/mysql/mysql.(*Extractor).StreamEvents.func1+0x1c4    /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:1020

1 @ 0x43d0c5 0x44e2d7 0x27e26bd 0x474801
#   0x27e26bc   github.com/actiontech/dtle/drivers/mysql.(*Driver).handleFingerprint+0x13c  /universe/src/github.com/actiontech/dtle/drivers/mysql/driver.go:404

1 @ 0x43d0c5 0x44e2d7 0x27e3df7 0x474801
#   0x27e3df6   github.com/actiontech/dtle/drivers/mysql.(*Driver).handleWait+0x176 /universe/src/github.com/actiontech/dtle/drivers/mysql/driver.go:578

1 @ 0x43d0c5 0x44e2d7 0x27e46c5 0x474801
#   0x27e46c4   github.com/actiontech/dtle/drivers/mysql.(*Driver).handleStats+0xc4 /universe/src/github.com/actiontech/dtle/drivers/mysql/driver.go:646

1 @ 0x43d0c5 0x44e2d7 0x27e9115 0x474801
#   0x27e9114   github.com/actiontech/dtle/drivers/mysql.(*taskHandle).run.func1+0xf4   /universe/src/github.com/actiontech/dtle/drivers/mysql/handle.go:139

1 @ 0x43d0c5 0x44e2d7 0x797e85 0x78c24b 0x771815 0x72d1b4 0x72cb9f 0x72ec1f 0x10398de 0x10398a6 0x104a374 0x1049545 0x105fe5f 0x474801
#   0x797e84    net/http.(*persistConn).roundTrip+0x764             /opt/go/src/net/http/transport.go:2610
#   0x78c24a    net/http.(*Transport).roundTrip+0xaca               /opt/go/src/net/http/transport.go:592
#   0x771814    net/http.(*Transport).RoundTrip+0x34                /opt/go/src/net/http/roundtrip.go:17
#   0x72d1b3    net/http.send+0x453                     /opt/go/src/net/http/client.go:251
#   0x72cb9e    net/http.(*Client).send+0xfe                    /opt/go/src/net/http/client.go:175
#   0x72ec1e    net/http.(*Client).do+0x45e                 /opt/go/src/net/http/client.go:717
#   0x10398dd   net/http.(*Client).Do+0xbd                  /opt/go/src/net/http/client.go:585
#   0x10398a5   github.com/hashicorp/consul/api.(*Client).doRequest+0x85    /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/consul/api/api.go:880
#   0x104a373   github.com/hashicorp/consul/api.(*KV).getInternal+0x2b3     /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/consul/api/kv.go:131
#   0x1049544   github.com/hashicorp/consul/api.(*KV).Get+0xa4          /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/consul/api/kv.go:65
#   0x105fe5e   github.com/docker/libkv/store/consul.(*Consul).Watch.func1+0xfe /universe/src/github.com/actiontech/dtle/vendor/github.com/docker/libkv/store/consul/consul.go:299

1 @ 0x43d0c5 0x44e2d7 0x854045 0x26338d8 0x265da68 0x266bdf0 0x474801
#   0x854044    github.com/go-mysql-org/go-mysql/replication.(*BinlogStreamer).GetEvent+0x104           /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/replication/binlogstreamer.go:32
#   0x26338d7   github.com/actiontech/dtle/drivers/mysql/mysql/binlog.(*BinlogReader).DataStreamEvents+0x177    /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/binlog/binlog_reader.go:899
#   0x265da67   github.com/actiontech/dtle/drivers/mysql/mysql.(*Extractor).StreamEvents+0x87           /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:1065
#   0x266bdef   github.com/actiontech/dtle/drivers/mysql/mysql.(*Extractor).initiateStreaming.func1+0x8f    /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:684

1 @ 0x43d0c5 0x44e2d7 0xb76b1f 0xb7747d 0xb99c37 0x474801
#   0xb76b1e    google.golang.org/grpc/internal/transport.(*controlBuffer).get+0xfe /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395
#   0xb7747c    google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x1dc  /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513
#   0xb99c36    google.golang.org/grpc/internal/transport.newHTTP2Server.func2+0xd6 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291

1 @ 0x43d0c5 0x44e2d7 0xb8fd65 0x474801
#   0xb8fd64    google.golang.org/grpc/internal/transport.(*http2Server).keepalive+0x264    /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966

1 @ 0x43d0c5 0x44e2d7 0xb95ce5 0xb95bb1 0xb96c15 0x4bb1a7 0xb96b52 0xb96b0f 0xbc8143 0xbc8d6d 0xbc967b 0xbdd476 0x135b9e2 0x136ed4a 0x135b8ad 0xbd1278 0xbd2948 0xbe0a0b 0x474801
#   0xb95ce4    google.golang.org/grpc/internal/transport.(*recvBufferReader).read+0xa4         /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:174
#   0xb95bb0    google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x210        /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:168
#   0xb96c14    google.golang.org/grpc/internal/transport.(*transportReader).Read+0x54          /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:479
#   0x4bb1a6    io.ReadAtLeast+0x86                                 /opt/go/src/io/io.go:328
#   0xb96b51    io.ReadFull+0xd1                                    /opt/go/src/io/io.go:347
#   0xb96b0e    google.golang.org/grpc/internal/transport.(*Stream).Read+0x8e               /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:463
#   0xbc8142    google.golang.org/grpc.(*parser).recvMsg+0x62                       /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/rpc_util.go:510
#   0xbc8d6c    google.golang.org/grpc.recvAndDecompress+0x4c                       /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/rpc_util.go:641
#   0xbc967a    google.golang.org/grpc.recv+0x9a                            /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/rpc_util.go:709
#   0xbdd475    google.golang.org/grpc.(*serverStream).RecvMsg+0x155                    /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/stream.go:1473
#   0x135b9e1   github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv+0x61 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/internal/plugin/grpc_broker.pb.go:167
#   0x136ed49   github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream+0x109            /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:84
#   0x135b8ac   github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler+0xac /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/internal/plugin/grpc_broker.pb.go:148
#   0xbd1277    google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329
#   0xbd2947    google.golang.org/grpc.(*Server).handleStream+0xc67                 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409
#   0xbe0a0a    google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa              /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746

1 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0xff1375 0x474801
#   0x4709ed    sync.runtime_notifyListWait+0xcd                        /opt/go/src/runtime/sema.go:513
#   0x48c838    sync.(*Cond).Wait+0x98                              /opt/go/src/sync/cond.go:56
#   0xff1374    github.com/nats-io/go-nats.(*asyncCallbacksHandler).asyncCBDispatcher+0x34  /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:1893

1 @ 0x43d0c5 0x471832 0x101e7af 0x474801
#   0x471831    time.Sleep+0xd1                         /opt/go/src/runtime/time.go:193
#   0x101e7ae   github.com/armon/go-metrics.(*Metrics).collectStats+0x2e    /universe/src/github.com/actiontech/dtle/vendor/github.com/armon/go-metrics/metrics.go:230

1 @ 0x46e31d 0x7b644e 0x7b6225 0x7b2db2 0x7c47c5 0x7c6017 0x77b024 0x77cead 0x25daedf 0x25dac62 0x25d301c 0x77e6a3 0x779b0d 0x474801
#   0x46e31c    runtime/pprof.runtime_goroutineProfileWithLabels+0x5c   /opt/go/src/runtime/mprof.go:716
#   0x7b644d    runtime/pprof.writeRuntimeProfile+0xcd          /opt/go/src/runtime/pprof/pprof.go:724
#   0x7b6224    runtime/pprof.writeGoroutine+0xa4           /opt/go/src/runtime/pprof/pprof.go:684
#   0x7b2db1    runtime/pprof.(*Profile).WriteTo+0x3f1          /opt/go/src/runtime/pprof/pprof.go:331
#   0x7c47c4    net/http/pprof.handler.ServeHTTP+0x384          /opt/go/src/net/http/pprof/pprof.go:253
#   0x7c6016    net/http/pprof.Index+0x8d6              /opt/go/src/net/http/pprof/pprof.go:371
#   0x77b023    net/http.HandlerFunc.ServeHTTP+0x43         /opt/go/src/net/http/server.go:2049
#   0x77ceac    net/http.(*ServeMux).ServeHTTP+0x1ac            /opt/go/src/net/http/server.go:2428
#   0x25daede   github.com/labstack/echo/v4.WrapHandler.func1+0x9e  /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:901
#   0x25dac61   github.com/labstack/echo/v4.(*Echo).add.func1+0x61  /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:544
#   0x25d301b   github.com/labstack/echo/v4.(*Echo).ServeHTTP+0x2bb /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:654
#   0x77e6a2    net/http.serverHandler.ServeHTTP+0xa2           /opt/go/src/net/http/server.go:2867
#   0x779b0c    net/http.(*conn).serve+0x8cc                /opt/go/src/net/http/server.go:1932
asiroliu commented 2 years ago

verify version: 9.9.9.9-master-6946fd3

  1. it's not a panic output
  2. after fixed log, just output error message and don't show stack 
    2022-03-22T16:53:16.866+0800 [ERROR] client.driver_mgr.dtle: onError: driver=dtle @module=dtle.extractor err="StreamEvents encountered unexpected error: dial tcp 172.100.9.3:3306: connect: connection refused" job=kill_scr_mysql_incr_repl-migration timestamp=2022-03-22T16:53:16.866+0800