search

actiontech / dtle

Distributed Data Transfer Service for MySQL
https://actiontech.github.io/dtle-docs-cn
Mozilla Public License 2.0
551 stars 133 forks source link

mysql-mysql job incr repl stage: stop or kill dest MySQL cause lost data #932

Open asiroliu opened 2 years ago

asiroliu commented 2 years ago

Description

mysql-mysql job incr repl stage: stop or kill dest MySQL cause lost data

Steps to reproduce the issue

  1. insert some data on src MySQL 
    shell> sysbench /usr/share/sysbench/oltp_common.lua --mysql-host=172.100.9.2 --mysql-port=3306 --mysql-user=test --mysql-password=test --create_secondary=off --report-interval=10 --time=0 --mysql-db=action_db_1 --tables=1 --table_size=5000 prepare
  2. create dtle job 
    {
"job_id": "kill_dest_mysql_incr_repl",
"is_password_encrypted": true,
"task_step_name": "all",
"failover": true,
"retry": 2,
"src_task": {
"task_name": "src",
"node_id": "5c3be52d-6136-a10f-7092-7e9317c76014",
"mysql_src_task_config": {
  "gtid": "",
  "binlog_relay": false
},
"drop_table_if_exists": true,
"skip_create_db_table": false,
"repl_chan_buffer_size": 120,
"chunk_size": 2000,
"group_max_size": 1,
"group_timeout": 100,
"connection_config": {
  "database_type": "MySQL",
  "host": "172.100.9.2",
  "port": 3306,
  "user": "test_src",
  "password": "test_dest"
},
"replicate_do_db": [
  {
    "table_schema": "action_db_1",
    "tables": [
      {
        "table_name": "sbtest1"
      }
    ]
  }
]
},
"dest_task": {
"task_name": "dest",
"node_id": "faa090cb-2859-bcc9-e1c4-dcfaef8af905",
"mysql_dest_task_config": {
  "use_my_sql_dependency": false,
  "dependency_history_size": 2500,
  "parallel_workers": 32
},
"connection_config": {
  "database_type": "MySQL",
  "host": "172.100.9.1",
  "port": 3306,
  "user": "test_dest",
  "password": "test_dest"
}
}
}
  3. insert data on src MySQL 
    shell> sysbench /usr/share/sysbench/oltp_write_only.lua --mysql-host=172.100.9.2 --mysql-port=3306 --mysql-user=test --mysql-password=test --create_secondary=off --report-interval=10 --time=0 --mysql-db=action_db_1 --tables=1 --table_size=5000 --events=5000 run
  4. kill dest MySQL when the incr repl is running 
    shell> pgrep mysql | xargs -r -n 1 kill -9
  5. wait dtle log report error 
    2022-03-31T13:40:00.446+0800 [ERROR] client.driver_mgr.dtle: onError: driver=dtle @module=dtle.applier err="invalid connection" job=kill_dest_mysql_incr_repl-migration timestamp=2022-03-31T13:40:00.431+0800
  6. start dest MySQL, wait for the incr repl is complete 
    2022-03-31T13:40:27.556+0800 [DEBUG] client.driver_mgr.dtle: after ApplyBinlogEvent.: driver=dtle @module=dtle.applier gno=19021 job=kill_dest_mysql_incr_repl-migration worker=13 timestamp=2022-03-31T13:40:27.550+0800
2022-03-31T13:40:27.556+0800 [DEBUG] client.driver_mgr.dtle: TimestampContext.Handle: got: driver=dtle @module=dtle.applier job=kill_dest_mysql_incr_repl-migration timestamp=1.6487052e+09 timestamp=2022-03-31T13:40:27.550+0800
2022-03-31T13:40:27.556+0800 [DEBUG] client.driver_mgr.dtle: event delay time: driver=dtle job=kill_dest_mysql_incr_repl-migration timestamp=1.6487052e+09 @module=dtle.applier.ApplyBinlogEvent timestamp=2022-03-31T13:40:27.550+0800
2022-03-31T13:40:27.556+0800 [DEBUG] client.driver_mgr.dtle: after ApplyBinlogEvent.: driver=dtle @module=dtle.applier gno=19020 job=kill_dest_mysql_incr_repl-migration worker=11 timestamp=2022-03-31T13:40:27.550+0800
2022-03-31T13:40:27.556+0800 [DEBUG] client.driver_mgr.dtle: TimestampContext.Handle: got: driver=dtle @module=dtle.applier job=kill_dest_mysql_incr_repl-migration timestamp=1.6487052e+09 timestamp=2022-03-31T13:40:27.550+0800
2022-03-31T13:40:34.038+0800 [DEBUG] client.driver_mgr.dtle: Stats: driver=dtle @module=dtle.applier job=kill_dest_mysql_incr_repl-migration timestamp=2022-03-31T13:40:34.038+0800
2022-03-31T13:40:34.038+0800 [DEBUG] client.driver_mgr.dtle: TimestampContext.GetDelay: driver=dtle delay=27 job=kill_dest_mysql_incr_repl-migration @module=dtle.applier timestamp=2022-03-31T13:40:34.038+0800
2022-03-31T13:40:34.084+0800 [DEBUG] client.driver_mgr.dtle: SaveGtidForJob: driver=dtle gtid=00003306-0000-0000-0000-000000003306:1-16,8cc4087f-b0b4-11ec-8859-0242ac640902:1-19021 job=kill_dest_mysql_incr_repl-migration @module=dtle.applier timestamp=2022-03-31T13:40:34.084+0800
2022-03-31T13:40:39.151+0800 [DEBUG] client.driver_mgr.dtle: no binlogEntry for 10s: driver=dtle @module=dtle.applier job=kill_dest_mysql_incr_repl-migration timestamp=2022-03-31T13:40:39.151+0800
2022-03-31T13:40:49.039+0800 [DEBUG] client.driver_mgr.dtle: Stats: driver=dtle @module=dtle.applier job=kill_dest_mysql_incr_repl-migration timestamp=2022-03-31T13:40:49.039+0800
2022-03-31T13:40:49.039+0800 [DEBUG] client.driver_mgr.dtle: TimestampContext.GetDelay: driver=dtle delay=27 job=kill_dest_mysql_incr_repl-migration @module=dtle.applier timestamp=2022-03-31T13:40:49.039+0800
2022-03-31T13:40:49.152+0800 [DEBUG] client.driver_mgr.dtle: no binlogEntry for 10s: driver=dtle @module=dtle.applier job=kill_dest_mysql_incr_repl-migration timestamp=2022-03-31T13:40:49.152+0800
  7. check data consistency 
    
shell> mysqldbcompare -q --server1=test:test@172.100.9.2:3306 --server2=test:test@172.100.9.1:3306 action_db_1 --disable-binary-logging
# WARNING: Using a password on the command line interface can be insecure.
# Checking databases action_db_1 on server1 and action_db_1 on server2
#

#

Data differences found among rows:

--- action_db_1.sbtest1 +++ action_db_1.sbtest1 @@ -1,7 +1,7 @@ +-------+-------+--------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------+ | id | k | c | pad | +-------+-------+--------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------+ -| 1927 | 2492 | 38636437694-16946271203-99572638155-22676375356-72767561962-23399335989-37498062844-58710815139-95717509828-32736846687 | 81958214744-99088791416-23592598286-39798795772-81352224804 | -| 2570 | 2674 | 90006434753-92906375060-77708757173-63713421542-76087366792-32321289719-01925904775-94165917653-25837822395-93092915234 | 08947527390-24943168235-35584429802-71991676277-25665988462 | -| 2255 | 2497 | 02074413568-26661077903-88777467029-14407985298-18352155983-55705355528-30682175664-33485910533-09730123957-99321965962 | 96986934587-79526090505-77921985956-33508253365-35153189677 | +| 1927 | 2065 | 61837777747-86547401501-93226206478-04376595682-79030838992-75532890733-23811762253-88524938350-74289620172-97639248661 | 76853786579-46464058787-66445416918-87789705025-72810043168 | +| 2570 | 2513 | 25159697695-13558960359-55970863362-77072116767-95900203114-62709764836-95602659780-45911811170-83974647187-58202211738 | 56865242726-08341493397-15112010757-79609641161-27675823353 | +| 2255 | 2497 | 68918655240-56417065264-74136324545-86809808623-53935908496-44505717353-22353245589-95428450265-00311452483-41827784671 | 96986934587-79526090505-77921985956-33508253365-35153189677 | +-------+-------+--------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------+


## Output of `./dtle version`:**

9.9.9.9-master-5279021

asiroliu commented 2 years ago

src /debug/pprof/goroutine?debug=1

``` goroutine profile: total 100 8 @ 0x43d0c5 0x40854f 0x4081cb 0x1477dbd 0x13f6456 0xbd1658 0xbd2d28 0xbe0deb 0x474801 # 0x1477dbc github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).TaskStats+0x19c /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:251 # 0x13f6455 github.com/hashicorp/nomad/plugins/drivers/proto._Driver_TaskStats_Handler+0x115 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4509 # 0xbd1657 google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329 # 0xbd2d27 google.golang.org/grpc.(*Server).handleStream+0xc67 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 8 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0x179a596 0x474801 # 0x4709ed sync.runtime_notifyListWait+0xcd /opt/go/src/runtime/sema.go:513 # 0x48c838 sync.(*Cond).Wait+0x98 /opt/go/src/sync/cond.go:56 # 0x179a595 github.com/nats-io/nats%2ego.(*Conn).waitForMsgs+0x635 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2626 4 @ 0x43d0c5 0x40854f 0x4081cb 0x179ba8f 0x474801 # 0x179ba8e github.com/nats-io/nats%2ego.(*Conn).flusher+0xee /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2978 4 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x150a08c 0x170fa25 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x150a08b github.com/nats-io/nats-server/v2/server.(*client).readLoop+0x110b /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/client.go:1187 # 0x170fa24 github.com/nats-io/nats-server/v2/server.(*Server).createClient.func1+0x44 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:2568 4 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x1795358 0x1799e08 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x1795357 github.com/nats-io/nats%2ego.(*natsReader).Read+0xb7 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:1649 # 0x1799e07 github.com/nats-io/nats%2ego.(*Conn).readLoop+0xc7 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2584 4 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x773819 0x544ce8 0x545a5d 0x545cb4 0x705c96 0x76da4a 0x76da4b 0x774cbd 0x779945 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x773818 net/http.(*connReader).Read+0x1b8 /opt/go/src/net/http/server.go:780 # 0x544ce7 bufio.(*Reader).fill+0x107 /opt/go/src/bufio/bufio.go:101 # 0x545a5c bufio.(*Reader).ReadSlice+0x3c /opt/go/src/bufio/bufio.go:360 # 0x545cb3 bufio.(*Reader).ReadLine+0x33 /opt/go/src/bufio/bufio.go:389 # 0x705c95 net/textproto.(*Reader).readLineSlice+0xd5 /opt/go/src/net/textproto/reader.go:57 # 0x76da49 net/textproto.(*Reader).ReadLine+0xa9 /opt/go/src/net/textproto/reader.go:38 # 0x76da4a net/http.readRequest+0xaa /opt/go/src/net/http/request.go:1027 # 0x774cbc net/http.(*conn).readRequest+0x19c /opt/go/src/net/http/server.go:966 # 0x779944 net/http.(*conn).serve+0x704 /opt/go/src/net/http/server.go:1858 4 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0x1508abe 0x170fa6a 0x474801 # 0x4709ed sync.runtime_notifyListWait+0xcd /opt/go/src/runtime/sema.go:513 # 0x48c838 sync.(*Cond).Wait+0x98 /opt/go/src/sync/cond.go:56 # 0x1508abd github.com/nats-io/nats-server/v2/server.(*client).writeLoop+0x23d /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/client.go:1061 # 0x170fa69 github.com/nats-io/nats-server/v2/server.(*Server).createClient.func2+0x29 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:2571 4 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0x1799a75 0x474801 # 0x4709ed sync.runtime_notifyListWait+0xcd /opt/go/src/runtime/sema.go:513 # 0x48c838 sync.(*Cond).Wait+0x98 /opt/go/src/sync/cond.go:56 # 0x1799a74 github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher+0x34 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2509 4 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0xff22d3 0x474801 # 0x4709ed sync.runtime_notifyListWait+0xcd /opt/go/src/runtime/sema.go:513 # 0x48c838 sync.(*Cond).Wait+0x98 /opt/go/src/sync/cond.go:56 # 0xff22d2 github.com/nats-io/go-nats.(*Conn).waitForMsgs+0x452 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:2014 3 @ 0x43d0c5 0x44e2d7 0x8a6275 0x474801 # 0x8a6274 database/sql.(*DB).connectionOpener+0xb4 /opt/go/src/database/sql/sql.go:1133 2 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x4f1cb7 0x4f1c87 0x545382 0x1372f25 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x4f1cb6 os.(*File).read+0x76 /opt/go/src/os/file_posix.go:31 # 0x4f1c86 os.(*File).Read+0x46 /opt/go/src/os/file.go:117 # 0x545381 bufio.(*Reader).Read+0x221 /opt/go/src/bufio/bufio.go:227 # 0x1372f24 github.com/hashicorp/go-plugin.copyChan+0xc4 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_stdio.go:181 2 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x794917 0x544ce8 0x544e6f 0x795608 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x794916 net/http.(*persistConn).Read+0x76 /opt/go/src/net/http/transport.go:1922 # 0x544ce7 bufio.(*Reader).fill+0x107 /opt/go/src/bufio/bufio.go:101 # 0x544e6e bufio.(*Reader).Peek+0x4e /opt/go/src/bufio/bufio.go:139 # 0x795607 net/http.(*persistConn).readLoop+0x1a7 /opt/go/src/net/http/transport.go:2083 2 @ 0x43d0c5 0x44e2d7 0x7971d7 0x474801 # 0x7971d6 net/http.(*persistConn).writeLoop+0xf6 /opt/go/src/net/http/transport.go:2382 1 @ 0x40e414 0x470f85 0x7c73e5 0x474801 # 0x470f84 os/signal.signal_recv+0xa4 /opt/go/src/runtime/sigqueue.go:168 # 0x7c73e4 os/signal.loop+0x24 /opt/go/src/os/signal/signal_unix.go:23 1 @ 0x43d0c5 0x40767a 0x4073d5 0x2645f3c 0x264c3ad 0x474801 # 0x2645f3b github.com/actiontech/dtle/drivers/mysql/mysql.(*Applier).onError+0x1bb /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/applier.go:1002 # 0x264c3ac github.com/actiontech/dtle/drivers/mysql/mysql.(*ApplierIncr).MtsWorker+0x60c /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/applier_incr.go:194 1 @ 0x43d0c5 0x40854f 0x40818b 0x1378ce9 0x474801 # 0x1378ce8 github.com/hashicorp/go-plugin.Serve.func3+0x88 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/server.go:433 1 @ 0x43d0c5 0x40854f 0x40818b 0x1478e98 0x13f6656 0xbd1658 0xbd2d28 0xbe0deb 0x474801 # 0x1478e97 github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).TaskEvents+0x97 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:358 # 0x13f6655 github.com/hashicorp/nomad/plugins/drivers/proto._Driver_TaskEvents_Handler+0x115 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4530 # 0xbd1657 google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329 # 0xbd2d27 google.golang.org/grpc.(*Server).handleStream+0xc67 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 1 @ 0x43d0c5 0x40854f 0x40818b 0x855dc9 0x474801 # 0x855dc8 github.com/actiontech/dtle/g.DumpLoop+0xa8 /universe/src/github.com/actiontech/dtle/g/dump.go:21 1 @ 0x43d0c5 0x40854f 0x40818b 0x856c2f 0x474801 # 0x856c2e github.com/actiontech/dtle/g.FreeMemoryWorker+0x4e /universe/src/github.com/actiontech/dtle/g/g.go:93 1 @ 0x43d0c5 0x40854f 0x40818b 0x85706b 0x474801 # 0x85706a github.com/actiontech/dtle/g.MemoryMonitor+0x6a /universe/src/github.com/actiontech/dtle/g/g.go:142 1 @ 0x43d0c5 0x40854f 0x4081cb 0xff2e6a 0x474801 # 0xff2e69 github.com/nats-io/go-nats.(*Conn).flusher+0xe9 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:2219 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x545382 0x4bb1a7 0x8c7b91 0x8c7b50 0x8c7729 0x8d6a2d 0x8d6a02 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x545381 bufio.(*Reader).Read+0x221 /opt/go/src/bufio/bufio.go:227 # 0x4bb1a6 io.ReadAtLeast+0x86 /opt/go/src/io/io.go:328 # 0x8c7b90 io.ReadFull+0x70 /opt/go/src/io/io.go:347 # 0x8c7b4f github.com/go-mysql-org/go-mysql/packet.(*Conn).ReadPacketTo+0x2f /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/packet/conn.go:132 # 0x8c7728 github.com/go-mysql-org/go-mysql/packet.(*Conn).ReadPacketReuseMem+0xa8 /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/packet/conn.go:98 # 0x8d6a2c github.com/go-mysql-org/go-mysql/packet.(*Conn).ReadPacket+0x30c /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/packet/conn.go:90 # 0x8d6a01 github.com/go-mysql-org/go-mysql/replication.(*BinlogSyncer).onStream+0x2e1 /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go:670 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x545382 0x4bb1a7 0xb51d69 0xb51d02 0xb525e5 0xb8bc3b 0xbcd282 0xbe0cff 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x545381 bufio.(*Reader).Read+0x221 /opt/go/src/bufio/bufio.go:227 # 0x4bb1a6 io.ReadAtLeast+0x86 /opt/go/src/io/io.go:328 # 0xb51d68 io.ReadFull+0x88 /opt/go/src/io/io.go:347 # 0xb51d01 golang.org/x/net/http2.readFrameHeader+0x21 /universe/src/github.com/actiontech/dtle/vendor/golang.org/x/net/http2/frame.go:237 # 0xb525e4 golang.org/x/net/http2.(*Framer).ReadFrame+0xa4 /universe/src/github.com/actiontech/dtle/vendor/golang.org/x/net/http2/frame.go:498 # 0xb8bc3a google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams+0x9a /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 # 0xbcd281 google.golang.org/grpc.(*Server).serveStreams+0xe1 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:742 # 0xbe0cfe google.golang.org/grpc.(*Server).handleRawConn.func1+0x3e /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:703 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x773819 0x544ce8 0x544e6f 0x7759e5 0x779945 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x773818 net/http.(*connReader).Read+0x1b8 /opt/go/src/net/http/server.go:780 # 0x544ce7 bufio.(*Reader).fill+0x107 /opt/go/src/bufio/bufio.go:101 # 0x544e6e bufio.(*Reader).Peek+0x4e /opt/go/src/bufio/bufio.go:139 # 0x7759e4 net/http.(*conn).readRequest+0xec4 /opt/go/src/net/http/server.go:963 # 0x779944 net/http.(*conn).serve+0x704 /opt/go/src/net/http/server.go:1858 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0xff1baa 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0xff1ba9 github.com/nats-io/go-nats.(*Conn).readLoop+0x189 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:1978 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4ea4b2 0x4ea494 0x6534e5 0x673172 0x671dc5 0x25d6f6f 0x77ea85 0x25d4b33 0x2917775 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4ea4b1 internal/poll.(*pollDesc).waitRead+0x211 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4ea493 internal/poll.(*FD).Accept+0x1f3 /opt/go/src/internal/poll/fd_unix.go:401 # 0x6534e4 net.(*netFD).accept+0x44 /opt/go/src/net/fd_unix.go:172 # 0x673171 net.(*TCPListener).accept+0x31 /opt/go/src/net/tcpsock_posix.go:139 # 0x671dc4 net.(*TCPListener).AcceptTCP+0x64 /opt/go/src/net/tcpsock.go:248 # 0x25d6f6e github.com/labstack/echo/v4.tcpKeepAliveListener.Accept+0x2e /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:963 # 0x77ea84 net/http.(*Server).Serve+0x284 /opt/go/src/net/http/server.go:2961 # 0x25d4b32 github.com/labstack/echo/v4.(*Echo).Start+0xf2 /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:671 # 0x2917774 github.com/actiontech/dtle/drivers/api.SetupApiServer.func3+0x74 /universe/src/github.com/actiontech/dtle/drivers/api/route.go:179 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4ea4b2 0x4ea494 0x6534e5 0x673172 0x671f65 0x16bc162 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4ea4b1 internal/poll.(*pollDesc).waitRead+0x211 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4ea493 internal/poll.(*FD).Accept+0x1f3 /opt/go/src/internal/poll/fd_unix.go:401 # 0x6534e4 net.(*netFD).accept+0x44 /opt/go/src/net/fd_unix.go:172 # 0x673171 net.(*TCPListener).accept+0x31 /opt/go/src/net/tcpsock_posix.go:139 # 0x671f64 net.(*TCPListener).Accept+0x64 /opt/go/src/net/tcpsock.go:261 # 0x16bc161 github.com/nats-io/nats-server/v2/server.(*Server).acceptConnections+0x41 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:2088 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4ea4b2 0x4ea494 0x6534e5 0x6797f2 0x677a45 0xbcc07f 0x137294e 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4ea4b1 internal/poll.(*pollDesc).waitRead+0x211 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4ea493 internal/poll.(*FD).Accept+0x1f3 /opt/go/src/internal/poll/fd_unix.go:401 # 0x6534e4 net.(*netFD).accept+0x44 /opt/go/src/net/fd_unix.go:172 # 0x6797f1 net.(*UnixListener).accept+0x31 /opt/go/src/net/unixsock_posix.go:162 # 0x677a44 net.(*UnixListener).Accept+0x64 /opt/go/src/net/unixsock.go:260 # 0xbcc07e google.golang.org/grpc.(*Server).Serve+0x27e /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:621 # 0x137294d github.com/hashicorp/go-plugin.(*GRPCServer).Serve+0x6d /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_server.go:138 1 @ 0x43d0c5 0x44e2d7 0x11a4d09 0x266d0f1 0x474801 # 0x11a4d08 github.com/actiontech/dtle/drivers/mysql/common.RegularlyUpdateJobStatus+0xe8 /universe/src/github.com/actiontech/dtle/drivers/mysql/common/common.go:170 # 0x266d0f0 github.com/actiontech/dtle/drivers/mysql/mysql.(*Extractor).initiateStreaming.func2+0x90 /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:691 1 @ 0x43d0c5 0x44e2d7 0x11b8474 0x474801 # 0x11b8473 github.com/actiontech/dtle/drivers/mysql/common.(*StoreManager).SrcWatchNats.func1+0x93 /universe/src/github.com/actiontech/dtle/drivers/mysql/common/store.go:219 1 @ 0x43d0c5 0x44e2d7 0x1370646 0x13713da 0x474801 # 0x1370645 github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv+0x85 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:121 # 0x13713d9 github.com/hashicorp/go-plugin.(*GRPCBroker).Run+0x79 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:411 1 @ 0x43d0c5 0x44e2d7 0x1372d25 0x135deb3 0xbd1658 0xbd2d28 0xbe0deb 0x474801 # 0x1372d24 github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio+0x144 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_stdio.go:58 # 0x135deb2 github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler+0x112 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/internal/plugin/grpc_stdio.pb.go:185 # 0xbd1657 google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329 # 0xbd2d27 google.golang.org/grpc.(*Server).handleStream+0xc67 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 1 @ 0x43d0c5 0x44e2d7 0x1376994 0x14762df 0x2918a28 0x29188be 0x2918d17 0x43cc96 0x474801 # 0x1376993 github.com/hashicorp/go-plugin.Serve+0x953 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/server.go:468 # 0x14762de github.com/hashicorp/nomad/plugins/drivers.Serve+0x23e /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/plugin.go:53 # 0x2918a27 github.com/hashicorp/nomad/plugins.serve+0x147 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/serve.go:47 # 0x29188bd github.com/hashicorp/nomad/plugins.Serve+0xbd /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/serve.go:26 # 0x2918d16 main.main+0x2d6 /universe/src/github.com/actiontech/dtle/cmd/nomad-plugin/main.go:30 # 0x43cc95 runtime.main+0x255 /opt/go/src/runtime/proc.go:225 1 @ 0x43d0c5 0x44e2d7 0x137878f 0x474801 # 0x137878e github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1+0x10e /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:70 1 @ 0x43d0c5 0x44e2d7 0x1476b25 0x13f5293 0xbd1658 0xbd2d28 0xbe0deb 0x474801 # 0x1476b24 github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).Fingerprint+0x124 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:75 # 0x13f5292 github.com/hashicorp/nomad/plugins/drivers/proto._Driver_Fingerprint_Handler+0x112 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4380 # 0xbd1657 google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329 # 0xbd2d27 google.golang.org/grpc.(*Server).handleStream+0xc67 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 1 @ 0x43d0c5 0x44e2d7 0x1477505 0x13f5ad7 0xbcec4b 0xbd2d8f 0xbe0deb 0x474801 # 0x1477504 github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).WaitTask+0x124 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:148 # 0x13f5ad6 github.com/hashicorp/nomad/plugins/drivers/proto._Driver_WaitTask_Handler+0x216 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4438 # 0xbcec4a google.golang.org/grpc.(*Server).processUnaryRPC+0x52a /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1082 # 0xbd2d8e google.golang.org/grpc.(*Server).handleStream+0xcce /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1405 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 1 @ 0x43d0c5 0x44e2d7 0x148090d 0x474801 # 0x148090c github.com/hashicorp/nomad/drivers/shared/eventer.(*Eventer).eventLoop+0xec /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/drivers/shared/eventer/eventer.go:66 1 @ 0x43d0c5 0x44e2d7 0x153f697 0x474801 # 0x153f696 github.com/nats-io/nats-server/v2/server.(*Server).internalSendLoop+0x116 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/events.go:319 1 @ 0x43d0c5 0x44e2d7 0x16afe85 0x474801 # 0x16afe84 github.com/nats-io/nats-server/v2/server.(*sendq).internalLoop+0x464 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/sendq.go:54 1 @ 0x43d0c5 0x44e2d7 0x16c82fc 0x474801 # 0x16c82fb github.com/nats-io/nats-server/v2/server.(*srvIPQueueLogger).run+0x11b /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:3645 1 @ 0x43d0c5 0x44e2d7 0x1700885 0x474801 # 0x1700884 github.com/nats-io/nats-server/v2/server.(*Server).startGWReplyMapExpiration.func1+0x124 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/gateway.go:3143 1 @ 0x43d0c5 0x44e2d7 0x1710349 0x474801 # 0x1710348 github.com/nats-io/nats-server/v2/server.(*Server).handleSignals.func1+0xa8 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/signal.go:48 1 @ 0x43d0c5 0x44e2d7 0x18459af 0x474801 # 0x18459ae github.com/nats-io/nats-streaming-server/server.(*StanServer).ioLoop+0x74e /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-streaming-server/server/server.go:4379 1 @ 0x43d0c5 0x44e2d7 0x184bf45 0x474801 # 0x184bf44 github.com/nats-io/nats-streaming-server/server.(*StanServer).processSubscriptionsStart+0xe4 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-streaming-server/server/server.go:5393 1 @ 0x43d0c5 0x44e2d7 0x242746d 0x474801 # 0x242746c go.opencensus.io/stats/view.(*worker).start+0xcc /universe/src/github.com/actiontech/dtle/vendor/go.opencensus.io/stats/view/worker.go:276 1 @ 0x43d0c5 0x44e2d7 0x265e8cf 0x474801 # 0x265e8ce github.com/actiontech/dtle/drivers/mysql/mysql.(*TimestampContext).Handle+0xce /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:952 1 @ 0x43d0c5 0x44e2d7 0x266da45 0x474801 # 0x266da44 github.com/actiontech/dtle/drivers/mysql/mysql.(*Extractor).StreamEvents.func1+0x1c4 /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:1020 1 @ 0x43d0c5 0x44e2d7 0x27e38dd 0x474801 # 0x27e38dc github.com/actiontech/dtle/drivers/mysql.(*Driver).handleFingerprint+0x13c /universe/src/github.com/actiontech/dtle/drivers/mysql/driver.go:404 1 @ 0x43d0c5 0x44e2d7 0x27e5017 0x474801 # 0x27e5016 github.com/actiontech/dtle/drivers/mysql.(*Driver).handleWait+0x176 /universe/src/github.com/actiontech/dtle/drivers/mysql/driver.go:578 1 @ 0x43d0c5 0x44e2d7 0x27e58e5 0x474801 # 0x27e58e4 github.com/actiontech/dtle/drivers/mysql.(*Driver).handleStats+0xc4 /universe/src/github.com/actiontech/dtle/drivers/mysql/driver.go:646 1 @ 0x43d0c5 0x44e2d7 0x27ea795 0x474801 # 0x27ea794 github.com/actiontech/dtle/drivers/mysql.(*taskHandle).run.func1+0xf4 /universe/src/github.com/actiontech/dtle/drivers/mysql/handle.go:139 1 @ 0x43d0c5 0x44e2d7 0x797e85 0x78c24b 0x771815 0x72d1b4 0x72cb9f 0x72ec1f 0x1039cbe 0x1039c86 0x104a754 0x1049925 0x106023f 0x474801 # 0x797e84 net/http.(*persistConn).roundTrip+0x764 /opt/go/src/net/http/transport.go:2610 # 0x78c24a net/http.(*Transport).roundTrip+0xaca /opt/go/src/net/http/transport.go:592 # 0x771814 net/http.(*Transport).RoundTrip+0x34 /opt/go/src/net/http/roundtrip.go:17 # 0x72d1b3 net/http.send+0x453 /opt/go/src/net/http/client.go:251 # 0x72cb9e net/http.(*Client).send+0xfe /opt/go/src/net/http/client.go:175 # 0x72ec1e net/http.(*Client).do+0x45e /opt/go/src/net/http/client.go:717 # 0x1039cbd net/http.(*Client).Do+0xbd /opt/go/src/net/http/client.go:585 # 0x1039c85 github.com/hashicorp/consul/api.(*Client).doRequest+0x85 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/consul/api/api.go:880 # 0x104a753 github.com/hashicorp/consul/api.(*KV).getInternal+0x2b3 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/consul/api/kv.go:131 # 0x1049924 github.com/hashicorp/consul/api.(*KV).Get+0xa4 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/consul/api/kv.go:65 # 0x106023e github.com/docker/libkv/store/consul.(*Consul).Watch.func1+0xfe /universe/src/github.com/actiontech/dtle/vendor/github.com/docker/libkv/store/consul/consul.go:299 1 @ 0x43d0c5 0x44e2d7 0x8d30e5 0x2635558 0x265eb88 0x266d010 0x474801 # 0x8d30e4 github.com/go-mysql-org/go-mysql/replication.(*BinlogStreamer).GetEvent+0x104 /universe/src/github.com/actiontech/dtle/vendor/github.com/go-mysql-org/go-mysql/replication/binlogstreamer.go:32 # 0x2635557 github.com/actiontech/dtle/drivers/mysql/mysql/binlog.(*BinlogReader).DataStreamEvents+0x177 /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/binlog/binlog_reader.go:927 # 0x265eb87 github.com/actiontech/dtle/drivers/mysql/mysql.(*Extractor).StreamEvents+0x87 /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:1065 # 0x266d00f github.com/actiontech/dtle/drivers/mysql/mysql.(*Extractor).initiateStreaming.func1+0x8f /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:684 1 @ 0x43d0c5 0x44e2d7 0xb76eff 0xb7785d 0xb9a017 0x474801 # 0xb76efe google.golang.org/grpc/internal/transport.(*controlBuffer).get+0xfe /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 # 0xb7785c google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x1dc /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 # 0xb9a016 google.golang.org/grpc/internal/transport.newHTTP2Server.func2+0xd6 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 1 @ 0x43d0c5 0x44e2d7 0xb90145 0x474801 # 0xb90144 google.golang.org/grpc/internal/transport.(*http2Server).keepalive+0x264 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 1 @ 0x43d0c5 0x44e2d7 0xb960c5 0xb95f91 0xb96ff5 0x4bb1a7 0xb96f32 0xb96eef 0xbc8523 0xbc914d 0xbc9a5b 0xbdd856 0x135ce42 0x13701aa 0x135cd0d 0xbd1658 0xbd2d28 0xbe0deb 0x474801 # 0xb960c4 google.golang.org/grpc/internal/transport.(*recvBufferReader).read+0xa4 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:174 # 0xb95f90 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x210 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:168 # 0xb96ff4 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x54 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:479 # 0x4bb1a6 io.ReadAtLeast+0x86 /opt/go/src/io/io.go:328 # 0xb96f31 io.ReadFull+0xd1 /opt/go/src/io/io.go:347 # 0xb96eee google.golang.org/grpc/internal/transport.(*Stream).Read+0x8e /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:463 # 0xbc8522 google.golang.org/grpc.(*parser).recvMsg+0x62 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/rpc_util.go:510 # 0xbc914c google.golang.org/grpc.recvAndDecompress+0x4c /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/rpc_util.go:641 # 0xbc9a5a google.golang.org/grpc.recv+0x9a /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/rpc_util.go:709 # 0xbdd855 google.golang.org/grpc.(*serverStream).RecvMsg+0x155 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/stream.go:1473 # 0x135ce41 github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv+0x61 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/internal/plugin/grpc_broker.pb.go:167 # 0x13701a9 github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream+0x109 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:84 # 0x135cd0c github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler+0xac /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/internal/plugin/grpc_broker.pb.go:148 # 0xbd1657 google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329 # 0xbd2d27 google.golang.org/grpc.(*Server).handleStream+0xc67 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 1 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0xff1755 0x474801 # 0x4709ed sync.runtime_notifyListWait+0xcd /opt/go/src/runtime/sema.go:513 # 0x48c838 sync.(*Cond).Wait+0x98 /opt/go/src/sync/cond.go:56 # 0xff1754 github.com/nats-io/go-nats.(*asyncCallbacksHandler).asyncCBDispatcher+0x34 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:1893 1 @ 0x43d0c5 0x471832 0x101eb8f 0x474801 # 0x471831 time.Sleep+0xd1 /opt/go/src/runtime/time.go:193 # 0x101eb8e github.com/armon/go-metrics.(*Metrics).collectStats+0x2e /universe/src/github.com/actiontech/dtle/vendor/github.com/armon/go-metrics/metrics.go:230 1 @ 0x46e31d 0x7b644e 0x7b6225 0x7b2db2 0x7c47c5 0x7c6017 0x77b024 0x77cead 0x25dc65f 0x25dc3e2 0x25d479c 0x77e6a3 0x779b0d 0x474801 # 0x46e31c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /opt/go/src/runtime/mprof.go:716 # 0x7b644d runtime/pprof.writeRuntimeProfile+0xcd /opt/go/src/runtime/pprof/pprof.go:724 # 0x7b6224 runtime/pprof.writeGoroutine+0xa4 /opt/go/src/runtime/pprof/pprof.go:684 # 0x7b2db1 runtime/pprof.(*Profile).WriteTo+0x3f1 /opt/go/src/runtime/pprof/pprof.go:331 # 0x7c47c4 net/http/pprof.handler.ServeHTTP+0x384 /opt/go/src/net/http/pprof/pprof.go:253 # 0x7c6016 net/http/pprof.Index+0x8d6 /opt/go/src/net/http/pprof/pprof.go:371 # 0x77b023 net/http.HandlerFunc.ServeHTTP+0x43 /opt/go/src/net/http/server.go:2049 # 0x77ceac net/http.(*ServeMux).ServeHTTP+0x1ac /opt/go/src/net/http/server.go:2428 # 0x25dc65e github.com/labstack/echo/v4.WrapHandler.func1+0x9e /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:901 # 0x25dc3e1 github.com/labstack/echo/v4.(*Echo).add.func1+0x61 /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:544 # 0x25d479b github.com/labstack/echo/v4.(*Echo).ServeHTTP+0x2bb /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:654 # 0x77e6a2 net/http.serverHandler.ServeHTTP+0xa2 /opt/go/src/net/http/server.go:2867 # 0x779b0c net/http.(*conn).serve+0x8cc /opt/go/src/net/http/server.go:1932 1 @ 0x4ca7c5 0x4c7efa 0x4e86a5 0x4e867b 0x4e8661 0x651eaf 0x666c31 0x773298 0x474801 # 0x4ca7c4 syscall.Syscall+0x4 /opt/go/src/syscall/asm_linux_amd64.s:20 # 0x4c7ef9 syscall.read+0x59 /opt/go/src/syscall/zsyscall_linux_amd64.go:686 # 0x4e86a4 syscall.Read+0x144 /opt/go/src/syscall/syscall_unix.go:187 # 0x4e867a internal/poll.ignoringEINTRIO+0x11a /opt/go/src/internal/poll/fd_unix.go:581 # 0x4e8660 internal/poll.(*FD).Read+0x100 /opt/go/src/internal/poll/fd_unix.go:162 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x773297 net/http.(*connReader).backgroundRead+0x57 /opt/go/src/net/http/server.go:672 ```
asiroliu commented 2 years ago

dest /debug/pprof/goroutine?debug=1

``` goroutine profile: total 171 32 @ 0x43d0c5 0x44e2d7 0x1881ca5 0x474801 # 0x1881ca4 github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1+0xa4 /universe/src/github.com/actiontech/dtle/vendor/github.com/go-sql-driver/mysql/connection.go:614 32 @ 0x43d0c5 0x44e2d7 0x264bf87 0x474801 # 0x264bf86 github.com/actiontech/dtle/drivers/mysql/mysql.(*ApplierIncr).MtsWorker+0x1e6 /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/applier_incr.go:187 15 @ 0x43d0c5 0x40854f 0x4081cb 0x1477dbd 0x13f6456 0xbd1658 0xbd2d28 0xbe0deb 0x474801 # 0x1477dbc github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).TaskStats+0x19c /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:251 # 0x13f6455 github.com/hashicorp/nomad/plugins/drivers/proto._Driver_TaskStats_Handler+0x115 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4509 # 0xbd1657 google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329 # 0xbd2d27 google.golang.org/grpc.(*Server).handleStream+0xc67 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 8 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0x179a596 0x474801 # 0x4709ed sync.runtime_notifyListWait+0xcd /opt/go/src/runtime/sema.go:513 # 0x48c838 sync.(*Cond).Wait+0x98 /opt/go/src/sync/cond.go:56 # 0x179a595 github.com/nats-io/nats%2ego.(*Conn).waitForMsgs+0x635 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2626 6 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x150a08c 0x170fa25 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x150a08b github.com/nats-io/nats-server/v2/server.(*client).readLoop+0x110b /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/client.go:1187 # 0x170fa24 github.com/nats-io/nats-server/v2/server.(*Server).createClient.func1+0x44 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:2568 6 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0x1508abe 0x170fa6a 0x474801 # 0x4709ed sync.runtime_notifyListWait+0xcd /opt/go/src/runtime/sema.go:513 # 0x48c838 sync.(*Cond).Wait+0x98 /opt/go/src/sync/cond.go:56 # 0x1508abd github.com/nats-io/nats-server/v2/server.(*client).writeLoop+0x23d /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/client.go:1061 # 0x170fa69 github.com/nats-io/nats-server/v2/server.(*Server).createClient.func2+0x29 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:2571 4 @ 0x43d0c5 0x40854f 0x4081cb 0x179ba8f 0x474801 # 0x179ba8e github.com/nats-io/nats%2ego.(*Conn).flusher+0xee /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2978 4 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x1795358 0x1799e08 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x1795357 github.com/nats-io/nats%2ego.(*natsReader).Read+0xb7 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:1649 # 0x1799e07 github.com/nats-io/nats%2ego.(*Conn).readLoop+0xc7 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2584 4 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0x1799a75 0x474801 # 0x4709ed sync.runtime_notifyListWait+0xcd /opt/go/src/runtime/sema.go:513 # 0x48c838 sync.(*Cond).Wait+0x98 /opt/go/src/sync/cond.go:56 # 0x1799a74 github.com/nats-io/nats%2ego.(*asyncCallbacksHandler).asyncCBDispatcher+0x34 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats.go/nats.go:2509 3 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0xff22d3 0x474801 # 0x4709ed sync.runtime_notifyListWait+0xcd /opt/go/src/runtime/sema.go:513 # 0x48c838 sync.(*Cond).Wait+0x98 /opt/go/src/sync/cond.go:56 # 0xff22d2 github.com/nats-io/go-nats.(*Conn).waitForMsgs+0x452 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:2014 2 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x4f1cb7 0x4f1c87 0x545382 0x1372f25 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x4f1cb6 os.(*File).read+0x76 /opt/go/src/os/file_posix.go:31 # 0x4f1c86 os.(*File).Read+0x46 /opt/go/src/os/file.go:117 # 0x545381 bufio.(*Reader).Read+0x221 /opt/go/src/bufio/bufio.go:227 # 0x1372f24 github.com/hashicorp/go-plugin.copyChan+0xc4 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_stdio.go:181 2 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x794917 0x544ce8 0x544e6f 0x795608 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x794916 net/http.(*persistConn).Read+0x76 /opt/go/src/net/http/transport.go:1922 # 0x544ce7 bufio.(*Reader).fill+0x107 /opt/go/src/bufio/bufio.go:101 # 0x544e6e bufio.(*Reader).Peek+0x4e /opt/go/src/bufio/bufio.go:139 # 0x795607 net/http.(*persistConn).readLoop+0x1a7 /opt/go/src/net/http/transport.go:2083 2 @ 0x43d0c5 0x44e2d7 0x7971d7 0x474801 # 0x7971d6 net/http.(*persistConn).writeLoop+0xf6 /opt/go/src/net/http/transport.go:2382 2 @ 0x43d0c5 0x44e2d7 0x797e85 0x78c24b 0x771815 0x72d1b4 0x72cb9f 0x72ec1f 0x1039cbe 0x1039c86 0x104a754 0x1049925 0x106023f 0x474801 # 0x797e84 net/http.(*persistConn).roundTrip+0x764 /opt/go/src/net/http/transport.go:2610 # 0x78c24a net/http.(*Transport).roundTrip+0xaca /opt/go/src/net/http/transport.go:592 # 0x771814 net/http.(*Transport).RoundTrip+0x34 /opt/go/src/net/http/roundtrip.go:17 # 0x72d1b3 net/http.send+0x453 /opt/go/src/net/http/client.go:251 # 0x72cb9e net/http.(*Client).send+0xfe /opt/go/src/net/http/client.go:175 # 0x72ec1e net/http.(*Client).do+0x45e /opt/go/src/net/http/client.go:717 # 0x1039cbd net/http.(*Client).Do+0xbd /opt/go/src/net/http/client.go:585 # 0x1039c85 github.com/hashicorp/consul/api.(*Client).doRequest+0x85 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/consul/api/api.go:880 # 0x104a753 github.com/hashicorp/consul/api.(*KV).getInternal+0x2b3 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/consul/api/kv.go:131 # 0x1049924 github.com/hashicorp/consul/api.(*KV).Get+0xa4 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/consul/api/kv.go:65 # 0x106023e github.com/docker/libkv/store/consul.(*Consul).Watch.func1+0xfe /universe/src/github.com/actiontech/dtle/vendor/github.com/docker/libkv/store/consul/consul.go:299 1 @ 0x40e414 0x470f85 0x7c73e5 0x474801 # 0x470f84 os/signal.signal_recv+0xa4 /opt/go/src/runtime/sigqueue.go:168 # 0x7c73e4 os/signal.loop+0x24 /opt/go/src/os/signal/signal_unix.go:23 1 @ 0x43d0c5 0x40854f 0x40818b 0x11a9b1b 0x2646794 0x474801 # 0x11a9b1a github.com/actiontech/dtle/drivers/mysql/common.(*StoreManager).WatchTargetGtid+0xfa /universe/src/github.com/actiontech/dtle/drivers/mysql/common/store.go:392 # 0x2646793 github.com/actiontech/dtle/drivers/mysql/mysql.(*Applier).watchTargetGtid+0x53 /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/applier.go:1046 1 @ 0x43d0c5 0x40854f 0x40818b 0x1378ce9 0x474801 # 0x1378ce8 github.com/hashicorp/go-plugin.Serve.func3+0x88 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/server.go:433 1 @ 0x43d0c5 0x40854f 0x40818b 0x1478e98 0x13f6656 0xbd1658 0xbd2d28 0xbe0deb 0x474801 # 0x1478e97 github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).TaskEvents+0x97 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:358 # 0x13f6655 github.com/hashicorp/nomad/plugins/drivers/proto._Driver_TaskEvents_Handler+0x115 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4530 # 0xbd1657 google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329 # 0xbd2d27 google.golang.org/grpc.(*Server).handleStream+0xc67 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 1 @ 0x43d0c5 0x40854f 0x40818b 0x266c63f 0x474801 # 0x266c63e github.com/actiontech/dtle/drivers/mysql/mysql.(*Extractor).Run.func2+0x3e /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:301 1 @ 0x43d0c5 0x40854f 0x40818b 0x855dc9 0x474801 # 0x855dc8 github.com/actiontech/dtle/g.DumpLoop+0xa8 /universe/src/github.com/actiontech/dtle/g/dump.go:21 1 @ 0x43d0c5 0x40854f 0x40818b 0x856c2f 0x474801 # 0x856c2e github.com/actiontech/dtle/g.FreeMemoryWorker+0x4e /universe/src/github.com/actiontech/dtle/g/g.go:93 1 @ 0x43d0c5 0x40854f 0x40818b 0x85706b 0x474801 # 0x85706a github.com/actiontech/dtle/g.MemoryMonitor+0x6a /universe/src/github.com/actiontech/dtle/g/g.go:142 1 @ 0x43d0c5 0x40854f 0x4081cb 0xff2e6a 0x474801 # 0xff2e69 github.com/nats-io/go-nats.(*Conn).flusher+0xe9 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:2219 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0x545382 0x4bb1a7 0xb51d69 0xb51d02 0xb525e5 0xb8bc3b 0xbcd282 0xbe0cff 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x545381 bufio.(*Reader).Read+0x221 /opt/go/src/bufio/bufio.go:227 # 0x4bb1a6 io.ReadAtLeast+0x86 /opt/go/src/io/io.go:328 # 0xb51d68 io.ReadFull+0x88 /opt/go/src/io/io.go:347 # 0xb51d01 golang.org/x/net/http2.readFrameHeader+0x21 /universe/src/github.com/actiontech/dtle/vendor/golang.org/x/net/http2/frame.go:237 # 0xb525e4 golang.org/x/net/http2.(*Framer).ReadFrame+0xa4 /universe/src/github.com/actiontech/dtle/vendor/golang.org/x/net/http2/frame.go:498 # 0xb8bc3a google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams+0x9a /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 # 0xbcd281 google.golang.org/grpc.(*Server).serveStreams+0xe1 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:742 # 0xbe0cfe google.golang.org/grpc.(*Server).handleRawConn.func1+0x3e /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:703 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4e8735 0x4e8717 0x651eaf 0x666c31 0xff1baa 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4e8734 internal/poll.(*pollDesc).waitRead+0x1d4 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4e8716 internal/poll.(*FD).Read+0x1b6 /opt/go/src/internal/poll/fd_unix.go:166 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0xff1ba9 github.com/nats-io/go-nats.(*Conn).readLoop+0x189 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:1978 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4ea4b2 0x4ea494 0x6534e5 0x673172 0x671dc5 0x25d6f6f 0x77ea85 0x25d4b33 0x2917775 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4ea4b1 internal/poll.(*pollDesc).waitRead+0x211 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4ea493 internal/poll.(*FD).Accept+0x1f3 /opt/go/src/internal/poll/fd_unix.go:401 # 0x6534e4 net.(*netFD).accept+0x44 /opt/go/src/net/fd_unix.go:172 # 0x673171 net.(*TCPListener).accept+0x31 /opt/go/src/net/tcpsock_posix.go:139 # 0x671dc4 net.(*TCPListener).AcceptTCP+0x64 /opt/go/src/net/tcpsock.go:248 # 0x25d6f6e github.com/labstack/echo/v4.tcpKeepAliveListener.Accept+0x2e /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:963 # 0x77ea84 net/http.(*Server).Serve+0x284 /opt/go/src/net/http/server.go:2961 # 0x25d4b32 github.com/labstack/echo/v4.(*Echo).Start+0xf2 /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:671 # 0x2917774 github.com/actiontech/dtle/drivers/api.SetupApiServer.func3+0x74 /universe/src/github.com/actiontech/dtle/drivers/api/route.go:179 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4ea4b2 0x4ea494 0x6534e5 0x673172 0x671f65 0x16bc162 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4ea4b1 internal/poll.(*pollDesc).waitRead+0x211 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4ea493 internal/poll.(*FD).Accept+0x1f3 /opt/go/src/internal/poll/fd_unix.go:401 # 0x6534e4 net.(*netFD).accept+0x44 /opt/go/src/net/fd_unix.go:172 # 0x673171 net.(*TCPListener).accept+0x31 /opt/go/src/net/tcpsock_posix.go:139 # 0x671f64 net.(*TCPListener).Accept+0x64 /opt/go/src/net/tcpsock.go:261 # 0x16bc161 github.com/nats-io/nats-server/v2/server.(*Server).acceptConnections+0x41 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:2088 1 @ 0x43d0c5 0x435331 0x46e775 0x4e7645 0x4ea4b2 0x4ea494 0x6534e5 0x6797f2 0x677a45 0xbcc07f 0x137294e 0x474801 # 0x46e774 internal/poll.runtime_pollWait+0x54 /opt/go/src/runtime/netpoll.go:227 # 0x4e7644 internal/poll.(*pollDesc).wait+0x44 /opt/go/src/internal/poll/fd_poll_runtime.go:87 # 0x4ea4b1 internal/poll.(*pollDesc).waitRead+0x211 /opt/go/src/internal/poll/fd_poll_runtime.go:92 # 0x4ea493 internal/poll.(*FD).Accept+0x1f3 /opt/go/src/internal/poll/fd_unix.go:401 # 0x6534e4 net.(*netFD).accept+0x44 /opt/go/src/net/fd_unix.go:172 # 0x6797f1 net.(*UnixListener).accept+0x31 /opt/go/src/net/unixsock_posix.go:162 # 0x677a44 net.(*UnixListener).Accept+0x64 /opt/go/src/net/unixsock.go:260 # 0xbcc07e google.golang.org/grpc.(*Server).Serve+0x27e /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:621 # 0x137294d github.com/hashicorp/go-plugin.(*GRPCServer).Serve+0x6d /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_server.go:138 1 @ 0x43d0c5 0x44e2d7 0x11b817d 0x474801 # 0x11b817c github.com/actiontech/dtle/drivers/mysql/common.(*StoreManager).DstPutNats.func1+0x9c /universe/src/github.com/actiontech/dtle/drivers/mysql/common/store.go:146 1 @ 0x43d0c5 0x44e2d7 0x1370646 0x13713da 0x474801 # 0x1370645 github.com/hashicorp/go-plugin.(*gRPCBrokerServer).Recv+0x85 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:121 # 0x13713d9 github.com/hashicorp/go-plugin.(*GRPCBroker).Run+0x79 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:411 1 @ 0x43d0c5 0x44e2d7 0x1372d25 0x135deb3 0xbd1658 0xbd2d28 0xbe0deb 0x474801 # 0x1372d24 github.com/hashicorp/go-plugin.(*grpcStdioServer).StreamStdio+0x144 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_stdio.go:58 # 0x135deb2 github.com/hashicorp/go-plugin/internal/plugin._GRPCStdio_StreamStdio_Handler+0x112 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/internal/plugin/grpc_stdio.pb.go:185 # 0xbd1657 google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329 # 0xbd2d27 google.golang.org/grpc.(*Server).handleStream+0xc67 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 1 @ 0x43d0c5 0x44e2d7 0x1376994 0x14762df 0x2918a28 0x29188be 0x2918d17 0x43cc96 0x474801 # 0x1376993 github.com/hashicorp/go-plugin.Serve+0x953 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/server.go:468 # 0x14762de github.com/hashicorp/nomad/plugins/drivers.Serve+0x23e /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/plugin.go:53 # 0x2918a27 github.com/hashicorp/nomad/plugins.serve+0x147 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/serve.go:47 # 0x29188bd github.com/hashicorp/nomad/plugins.Serve+0xbd /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/serve.go:26 # 0x2918d16 main.main+0x2d6 /universe/src/github.com/actiontech/dtle/cmd/nomad-plugin/main.go:30 # 0x43cc95 runtime.main+0x255 /opt/go/src/runtime/proc.go:225 1 @ 0x43d0c5 0x44e2d7 0x137878f 0x474801 # 0x137878e github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream.func1+0x10e /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:70 1 @ 0x43d0c5 0x44e2d7 0x1476b25 0x13f5293 0xbd1658 0xbd2d28 0xbe0deb 0x474801 # 0x1476b24 github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).Fingerprint+0x124 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:75 # 0x13f5292 github.com/hashicorp/nomad/plugins/drivers/proto._Driver_Fingerprint_Handler+0x112 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4380 # 0xbd1657 google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329 # 0xbd2d27 google.golang.org/grpc.(*Server).handleStream+0xc67 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 1 @ 0x43d0c5 0x44e2d7 0x1477505 0x13f5ad7 0xbcec4b 0xbd2d8f 0xbe0deb 0x474801 # 0x1477504 github.com/hashicorp/nomad/plugins/drivers.(*driverPluginServer).WaitTask+0x124 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/server.go:148 # 0x13f5ad6 github.com/hashicorp/nomad/plugins/drivers/proto._Driver_WaitTask_Handler+0x216 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/plugins/drivers/proto/driver.pb.go:4438 # 0xbcec4a google.golang.org/grpc.(*Server).processUnaryRPC+0x52a /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1082 # 0xbd2d8e google.golang.org/grpc.(*Server).handleStream+0xcce /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1405 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 1 @ 0x43d0c5 0x44e2d7 0x148090d 0x474801 # 0x148090c github.com/hashicorp/nomad/drivers/shared/eventer.(*Eventer).eventLoop+0xec /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/nomad/drivers/shared/eventer/eventer.go:66 1 @ 0x43d0c5 0x44e2d7 0x153f697 0x474801 # 0x153f696 github.com/nats-io/nats-server/v2/server.(*Server).internalSendLoop+0x116 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/events.go:319 1 @ 0x43d0c5 0x44e2d7 0x16afe85 0x474801 # 0x16afe84 github.com/nats-io/nats-server/v2/server.(*sendq).internalLoop+0x464 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/sendq.go:54 1 @ 0x43d0c5 0x44e2d7 0x16c82fc 0x474801 # 0x16c82fb github.com/nats-io/nats-server/v2/server.(*srvIPQueueLogger).run+0x11b /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/server.go:3645 1 @ 0x43d0c5 0x44e2d7 0x1700885 0x474801 # 0x1700884 github.com/nats-io/nats-server/v2/server.(*Server).startGWReplyMapExpiration.func1+0x124 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/gateway.go:3143 1 @ 0x43d0c5 0x44e2d7 0x1710349 0x474801 # 0x1710348 github.com/nats-io/nats-server/v2/server.(*Server).handleSignals.func1+0xa8 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-server/v2/server/signal.go:48 1 @ 0x43d0c5 0x44e2d7 0x18459af 0x474801 # 0x18459ae github.com/nats-io/nats-streaming-server/server.(*StanServer).ioLoop+0x74e /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-streaming-server/server/server.go:4379 1 @ 0x43d0c5 0x44e2d7 0x184bf45 0x474801 # 0x184bf44 github.com/nats-io/nats-streaming-server/server.(*StanServer).processSubscriptionsStart+0xe4 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/nats-streaming-server/server/server.go:5393 1 @ 0x43d0c5 0x44e2d7 0x242746d 0x474801 # 0x242746c go.opencensus.io/stats/view.(*worker).start+0xcc /universe/src/github.com/actiontech/dtle/vendor/go.opencensus.io/stats/view/worker.go:276 1 @ 0x43d0c5 0x44e2d7 0x2640545 0x474801 # 0x2640544 github.com/actiontech/dtle/drivers/mysql/mysql.(*Applier).updateGtidLoop+0x264 /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/applier.go:206 1 @ 0x43d0c5 0x44e2d7 0x264debb 0x474801 # 0x264deba github.com/actiontech/dtle/drivers/mysql/mysql.(*ApplierIncr).heterogeneousReplay+0x1ba /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/applier_incr.go:385 1 @ 0x43d0c5 0x44e2d7 0x2652e1d 0x474801 # 0x2652e1c github.com/actiontech/dtle/drivers/mysql/mysql.(*MtsManager).LcUpdater+0x7c /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/applier_mts.go:101 1 @ 0x43d0c5 0x44e2d7 0x265e8cf 0x474801 # 0x265e8ce github.com/actiontech/dtle/drivers/mysql/mysql.(*TimestampContext).Handle+0xce /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/extractor.go:952 1 @ 0x43d0c5 0x44e2d7 0x2667e7b 0x474801 # 0x2667e7a github.com/actiontech/dtle/drivers/mysql/mysql.(*Applier).doFullCopy.func1+0x13a /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/applier.go:350 1 @ 0x43d0c5 0x44e2d7 0x266aaf3 0x474801 # 0x266aaf2 github.com/actiontech/dtle/drivers/mysql/mysql.(*ApplierIncr).heterogeneousReplay.func1+0xb2 /universe/src/github.com/actiontech/dtle/drivers/mysql/mysql/applier_incr.go:362 1 @ 0x43d0c5 0x44e2d7 0x27e38dd 0x474801 # 0x27e38dc github.com/actiontech/dtle/drivers/mysql.(*Driver).handleFingerprint+0x13c /universe/src/github.com/actiontech/dtle/drivers/mysql/driver.go:404 1 @ 0x43d0c5 0x44e2d7 0x27e5017 0x474801 # 0x27e5016 github.com/actiontech/dtle/drivers/mysql.(*Driver).handleWait+0x176 /universe/src/github.com/actiontech/dtle/drivers/mysql/driver.go:578 1 @ 0x43d0c5 0x44e2d7 0x27e58e5 0x474801 # 0x27e58e4 github.com/actiontech/dtle/drivers/mysql.(*Driver).handleStats+0xc4 /universe/src/github.com/actiontech/dtle/drivers/mysql/driver.go:646 1 @ 0x43d0c5 0x44e2d7 0x27ea795 0x474801 # 0x27ea794 github.com/actiontech/dtle/drivers/mysql.(*taskHandle).run.func1+0xf4 /universe/src/github.com/actiontech/dtle/drivers/mysql/handle.go:139 1 @ 0x43d0c5 0x44e2d7 0x8a57f4 0x474801 # 0x8a57f3 database/sql.(*DB).connectionCleaner+0x113 /opt/go/src/database/sql/sql.go:1005 1 @ 0x43d0c5 0x44e2d7 0x8a6275 0x474801 # 0x8a6274 database/sql.(*DB).connectionOpener+0xb4 /opt/go/src/database/sql/sql.go:1133 1 @ 0x43d0c5 0x44e2d7 0xb76eff 0xb7785d 0xb9a017 0x474801 # 0xb76efe google.golang.org/grpc/internal/transport.(*controlBuffer).get+0xfe /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 # 0xb7785c google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x1dc /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 # 0xb9a016 google.golang.org/grpc/internal/transport.newHTTP2Server.func2+0xd6 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 1 @ 0x43d0c5 0x44e2d7 0xb90145 0x474801 # 0xb90144 google.golang.org/grpc/internal/transport.(*http2Server).keepalive+0x264 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 1 @ 0x43d0c5 0x44e2d7 0xb960c5 0xb95f91 0xb96ff5 0x4bb1a7 0xb96f32 0xb96eef 0xbc8523 0xbc914d 0xbc9a5b 0xbdd856 0x135ce42 0x13701aa 0x135cd0d 0xbd1658 0xbd2d28 0xbe0deb 0x474801 # 0xb960c4 google.golang.org/grpc/internal/transport.(*recvBufferReader).read+0xa4 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:174 # 0xb95f90 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x210 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:168 # 0xb96ff4 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x54 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:479 # 0x4bb1a6 io.ReadAtLeast+0x86 /opt/go/src/io/io.go:328 # 0xb96f31 io.ReadFull+0xd1 /opt/go/src/io/io.go:347 # 0xb96eee google.golang.org/grpc/internal/transport.(*Stream).Read+0x8e /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/internal/transport/transport.go:463 # 0xbc8522 google.golang.org/grpc.(*parser).recvMsg+0x62 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/rpc_util.go:510 # 0xbc914c google.golang.org/grpc.recvAndDecompress+0x4c /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/rpc_util.go:641 # 0xbc9a5a google.golang.org/grpc.recv+0x9a /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/rpc_util.go:709 # 0xbdd855 google.golang.org/grpc.(*serverStream).RecvMsg+0x155 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/stream.go:1473 # 0x135ce41 github.com/hashicorp/go-plugin/internal/plugin.(*gRPCBrokerStartStreamServer).Recv+0x61 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/internal/plugin/grpc_broker.pb.go:167 # 0x13701a9 github.com/hashicorp/go-plugin.(*gRPCBrokerServer).StartStream+0x109 /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/grpc_broker.go:84 # 0x135cd0c github.com/hashicorp/go-plugin/internal/plugin._GRPCBroker_StartStream_Handler+0xac /universe/src/github.com/actiontech/dtle/vendor/github.com/hashicorp/go-plugin/internal/plugin/grpc_broker.pb.go:148 # 0xbd1657 google.golang.org/grpc.(*Server).processStreamingRPC+0xcd7 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1329 # 0xbd2d27 google.golang.org/grpc.(*Server).handleStream+0xc67 /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:1409 # 0xbe0dea google.golang.org/grpc.(*Server).serveStreams.func1.1+0xaa /universe/src/github.com/actiontech/dtle/vendor/google.golang.org/grpc/server.go:746 1 @ 0x43d0c5 0x470a18 0x4709ee 0x48c839 0xff1755 0x474801 # 0x4709ed sync.runtime_notifyListWait+0xcd /opt/go/src/runtime/sema.go:513 # 0x48c838 sync.(*Cond).Wait+0x98 /opt/go/src/sync/cond.go:56 # 0xff1754 github.com/nats-io/go-nats.(*asyncCallbacksHandler).asyncCBDispatcher+0x34 /universe/src/github.com/actiontech/dtle/vendor/github.com/nats-io/go-nats/nats.go:1893 1 @ 0x43d0c5 0x471832 0x101eb8f 0x474801 # 0x471831 time.Sleep+0xd1 /opt/go/src/runtime/time.go:193 # 0x101eb8e github.com/armon/go-metrics.(*Metrics).collectStats+0x2e /universe/src/github.com/actiontech/dtle/vendor/github.com/armon/go-metrics/metrics.go:230 1 @ 0x46e31d 0x7b644e 0x7b6225 0x7b2db2 0x7c47c5 0x7c6017 0x77b024 0x77cead 0x25dc65f 0x25dc3e2 0x25d479c 0x77e6a3 0x779b0d 0x474801 # 0x46e31c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /opt/go/src/runtime/mprof.go:716 # 0x7b644d runtime/pprof.writeRuntimeProfile+0xcd /opt/go/src/runtime/pprof/pprof.go:724 # 0x7b6224 runtime/pprof.writeGoroutine+0xa4 /opt/go/src/runtime/pprof/pprof.go:684 # 0x7b2db1 runtime/pprof.(*Profile).WriteTo+0x3f1 /opt/go/src/runtime/pprof/pprof.go:331 # 0x7c47c4 net/http/pprof.handler.ServeHTTP+0x384 /opt/go/src/net/http/pprof/pprof.go:253 # 0x7c6016 net/http/pprof.Index+0x8d6 /opt/go/src/net/http/pprof/pprof.go:371 # 0x77b023 net/http.HandlerFunc.ServeHTTP+0x43 /opt/go/src/net/http/server.go:2049 # 0x77ceac net/http.(*ServeMux).ServeHTTP+0x1ac /opt/go/src/net/http/server.go:2428 # 0x25dc65e github.com/labstack/echo/v4.WrapHandler.func1+0x9e /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:901 # 0x25dc3e1 github.com/labstack/echo/v4.(*Echo).add.func1+0x61 /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:544 # 0x25d479b github.com/labstack/echo/v4.(*Echo).ServeHTTP+0x2bb /universe/src/github.com/actiontech/dtle/vendor/github.com/labstack/echo/v4/echo.go:654 # 0x77e6a2 net/http.serverHandler.ServeHTTP+0xa2 /opt/go/src/net/http/server.go:2867 # 0x779b0c net/http.(*conn).serve+0x8cc /opt/go/src/net/http/server.go:1932 1 @ 0x4ca7c5 0x4c7efa 0x4e86a5 0x4e867b 0x4e8661 0x651eaf 0x666c31 0x773298 0x474801 # 0x4ca7c4 syscall.Syscall+0x4 /opt/go/src/syscall/asm_linux_amd64.s:20 # 0x4c7ef9 syscall.read+0x59 /opt/go/src/syscall/zsyscall_linux_amd64.go:686 # 0x4e86a4 syscall.Read+0x144 /opt/go/src/syscall/syscall_unix.go:187 # 0x4e867a internal/poll.ignoringEINTRIO+0x11a /opt/go/src/internal/poll/fd_unix.go:581 # 0x4e8660 internal/poll.(*FD).Read+0x100 /opt/go/src/internal/poll/fd_unix.go:162 # 0x651eae net.(*netFD).Read+0x4e /opt/go/src/net/fd_posix.go:55 # 0x666c30 net.(*conn).Read+0x90 /opt/go/src/net/net.go:183 # 0x773297 net/http.(*connReader).backgroundRead+0x57 /opt/go/src/net/http/server.go:672 ```
ghost commented 2 years ago

需要记录