search

talariadb / talaria

TalariaDB is a distributed, highly available, and low latency time-series database for Presto
MIT License
215 stars 31 forks source link

graceful shutdown panic #116

Closed atlas-comstock closed 2 years ago

atlas-comstock commented 2 years ago 
2022/07/13 06:44:44 received signal terminated, start graceful shutdown
2022/07/13 06:44:44 Cancel ctx done
2022/07/13 06:44:44 Close gossip done
2022/07/13 06:44:44 server: wait exit...
2022/07/13 06:44:44 GRPC GracefulStop done, it will wait all request finished
2022/07/13 06:44:44 Close table nodes done, it will compact all if compact is enable
2022/07/13 06:44:44 Close table logs done, it will compact all if compact is enable
2022/07/13 06:44:44 Compact start closing, will compact all data
unexpected fault address 0xffff1c04d30f
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0xffff1c04d30f pc=0x18897b8]

goroutine 994 [running]:
runtime.throw({0x1f02823, 0x5})
        /usr/local/go/src/runtime/panic.go:1198 +0x54 fp=0x40007354c0 sp=0x4000735490 pc=0xb94544
runtime.sigpanic()
        /usr/local/go/src/runtime/signal_unix.go:742 +0x1e4 fp=0x4000735500 sp=0x40007354c0 pc=0xbadc64
github.com/google/flatbuffers/go.GetInt32(...)
        /go/src/talaria/vendor/github.com/google/flatbuffers/go/encode.go:89
github.com/google/flatbuffers/go.GetSOffsetT(...)
        /go/src/talaria/vendor/github.com/google/flatbuffers/go/encode.go:126
github.com/google/flatbuffers/go.(*Table).GetSOffsetT(...)
        /go/src/talaria/vendor/github.com/google/flatbuffers/go/table.go:139
github.com/google/flatbuffers/go.(*Table).Offset(0x40005ba020, 0x4)
        /go/src/talaria/vendor/github.com/google/flatbuffers/go/table.go:15 +0x58 fp=0x4000735530 sp=0x4000735510 pc=0x18897b8
github.com/dgraph-io/badger/v3/fb.(*TableIndex).Offsets(0x40005ba020, 0x40007356d0, 0xeb1)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/fb/TableIndex.go:30 +0x34 fp=0x4000735560 sp=0x4000735530 pc=0x188c994
github.com/dgraph-io/badger/v3/table.(*Table).offsets(0x4016c7c300, 0x40007356d0, 0xeb1)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/table.go:531 +0x44 fp=0x4000735590 sp=0x4000735560 pc=0x189e5f4
github.com/dgraph-io/badger/v3/table.(*Table).block(0x4016c7c300, 0xeb1, 0x1)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/table.go:556 +0x1e0 fp=0x40007357a0 sp=0x4000735590 pc=0x189e800
github.com/dgraph-io/badger/v3/table.(*Iterator).next(0x4005a569a0)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/iterator.go:331 +0xe0 fp=0x40007357f0 sp=0x40007357a0 pc=0x1899c70
github.com/dgraph-io/badger/v3/table.(*Iterator).next(0x4005a569a0)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/iterator.go:348 +0x204 fp=0x4000735840 sp=0x40007357f0 pc=0x1899d94
github.com/dgraph-io/badger/v3/table.(*Iterator).Next(0x4005a569a0)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/iterator.go:406 +0x40 fp=0x4000735860 sp=0x4000735840 pc=0x189a170
github.com/dgraph-io/badger/v3/table.(*node).next(0x4002f189e0)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/merge_iterator.go:83 +0x50 fp=0x4000735880 sp=0x4000735860 pc=0x189b4d0
github.com/dgraph-io/badger/v3/table.(*MergeIterator).Next(0x4002f189a0)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/merge_iterator.go:157 +0x30 fp=0x40007358f0 sp=0x4000735880 pc=0x189b990
github.com/dgraph-io/badger/v3/table.(*node).next(0x4002f18b00)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/merge_iterator.go:79 +0x5c fp=0x4000735910 sp=0x40007358f0 pc=0x189b4dc
github.com/dgraph-io/badger/v3/table.(*MergeIterator).Next(0x4002f18b00)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/merge_iterator.go:157 +0x30 fp=0x4000735980 sp=0x4000735910 pc=0x189b990
github.com/dgraph-io/badger/v3/table.(*node).next(0x4002f18bf0)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/merge_iterator.go:79 +0x5c fp=0x40007359a0 sp=0x4000735980 pc=0x189b4dc
github.com/dgraph-io/badger/v3/table.(*MergeIterator).Next(0x4002f18bb0)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/table/merge_iterator.go:157 +0x30 fp=0x4000735a10 sp=0x40007359a0 pc=0x189b990
github.com/dgraph-io/badger/v3.(*Iterator).parseItem(0x4002f18c60)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/iterator.go:666 +0x500 fp=0x4000735b00 sp=0x4000735a10 pc=0x18bb290
github.com/dgraph-io/badger/v3.(*Iterator).prefetch(0x4002f18c60)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/iterator.go:739 +0xa8 fp=0x4000735b40 sp=0x4000735b00 pc=0x18bbbb8
github.com/dgraph-io/badger/v3.(*Iterator).Seek(0x4002f18c60, {0x4001ab82f0, 0x10, 0x10})
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/iterator.go:780 +0x338 fp=0x4000735ba0 sp=0x4000735b40 pc=0x18bbf38
github.com/kelindar/talaria/internal/storage/disk.(*Storage).Range.func1(0x4000636000)
        /go/src/talaria/internal/storage/disk/disk.go:187 +0x1bc fp=0x4000735cc0 sp=0x4000735ba0 pc=0x18f1a9c
github.com/dgraph-io/badger/v3.(*DB).View(0x4000459b00, 0x4036d39d60)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/txn.go:806 +0xac fp=0x4000735d00 sp=0x4000735cc0 pc=0x18e2f3c
github.com/kelindar/talaria/internal/storage/disk.(*Storage).Range(0x4000e9d080, {0x4001ab82f0, 0x10, 0x10}, {0x4001ab8300, 0x10, 0x10}, 0x4003106000)
        /go/src/talaria/internal/storage/disk/disk.go:179 +0xe4 fp=0x4000735db0 sp=0x4000735d00 pc=0x18f18b4
github.com/kelindar/talaria/internal/storage/compact.(*Storage).Compact(0x4002f3e8c0, {0x21eda58, 0x400004a058})
        /go/src/talaria/internal/storage/compact/compact.go:99 +0x290 fp=0x4000735eb0 sp=0x4000735db0 pc=0x18f40e0
github.com/kelindar/talaria/internal/storage/compact.(*Storage).Compact-fm({0x21eda58, 0x400004a058})
        /go/src/talaria/internal/storage/compact/compact.go:87 +0x40 fp=0x4000735f00 sp=0x4000735eb0 pc=0x18f50a0
github.com/kelindar/talaria/internal/storage/compact.compactEvery.func1({0x21eda58, 0x400004a058})
        /go/src/talaria/internal/storage/compact/compact.go:58 +0x58 fp=0x4000735f50 sp=0x4000735f00 pc=0x18f3bb8
github.com/grab/async.(*task).run.func1(0x4002ddd450, {0x21eda58, 0x400004a058}, 0x40005322a0)
        /go/src/talaria/vendor/github.com/grab/async/task.go:139 +0x3c fp=0x4000735fb0 sp=0x4000735f50 pc=0x100acac
runtime.goexit()
        /usr/local/go/src/runtime/asm_arm64.s:1133 +0x4 fp=0x4000735fb0 sp=0x4000735fb0 pc=0xbcc534
created by github.com/grab/async.(*task).run
        /go/src/talaria/vendor/github.com/grab/async/task.go:138 +0x14c

goroutine 1 [chan receive]:
main.main()
        /go/src/talaria/main.go:108 +0x1064

goroutine 6 [select]:
go.opencensus.io/stats/view.(*worker).start(0x40002ac700)
        /go/src/talaria/vendor/go.opencensus.io/stats/view/worker.go:276 +0x98
created by go.opencensus.io/stats/view.init.0
        /go/src/talaria/vendor/go.opencensus.io/stats/view/worker.go:34 +0x80

goroutine 118 [chan receive, 1 minutes]:
github.com/golang/glog.(*loggingT).flushDaemon(0x2da36c0)
        /go/src/talaria/vendor/github.com/golang/glog/glog.go:882 +0x70
created by github.com/golang/glog.init.0
        /go/src/talaria/vendor/github.com/golang/glog/glog.go:410 +0x2a8

goroutine 913 [select, 7 minutes]:
github.com/dgraph-io/badger/v3.(*vlogThreshold).listenForValueThresholdUpdate(0x40003aafc0)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/value.go:1172 +0xd4
created by github.com/dgraph-io/badger/v3.Open
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/db.go:380 +0x123c

goroutine 299 [select]:
github.com/DataDog/datadog-go/statsd.(*sender).sendLoop(0x4000492c40)
        /go/src/talaria/vendor/github.com/DataDog/datadog-go/statsd/sender.go:92 +0xb0
created by github.com/DataDog/datadog-go/statsd.newSender
        /go/src/talaria/vendor/github.com/DataDog/datadog-go/statsd/sender.go:46 +0xf8

goroutine 300 [select]:
github.com/DataDog/datadog-go/statsd.(*Client).watch(0x4000a02800)
        /go/src/talaria/vendor/github.com/DataDog/datadog-go/statsd/statsd.go:384 +0x80
github.com/DataDog/datadog-go/statsd.newWithWriter.func1(0x4000a02800)
        /go/src/talaria/vendor/github.com/DataDog/datadog-go/statsd/statsd.go:351 +0x50
created by github.com/DataDog/datadog-go/statsd.newWithWriter
        /go/src/talaria/vendor/github.com/DataDog/datadog-go/statsd/statsd.go:349 +0xa88

goroutine 301 [select]:
github.com/DataDog/datadog-go/statsd.(*Client).telemetry(0x4000a02800)
        /go/src/talaria/vendor/github.com/DataDog/datadog-go/statsd/statsd.go:399 +0x8c
github.com/DataDog/datadog-go/statsd.newWithWriter.func2(0x4000a02800, 0x4000642090)
        /go/src/talaria/vendor/github.com/DataDog/datadog-go/statsd/statsd.go:357 +0x5c
created by github.com/DataDog/datadog-go/statsd.newWithWriter
        /go/src/talaria/vendor/github.com/DataDog/datadog-go/statsd/statsd.go:354 +0xac4

goroutine 927 [select]:
github.com/grab/async.Repeat.func2({0x21eda58, 0x400004a058})
        /go/src/talaria/vendor/github.com/grab/async/repeat.go:24 +0xb0
github.com/grab/async.(*task).run.func1(0x4000a8ec80, {0x21eda58, 0x400004a058}, 0x4000eabce0)
        /go/src/talaria/vendor/github.com/grab/async/task.go:139 +0x3c
created by github.com/grab/async.(*task).run
        /go/src/talaria/vendor/github.com/grab/async/task.go:138 +0x14c

goroutine 216 [select, 1 minutes]:
github.com/grab/async.Repeat.func2({0x21eda58, 0x400004a058})
        /go/src/talaria/vendor/github.com/grab/async/repeat.go:24 +0xb0
github.com/grab/async.(*task).run.func1(0x4000a8e0f0, {0x21eda58, 0x400004a058}, 0x4000d243c0)
        /go/src/talaria/vendor/github.com/grab/async/task.go:139 +0x3c
created by github.com/grab/async.(*task).run
        /go/src/talaria/vendor/github.com/grab/async/task.go:138 +0x14c

goroutine 1896 [select]:
github.com/grab/async.(*task).run(0x4001aba050, {0x21eda58, 0x400004a058})
        /go/src/talaria/vendor/github.com/grab/async/task.go:143 +0x1c8
created by github.com/grab/async.(*task).Run
        /go/src/talaria/vendor/github.com/grab/async/task.go:100 +0x4c

goroutine 348 [chan receive, 3 minutes]:
github.com/kelindar/loader.(*watcher).Start.func1(0x4000a8e460, {0x21eda58, 0x400004a058})
        /go/src/talaria/vendor/github.com/kelindar/loader/watcher.go:78 +0x70
created by github.com/kelindar/loader.(*watcher).Start
        /go/src/talaria/vendor/github.com/kelindar/loader/watcher.go:77 +0x144

goroutine 351 [select]:
github.com/dgraph-io/badger/v3/y.(*WaterMark).process(0x4000e9d590, 0x4000e9d560)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/y/watermark.go:214 +0x1c4
created by github.com/dgraph-io/badger/v3/y.(*WaterMark).Init
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/y/watermark.go:72 +0x84

goroutine 352 [select, 7 minutes]:
github.com/dgraph-io/badger/v3/y.(*WaterMark).process(0x4000e9d5c0, 0x4000e9d560)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/y/watermark.go:214 +0x1c4
created by github.com/dgraph-io/badger/v3/y.(*WaterMark).Init
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/y/watermark.go:72 +0x84

goroutine 353 [select]:
github.com/dgraph-io/ristretto/z.(*AllocatorPool).freeupAllocators(0x400048ea50)
        /go/src/talaria/vendor/github.com/dgraph-io/ristretto/z/allocator.go:385 +0x12c
created by github.com/dgraph-io/ristretto/z.NewAllocatorPool
        /go/src/talaria/vendor/github.com/dgraph-io/ristretto/z/allocator.go:324 +0xc0

goroutine 386 [select]:
github.com/dgraph-io/ristretto.(*lfuPolicy).processItems(0x40003ab040)
        /go/src/talaria/vendor/github.com/dgraph-io/ristretto/policy.go:67 +0x70
created by github.com/dgraph-io/ristretto.newPolicy
        /go/src/talaria/vendor/github.com/dgraph-io/ristretto/policy.go:51 +0x158

goroutine 387 [select]:
github.com/dgraph-io/ristretto.(*Cache).processItems(0x4000492fc0)
        /go/src/talaria/vendor/github.com/dgraph-io/ristretto/cache.go:476 +0x10c
created by github.com/dgraph-io/ristretto.NewCache
        /go/src/talaria/vendor/github.com/dgraph-io/ristretto/cache.go:213 +0x590

goroutine 389 [select, 1 minutes]:
github.com/dgraph-io/badger/v3.(*DB).updateSize(0x4000459b00, 0x400099c750)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/db.go:1171 +0x13c
created by github.com/dgraph-io/badger/v3.Open
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/db.go:331 +0xbf4

goroutine 1015 [chan receive, 7 minutes]:
github.com/kelindar/talaria/internal/scripting.(*Loader).watch.func1({0x21eda58, 0x400004a058})
        /go/src/talaria/internal/scripting/loader.go:76 +0x50
github.com/grab/async.(*task).run.func1(0x4000c6a7d0, {0x21eda58, 0x400004a058}, 0x4000a28840)
        /go/src/talaria/vendor/github.com/grab/async/task.go:139 +0x3c
created by github.com/grab/async.(*task).run
        /go/src/talaria/vendor/github.com/grab/async/task.go:138 +0x14c

goroutine 1083 [select, 7 minutes]:
github.com/grab/async.(*task).run(0x4000c6a7d0, {0x21eda58, 0x400004a058})
        /go/src/talaria/vendor/github.com/grab/async/task.go:143 +0x1c8
created by github.com/grab/async.(*task).Run
        /go/src/talaria/vendor/github.com/grab/async/task.go:100 +0x4c

goroutine 1082 [chan receive, 2 minutes]:
github.com/kelindar/loader.(*watcher).Start.func1(0x4000c6a6e0, {0x21eda58, 0x400004a058})
        /go/src/talaria/vendor/github.com/kelindar/loader/watcher.go:78 +0x70
created by github.com/kelindar/loader.(*watcher).Start
        /go/src/talaria/vendor/github.com/kelindar/loader/watcher.go:77 +0x144

goroutine 1081 [select, 7 minutes]:
github.com/grab/async.(*task).run(0x4000c6a1e0, {0x21eda58, 0x400004a058})
        /go/src/talaria/vendor/github.com/grab/async/task.go:143 +0x1c8
created by github.com/grab/async.(*task).Run
        /go/src/talaria/vendor/github.com/grab/async/task.go:100 +0x4c

goroutine 1095 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0xffff80953de8, 0x72)
        /usr/local/go/src/runtime/netpoll.go:303 +0xb4
internal/poll.(*pollDesc).wait(0x4003099818, 0x72, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x38
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0x4003099800)
        /usr/local/go/src/internal/poll/fd_unix.go:402 +0x1f4
net.(*netFD).accept(0x4003099800)
        /usr/local/go/src/net/fd_unix.go:173 +0x2c
net.(*TCPListener).accept(0x40030dcc48)
        /usr/local/go/src/net/tcpsock_posix.go:140 +0x2c
net.(*TCPListener).Accept(0x40030dcc48)
        /usr/local/go/src/net/tcpsock.go:262 +0x34
net/http.(*Server).Serve(0x4000b32000, {0x21db748, 0x40030dcc48})
        /usr/local/go/src/net/http/server.go:3002 +0x364
net/http.(*Server).ListenAndServe(0x4000b32000)
        /usr/local/go/src/net/http/server.go:2931 +0xb0
main.startHTTPServerAsync.func1(0x17ac)
        /go/src/talaria/main.go:166 +0x240
created by main.startHTTPServerAsync
        /go/src/talaria/main.go:155 +0x3c

goroutine 1005 [syscall]:
syscall.Syscall(0xd7, 0xfffefb42c000, 0x206b90, 0x0)
        /usr/local/go/src/syscall/asm_linux_arm64.s:9 +0x10
github.com/dgraph-io/ristretto/z.munmap({0xfffefb42c000, 0x206b90, 0x206b90})
        /go/src/talaria/vendor/github.com/dgraph-io/ristretto/z/mmap_linux.go:71 +0x78
github.com/dgraph-io/ristretto/z.Munmap(...)
        /go/src/talaria/vendor/github.com/dgraph-io/ristretto/z/mmap.go:31
github.com/dgraph-io/ristretto/z.(*MmapFile).Close(0x4000245220, 0xffffffffffffffff)
        /go/src/talaria/vendor/github.com/dgraph-io/ristretto/z/file.go:194 +0x12c
github.com/dgraph-io/badger/v3.(*levelHandler).close(0x40118b67e0)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/level_handler.go:233 +0x108
github.com/dgraph-io/badger/v3.(*levelsController).cleanupLevels(0x4000493030)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/levels.go:208 +0x78
github.com/dgraph-io/badger/v3.(*levelsController).close(0x4000493030)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/levels.go:1564 +0x28
github.com/dgraph-io/badger/v3.(*DB).close(0x4000459b00)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/db.go:623 +0x46c
github.com/dgraph-io/badger/v3.(*DB).Close.func1()
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/db.go:529 +0x30
sync.(*Once).doSlow(0x4000459ee0, 0x400dd5dcd0)
        /usr/local/go/src/sync/once.go:68 +0x108
sync.(*Once).Do(...)
        /usr/local/go/src/sync/once.go:59
github.com/dgraph-io/badger/v3.(*DB).Close(0x4000459b00)
        /go/src/talaria/vendor/github.com/dgraph-io/badger/v3/db.go:528 +0x68
github.com/kelindar/talaria/internal/storage/disk.(*Storage).Close(0x4000e9d080)
        /go/src/talaria/internal/storage/disk/disk.go:299 +0x50
github.com/kelindar/talaria/internal/storage.Close({0x400dd5dde8, 0x2, 0x2})
        /go/src/talaria/internal/storage/storage.go:49 +0xc0
github.com/kelindar/talaria/internal/storage/compact.(*Storage).Close(0x4002f3e8c0)
        /go/src/talaria/internal/storage/compact/compact.go:186 +0xe4
github.com/kelindar/talaria/internal/table/timeseries.(*Table).Close(0x4000b443c0)
        /go/src/talaria/internal/table/timeseries/timeseries.go:78 +0x34
github.com/kelindar/talaria/internal/server.(*Server).Close(0x4000c6a230, {0x2208f80, 0x40003aa500})
        /go/src/talaria/internal/server/server.go:164 +0x15c
main.main.func1({0x21d98b0, 0x2d4a5f8})
        /go/src/talaria/main.go:84 +0x16c
main.onSignal.func1(0x40040ff800, 0x4004361040)
        /go/src/talaria/main.go:149 +0x44
created by main.onSignal
        /go/src/talaria/main.go:147 +0xc4

goroutine 1004 [syscall]:
os/signal.signal_recv()
        /usr/local/go/src/runtime/sigqueue.go:169 +0x108
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:24 +0x20
created by os/signal.Notify.func1.1
        /usr/local/go/src/os/signal/signal.go:151 +0x38

goroutine 1003 [chan receive, 7 minutes]:
github.com/kelindar/talaria/internal/table/timeseries.(*Table).loadStaticSchema.func1({0x21eda58, 0x400004a058})
        /go/src/talaria/internal/table/timeseries/timeseries.go:136 +0x50
github.com/grab/async.(*task).run.func1(0x4000c6a1e0, {0x21eda58, 0x400004a058}, 0x40005338c0)
        /go/src/talaria/vendor/github.com/grab/async/task.go:139 +0x3c
created by github.com/grab/async.(*task).run
        /go/src/talaria/vendor/github.com/grab/async/task.go:138 +0x14c

goroutine 1080 [chan receive, 2 minutes]:
github.com/kelindar/loader.(*watcher).Start.func1(0x4002ddda90, {0x21eda58, 0x400004a058})
        /go/src/talaria/vendor/github.com/kelindar/loader/watcher.go:78 +0x70
created by github.com/kelindar/loader.(*watcher).Start
        /go/src/talaria/vendor/github.com/kelindar/loader/watcher.go:77 +0x144

goroutine 1278 [select]:
github.com/grab/async.Consume.func1({0x21eda58, 0x400004a058})
        /go/src/talaria/vendor/github.com/grab/async/consume.go:34 +0x1b0
github.com/grab/async.(*task).run.func1(0x4001aba050, {0x21eda58, 0x400004a058}, 0x4000c884e0)
        /go/src/talaria/vendor/github.com/grab/async/task.go:139 +0x3c
created by github.com/grab/async.(*task).run
tardunge commented 2 years ago

fixed by https://github.com/talariadb/talaria/pull/110#issue-1292698742