Open vponomaryov opened 2 years ago
scylla-bench version: v0.1.9
In one of the CI jobs [1] we got following go failure of 1 of the 4 loaders (full logs are at [2]):
go failure
3h52m47.4s 462 4620 0 1.2s 1.2s 1s 681ms 538ms 199ms 245ms 3h52m53.2s 493 4930 0 1.1s 1.1s 794ms 599ms 476ms 193ms 229ms unexpected fault address 0x0 fatal error: fault 3h52m59s 543 5430 0 1s 909ms 748ms 579ms 393ms 171ms 203ms [signal SIGSEGV: segmentation violation code=0x80 addr=0x0 pc=0x5fb2e5] goroutine 1 [running]: runtime.throw({0x712b0d?, 0xdcd391ca2a8e96b2?}) /usr/local/go/src/runtime/panic.go:992 +0x71 fp=0xc0004f9390 sp=0xc0004f9360 pc=0x434c31 runtime.sigpanic() /usr/local/go/src/runtime/signal_unix.go:825 +0x2ec fp=0xc0004f93e0 sp=0xc0004f9390 pc=0x44a1ec github.com/HdrHistogram/hdrhistogram-go.(*Histogram).getCountAtIndex(...) /go/pkg/mod/github.com/!hdr!histogram/hdrhistogram-go@v1.1.2/hdr.go:595 github.com/HdrHistogram/hdrhistogram-go.(*iterator).nextCountAtIdx(0x203037?, 0x7f911e0199a8?) /go/pkg/mod/github.com/!hdr!histogram/hdrhistogram-go@v1.1.2/hdr.go:662 +0x65 fp=0xc0004f9400 sp=0xc0004f93e0 pc=0x5fb2e5 github.com/HdrHistogram/hdrhistogram-go.(*iterator).next(0xc0004f9450) /go/pkg/mod/github.com/!hdr!histogram/hdrhistogram-go@v1.1.2/hdr.go:670 +0x25 fp=0xc0004f9420 sp=0xc0004f9400 pc=0x5fb385 github.com/HdrHistogram/hdrhistogram-go.(*rIterator).next(...) /go/pkg/mod/github.com/!hdr!histogram/hdrhistogram-go@v1.1.2/hdr.go:683 github.com/HdrHistogram/hdrhistogram-go.(*Histogram).Merge(0xf0000000c?, 0x4000000000a?) /go/pkg/mod/github.com/!hdr!histogram/hdrhistogram-go@v1.1.2/hdr.go:177 +0x8d fp=0xc0004f9498 sp=0xc0004f9420 pc=0x5fab6d github.com/scylladb/scylla-bench/pkg/results.(*MergedResult).AddResult(0xc0dd0ebb00, {0x0, 0x0, 0x4, 0x28, 0x0, {0x0, 0x0, 0x0}, 0xc0457f6a00, ...}) /go/scylla-bench-0.1.9/pkg/results/merged_result.go:53 +0x1b0 fp=0xc0004f9538 sp=0xc0004f9498 pc=0x5fceb0 github.com/scylladb/scylla-bench/pkg/results.(*TestResults).GetResultsFromThreadsAndMerge(0xc000328380) /go/scylla-bench-0.1.9/pkg/results/thread_results.go:60 +0x89 fp=0xc0004f9688 sp=0xc0004f9538 pc=0x5fe449 github.com/scylladb/scylla-bench/pkg/results.(*TestResults).GetTotalResults(0xc000328380) /go/scylla-bench-0.1.9/pkg/results/thread_results.go:82 +0xcc fp=0xc0004f96d8 sp=0xc0004f9688 pc=0x5fe86c main.main() /go/scylla-bench-0.1.9/main.go:587 +0x34fd fp=0xc0004f9f80 sp=0xc0004f96d8 pc=0x6967dd runtime.main() /usr/local/go/src/runtime/proc.go:250 +0x212 fp=0xc0004f9fe0 sp=0xc0004f9f80 pc=0x437352 runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc0004f9fe8 sp=0xc0004f9fe0 pc=0x4643c1 goroutine 9 [select, 10 minutes]: github.com/gocql/gocql.(*eventDebouncer).flusher(0xc000060960) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/events.go:42 +0x65 created by github.com/gocql/gocql.newEventDebouncer /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/events.go:30 +0x158 ... + 1 more as above goroutine 485 [select]: github.com/gocql/gocql.(*writeCoalescer).writeFlusher(0xc0129916e0, 0xc0002e4788?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:880 +0x138 created by github.com/gocql/gocql.newWriteCoalescer /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:785 +0x173 ... + 28 more as above goroutine 1973 [runnable]: io.ReadAtLeast({0x795e00, 0xc0001ae600}, {0xc0e5b80000, 0x38d5, 0x38d5}, 0x38d5) /usr/local/go/src/io/io.go:325 +0x19c io.ReadFull(...) /usr/local/go/src/io/io.go:350 github.com/gocql/gocql.(*framer).readFrame(0xc0e2274240, 0xc0d9821e00) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/frame.go:514 +0x145 github.com/gocql/gocql.(*Conn).recv(0xc0001ae600, {0x797dd0, 0xc0210ff3c0}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:717 +0x474 github.com/gocql/gocql.(*Conn).serve(0xc0000317d0?, {0x797dd0, 0xc0210ff3c0}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:576 +0x2e created by github.com/gocql/gocql.(*Conn).init /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:325 +0x265 goroutine 2295 [select]: net.(*netFD).connect.func2() /usr/local/go/src/net/fd_unix.go:118 +0x86 created by net.(*netFD).connect /usr/local/go/src/net/fd_unix.go:117 +0x37b goroutine 54 [select]: github.com/gocql/gocql.(*controlConn).heartBeat(0x0?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/control.go:76 +0x107 created by github.com/gocql/gocql.(*controlConn).connect /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/control.go:265 +0x136 goroutine 123 [IO wait]: internal/poll.runtime_pollWait(0x7f914b7c9540, 0x77) /usr/local/go/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc0db05bd00?, 0x0?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitWrite(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).WaitWrite(...) /usr/local/go/src/internal/poll/fd_unix.go:741 net.(*netFD).connect(0xc0db05bd00, {0x797e40?, 0xc0e2bbaf00}, {0xc0004fb468?, 0x40e405?}, {0x796680?, 0xc0e3588d20?}) /usr/local/go/src/net/fd_unix.go:141 +0x716 net.(*netFD).dial(0xc0db05bd00, {0x797e40, 0xc0e2bbaf00}, {0x7987c0?, 0xc0e2f7fe30?}, {0x7987c0?, 0xc0e2f7fe90}, 0x0?) /usr/local/go/src/net/sock_posix.go:149 +0x394 net.socket({0x797e40, 0xc0e2bbaf00}, {0x712793, 0x3}, 0x2, 0x1, 0x0?, 0x58?, {0x7987c0, 0xc0e2f7fe30}, ...) /usr/local/go/src/net/sock_posix.go:70 +0x2b2 net.internetSocket({0x797e40, 0xc0e2bbaf00}, {0x712793, 0x3}, {0x7987c0, 0xc0e2f7fe30}, {0x7987c0, 0xc0e2f7fe90}, 0xc0e35fa850?, 0x0, ...) /usr/local/go/src/net/ipsock_posix.go:142 +0xf8 net.(*sysDialer).doDialTCP(0xc0db05bc80, {0x797e40, 0xc0e2bbaf00}, 0xc0e2f7fe30, 0x6c9aa0?) /usr/local/go/src/net/tcpsock_posix.go:65 +0xa5 net.(*sysDialer).dialTCP(0x95d951?, {0x797e40?, 0xc0e2bbaf00?}, 0xc0004fb6f8?, 0x467a1d?) /usr/local/go/src/net/tcpsock_posix.go:61 +0x59 net.(*sysDialer).dialSingle(0xc0db05bc80, {0x797e40, 0xc0e2bbaf00}, {0x797508?, 0xc0e2f7fe90}) /usr/local/go/src/net/dial.go:583 +0x28b net.(*sysDialer).dialSerial(0xc0db05bc80, {0x797e40, 0xc0e2bbaf00}, {0xc0d4a9bd60?, 0x1, 0x7128f2?}) /usr/local/go/src/net/dial.go:551 +0x312 net.(*Dialer).DialContext(0xc0004fba28, {0x797dd0, 0xc00005f500}, {0x712793, 0x3}, {0xc0e35fa840, 0xe}) /usr/local/go/src/net/dial.go:428 +0x736 github.com/gocql/gocql.(*ScyllaShardAwareDialer).DialContext(0xc000060a80, {0x797dd0, 0xc00005f500}, {0x712793, 0x3}, {0xc0e35fa840, 0xe}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/scylla.go:657 +0x17e github.com/gocql/gocql.(*scyllaDialer).DialHost(0xc000160270, {0x797dd0, 0xc00005f500}, 0xc000082340?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/scylla.go:541 +0xff github.com/gocql/gocql.(*Session).dialWithoutObserver(0xc00015e400, {0x797dd0, 0xc00005f500}, 0xc000082340, 0xc000102460, {0x796060?, 0xc0e08fee00}, 0xef915c93a46?, 0x0) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:258 +0xfb github.com/gocql/gocql.(*Session).dialShard(0xc00015e400, {0x797dd0?, 0xc00005f500?}, 0x0?, 0x40e405?, {0x796060?, 0xc0e08fee00?}, 0x10?, 0x30?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:231 +0x126 github.com/gocql/gocql.(*Session).connectShard(...) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:213 github.com/gocql/gocql.(*hostConnPool).connect(0xc0e08fee00?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/connectionpool.go:501 +0x14a github.com/gocql/gocql.(*hostConnPool).fill(0xc0e08fee00) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/connectionpool.go:396 +0x17c github.com/gocql/gocql.(*policyConnPool).addHost(0xc00006a460, 0xc000082340) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/connectionpool.go:254 +0x1cf github.com/gocql/gocql.(*Session).reconnectDownedHosts(0xc00015e400, 0x0?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:392 +0x198 created by github.com/gocql/gocql.(*Session).init /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:321 +0x9b6 goroutine 312 [chan receive, 233 minutes]: main.main.func2() /go/scylla-bench-0.1.9/main.go:522 +0x28 created by main.main /go/scylla-bench-0.1.9/main.go:521 +0x2525 goroutine 1929 [select]: github.com/gocql/gocql.(*Conn).heartBeat(0xc0002fcfd0?, {0x797dd0, 0xc02164dc40}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:616 +0x11d created by github.com/gocql/gocql.(*Conn).init /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:326 +0x2cd ... + 28 more as above goroutine 546 [IO wait]: internal/poll.runtime_pollWait(0x7f915082a788, 0x72) /usr/local/go/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc0120df100?, 0xc012522000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc0120df100, {0xc012522000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc0120df100, {0xc012522000?, 0x0?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc000187010, {0xc012522000?, 0x0?, 0x203036?}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc0118fb740, {0xc0000a4dc8, 0x1, 0x4bf873?}) /usr/local/go/src/bufio/bufio.go:236 +0x1b4 io.ReadAtLeast({0x795ba0, 0xc0118fb740}, {0xc0000a4dc8, 0x1, 0x9}, 0x1) /usr/local/go/src/io/io.go:331 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:350 github.com/gocql/gocql.readHeader({0x795ba0?, 0xc0118fb740?}, {0xc0000a4dc8?, 0x9?, 0x9?}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/frame.go:436 +0x9f github.com/gocql/gocql.(*Conn).recv(0xc0000a4d80, {0x797dd0, 0xc011c7f000}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:659 +0xe5 github.com/gocql/gocql.(*Conn).serve(0x636e40?, {0x797dd0, 0xc011c7f000}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:576 +0x2e created by github.com/gocql/gocql.(*Conn).init /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:325 +0x265 ... + 24 more as above goroutine 323 [select]: github.com/gocql/gocql.(*Conn).exec(0xc0001ae600, {0x797e08, 0xc000018080}, {0x7962c0, 0xc0ce7c4100}, {0x0, 0x0?}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:972 +0x4d7 github.com/gocql/gocql.(*Conn).executeQuery(0xc0001ae600, {0x797e08, 0xc000018080}, 0x4b6b37?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:1200 +0x585 github.com/gocql/gocql.(*Query).execute(0xc066d7f870?, {0x797e08?, 0xc000018080?}, 0xc00016c340?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:1107 +0x2c github.com/gocql/gocql.(*queryExecutor).attemptQuery(0xc00000e1c8, {0x797e08, 0xc000018080}, {0x7996a8, 0xc0e1385900}, 0xc0001ae600) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:32 +0x8a github.com/gocql/gocql.(*queryExecutor).do(0xc00000e1c8, {0x797e08, 0xc000018080}, {0x7996a8?, 0xc0e1385900?}, 0xc0deceeee0) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:127 +0x1a5 github.com/gocql/gocql.(*queryExecutor).executeQuery(0xc00000e1c8, {0x7996a8, 0xc0e1385900}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:66 +0x10d github.com/gocql/gocql.(*Session).executeQuery(0xc00015e400, 0x0?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:524 +0x8c github.com/gocql/gocql.(*Query).Iter(0xc0e1385900?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:1315 +0xa5 main.DoReadsFromTable.func1(0xc000278030) /go/scylla-bench-0.1.9/modes.go:450 +0x6ea main.RunTest(0xc000278030, {0x798658, 0xc0002382d0}, {0x797468, 0x95dc08}, 0xc066d7fe50) /go/scylla-bench-0.1.9/modes.go:126 +0x1b0 main.DoReadsFromTable({0x71299a?, 0x92d9a0?}, 0xc00026dcb0?, 0xc00009c0e8?, {0x798658?, 0xc0002382d0?}, {0x797468?, 0x95dc08?}) /go/scylla-bench-0.1.9/modes.go:418 +0xb5 main.DoReads(0x7ffd4290dd65?, 0x407d80?, {0x798658?, 0xc0002382d0?}, {0x797468?, 0x95dc08?}) /go/scylla-bench-0.1.9/modes.go:378 +0x53 main.main.func4(0x0?, 0xc000256cc0?, {0x797468, 0x95dc08}) /go/scylla-bench-0.1.9/main.go:584 +0x105 main.RunConcurrently.func1(0x3) /go/scylla-bench-0.1.9/modes.go:78 +0x66 created by main.RunConcurrently /go/scylla-bench-0.1.9/modes.go:76 +0x19c ... + 95 more as above goroutine 313 [sleep, 233 minutes]: time.Sleep(0x1d77b67da000) /usr/local/go/src/runtime/time.go:194 +0x12e main.main.func3() /go/scylla-bench-0.1.9/main.go:533 +0x25 created by main.main /go/scylla-bench-0.1.9/main.go:532 +0x253b goroutine 311 [syscall, 233 minutes]: os/signal.signal_recv() /usr/local/go/src/runtime/sigqueue.go:151 +0x2f os/signal.loop() /usr/local/go/src/os/signal/signal_unix.go:23 +0x19 created by os/signal.Notify.func1.1 /usr/local/go/src/os/signal/signal.go:151 +0x2a goroutine 2109 [runnable]: io.ReadAtLeast({0x795e00, 0xc01735f080}, {0xc0e5b1e000, 0x3845, 0x3845}, 0x3845) /usr/local/go/src/io/io.go:325 +0x19c io.ReadFull(...) /usr/local/go/src/io/io.go:350 github.com/gocql/gocql.(*framer).readFrame(0xc0e2c73b00, 0xc0e4ab6140) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/frame.go:514 +0x145 github.com/gocql/gocql.(*Conn).recv(0xc01735f080, {0x797dd0, 0xc000096f80}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:717 +0x474 github.com/gocql/gocql.(*Conn).serve(0x636e40?, {0x797dd0, 0xc000096f80}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:576 +0x2e created by github.com/gocql/gocql.(*Conn).init /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:325 +0x265 goroutine 2140 [IO wait]: internal/poll.runtime_pollWait(0x7f915082ad28, 0x72) /usr/local/go/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc018143000?, 0xc01077a000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc018143000, {0xc01077a000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc018143000, {0xc01077a000?, 0x0?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc021ba4b80, {0xc01077a000?, 0x0?, 0x203036?}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc01e3122a0, {0xc019e670c8, 0x1, 0x4bf873?}) /usr/local/go/src/bufio/bufio.go:236 +0x1b4 io.ReadAtLeast({0x795ba0, 0xc01e3122a0}, {0xc019e670c8, 0x1, 0x9}, 0x1) /usr/local/go/src/io/io.go:331 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:350 github.com/gocql/gocql.readHeader({0x795ba0?, 0xc01e3122a0?}, {0xc019e670c8?, 0x9?, 0x9?}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/frame.go:436 +0x9f github.com/gocql/gocql.(*Conn).recv(0xc019e67080, {0x797dd0, 0xc00018dc00}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:659 +0xe5 github.com/gocql/gocql.(*Conn).serve(0x0?, {0x797dd0, 0xc00018dc00}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:576 +0x2e created by github.com/gocql/gocql.(*Conn).init /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:325 +0x265 goroutine 1948 [IO wait]: internal/poll.runtime_pollWait(0x7f914b7c9450, 0x72) /usr/local/go/src/runtime/netpoll.go:302 +0x89 internal/poll.(*pollDesc).wait(0xc018142f00?, 0xc008384000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:88 internal/poll.(*FD).Read(0xc018142f00, {0xc008384000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a net.(*netFD).Read(0xc018142f00, {0xc008384000?, 0x0?, 0x8?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc000202310, {0xc008384000?, 0x0?, 0x203036?}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc022492960, {0xc0001ae4c8, 0x1, 0x4bf873?}) /usr/local/go/src/bufio/bufio.go:236 +0x1b4 io.ReadAtLeast({0x795ba0, 0xc022492960}, {0xc0001ae4c8, 0x1, 0x9}, 0x1) /usr/local/go/src/io/io.go:331 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:350 github.com/gocql/gocql.readHeader({0x795ba0?, 0xc022492960?}, {0xc0001ae4c8?, 0x9?, 0x9?}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/frame.go:436 +0x9f github.com/gocql/gocql.(*Conn).recv(0xc0001ae480, {0x797dd0, 0xc0210ff300}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:659 +0xe5 github.com/gocql/gocql.(*Conn).serve(0xc0002f87d0?, {0x797dd0, 0xc0210ff300}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:576 +0x2e created by github.com/gocql/gocql.(*Conn).init /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:325 +0x265 goroutine 334 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0210ff610, 0xa9f04a0) /usr/local/go/src/runtime/sema.go:513 +0x13d sync.(*Cond).Wait(0x6b8600?) /usr/local/go/src/sync/cond.go:56 +0x8c github.com/gocql/gocql.(*writeCoalescer).Write(0xc022493020, {0xc0e3737800, 0x3b, 0x80}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:857 +0x1aa github.com/gocql/gocql.(*Conn).Write(0x80?, {0xc0e3737800?, 0x2?, 0xc0e37394d0?}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:332 +0x27 github.com/gocql/gocql.(*framer).finishWrite(0xc0e37eab40) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/frame.go:779 +0x264 github.com/gocql/gocql.(*framer).writeExecuteFrame(0xc0e37eab40, 0x1000, {0xc01c1c00a0, 0x10, 0x412436?}, 0xc0ce7c5018, 0xc0ce7c5078) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/frame.go:1634 +0x667 github.com/gocql/gocql.(*writeExecuteFrame).writeFrame(0x6c77c0?, 0xc00822aff0?, 0x1000?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/frame.go:1609 +0x35 github.com/gocql/gocql.(*Conn).exec(0xc0001ae600, {0x797e08, 0xc000018080}, {0x7962c0, 0xc0ce7c5000}, {0x0, 0x0?}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:934 +0x30a github.com/gocql/gocql.(*Conn).executeQuery(0xc0001ae600, {0x797e08, 0xc000018080}, 0x4b6b37?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:1200 +0x585 github.com/gocql/gocql.(*Query).execute(0xc011d5d870?, {0x797e08?, 0xc000018080?}, 0xc00016c340?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:1107 +0x2c github.com/gocql/gocql.(*queryExecutor).attemptQuery(0xc00000e1c8, {0x797e08, 0xc000018080}, {0x7996a8, 0xc0e3fa0a00}, 0xc0001ae600) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:32 +0x8a github.com/gocql/gocql.(*queryExecutor).do(0xc00000e1c8, {0x797e08, 0xc000018080}, {0x7996a8?, 0xc0e3fa0a00?}, 0xc0decef490) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:127 +0x1a5 github.com/gocql/gocql.(*queryExecutor).executeQuery(0xc00000e1c8, {0x7996a8, 0xc0e3fa0a00}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:66 +0x10d github.com/gocql/gocql.(*Session).executeQuery(0xc00015e400, 0x0?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:524 +0x8c github.com/gocql/gocql.(*Query).Iter(0xc0e3fa0a00?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:1315 +0xa5 main.DoReadsFromTable.func1(0xc0002ec2d0) /go/scylla-bench-0.1.9/modes.go:450 +0x6ea main.RunTest(0xc0002ec2d0, {0x798658, 0xc0002384e0}, {0x797468, 0x95dc08}, 0xc011d5de50) /go/scylla-bench-0.1.9/modes.go:126 +0x1b0 main.DoReadsFromTable({0x71299a?, 0x92d9a0?}, 0xc0001d7410?, 0xc0001860d8?, {0x798658?, 0xc0002384e0?}, {0x797468?, 0x95dc08?}) /go/scylla-bench-0.1.9/modes.go:418 +0xb5 main.DoReads(0x7ffd4290dd65?, 0x0?, {0x798658?, 0xc0002384e0?}, {0x797468?, 0x95dc08?}) /go/scylla-bench-0.1.9/modes.go:378 +0x53 main.main.func4(0x0?, 0xc000302d80?, {0x797468, 0x95dc08}) /go/scylla-bench-0.1.9/main.go:584 +0x105 main.RunConcurrently.func1(0xe) /go/scylla-bench-0.1.9/modes.go:78 +0x66 created by main.RunConcurrently /go/scylla-bench-0.1.9/modes.go:76 +0x19c goroutine 337 [runnable]: github.com/gocql/gocql.(*Conn).exec(0xc0001ae600, {0x797e08, 0xc000018080}, {0x7962c0, 0xc0b6c68980}, {0x0, 0x0?}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:972 +0x4d7 github.com/gocql/gocql.(*Conn).executeQuery(0xc0001ae600, {0x797e08, 0xc000018080}, 0x4b6b37?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:1200 +0x585 github.com/gocql/gocql.(*Query).execute(0xc000367870?, {0x797e08?, 0xc000018080?}, 0xc00016c340?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:1107 +0x2c github.com/gocql/gocql.(*queryExecutor).attemptQuery(0xc00000e1c8, {0x797e08, 0xc000018080}, {0x7996a8, 0xc0e1384b40}, 0xc0001ae600) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:32 +0x8a github.com/gocql/gocql.(*queryExecutor).do(0xc00000e1c8, {0x797e08, 0xc000018080}, {0x7996a8?, 0xc0e1384b40?}, 0xc0deceea80) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:127 +0x1a5 github.com/gocql/gocql.(*queryExecutor).executeQuery(0xc00000e1c8, {0x7996a8, 0xc0e1384b40}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:66 +0x10d github.com/gocql/gocql.(*Session).executeQuery(0xc00015e400, 0x0?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:524 +0x8c github.com/gocql/gocql.(*Query).Iter(0xc0e1384b40?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:1315 +0xa5 main.DoReadsFromTable.func1(0xc0002ec360) /go/scylla-bench-0.1.9/modes.go:450 +0x6ea main.RunTest(0xc0002ec360, {0x798658, 0xc000238570}, {0x797468, 0x95dc08}, 0xc000367e50) /go/scylla-bench-0.1.9/modes.go:126 +0x1b0 main.DoReadsFromTable({0x71299a?, 0x92d9a0?}, 0xc0003e2870?, 0xc000010158?, {0x798658?, 0xc000238570?}, {0x797468?, 0x95dc08?}) /go/scylla-bench-0.1.9/modes.go:418 +0xb5 main.DoReads(0x7ffd4290dd65?, 0x0?, {0x798658?, 0xc000238570?}, {0x797468?, 0x95dc08?}) /go/scylla-bench-0.1.9/modes.go:378 +0x53 main.main.func4(0x0?, 0xc000389b60?, {0x797468, 0x95dc08}) /go/scylla-bench-0.1.9/main.go:584 +0x105 main.RunConcurrently.func1(0x11) /go/scylla-bench-0.1.9/modes.go:78 +0x66 created by main.RunConcurrently /go/scylla-bench-0.1.9/modes.go:76 +0x19c goroutine 350 [runnable]: github.com/gocql/gocql.(*Conn).exec(0xc0001ae600, {0x797e08, 0xc000018080}, {0x7962c0, 0xc0be359d80}, {0x0, 0x0?}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:972 +0x4d7 github.com/gocql/gocql.(*Conn).executeQuery(0xc0001ae600, {0x797e08, 0xc000018080}, 0x4b6b37?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/conn.go:1200 +0x585 github.com/gocql/gocql.(*Query).execute(0xc068c4f870?, {0x797e08?, 0xc000018080?}, 0xc00016c340?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:1107 +0x2c github.com/gocql/gocql.(*queryExecutor).attemptQuery(0xc00000e1c8, {0x797e08, 0xc000018080}, {0x7996a8, 0xc0e193a640}, 0xc0001ae600) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:32 +0x8a github.com/gocql/gocql.(*queryExecutor).do(0xc00000e1c8, {0x797e08, 0xc000018080}, {0x7996a8?, 0xc0e193a640?}, 0xc0e17f83f0) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:127 +0x1a5 github.com/gocql/gocql.(*queryExecutor).executeQuery(0xc00000e1c8, {0x7996a8, 0xc0e193a640}) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/query_executor.go:66 +0x10d github.com/gocql/gocql.(*Session).executeQuery(0xc00015e400, 0x0?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:524 +0x8c github.com/gocql/gocql.(*Query).Iter(0xc0e193a640?) /go/pkg/mod/github.com/scylladb/gocql@v1.7.2/session.go:1315 +0xa5 main.DoReadsFromTable.func1(0xc000278570) /go/scylla-bench-0.1.9/modes.go:450 +0x6ea main.RunTest(0xc000278570, {0x798658, 0xc00010c240}, {0x797468, 0x95dc08}, 0xc068c4fe50) /go/scylla-bench-0.1.9/modes.go:126 +0x1b0 main.DoReadsFromTable({0x71299a?, 0x92d9a0?}, 0xc0001d7890?, 0xc0002020a8?, {0x798658?, 0xc00010c240?}, {0x797468?, 0x95dc08?}) /go/scylla-bench-0.1.9/modes.go:418 +0xb5 main.DoReads(0x7ffd4290dd65?, 0x0?, {0x798658?, 0xc00010c240?}, {0x797468?, 0x95dc08?}) /go/scylla-bench-0.1.9/modes.go:378 +0x53 main.main.func4(0x0?, 0xc000303200?, {0x797468, 0x95dc08}) /go/scylla-bench-0.1.9/main.go:584 +0x105 main.RunConcurrently.func1(0x1e) /go/scylla-bench-0.1.9/modes.go:78 +0x66 created by main.RunConcurrently /go/scylla-bench-0.1.9/modes.go:76 +0x19c goroutine 356 [runnable]: io.ReadAtLeast({0x795bc0, 0xc0e54f1b00}, {0xc0e5af2e00, 0x60e, 0x60e}, 0x60e) /usr/local/go/src/io/io.go:325 +0x19c io.ReadFull(...) /usr/local/go/src/io/io.go:350 encoding/binary.Read({0x795bc0, 0xc0e54f1b00}, {0x7990c8, 0x95dc08}, {0x6b8600?, 0xc0e4ad7bd8}) /usr/local/go/src/encoding/binary/binary.go:166 +0xc8 main.ValidateData(0x1a, 0x15ad, {0xc0e58fd000, 0x646, 0x800}) /go/scylla-bench-0.1.9/modes.go:277 +0x465 main.DoReadsFromTable.func1(0xc000368030) /go/scylla-bench-0.1.9/modes.go:455 +0xda5 main.RunTest(0xc000368030, {0x798658, 0xc000238750}, {0x797468, 0x95dc08}, 0xc066a2de50) /go/scylla-bench-0.1.9/modes.go:126 +0x1b0 main.DoReadsFromTable({0x71299a?, 0x0?}, 0x0?, 0x0?, {0x798658?, 0xc000238750?}, {0x797468?, 0x95dc08?}) /go/scylla-bench-0.1.9/modes.go:418 +0xb5 main.DoReads(0x7ffd4290dd65?, 0x0?, {0x798658?, 0xc000238750?}, {0x797468?, 0x95dc08?}) /go/scylla-bench-0.1.9/modes.go:378 +0x53 main.main.func4(0x0?, 0x0?, {0x797468, 0x95dc08}) /go/scylla-bench-0.1.9/main.go:584 +0x105 main.RunConcurrently.func1(0x24) /go/scylla-bench-0.1.9/modes.go:78 +0x66 created by main.RunConcurrently /go/scylla-bench-0.1.9/modes.go:76 +0x19c
Looks like we have, in some place, a race condition where we are able to reach a non-existing stuff.
[1] https://jenkins.scylladb.com/job/enterprise-2022.2/job/longevity/job/longevity-large-partition-8h-test/6 [2] https://cloudius-jenkins-test.s3.amazonaws.com/6293270c-6a9d-4d9d-a90e-345cc7c4dbc8/20220907_204044/loader-set-6293270c.tar.gz
scylla-bench version: v0.1.9
In one of the CI jobs [1] we got following
go failureof 1 of the 4 loaders (full logs are at [2]):Looks like we have, in some place, a race condition where we are able to reach a non-existing stuff.
[1] https://jenkins.scylladb.com/job/enterprise-2022.2/job/longevity/job/longevity-large-partition-8h-test/6 [2] https://cloudius-jenkins-test.s3.amazonaws.com/6293270c-6a9d-4d9d-a90e-345cc7c4dbc8/20220907_204044/loader-set-6293270c.tar.gz