InfluxDB 2.0 docker crash

[bkak]

Steps to reproduce: Here is my docker-compose file:

influxdb2:
    container_name: influxdb2
    ports:
      - '9999:9999'
    image: 'quay.io/influxdb/influxdb:2.0.0-beta'
    environment:
      - VAULT_CLIENT_TIMEOUT=300s
    volumes:
      - /mnt/bde/influx_data2:/var/lib/influxdb2
    command: influxd run --bolt-path /var/lib/influxdb2/influxd.bolt --engine-path /var/lib/influxdb2/engine --store bolt --vault-client-timeout=300s

Expected behavior: Flux query should execute

Actual behavior: It crashes when some flux query executes. Not sure which one it is. I can find out which one it is, if that helps to find this bug.

Environment info:

• System info: Linux 4.9.0-6-amd64 x86_64
• InfluxDB version: InfluxDB 2.0.0-beta.9 (git: eb6a5520df) build_date: 2020-05-20T14:25:48Z
• Other relevant environment details: 32 GB RAM

Logs: github.com/influxdata/flux/plan.walk(0x2f177c0, 0xc060524870, 0x2bbd800, 0xc00171bb58, 0xc00171b9b0, 0x0, 0x0) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/plan/walk.go:82 +0x15d fp=0xc00171b748 sp=0xc00171b6d8 pc=0xc4574d github.com/influxdata/flux/plan.walk(0x2f177c0, 0xc0605247e0, 0x2bbd800, 0xc00171bb58, 0xc00171b9b0, 0x0, 0x0) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/plan/walk.go:82 +0x15d fp=0xc00171b7b8 sp=0xc00171b748 pc=0xc4574d github.com/influxdata/flux/plan.walk(0x2f177c0, 0xc0605242d0, 0x2bbd800, 0xc00171bb58, 0xc00171b9b0, 0xc00171b8e8, 0x0) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/plan/walk.go:82 +0x15d fp=0xc00171b828 sp=0xc00171b7b8 pc=0xc4574d github.com/influxdata/flux/plan.walk(0x2f177c0, 0xc060524240, 0x2bbd800, 0xc00171bb58, 0xc00171b9b0, 0x1, 0x2) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/plan/walk.go:82 +0x15d fp=0xc00171b898 sp=0xc00171b828 pc=0xc4574d github.com/influxdata/flux/plan.walk(0x2f177c0, 0xc0605241b0, 0x2bbd800, 0xc00171bb58, 0xc00171b9b0, 0xc0068d06c0, 0x60) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/plan/walk.go:82 +0x15d fp=0xc00171b908 sp=0xc00171b898 pc=0xc4574d github.com/influxdata/flux/plan.(Spec).BottomUpWalk(0xc00181da40, 0xc00171bb58, 0xc0068d06c0, 0x1b782e0) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/plan/walk.go:60 +0x2fd fp=0xc00171bae0 sp=0xc00171b908 pc=0xc4557d github.com/influxdata/flux/execute.(executor).createExecutionState(0xc002cd2e90, 0x2eec740, 0xc013167c20, 0xc00181da40, 0xc09701b2c0, 0x1b782e0, 0xc002cd2e90, 0xb0) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:104 +0x18c fp=0xc00171bb88 sp=0xc00171bae0 pc=0xc84cdc github.com/influxdata/flux/execute.(executor).Execute(0xc002cd2e90, 0x2eec740, 0xc013167c20, 0xc00181da40, 0xc09701b2c0, 0x7, 0x0, 0x0, 0x0) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:70 +0x5d fp=0xc00171bbf0 sp=0xc00171bb88 pc=0xc84a1d github.com/influxdata/flux/lang.(Program).Start(0xc0100205a0, 0x2eec740, 0xc013167bc0, 0xc09701b2c0, 0x1e4576a, 0xd, 0x0, 0x0) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/lang/compiler.go:251 +0x1f8 fp=0xc00171bcd0 sp=0xc00171bbf0 pc=0xcbda38 github.com/influxdata/flux/lang.(AstProgram).Start(0xc005fe64e0, 0x2eec740, 0xc005fe65d0, 0xc09701b2c0, 0x0, 0x0, 0x0, 0x0) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/lang/compiler.go:361 +0x30b fp=0xc00171be00 sp=0xc00171bcd0 pc=0xcbed0b github.com/influxdata/influxdb/v2/query/control.(Controller).executeQuery(0xc000a93c20, 0xc0001a9c80) /go/src/github.com/influxdata/influxdb/query/control/controller.go:425 +0x16d fp=0xc00171bef0 sp=0xc00171be00 pc=0x157557d github.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:383 +0x4c fp=0xc00171bf88 sp=0xc00171bef0 pc=0x157534c github.com/influxdata/influxdb/v2/query/control.New.func1(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:190 +0x64 fp=0xc00171bfd8 sp=0xc00171bf88 pc=0x1578ef4 runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc00171bfe0 sp=0xc00171bfd8 pc=0x45f211 created by github.com/influxdata/influxdb/v2/query/control.New /go/src/github.com/influxdata/influxdb/query/control/controller.go:188 +0x7ed

goroutine 1 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/cmd/influxd/launcher.NewCommand.func1(0xc0000dcc80, 0xc0001c5d50, 0x0, 0x7) /go/src/github.com/influxdata/influxdb/cmd/influxd/launcher/launcher.go:107 +0x225 github.com/spf13/cobra.(Command).execute(0xc0000dcc80, 0xc0001c5ce0, 0x7, 0x7, 0xc0000dcc80, 0xc0001c5ce0) /go/pkg/mod/github.com/spf13/cobra@v0.0.5/command.go:830 +0x2aa github.com/spf13/cobra.(Command).ExecuteC(0x4e3c220, 0x8, 0x8, 0x4e3c220) /go/pkg/mod/github.com/spf13/cobra@v0.0.5/command.go:914 +0x2fb github.com/spf13/cobra.(*Command).Execute(...) /go/pkg/mod/github.com/spf13/cobra@v0.0.5/command.go:864 main.main() /go/src/github.com/influxdata/influxdb/cmd/influxd/main.go:75 +0x76

goroutine 34 [chan receive, 4 minutes]: github.com/influxdata/influxdb/v2/bolt.init.0.func1() /go/src/github.com/influxdata/influxdb/bolt/metrics.go:115 +0x8a created by github.com/influxdata/influxdb/v2/bolt.init.0 /go/src/github.com/influxdata/influxdb/bolt/metrics.go:114 +0xdc

goroutine 35 [syscall, 63 minutes]: os/signal.signal_recv(0x0) /usr/local/go/src/runtime/sigqueue.go:147 +0x9c os/signal.loop() /usr/local/go/src/os/signal/signal_unix.go:23 +0x22 created by os/signal.init.0 /usr/local/go/src/os/signal/signal_unix.go:29 +0x41

goroutine 20 [select]: go.opencensus.io/stats/view.(*worker).start(0xc000166f00) /go/pkg/mod/go.opencensus.io@v0.22.0/stats/view/worker.go:154 +0x100 created by go.opencensus.io/stats/view.init.0 /go/pkg/mod/go.opencensus.io@v0.22.0/stats/view/worker.go:32 +0x57

goroutine 6952 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cf00) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 50 [select, 63 minutes]: github.com/influxdata/influxdb/v2/kit/signals.WithSignals.func1(0xc00065c000, 0x2eec680, 0xc000aac000, 0xc0007b8c60) /go/src/github.com/influxdata/influxdb/kit/signals/context.go:18 +0xe8 created by github.com/influxdata/influxdb/v2/kit/signals.WithSignals /go/src/github.com/influxdata/influxdb/kit/signals/context.go:16 +0xcd

goroutine 6898 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5aa0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 41 [select]: github.com/influxdata/influxdb/v2/tsdb/tsm1.(Engine).compactCache(0xc000c2c120) /go/src/github.com/influxdata/influxdb/tsdb/tsm1/engine.go:906 +0x13a github.com/influxdata/influxdb/v2/tsdb/tsm1.(Engine).enableSnapshotCompactions.func1(0xc00074ed10, 0xc000c2c120) /go/src/github.com/influxdata/influxdb/tsdb/tsm1/engine.go:417 +0x5b created by github.com/influxdata/influxdb/v2/tsdb/tsm1.(*Engine).enableSnapshotCompactions /go/src/github.com/influxdata/influxdb/tsdb/tsm1/engine.go:417 +0x155

goroutine 7073 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6f988, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc004072118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc004072100, 0xc001542000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc004072100, 0xc001542000, 0x8000, 0x8000, 0xc0011a8e40, 0x445c2c, 0x8000) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00011c088, 0xc001542000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 github.com/nats-io/gnatsd/server.(client).readLoop(0xc000ec3200) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:420 +0x42c created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 6958 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d140) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6932 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c780) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6867 [select]: github.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:379 +0xd6 github.com/influxdata/influxdb/v2/query/control.New.func1(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:190 +0x64 created by github.com/influxdata/influxdb/v2/query/control.New /go/src/github.com/influxdata/influxdb/query/control/controller.go:188 +0x7ed

goroutine 7049 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000851cd0, 0xc0000000f6) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000851cc0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282000, 0xc001f3a0c0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6957 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d0e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6937 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c960) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6927 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c5a0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6965 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d3e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7084 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00046c510, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00046c500) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc00032e840) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6896 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d59e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6750 [IO wait, 3 minutes]: internal/poll.runtime_pollWait(0x7fdee3f6fb28, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc004072098, 0x72, 0x0, 0x40, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc004072080, 0xc000ce81c0, 0x40, 0x40, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc004072080, 0xc000ce81c0, 0x40, 0x40, 0x5ec50394, 0xc004f1de40, 0x4cc766) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00011c060, 0xc000ce81c0, 0x40, 0x40, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 github.com/nats-io/gnatsd/server.(client).readLoop(0xc000ec2900) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:420 +0x42c created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 6956 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d080) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6863 [select, 1 minutes]: github.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:379 +0xd6 github.com/influxdata/influxdb/v2/query/control.New.func1(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:190 +0x64 created by github.com/influxdata/influxdb/v2/query/control.New /go/src/github.com/influxdata/influxdb/query/control/controller.go:188 +0x7ed

goroutine 6916 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c180) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6914 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c0c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6893 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d58c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6907 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5e00) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7052 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e850, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e840) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(asyncCallbacksHandler).asyncCBDispatcher(0xc0001236c0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1857 +0x35 created by github.com/nats-io/go-nats.Options.Connect /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:941 +0x200

goroutine 6900 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5b60) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7044 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e4d0, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e4c0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(asyncCallbacksHandler).asyncCBDispatcher(0xc000123060) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1857 +0x35 created by github.com/nats-io/go-nats.Options.Connect /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:941 +0x200

goroutine 6814 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000850410, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000850400) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/gnatsd/server.(client).writeLoop(0xc000f1e000) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:386 +0x142 created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 7048 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0008508d0, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0008508c0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(asyncCallbacksHandler).asyncCBDispatcher(0xc00010b9a0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1857 +0x35 created by github.com/nats-io/go-nats.Options.Connect /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:941 +0x200

goroutine 6869 [select, 1 minutes]: github.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:379 +0xd6 github.com/influxdata/influxdb/v2/query/control.New.func1(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:190 +0x64 created by github.com/influxdata/influxdb/v2/query/control.New /go/src/github.com/influxdata/influxdb/query/control/controller.go:188 +0x7ed

goroutine 6962 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d2c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6934 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c840) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6944 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cc00) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7079 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0008eded0, 0xc000000027) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0008edec0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc00032e300) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6751 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0008eda10, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0008eda00) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/gnatsd/server.(client).writeLoop(0xc000ec2900) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:386 +0x142 created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 6922 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c3c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6935 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c8a0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7083 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00046c450, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00046c440) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc00032e6c0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6919 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c2a0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6955 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d020) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6930 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c6c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6923 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c420) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6948 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cd80) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6904 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5ce0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6913 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c060) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7085 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00046c710, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00046c700) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc00032e9c0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7047 [chan receive, 63 minutes]: github.com/nats-io/go-nats.(Conn).flusher(0xc0024e4580) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2184 +0x10b created by github.com/nats-io/go-nats.(Conn).processConnectInit /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1336 +0x1e9

goroutine 6954 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cfc0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6860 [select, 3 minutes]: github.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:379 +0xd6 github.com/influxdata/influxdb/v2/query/control.New.func1(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:190 +0x64 created by github.com/influxdata/influxdb/v2/query/control.New /go/src/github.com/influxdata/influxdb/query/control/controller.go:188 +0x7ed

goroutine 7041 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000850550, 0x1e) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000850540) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/gnatsd/server.(client).writeLoop(0xc000f1f200) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:386 +0x142 created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 7082 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00046c350, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00046c340) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc00032e600) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6942 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cb40) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6865 [select, 1 minutes]: github.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:379 +0xd6 github.com/influxdata/influxdb/v2/query/control.New.func1(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:190 +0x64 created by github.com/influxdata/influxdb/v2/query/control.New /go/src/github.com/influxdata/influxdb/query/control/controller.go:188 +0x7ed

goroutine 6925 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c4e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6915 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c120) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7090 [sync.Cond.Wait, 1 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000a04710, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000a04700) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc003e42480) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6892 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5860) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7046 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6fbf8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc000bda418, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000bda400, 0xc0011b0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc000bda400, 0xc0011b0000, 0x8000, 0x8000, 0x36, 0xc000264f00, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00000ffe0, 0xc0011b0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 github.com/nats-io/go-nats.(Conn).readLoop(0xc0024e4580) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1942 +0x1b2 created by github.com/nats-io/go-nats.(*Conn).processConnectInit /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1335 +0x1c4

goroutine 6960 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d200) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6868 [select]: github.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:379 +0xd6 github.com/influxdata/influxdb/v2/query/control.New.func1(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:190 +0x64 created by github.com/influxdata/influxdb/v2/query/control.New /go/src/github.com/influxdata/influxdb/query/control/controller.go:188 +0x7ed

goroutine 7089 [sync.Cond.Wait, 1 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000a04490, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000a04480) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc003e423c0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6813 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6fe68, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc000ba8498, 0x72, 0x10000, 0x10000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000ba8480, 0xc00136a000, 0x10000, 0x10000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc000ba8480, 0xc00136a000, 0x10000, 0x10000, 0x1ad76c0, 0x2013d6f3c40ead42, 0xc000a229c0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00010f370, 0xc00136a000, 0x10000, 0x10000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 github.com/nats-io/gnatsd/server.(client).readLoop(0xc000f1e000) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:420 +0x42c created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 6933 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c7e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7081 [sync.Cond.Wait, 1 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00046c310, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00046c300) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc00032e480) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6901 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5bc0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6928 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c600) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6940 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6ca80) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6897 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5a40) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6799 [sync.Cond.Wait, 3 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000a04050, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000a04040) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc003e420c0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6816 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6fcc8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc000bda298, 0x72, 0x0, 0x80, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000bda280, 0xc016b02000, 0x80, 0x80, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc000bda280, 0xc016b02000, 0x80, 0x80, 0x1ad76c0, 0xe6acf5cd38d23d20, 0xc000a22bd0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00000ffd8, 0xc016b02000, 0x80, 0x80, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 github.com/nats-io/gnatsd/server.(client).readLoop(0xc000f1f200) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:420 +0x42c created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 6947 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cd20) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7078 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0008eddd0, 0xc00000017d) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0008eddc0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc00032e240) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6726 [select, 3 minutes]: github.com/influxdata/influxdb/v2/storage.(Engine).runRetentionEnforcer.func1(0xc000001c80, 0xc001b53740, 0xc000024e10) /go/src/github.com/influxdata/influxdb/storage/engine.go:357 +0xff created by github.com/influxdata/influxdb/v2/storage.(Engine).runRetentionEnforcer /go/src/github.com/influxdata/influxdb/storage/engine.go:352 +0x3f9

goroutine 6918 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c240) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6936 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c900) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6912 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c000) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6945 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cc60) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6946 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6ccc0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6911 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5f80) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6949 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cde0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6908 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5e60) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6921 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c360) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6866 [select]: github.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:379 +0xd6 github.com/influxdata/influxdb/v2/query/control.New.func1(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:190 +0x64 created by github.com/influxdata/influxdb/v2/query/control.New /go/src/github.com/influxdata/influxdb/query/control/controller.go:188 +0x7ed

goroutine 7080 [sync.Cond.Wait, 3 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0008edfd0, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0008edfc0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc00032e3c0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6963 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d320) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6920 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c300) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7051 [chan receive]: github.com/nats-io/go-nats.(Conn).flusher(0xc000282b00) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2184 +0x10b created by github.com/nats-io/go-nats.(Conn).processConnectInit /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1336 +0x1e9

goroutine 6905 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5d40) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7091 [sync.Cond.Wait, 1 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000a04750, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000a04740) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc003e42600) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6964 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d380) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6902 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5c20) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6961 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d260) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6931 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c720) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7092 [sync.Cond.Wait, 1 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000a04810, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000a04800) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc003e426c0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6891 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5800) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6909 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5ec0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6941 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cae0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6939 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6ca20) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7074 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0008edb10, 0xc00000000b) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0008edb00) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/gnatsd/server.(client).writeLoop(0xc000ec3200) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:386 +0x142 created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 6862 [select, 3 minutes]: github.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:379 +0xd6 github.com/influxdata/influxdb/v2/query/control.New.func1(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:190 +0x64 created by github.com/influxdata/influxdb/v2/query/control.New /go/src/github.com/influxdata/influxdb/query/control/controller.go:188 +0x7ed

goroutine 6953 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cf60) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6950 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6ce40) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6903 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5c80) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6894 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5920) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6938 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c9c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6910 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5f20) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6951 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cea0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6924 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c480) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6926 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c540) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7075 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0008edb50, 0xc0000002fa) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0008edb40) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282b00, 0xc00032e000) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6906 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5da0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6929 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c660) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6899 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5b00) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7077 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0008edd90, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0008edd80) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc00032e0c0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7043 [chan receive]: github.com/nats-io/go-nats.(Conn).flusher(0xc000282000) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2184 +0x10b created by github.com/nats-io/go-nats.(Conn).processConnectInit /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1336 +0x1e9

goroutine 6861 [select, 1 minutes]: github.com/influxdata/influxdb/v2/query/control.(*Controller).processQueryQueue(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:379 +0xd6 github.com/influxdata/influxdb/v2/query/control.New.func1(0xc000a93c20) /go/src/github.com/influxdata/influxdb/query/control/controller.go:190 +0x64 created by github.com/influxdata/influxdb/v2/query/control.New /go/src/github.com/influxdata/influxdb/query/control/controller.go:188 +0x7ed

goroutine 6959 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d1a0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6917 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6c1e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6895 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc0045d5980) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6943 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6cba0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6800 [sync.Cond.Wait, 1 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000a04190, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000a04180) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc003e42240) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 6966 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d440) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6967 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d4a0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6968 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d500) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6969 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d560) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6970 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d5c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6971 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d620) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6972 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d680) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6973 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d6e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6974 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d740) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6975 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d7a0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6976 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d800) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6977 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d860) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6978 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d8c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6979 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d920) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6980 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d980) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6981 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6d9e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6982 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6da40) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6983 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6daa0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6984 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6db00) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6985 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6db60) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6986 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6dbc0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6987 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6dc20) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6988 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6dc80) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6989 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6dce0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6990 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6dd40) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6991 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6dda0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6992 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6de00) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6993 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6de60) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6994 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6dec0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6995 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6df20) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6996 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d6df80) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6997 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a000) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6998 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a060) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 6999 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a0c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7000 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a120) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7001 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a180) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7002 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a1e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7003 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a240) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7004 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a2a0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7005 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a300) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7006 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a360) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7007 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a3c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7008 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a420) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7009 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a480) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7010 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a4e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7011 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a540) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7012 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a5a0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7013 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a600) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7014 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a660) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7015 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a6c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7016 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a720) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7017 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a780) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7018 [chan receive, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.(*TreeScheduler).work(0xc0041403c0, 0x2eec6c0, 0xc0000520a8, 0xc004d9a7e0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:312 +0xb8 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:139 +0x2cf

goroutine 7019 [select, 63 minutes]: github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler.func2(0xc0041403c0) /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:156 +0x4e4 created by github.com/influxdata/influxdb/v2/task/backend/scheduler.NewScheduler /go/src/github.com/influxdata/influxdb/task/backend/scheduler/treescheduler.go:152 +0x43e

goroutine 6748 [select]: github.com/nats-io/nats-streaming-server/server.(StanServer).ioLoop(0xc0038f5200, 0xc003844330) /go/pkg/mod/github.com/nats-io/nats-streaming-server@v0.11.2/server/server.go:3690 +0x82c created by github.com/nats-io/nats-streaming-server/server.(StanServer).startIOLoop /go/pkg/mod/github.com/nats-io/nats-streaming-server@v0.11.2/server/server.go:3589 +0xcc

goroutine 7029 [IO wait, 63 minutes]: internal/poll.runtime_pollWait(0x7fdee3f70008, 0x72, 0x0) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc000ba8318, 0x72, 0x0, 0x0, 0x1e3923b) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0xc000ba8300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8 net.(netFD).accept(0xc000ba8300, 0x438aae, 0xc0011a9d70, 0xc0011a9db8) /usr/local/go/src/net/fd_unix.go:238 +0x42 net.(TCPListener).accept(0xc000122ca0, 0x0, 0x0, 0xc0011a9e88) /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 net.(TCPListener).Accept(0xc000122ca0, 0xc00047e301, 0xc00000fff0, 0x0, 0x0) /usr/local/go/src/net/tcpsock.go:261 +0x47 github.com/nats-io/gnatsd/server.(Server).AcceptLoop(0xc000186380, 0x0) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:488 +0x6c8 github.com/nats-io/gnatsd/server.(Server).Start(0xc000186380) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:329 +0x39c created by github.com/nats-io/nats-streaming-server/server.(*StanServer).startNATSServer /go/pkg/mod/github.com/nats-io/nats-streaming-server@v0.11.2/server/server.go:2086 +0xf8

goroutine 7042 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6fd98, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc000ba8698, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000ba8680, 0xc001170000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc000ba8680, 0xc001170000, 0x8000, 0x8000, 0x32, 0xc000356500, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00010f6c8, 0xc001170000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 github.com/nats-io/go-nats.(Conn).readLoop(0xc000282000) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1942 +0x1b2 created by github.com/nats-io/go-nats.(*Conn).processConnectInit /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1335 +0x1c4

goroutine 7033 [IO wait, 3 minutes]: internal/poll.runtime_pollWait(0x7fdee3f6ff38, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc000bda218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000bda200, 0xc001114000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc000bda200, 0xc001114000, 0x8000, 0x8000, 0x6, 0xc000264500, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00000fe68, 0xc001114000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 github.com/nats-io/go-nats.(Conn).readLoop(0xc0024e4000) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1942 +0x1b2 created by github.com/nats-io/go-nats.(*Conn).processConnectInit /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1335 +0x1c4

goroutine 7034 [chan receive]: github.com/nats-io/go-nats.(Conn).flusher(0xc0024e4000) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2184 +0x10b created by github.com/nats-io/go-nats.(Conn).processConnectInit /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1336 +0x1e9

goroutine 7035 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000850090, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000850080) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(asyncCallbacksHandler).asyncCBDispatcher(0xc00010a660) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1857 +0x35 created by github.com/nats-io/go-nats.Options.Connect /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:941 +0x200

goroutine 7037 [select, 63 minutes]: github.com/nats-io/nats-streaming-server/server.(StanServer).processSubscriptionsStart(0xc0038f5200) /go/pkg/mod/github.com/nats-io/nats-streaming-server@v0.11.2/server/server.go:4631 +0x114 created by github.com/nats-io/nats-streaming-server/server.(StanServer).start /go/pkg/mod/github.com/nats-io/nats-streaming-server@v0.11.2/server/server.go:1730 +0x509

goroutine 7038 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e5d0, 0xc000000002) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e5c0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282000, 0xc000450c00) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7039 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e610, 0xc0000002fa) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e600) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282000, 0xc000450cc0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7040 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e650, 0xc00000000b) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e640) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282000, 0xc000450d80) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7057 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e690, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e680) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282000, 0xc000450e40) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7058 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e6d0, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e6c0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282000, 0xc000450f00) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7059 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e710, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e700) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282000, 0xc000450fc0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7060 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e750, 0xc0000005dc) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e740) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282000, 0xc000451080) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7050 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6fa58, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc000ba8a18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000ba8a00, 0xc0013b2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc000ba8a00, 0xc0013b2000, 0x8000, 0x8000, 0x50, 0xc0002f0000, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00010f7a8, 0xc0013b2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 github.com/nats-io/go-nats.(Conn).readLoop(0xc000282b00) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1942 +0x1b2 created by github.com/nats-io/go-nats.(*Conn).processConnectInit /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1335 +0x1c4

goroutine 7063 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6f7e8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc000bda598, 0x72, 0x0, 0x80, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000bda580, 0xc000198100, 0x80, 0x80, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc000bda580, 0xc000198100, 0x80, 0x80, 0x1ad76c0, 0x21c94cfaeea5b31d, 0xc0007f9440) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00000fff0, 0xc000198100, 0x80, 0x80, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 github.com/nats-io/gnatsd/server.(client).readLoop(0xc000f57200) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:420 +0x42c created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 7053 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000851d90, 0xc00000007f) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000851d80) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282b00, 0xc001f3a180) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7054 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000851dd0, 0xc0000002f9) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000851dc0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282b00, 0xc001f3a240) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7055 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc000851fd0, 0xc000000001) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc000851fc0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000282b00, 0xc001f3a300) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7065 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6f8b8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc004072318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc004072300, 0xc0013ce000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc004072300, 0xc0013ce000, 0x8000, 0x8000, 0x50, 0xc000356a00, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00011c110, 0xc0013ce000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 github.com/nats-io/go-nats.(Conn).readLoop(0xc000224580) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1942 +0x1b2 created by github.com/nats-io/go-nats.(*Conn).processConnectInit /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1335 +0x1c4

goroutine 7064 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e910, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e900) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/gnatsd/server.(client).writeLoop(0xc000f57200) /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:386 +0x142 created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 7066 [chan receive]: github.com/nats-io/go-nats.(Conn).flusher(0xc000224580) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2184 +0x10b created by github.com/nats-io/go-nats.(Conn).processConnectInit /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1336 +0x1e9

goroutine 7067 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0008edc10, 0x0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0008edc00) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(asyncCallbacksHandler).asyncCBDispatcher(0xc000885240) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1857 +0x35 created by github.com/nats-io/go-nats.Options.Connect /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:941 +0x200

goroutine 7068 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e950, 0xc00000007f) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e940) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc000451140) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7069 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055e990, 0xc0000002f9) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055e980) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc000451200) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7070 [sync.Cond.Wait, 63 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055ea10, 0xc00000000c) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055ea00) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc0004512c0) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7071 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055ea90, 0xc00000017d) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055ea80) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc000451380) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7072 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055eb10, 0xc000000027) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055eb00) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc000451500) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7105 [sync.Cond.Wait, 1 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0009cc010, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0009cc000) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc001f3a480) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7106 [sync.Cond.Wait, 1 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0009cc1d0, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0009cc1c0) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc001f3a540) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7121 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055eb90, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055eb80) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc000224580, 0xc000451680) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7122 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055ec10, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055ec00) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc000451740) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7123 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc00055ec50, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc00055ec40) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc000451800) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7107 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0009cc250, 0xc000000026) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(Cond).Wait(0xc0009cc240) /usr/local/go/src/sync/cond.go:56 +0x9d github.com/nats-io/go-nats.(Conn).waitForMsgs(0xc0024e4580, 0xc001f3a600) /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b created by github.com/nats-io/go-nats.(*Conn).subscribe /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 7108 [select]: github.com/influxdata/influxdb/v2/gather.(Scheduler).run(0xc00055eac0, 0x2eec680, 0xc000aac040, 0x2eec680, 0xc000aac040) /go/src/github.com/influxdata/influxdb/gather/scheduler.go:95 +0xe8 github.com/influxdata/influxdb/v2/gather.(Scheduler).Run(0xc00055eac0, 0x2eec680, 0xc000aac040, 0x1, 0xc004e2be00) /go/src/github.com/influxdata/influxdb/gather/scheduler.go:90 +0x79 github.com/influxdata/influxdb/v2/cmd/influxd/launcher.(Launcher).run.func3(0xc0000d0000, 0xc00055eac0, 0x2eec680, 0xc000aac040, 0xc004e2be00) /go/src/github.com/influxdata/influxdb/cmd/influxd/launcher/launcher.go:857 +0x1dd created by github.com/influxdata/influxdb/v2/cmd/influxd/launcher.(Launcher).run /go/src/github.com/influxdata/influxdb/cmd/influxd/launcher/launcher.go:854 +0x611a

goroutine 7139 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6f718, 0x72, 0x0) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc000bda918, 0x72, 0x0, 0x0, 0x1e3923b) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0xc000bda900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8 net.(netFD).accept(0xc000bda900, 0xc016591950, 0x74cfb4, 0xc0001b2180) /usr/local/go/src/net/fd_unix.go:238 +0x42 net.(TCPListener).accept(0xc000899640, 0x5ec503f1, 0xc016591950, 0x4cc766) /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 net.(TCPListener).Accept(0xc000899640, 0xc0165919a0, 0x18, 0xc001bb3200, 0x74c4e4) /usr/local/go/src/net/tcpsock.go:261 +0x47 net/http.(Server).Serve(0xc0001b20e0, 0x2ee1c80, 0xc000899640, 0x0, 0x0) /usr/local/go/src/net/http/server.go:2896 +0x280 github.com/influxdata/influxdb/v2/cmd/influxd/launcher.(Launcher).run.func4(0xc0000d0000, 0x1e34c7c, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/influxdata/influxdb/cmd/influxd/launcher/launcher.go:1016 +0x7a1 created by github.com/influxdata/influxdb/v2/cmd/influxd/launcher.(*Launcher).run /go/src/github.com/influxdata/influxdb/cmd/influxd/launcher/launcher.go:1007 +0x80c0

goroutine 7124 [select]: github.com/influxdata/influxdb/v2/gather.(Scheduler).Run.func1(0xc00055eac0, 0x2eec680, 0xc000aac040) /go/src/github.com/influxdata/influxdb/gather/scheduler.go:82 +0xf7 created by github.com/influxdata/influxdb/v2/gather.(Scheduler).Run /go/src/github.com/influxdata/influxdb/gather/scheduler.go:80 +0x57

goroutine 7140 [select, 63 minutes]: github.com/influxdata/influxdb/v2/telemetry.(*Reporter).Report(0xc000899680, 0x2eec680, 0xc000aac000) /go/src/github.com/influxdata/influxdb/telemetry/reporter.go:44 +0x53b github.com/influxdata/influxdb/v2/cmd/influxd/launcher.NewCommand.func1.1(0xc003211060, 0xc000899680, 0xc0008faab0) /go/src/github.com/influxdata/influxdb/cmd/influxd/launcher/launcher.go:103 +0x75 created by github.com/influxdata/influxdb/v2/cmd/influxd/launcher.NewCommand.func1 /go/src/github.com/influxdata/influxdb/cmd/influxd/launcher/launcher.go:101 +0x505

goroutine 7173 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6f578, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc000404f98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000404f80, 0xc00220d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc000404f80, 0xc00220d000, 0x1000, 0x1000, 0x43142c, 0xc004f19b20, 0x45be90) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0009116b8, 0xc00220d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 net/http.(persistConn).Read(0xc000992d80, 0xc00220d000, 0x1000, 0x1000, 0xc000af9da0, 0xc004f19c20, 0x404d15) /usr/local/go/src/net/http/transport.go:1758 +0x75 bufio.(Reader).fill(0xc00217ee40) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).Peek(0xc00217ee40, 0x1, 0x0, 0x0, 0x1, 0xc003f8a000, 0x0) /usr/local/go/src/bufio/bufio.go:138 +0x4f net/http.(persistConn).readLoop(0xc000992d80) /usr/local/go/src/net/http/transport.go:1911 +0x1d6 created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 7174 [select]: net/http.(persistConn).writeLoop(0xc000992d80) /usr/local/go/src/net/http/transport.go:2210 +0x123 created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 7130 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6f4a8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc004073f18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc004073f00, 0xc002290000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc004073f00, 0xc002290000, 0x1000, 0x1000, 0x0, 0x5, 0x40907b) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc000ea24d8, 0xc002290000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 net/http.(connReader).Read(0xc002234540, 0xc002290000, 0x1000, 0x1000, 0x0, 0x0, 0x203000) /usr/local/go/src/net/http/server.go:785 +0xf4 bufio.(Reader).fill(0xc003de1980) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).ReadSlice(0xc003de1980, 0xa, 0x28, 0xc0000799a8, 0x40c1e6, 0xc0001c6e00, 0x100) /usr/local/go/src/bufio/bufio.go:359 +0x3d bufio.(Reader).ReadLine(0xc003de1980, 0xc0000799b0, 0xc000095880, 0x7fdee622e6d0, 0x0, 0x40ca58, 0x30) /usr/local/go/src/bufio/bufio.go:388 +0x34 net/textproto.(Reader).readLineSlice(0xc003e63e00, 0xc0001c6e00, 0xc004073f00, 0x0, 0x0, 0x43142c) /usr/local/go/src/net/textproto/reader.go:57 +0x6c net/textproto.(Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:38 net/http.readRequest(0xc003de1980, 0x0, 0xc0001c6e00, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1012 +0x92 net/http.(conn).readRequest(0xc001793720, 0x2eec680, 0xc000533400, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:965 +0x15f net/http.(conn).serve(0xc001793720, 0x2eec680, 0xc000533400) /usr/local/go/src/net/http/server.go:1817 +0x6d4 created by net/http.(Server).Serve /usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 28316 [IO wait]: internal/poll.runtime_pollWait(0x7fde2f286980, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc004a5d718, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc004a5d700, 0xc004f14000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc004a5d700, 0xc004f14000, 0x1000, 0x1000, 0x0, 0x3030320000000000, 0xed656fac7) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00712b9c0, 0xc004f14000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 net/http.(connReader).Read(0xc06bfee270, 0xc004f14000, 0x1000, 0x1000, 0xc0009e3440, 0x3e11074, 0x2) /usr/local/go/src/net/http/server.go:785 +0xf4 bufio.(Reader).fill(0xc0169aef60) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).ReadSlice(0xc0169aef60, 0xa, 0xc56aac3b28, 0xc56aac39a8, 0x40c1e6, 0xc02fb42900, 0x100) /usr/local/go/src/bufio/bufio.go:359 +0x3d bufio.(Reader).ReadLine(0xc0169aef60, 0xc56aac39b0, 0xc00006d180, 0x7fdee622e6d0, 0x0, 0x1, 0xc56aac39f0) /usr/local/go/src/bufio/bufio.go:388 +0x34 net/textproto.(Reader).readLineSlice(0xc56862f260, 0xc02fb42900, 0xc004a5d700, 0x0, 0x0, 0x43142c) /usr/local/go/src/net/textproto/reader.go:57 +0x6c net/textproto.(Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:38 net/http.readRequest(0xc0169aef60, 0x0, 0xc02fb42900, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1012 +0x92 net/http.(conn).readRequest(0xc0011ba1e0, 0x2eec680, 0xc0009e3380, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:965 +0x15f net/http.(conn).serve(0xc0011ba1e0, 0x2eec680, 0xc0009e3380) /usr/local/go/src/net/http/server.go:1817 +0x6d4 created by net/http.(Server).Serve /usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 144476 [IO wait]: internal/poll.runtime_pollWait(0x7fdee3f6f168, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(pollDesc).wait(0xc004072598, 0x72, 0x0, 0x1, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc004072580, 0xc007b0e461, 0x1, 0x1, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf net.(netFD).Read(0xc004072580, 0xc007b0e461, 0x1, 0x1, 0x0, 0x27192, 0x2ed68) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0122c4090, 0xc007b0e461, 0x1, 0x1, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x68 net/http.(connReader).backgroundRead(0xc007b0e450) /usr/local/go/src/net/http/server.go:677 +0x58 created by net/http.(*connReader).startBackgroundRead /usr/local/go/src/net/http/server.go:673 +0xd4

goroutine 144475 [chan receive]: github.com/influxdata/flux.(queryResultIterator).More(0xc005fe6540, 0xc010020620) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/result_iterator.go:61 +0x6c github.com/influxdata/flux.(DelimitedMultiResultEncoder).Encode(0xc005fe65a0, 0x2ea6b40, 0xc00117a8e0, 0x2ef4a80, 0xc005fe6540, 0xc0001a9c80, 0x0, 0x0) /go/pkg/mod/github.com/influxdata/flux@v0.66.1/result.go:292 +0x73 github.com/influxdata/influxdb/v2/query.ProxyQueryServiceAsyncBridge.Query(0x2ea8420, 0xc000a93c20, 0x2eec740, 0xc007b0ea20, 0x2ea6b40, 0xc00117a8e0, 0xc0100761c0, 0x0, 0x0, 0x0, ...) /go/src/github.com/influxdata/influxdb/query/bridges.go:145 +0x250 github.com/influxdata/influxdb/v2/http.routingQueryService.Query(0x2ecf180, 0xc0045c9970, 0x2ecf180, 0xc0045c9970, 0x2eec740, 0xc007b0e9f0, 0x2ea6b40, 0xc00117a8e0, 0xc0100761c0, 0x0, ...) /go/src/github.com/influxdata/influxdb/http/query_handler.go:649 +0xf5 github.com/influxdata/influxdb/v2/http.(FluxHandler).handleQuery(0xc001c76060, 0x2edab40, 0xc00117a420, 0xc000103400) /go/src/github.com/influxdata/influxdb/http/query_handler.go:184 +0xa01 net/http.HandlerFunc.ServeHTTP(0xc000b15520, 0x2edab40, 0xc00117a420, 0xc000103400) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/NYTimes/gziphandler.GzipHandlerWithOpts.func1.1(0x2edab40, 0xc00117a420, 0xc000103400) /go/pkg/mod/github.com/!n!y!times/gziphandler@v1.0.1/gzip.go:293 +0x323 net/http.HandlerFunc.ServeHTTP(0xc000b49080, 0x2edab40, 0xc00117a420, 0xc000103400) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/influxdata/httprouter.(Router).Handler.func1(0x2edab40, 0xc00117a420, 0xc000103400, 0x0, 0x0, 0x0) /go/pkg/mod/github.com/influxdata/httprouter@v1.3.1-0.20191122104820-ee83e2772f69/router.go:325 +0x1c8 github.com/influxdata/httprouter.(Router).ServeHTTP(0xc004ccdcb0, 0x2edab40, 0xc00117a420, 0xc000103300) /go/pkg/mod/github.com/influxdata/httprouter@v1.3.1-0.20191122104820-ee83e2772f69/router.go:453 +0x896 github.com/go-chi/chi.(Mux).Mount.func1(0x2edab40, 0xc00117a420, 0xc000103300) /go/pkg/mod/github.com/go-chi/chi@v4.1.0+incompatible/mux.go:298 +0x121 net/http.HandlerFunc.ServeHTTP(0xc000987160, 0x2edab40, 0xc00117a420, 0xc000103300) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/go-chi/chi.(Mux).routeHTTP(0xc000373140, 0x2edab40, 0xc00117a420, 0xc000103300) /go/pkg/mod/github.com/go-chi/chi@v4.1.0+incompatible/mux.go:431 +0x278 net/http.HandlerFunc.ServeHTTP(0xc000b14c70, 0x2edab40, 0xc00117a420, 0xc000103300) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/influxdata/influxdb/v2/kit/transport/http.SetCORS.func1(0x2edab40, 0xc00117a420, 0xc000103300) /go/src/github.com/influxdata/influxdb/kit/transport/http/middleware.go:31 +0xcb net/http.HandlerFunc.ServeHTTP(0xc0004b7240, 0x2edab40, 0xc00117a420, 0xc000103300) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/go-chi/chi/middleware.StripSlashes.func1(0x2edab40, 0xc00117a420, 0xc000103300) /go/pkg/mod/github.com/go-chi/chi@v4.1.0+incompatible/middleware/strip.go:25 +0xeb net/http.HandlerFunc.ServeHTTP(0xc0004b7280, 0x2edab40, 0xc00117a420, 0xc000103300) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/influxdata/influxdb/v2/kit/transport/http.SkipOptions.func1(0x2edab40, 0xc00117a420, 0xc000103300) /go/src/github.com/influxdata/influxdb/kit/transport/http/middleware.go:68 +0xbd net/http.HandlerFunc.ServeHTTP(0xc0004b72c0, 0x2edab40, 0xc00117a420, 0xc000103300) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/go-chi/chi.(Mux).ServeHTTP(0xc000373140, 0x2edab40, 0xc00117a420, 0xc000103300) /go/pkg/mod/github.com/go-chi/chi@v4.1.0+incompatible/mux.go:70 +0x513 github.com/influxdata/influxdb/v2/http.(AuthenticationHandler).ServeHTTP(0xc000dd7ab0, 0x2edab40, 0xc00117a420, 0xc000103100) /go/src/github.com/influxdata/influxdb/http/authentication_middleware.go:125 +0x2d2 github.com/influxdata/influxdb/v2/kit/transport/http.SetCORS.func1(0x2edab40, 0xc00117a420, 0xc000103100) /go/src/github.com/influxdata/influxdb/kit/transport/http/middleware.go:31 +0xcb net/http.HandlerFunc.ServeHTTP(0xc000898e60, 0x2edab40, 0xc00117a420, 0xc000103100) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/influxdata/influxdb/v2/kit/transport/http.SkipOptions.func1(0x2edab40, 0xc00117a420, 0xc000103100) /go/src/github.com/influxdata/influxdb/kit/transport/http/middleware.go:68 +0xbd net/http.HandlerFunc.ServeHTTP(0xc000898e80, 0x2edab40, 0xc00117a420, 0xc000103100) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/influxdata/influxdb/v2/http.(PlatformHandler).ServeHTTP(0xc000898ec0, 0x2edab40, 0xc00117a420, 0xc000103100) /go/src/github.com/influxdata/influxdb/http/platform_handler.go:63 +0xe6 github.com/go-chi/chi.(Mux).Mount.func1(0x2edab40, 0xc00117a420, 0xc000103100) /go/pkg/mod/github.com/go-chi/chi@v4.1.0+incompatible/mux.go:298 +0x121 net/http.HandlerFunc.ServeHTTP(0xc0008993e0, 0x2edab40, 0xc00117a420, 0xc000103100) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/go-chi/chi.(Mux).routeHTTP(0xc0007b91a0, 0x2edab40, 0xc00117a420, 0xc000103100) /go/pkg/mod/github.com/go-chi/chi@v4.1.0+incompatible/mux.go:431 +0x278 net/http.HandlerFunc.ServeHTTP(0xc0007c8a10, 0x2edab40, 0xc00117a420, 0xc000103100) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/influxdata/influxdb/v2/kit/transport/http.Metrics.func1.1(0x2ee1f40, 0xc00154a000, 0xc000103100) /go/src/github.com/influxdata/influxdb/kit/transport/http/middleware.go:53 +0x19f net/http.HandlerFunc.ServeHTTP(0xc000599340, 0x2ee1f40, 0xc00154a000, 0xc000103100) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/influxdata/influxdb/v2/kit/transport/http.Trace.func1.1(0x2ee1f40, 0xc00154a000, 0xc000103100) /go/src/github.com/influxdata/influxdb/kit/transport/http/middleware.go:93 +0x3a5 net/http.HandlerFunc.ServeHTTP(0xc0006a8330, 0x2ee1f40, 0xc00154a000, 0xc000102f00) /usr/local/go/src/net/http/server.go:2007 +0x44 github.com/go-chi/chi.(Mux).ServeHTTP(0xc0007b91a0, 0x2ee1f40, 0xc00154a000, 0xc000102e00) /go/pkg/mod/github.com/go-chi/chi@v4.1.0+incompatible/mux.go:86 +0x2b2 github.com/influxdata/influxdb/v2/http.(Handler).ServeHTTP(0xc0005992c0, 0x2ee1f40, 0xc00154a000, 0xc000102e00) /go/src/github.com/influxdata/influxdb/http/handler.go:130 +0x55 net/http.serverHandler.ServeHTTP(0xc0001b20e0, 0x2ee1f40, 0xc00154a000, 0xc000102e00) /usr/local/go/src/net/http/server.go:2802 +0xa4 net/http.(conn).serve(0xc00059e780, 0x2eec680, 0xc0008f2200) /usr/local/go/src/net/http/server.go:1890 +0x875 created by net/http.(Server).Serve /usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 143021 [select]: github.com/influxdata/influxdb/v2/tsdb/tsm1.(Engine).compact(0xc000c2c120, 0xc007310610) /go/src/github.com/influxdata/influxdb/tsdb/tsm1/engine.go:995 +0x141 github.com/influxdata/influxdb/v2/tsdb/tsm1.(Engine).enableLevelCompactions.func1(0xc007310610, 0xc000c2c120) /go/src/github.com/influxdata/influxdb/tsdb/tsm1/engine.go:338 +0x65 created by github.com/influxdata/influxdb/v2/tsdb/tsm1.(*Engine).enableLevelCompactions /go/src/github.com/influxdata/influxdb/tsdb/tsm1/engine.go:338 +0x131 .

[bkak]

This is very crucial. InfluxDB crashes when I try to use aggregateWindow function.

This is the task I created:

from(bucket: "my_bucket")
    |> range(start: -duration(v: int(v: task.every) * 2))
    |> filter(fn: (r) =>
        (r["_measurement"] == "phf"))
    |> aggregateWindow(fn: mean, every: 1ms)
    |> to(bucket: "phf_downsampled", org: "downsampled")

Here is the log if that helps:

    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:329 +0x39c
created by github.com/nats-io/nats-streaming-server/server.(*StanServer).startNATSServer
    /go/pkg/mod/github.com/nats-io/nats-streaming-server@v0.11.2/server/server.go:2086 +0xf8

goroutine 10821 [IO wait]:
internal/poll.runtime_pollWait(0x7f49575b3f60, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc000750f18, 0x72, 0x100, 0x100, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000750f00, 0xc3020e4100, 0x100, 0x100, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc000750f00, 0xc3020e4100, 0x100, 0x100, 0x1ad76c0, 0xc3bc64b4cd7bbd48, 0xc00072bec0)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000e208, 0xc3020e4100, 0x100, 0x100, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x68
github.com/nats-io/gnatsd/server.(*client).readLoop(0xc000a17600)
    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:420 +0x42c
created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine
    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 10822 [sync.Cond.Wait]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051e990, 0xc000000004)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051e980)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/gnatsd/server.(*client).writeLoop(0xc000a17600)
    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:386 +0x142
created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine
    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 10824 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f49575b3e90, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00004db18, 0x72, 0x0, 0x40, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00004db00, 0xc0052e8740, 0x40, 0x40, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00004db00, 0xc0052e8740, 0x40, 0x40, 0x5ecbba03, 0xc001543e40, 0x4cc766)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000128d18, 0xc0052e8740, 0x40, 0x40, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x68
github.com/nats-io/gnatsd/server.(*client).readLoop(0xc00023fb00)
    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:420 +0x42c
created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine
    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 10825 [sync.Cond.Wait, 1 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051e9d0, 0x6d6f726800000000)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051e9c0)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/gnatsd/server.(*client).writeLoop(0xc00023fb00)
    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:386 +0x142
created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine
    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 10827 [sync.Cond.Wait, 1 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051ea50, 0xc000000002)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051ea40)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/go-nats.(*Conn).waitForMsgs(0xc0001d7b80, 0xc004593140)
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b
created by github.com/nats-io/go-nats.(*Conn).subscribe
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 10828 [sync.Cond.Wait]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051ea90, 0xc00000001a)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051ea80)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/go-nats.(*Conn).waitForMsgs(0xc0001d7b80, 0xc004593200)
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b
created by github.com/nats-io/go-nats.(*Conn).subscribe
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 10829 [sync.Cond.Wait, 1 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051ead0, 0xc00000000b)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051eac0)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/go-nats.(*Conn).waitForMsgs(0xc0001d7b80, 0xc0045932c0)
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b
created by github.com/nats-io/go-nats.(*Conn).subscribe
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 10830 [sync.Cond.Wait, 1 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051eb10, 0x0)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051eb00)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/go-nats.(*Conn).waitForMsgs(0xc0001d7b80, 0xc004593380)
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b
created by github.com/nats-io/go-nats.(*Conn).subscribe
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 10831 [sync.Cond.Wait, 1 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051eb50, 0x0)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051eb40)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/go-nats.(*Conn).waitForMsgs(0xc0001d7b80, 0xc004593440)
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b
created by github.com/nats-io/go-nats.(*Conn).subscribe
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 10832 [sync.Cond.Wait, 1 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051eb90, 0x0)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051eb80)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/go-nats.(*Conn).waitForMsgs(0xc0001d7b80, 0xc004593500)
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b
created by github.com/nats-io/go-nats.(*Conn).subscribe
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 10849 [sync.Cond.Wait]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051ebd0, 0xc000000025)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051ebc0)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/go-nats.(*Conn).waitForMsgs(0xc0001d7b80, 0xc0045935c0)
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b
created by github.com/nats-io/go-nats.(*Conn).subscribe
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 10851 [sync.Cond.Wait, 1 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051ec10, 0x0)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051ec00)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/gnatsd/server.(*client).writeLoop(0xc0009ae000)
    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/client.go:386 +0x142
created by github.com/nats-io/gnatsd/server.(*Server).startGoRoutine
    /go/pkg/mod/github.com/nats-io/gnatsd@v1.3.0/server/server.go:1143 +0xad

goroutine 10853 [sync.Cond.Wait]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00051ecd0, 0xc000000002)
    /usr/local/go/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00051ecc0)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/nats-io/go-nats.(*Conn).waitForMsgs(0xc000258680, 0xc004593800)
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:1979 +0x45b
created by github.com/nats-io/go-nats.(*Conn).subscribe
    /go/pkg/mod/github.com/nats-io/go-nats@v1.7.0/nats.go:2775 +0x489

goroutine 10950 [IO wait]:
internal/poll.runtime_pollWait(0x7f49575b3740, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc000823498, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000823480, 0xc0032d2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc000823480, 0xc0032d2000, 0x1000, 0x1000, 0x43142c, 0xc005588b20, 0x45be90)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0029f6388, 0xc0032d2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc002f36900, 0xc0032d2000, 0x1000, 0x1000, 0xc0018e1c20, 0xc005588c20, 0x404d15)
    /usr/local/go/src/net/http/transport.go:1758 +0x75
bufio.(*Reader).fill(0xc00561bc80)
    /usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00561bc80, 0x1, 0x0, 0x0, 0x1, 0xc0da592f00, 0x0)
    /usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc002f36900)
    /usr/local/go/src/net/http/transport.go:1911 +0x1d6
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1580 +0xb0d

goroutine 13999 [select]:
github.com/influxdata/influxdb/v2/storage/flux.(*filterIterator).handleRead(0xc0eb75c0e0, 0xc080ec9620, 0x2f02120, 0xc0e6e73080, 0x0, 0x0)
    /go/src/github.com/influxdata/influxdb/storage/flux/reader.go:231 +0x58e
github.com/influxdata/influxdb/v2/storage/flux.(*filterIterator).Do(0xc0eb75c0e0, 0xc080ec9620, 0xc060a04501, 0xc0eb75c0e0)
    /go/src/github.com/influxdata/influxdb/storage/flux/reader.go:171 +0x148
github.com/influxdata/influxdb/v2/query/stdlib/influxdata/influxdb.(*Source).processTables(0xc0769ad760, 0x2eec740, 0xc080dd3b00, 0x2ecaf40, 0xc0eb75c0e0, 0x1612454d992f0400, 0x1612454d992f0400, 0xc08f00d440)
    /go/src/github.com/influxdata/influxdb/query/stdlib/influxdata/influxdb/source.go:75 +0x94
github.com/influxdata/influxdb/v2/query/stdlib/influxdata/influxdb.(*readFilterSource).run(0xc0769ad760, 0x2eec740, 0xc080dd3b00, 0x59e7cb8e5469000, 0x1e3f8d4)
    /go/src/github.com/influxdata/influxdb/query/stdlib/influxdata/influxdb/source.go:152 +0x138
github.com/influxdata/influxdb/v2/query/stdlib/influxdata/influxdb.(*Source).Run(0xc0769ad760, 0x2eec740, 0xc080dd3b00)
    /go/src/github.com/influxdata/influxdb/query/stdlib/influxdata/influxdb/source.go:55 +0x31d
github.com/influxdata/flux/execute.(*executionState).do.func1(0xc060a04510, 0xc0e3842880, 0x2eec740, 0xc080dd3b00, 0x2ecad40, 0xc0769ad760)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:275 +0xd7
created by github.com/influxdata/flux/execute.(*executionState).do
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:250 +0xf8

goroutine 15151 [semacquire]:
runtime.ReadMemStats(0xc0da74c000)
    /usr/local/go/src/runtime/mstats.go:446 +0x36
github.com/prometheus/client_golang/prometheus.(*goCollector).Collect.func1(0xc000984500, 0xc0da74c000, 0xc0b214ef00)
    /go/pkg/mod/github.com/prometheus/client_golang@v1.0.0/prometheus/go_collector.go:307 +0x38
created by github.com/prometheus/client_golang/prometheus.(*goCollector).Collect
    /go/pkg/mod/github.com/prometheus/client_golang@v1.0.0/prometheus/go_collector.go:306 +0xac

goroutine 10951 [select]:
net/http.(*persistConn).writeLoop(0xc002f36900)
    /usr/local/go/src/net/http/transport.go:2210 +0x123
created by net/http.(*Transport).dialConn
    /usr/local/go/src/net/http/transport.go:1581 +0xb32

goroutine 10977 [IO wait]:
internal/poll.runtime_pollWait(0x7f49575b3810, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc000985998, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000985980, 0xc006476000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc000985980, 0xc006476000, 0x1000, 0x1000, 0x0, 0x3030320000000000, 0xed65db160)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc002f79450, 0xc006476000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x68
net/http.(*connReader).Read(0xc0031ebdd0, 0xc006476000, 0x1000, 0x1000, 0xc2e5eb3d00, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc006338060)
    /usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).ReadSlice(0xc006338060, 0xc071bbaf0a, 0xd, 0xc2ddc0d9a8, 0x40c3d4, 0xc31f1a0d00, 0x100)
    /usr/local/go/src/bufio/bufio.go:359 +0x3d
bufio.(*Reader).ReadLine(0xc006338060, 0xc2ddc0d9b0, 0xc00006ca80, 0x7f4959a166d0, 0xc006e34900, 0x1d958a0, 0xc006efc900)
    /usr/local/go/src/bufio/bufio.go:388 +0x34
net/textproto.(*Reader).readLineSlice(0xc006efc900, 0xc31f1a0d00, 0xc000985980, 0x0, 0x0, 0x2e4dde6)
    /usr/local/go/src/net/textproto/reader.go:57 +0x6c
net/textproto.(*Reader).ReadLine(...)
    /usr/local/go/src/net/textproto/reader.go:38
net/http.readRequest(0xc006338060, 0x0, 0xc31f1a0d00, 0x0, 0x0)
    /usr/local/go/src/net/http/request.go:1012 +0x92
net/http.(*conn).readRequest(0xc0063bf360, 0x2eec680, 0xc0032519c0, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:965 +0x15f
net/http.(*conn).serve(0xc0063bf360, 0x2eec680, 0xc0032519c0)
    /usr/local/go/src/net/http/server.go:1817 +0x6d4
created by net/http.(*Server).Serve
    /usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 15176 [IO wait]:
internal/poll.runtime_pollWait(0x7f488f6496d8, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0df80ca18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0df80ca00, 0xc3653da000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0df80ca00, 0xc3653da000, 0x1000, 0x1000, 0xc2d2e7d9e8, 0x4d501d, 0xc0df80ca00)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0daface78, 0xc3653da000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x68
net/http.(*connReader).Read(0xc0e02a38c0, 0xc3653da000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc3652c2900)
    /usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc3652c2900, 0x4, 0x0, 0x0, 0x0, 0x0, 0x2e4dde6)
    /usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).readRequest(0xc0bd8c0820, 0x2eec680, 0xc0e02a6800, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:962 +0xb3b
net/http.(*conn).serve(0xc0bd8c0820, 0x2eec680, 0xc0e02a6800)
    /usr/local/go/src/net/http/server.go:1817 +0x6d4
created by net/http.(*Server).Serve
    /usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 14483 [select]:
github.com/influxdata/flux/execute.(*result).Do(0xc309e4f9e0, 0x2bbf750, 0xc30c1b1100, 0xc309e4f9e0)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/result.go:60 +0xd9
github.com/influxdata/influxdb/v2/query/control.(*errorCollectingTableIterator).Do(0xc30c1b1100, 0x2bbf750, 0xc30c1b1100, 0xc30c1b10e0)
    /go/src/github.com/influxdata/influxdb/query/control/controller.go:908 +0x41
github.com/influxdata/influxdb/v2/task/backend/executor.exhaustResultIterators(0x2eca980, 0xc30c1b10e0, 0xc30c1b10e0, 0xc302159950)
    /go/src/github.com/influxdata/influxdb/task/backend/executor/executor.go:513 +0x51
github.com/influxdata/influxdb/v2/task/backend/executor.(*worker).executeQuery(0xc3021652a0, 0xc304196e00)
    /go/src/github.com/influxdata/influxdb/task/backend/executor/executor.go:424 +0x534
github.com/influxdata/influxdb/v2/task/backend/executor.(*worker).work(0xc3021652a0)
    /go/src/github.com/influxdata/influxdb/task/backend/executor/executor.go:319 +0x538
github.com/influxdata/influxdb/v2/task/backend/executor.(*Executor).startWorker.func1(0xc004592d80, 0xc3021652a0)
    /go/src/github.com/influxdata/influxdb/task/backend/executor/executor.go:205 +0x7e
created by github.com/influxdata/influxdb/v2/task/backend/executor.(*Executor).startWorker
    /go/src/github.com/influxdata/influxdb/task/backend/executor/executor.go:202 +0xa8

goroutine 15179 [IO wait]:
internal/poll.runtime_pollWait(0x7f488f649468, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0df80cb98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0df80cb80, 0xc006b52000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0df80cb80, 0xc006b52000, 0x1000, 0x1000, 0xc2d3f119e8, 0x4d501d, 0xc0df80cb80)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0daface90, 0xc006b52000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x68
net/http.(*connReader).Read(0xc0df80b2f0, 0xc006b52000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc0055f2840)
    /usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc0055f2840, 0x4, 0x0, 0x0, 0x0, 0x0, 0x2e4dde6)
    /usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).readRequest(0xc0bd8c0a00, 0x2eec680, 0xc0df812140, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:962 +0xb3b
net/http.(*conn).serve(0xc0bd8c0a00, 0x2eec680, 0xc0df812140)
    /usr/local/go/src/net/http/server.go:1817 +0x6d4
created by net/http.(*Server).Serve
    /usr/local/go/src/net/http/server.go:2928 +0x384

goroutine 14482 [IO wait]:
internal/poll.runtime_pollWait(0x7f488fe97c00, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc007af0298, 0x72, 0x0, 0x1, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc007af0280, 0xc00a48ce81, 0x1, 0x1, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc007af0280, 0xc00a48ce81, 0x1, 0x1, 0xc0deaa67b8, 0x12ef03d, 0xc000258680)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0035de450, 0xc00a48ce81, 0x1, 0x1, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x68
net/http.(*connReader).backgroundRead(0xc00a48ce70)
    /usr/local/go/src/net/http/server.go:677 +0x58
created by net/http.(*connReader).startBackgroundRead
    /usr/local/go/src/net/http/server.go:673 +0xd4

goroutine 14486 [runnable]:
github.com/influxdata/flux/execute.(*GroupLookup).newKeyGroup(...)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/group_lookup.go:263
github.com/influxdata/flux/execute.(*GroupLookup).createOrSetInGroup.func1(0xc2d340e840, 0x1, 0xc309e4f650, 0xd)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/group_lookup.go:245 +0x1b6
github.com/influxdata/flux/execute.(*GroupLookup).createOrSetInGroup(0xc309e4f650, 0x1f1, 0x2f1ba20, 0xc2cc967a90, 0x1bdb060, 0xc07a454be0)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/group_lookup.go:246 +0x62a
github.com/influxdata/flux/execute.(*GroupLookup).Set(0xc309e4f650, 0x2f1ba20, 0xc2cc967a90, 0x1bdb060, 0xc07a454be0)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/group_lookup.go:148 +0x7f
github.com/influxdata/flux/execute.(*tableBuilderCache).TableBuilder(0xc309e35cc0, 0x2f1ba20, 0xc2cc967a90, 0xc36edaa1c0, 0x5, 0x9)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/table.go:2146 +0x212
github.com/influxdata/flux/stdlib/universe.(*fixedWindowTransformation).Process(0xc3099fcb60, 0x4d5eccd93a5f1dca, 0xb8e093295a108e, 0x7f488facf740, 0xc3c25e8d80, 0x1, 0xa)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/stdlib/universe/window.go:300 +0x81e
github.com/influxdata/flux/execute.processMessage(0x2eec740, 0xc309e4f380, 0x2ef6300, 0xc3099fcb60, 0x2ec7780, 0xc3c2607e60, 0xc005189e9a, 0xc006e08000, 0xc005189ef8)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/transport.go:206 +0x2d6
github.com/influxdata/flux/execute.(*consecutiveTransport).processMessages(0xc309df6d20, 0x2eec740, 0xc309e4f380, 0xa)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/transport.go:159 +0xc6
github.com/influxdata/flux/execute.(*poolDispatcher).run(0xc309df6cc0, 0x2eec740, 0xc309e4f380)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/dispatcher.go:132 +0x65
github.com/influxdata/flux/execute.(*poolDispatcher).Start.func1(0xc309df6cc0, 0x2eec740, 0xc309e4f380)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/dispatcher.go:86 +0xcd
created by github.com/influxdata/flux/execute.(*poolDispatcher).Start
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/dispatcher.go:63 +0x7e

goroutine 13441 [select]:
github.com/influxdata/flux/execute.(*executionState).do.func3(0xc006c6fb80, 0x2eec740, 0xc003535470)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:292 +0x16a
created by github.com/influxdata/flux/execute.(*executionState).do
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:289 +0x1b4

goroutine 13385 [select]:
github.com/influxdata/flux/execute.(*result).Do(0xc003535ad0, 0x2bbf750, 0xc0000ca340, 0xc003535ad0)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/result.go:60 +0xd9
github.com/influxdata/influxdb/v2/query/control.(*errorCollectingTableIterator).Do(0xc0000ca340, 0x2bbf750, 0xc0000ca340, 0xc0000ca320)
    /go/src/github.com/influxdata/influxdb/query/control/controller.go:908 +0x41
github.com/influxdata/influxdb/v2/task/backend/executor.exhaustResultIterators(0x2eca980, 0xc0000ca320, 0xc0000ca320, 0xc007d28370)
    /go/src/github.com/influxdata/influxdb/task/backend/executor/executor.go:513 +0x51
github.com/influxdata/influxdb/v2/task/backend/executor.(*worker).executeQuery(0xc0053be550, 0xc0068bc400)
    /go/src/github.com/influxdata/influxdb/task/backend/executor/executor.go:424 +0x534
github.com/influxdata/influxdb/v2/task/backend/executor.(*worker).work(0xc0053be550)
    /go/src/github.com/influxdata/influxdb/task/backend/executor/executor.go:319 +0x538
github.com/influxdata/influxdb/v2/task/backend/executor.(*Executor).startWorker.func1(0xc004592d80, 0xc0053be550)
    /go/src/github.com/influxdata/influxdb/task/backend/executor/executor.go:205 +0x7e
created by github.com/influxdata/influxdb/v2/task/backend/executor.(*Executor).startWorker
    /go/src/github.com/influxdata/influxdb/task/backend/executor/executor.go:202 +0xa8

goroutine 13443 [chan receive]:
github.com/influxdata/flux/lang.(*Program).readMetadata(0xc003d53cc0, 0xc000165ad0, 0xc007597b00)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/lang/compiler.go:284 +0xa2
created by github.com/influxdata/flux/lang.(*Program).Start
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/lang/compiler.go:263 +0x355

goroutine 14258 [select]:
github.com/influxdata/flux/execute.(*executionState).do.func3(0xc0e3842880, 0x2eec740, 0xc080dd3b00)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:292 +0x16a
created by github.com/influxdata/flux/execute.(*executionState).do
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:289 +0x1b4

goroutine 14260 [chan receive]:
github.com/influxdata/flux/lang.(*Program).readMetadata(0xc0e0f23860, 0xc0769ad600, 0xc0031e6420)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/lang/compiler.go:284 +0xa2
created by github.com/influxdata/flux/lang.(*Program).Start
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/lang/compiler.go:263 +0x355

goroutine 14487 [select]:
github.com/influxdata/flux/execute.(*executionState).do.func3(0xc309e54100, 0x2eec740, 0xc309e4f380)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:292 +0x16a
created by github.com/influxdata/flux/execute.(*executionState).do
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:289 +0x1b4

goroutine 14257 [runnable]:
github.com/influxdata/flux/execute.(*GroupLookup).newKeyGroup(...)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/group_lookup.go:263
github.com/influxdata/flux/execute.(*GroupLookup).createOrSetInGroup.func1(0xc2d340d500, 0x1, 0xc080ddb1a0, 0x1d6)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/group_lookup.go:245 +0x1b6
github.com/influxdata/flux/execute.(*GroupLookup).createOrSetInGroup(0xc080ddb1a0, 0x228, 0x2f1ba20, 0xc2d33f8e60, 0x1bdb060, 0xc07a4657c0)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/group_lookup.go:246 +0x62a
github.com/influxdata/flux/execute.(*GroupLookup).Set(0xc080ddb1a0, 0x2f1ba20, 0xc2d33f8e60, 0x1bdb060, 0xc07a4657c0)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/group_lookup.go:148 +0x7f
github.com/influxdata/flux/execute.(*tableBuilderCache).TableBuilder(0xc080ec8fc0, 0x2f1ba20, 0xc2d33f8e60, 0xc0001849a0, 0x5, 0x9)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/table.go:2146 +0x212
github.com/influxdata/flux/stdlib/universe.(*fixedWindowTransformation).Process(0xc0e3496000, 0x4d5eccd93a5f1dca, 0xb8e093295a108e, 0x7f488facf740, 0xc32b542840, 0x1, 0xa)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/stdlib/universe/window.go:300 +0x81e
github.com/influxdata/flux/execute.processMessage(0x2eec740, 0xc080dd3b00, 0x2ef6300, 0xc0e3496000, 0x2ec7780, 0xc32b5bf380, 0xc00188fe9a, 0xc002df8f00, 0xc00188fef8)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/transport.go:206 +0x2d6
github.com/influxdata/flux/execute.(*consecutiveTransport).processMessages(0xc0031e6120, 0x2eec740, 0xc080dd3b00, 0xa)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/transport.go:159 +0xc6
github.com/influxdata/flux/execute.(*poolDispatcher).run(0xc0031e60c0, 0x2eec740, 0xc080dd3b00)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/dispatcher.go:132 +0x65
github.com/influxdata/flux/execute.(*poolDispatcher).Start.func1(0xc0031e60c0, 0x2eec740, 0xc080dd3b00)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/dispatcher.go:86 +0xcd
created by github.com/influxdata/flux/execute.(*poolDispatcher).Start
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/dispatcher.go:63 +0x7e

goroutine 14000 [semacquire]:
sync.runtime_Semacquire(0xc060a04518)
    /usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc060a04510)
    /usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/influxdata/flux/execute.(*executionState).do.func2(0xc0e3842880, 0xc060a04510)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:285 +0x5f
created by github.com/influxdata/flux/execute.(*executionState).do
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:283 +0x13d

goroutine 14485 [semacquire]:
sync.runtime_Semacquire(0xc309e39078)
    /usr/local/go/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc309e39070)
    /usr/local/go/src/sync/waitgroup.go:130 +0x64
github.com/influxdata/flux/execute.(*executionState).do.func2(0xc309e54100, 0xc309e39070)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:285 +0x5f
created by github.com/influxdata/flux/execute.(*executionState).do
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:283 +0x13d

goroutine 13997 [IO wait]:
internal/poll.runtime_pollWait(0x7f488fe97cd0, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0065ed318, 0x72, 0x0, 0x1, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0065ed300, 0xc007b55361, 0x1, 0x1, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0065ed300, 0xc007b55361, 0x1, 0x1, 0x0, 0x2bbd778, 0x0)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0029f64a0, 0xc007b55361, 0x1, 0x1, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x68
net/http.(*connReader).backgroundRead(0xc007b55350)
    /usr/local/go/src/net/http/server.go:677 +0x58
created by net/http.(*connReader).startBackgroundRead
    /usr/local/go/src/net/http/server.go:673 +0xd4

goroutine 13438 [select]:
github.com/influxdata/influxdb/v2/storage/flux.(*filterIterator).handleRead(0xc005c99500, 0xc0064082a0, 0x2f02120, 0xc0052520c0, 0x0, 0x0)
    /go/src/github.com/influxdata/influxdb/storage/flux/reader.go:231 +0x58e
github.com/influxdata/influxdb/v2/storage/flux.(*filterIterator).Do(0xc005c99500, 0xc0064082a0, 0xc00640aa01, 0xc005c99500)
    /go/src/github.com/influxdata/influxdb/storage/flux/reader.go:171 +0x148
github.com/influxdata/influxdb/v2/query/stdlib/influxdata/influxdb.(*Source).processTables(0xc000165c30, 0x2eec740, 0xc003535470, 0x2ecaf40, 0xc005c99500, 0x16124549684cd000, 0x16124549684cd000, 0xc007597500)
    /go/src/github.com/influxdata/influxdb/query/stdlib/influxdata/influxdb/source.go:75 +0x94
github.com/influxdata/influxdb/v2/query/stdlib/influxdata/influxdb.(*readFilterSource).run(0xc000165c30, 0x2eec740, 0xc003535470, 0x59e7cb8e5469000, 0x1e3f8d4)
    /go/src/github.com/influxdata/influxdb/query/stdlib/influxdata/influxdb/source.go:152 +0x138
github.com/influxdata/influxdb/v2/query/stdlib/influxdata/influxdb.(*Source).Run(0xc000165c30, 0x2eec740, 0xc003535470)
    /go/src/github.com/influxdata/influxdb/query/stdlib/influxdata/influxdb/source.go:55 +0x31d
github.com/influxdata/flux/execute.(*executionState).do.func1(0xc00640aac0, 0xc006c6fb80, 0x2eec740, 0xc003535470, 0x2ecad40, 0xc000165c30)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:275 +0xd7
created by github.com/influxdata/flux/execute.(*executionState).do
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/execute/executor.go:250 +0xf8

goroutine 14489 [chan receive]:
github.com/influxdata/flux/lang.(*Program).readMetadata(0xc3094f2a20, 0xc308eaa840, 0xc309df7020)
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/lang/compiler.go:284 +0xa2
created by github.com/influxdata/flux/lang.(*Program).Start
    /go/pkg/mod/github.com/influxdata/flux@v0.66.1/lang/compiler.go:263 +0x355```

[danxmoran]

I believe this is/was a Flux bug unrelated to Docker.

@bkak do you still see the panic if you run with v2.0.3 instead of 2.0.0-beta? Flux has gone through many revisions in-between.

[danxmoran]

Closing this for now, feel free to reopen / create a new issue if you see the problem again with the latest builds.