search

cockroachdb / cockroach

CockroachDB — the cloud native, distributed SQL database designed for high availability, effortless scale, and control over data placement.
https://www.cockroachlabs.com
Other
30.2k stars 3.82k forks source link

logictest: racy partitioning value modification in indexSplitAndScatter #116754

Closed cockroach-teamcity closed 11 months ago

cockroach-teamcity commented 11 months ago

pkg/ccl/logictestccl/tests/5node/5node_test.TestCCLLogic_partitioning_hash_sharded_index failed on master @ 5f76d93b2ad8d29c14c9b176753ec33cbf81e3fe:

      github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1918 +0x26b
  github.com/cockroachdb/cockroach/pkg/util/netutil.(*TCPServer).ServeWith.func1()
      github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:191 +0x1c6
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 536 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1a4
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.GRPCTransportFactory()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport_race.go:98 +0x133
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2312 +0xdba
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1925 +0x9ab
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1493 +0x1157
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*RangeIterator).Seek()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/range_iter.go:208 +0x75a
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1487 +0x29a
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1109 +0xa4f
  github.com/cockroachdb/cockroach/pkg/kv.lookupRangeFwdScan()
      github.com/cockroachdb/cockroach/pkg/kv/range_lookup.go:331 +0x861
  github.com/cockroachdb/cockroach/pkg/kv.RangeLookup()
      github.com/cockroachdb/cockroach/pkg/kv/range_lookup.go:206 +0x349
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeLookup()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:825 +0x12a
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache.(*RangeCache).performRangeLookup()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache/range_cache.go:1041 +0x3f0
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache.tryLookupImpl.func1()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache/range_cache.go:929 +0xb9
  github.com/cockroachdb/cockroach/pkg/util/timeutil.RunWithTimeout()
      github.com/cockroachdb/cockroach/pkg/util/timeutil/timeout.go:29 +0x12d
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache.tryLookupImpl()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache/range_cache.go:926 +0x206
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache.(*RangeCache).tryLookup.func3()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache/range_cache.go:824 +0xcd
  github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).doCall.func1()
      github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:389 +0x4b
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:319 +0x185
  github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).doCall()
      github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:388 +0x2c4
  github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).DoChan.func1()
      github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:358 +0xc6
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/sql-queries

This test on roachdash | Improve this report!

Jira issue: CRDB-34757

yuzefovich commented 11 months ago

This seems like unintended modification of the key coming from partitioning descriptor due to memory aliasing

WARNING: DATA RACE
Write at 0x00c008ec9c8d by goroutine 9158:
  github.com/cockroachdb/cockroach/pkg/util/encoding.EncodeUvarintAscending()
      github.com/cockroachdb/cockroach/pkg/util/encoding/encoding.go:414 +0xb8
  github.com/cockroachdb/cockroach/pkg/util/encoding.EncodeVarintAscending()
      github.com/cockroachdb/cockroach/pkg/util/encoding/encoding.go:336 +0xc1b
  github.com/cockroachdb/cockroach/pkg/sql.(*indexSplitAndScatter).MaybeSplitIndexSpansForPartitioning()
      github.com/cockroachdb/cockroach/pkg/sql/index_split_scatter.go:130 +0x78a
  github.com/cockroachdb/cockroach/pkg/sql.(*indexSplitAndScatter).MaybeSplitIndexSpans()
      github.com/cockroachdb/cockroach/pkg/sql/index_split_scatter.go:48 +0xa8
  github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec.(*deferredState).exec()
      github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec/exec_deferred_mutation.go:206 +0x613
  github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec.executeMutationOps()
      github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec/exec_mutation.go:54 +0x8e8
  github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec.ExecuteStage()
      github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scexec/executor.go:28 +0x1eb
  github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scrun.executeStage()
      github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scrun/scrun.go:193 +0x66b
  github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scrun.runTransactionPhase()
      github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scrun/scrun.go:103 +0x8e7
  github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scrun.RunPreCommitPhase()
      github.com/cockroachdb/cockroach/pkg/sql/schemachanger/scrun/scrun.go:67 +0x605
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).runPreCommitStages()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:4242 +0x553
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1489 +0x57e
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal-fm()
      <autogenerated>:1 +0x47
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransaction()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1391 +0x3c1
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3075 +0x269
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState.func14()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:757 +0x330
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState.func23()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:759 +0x4f
  runtime.deferreturn()
      GOROOT/src/runtime/panic.go:477 +0x30
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:146 +0x184
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3372 +0x579
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x75d
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2227 +0x7e4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2232 +0x2452
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:146 +0x184
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3372 +0x579
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x75d
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2227 +0x7e4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2232 +0x2452
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:146 +0x184
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3372 +0x579
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x75d
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2227 +0x7e4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2232 +0x2452
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:146 +0x184
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3372 +0x579
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x75d
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2227 +0x7e4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2232 +0x2452
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:146 +0x184
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3372 +0x579
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x75d
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2227 +0x7e4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2232 +0x2452
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*planner).SchemaChange()
      github.com/cockroachdb/cockroach/pkg/sql/schema_change_plan_node.go:72 +0x478
  github.com/cockroachdb/cockroach/pkg/sql.buildOpaque()
      github.com/cockroachdb/cockroach/pkg/sql/opaque.go:57 +0x3a5
  github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder.(*Builder).tryBuildOpaque()
      github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/opaque.go:70 +0x23b
  github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder.(*Builder).buildStmt()
      github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/builder.go:457 +0xfb7
  github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder.(*Builder).buildStmtAtRootWithScope()
      github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/builder.go:290 +0x2fa
  github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder.(*Builder).buildStmtAtRoot()
      github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/builder.go:271 +0x1ea
  github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder.(*Builder).Build()
      github.com/cockroachdb/cockroach/pkg/sql/opt/optbuilder/builder.go:250 +0x66e
  github.com/cockroachdb/cockroach/pkg/sql.(*optPlanningCtx).buildExecMemo()
      github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:585 +0xba5
  github.com/cockroachdb/cockroach/pkg/sql.(*planner).makeOptimizerPlan()
      github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:245 +0x211
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).makeExecPlan()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2274 +0x125
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1744 +0xd04
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1108 +0x7aa7
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:146 +0x184
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3372 +0x579
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x75d
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2227 +0x7e4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2232 +0x2452
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).makeExecPlan()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2274 +0x125
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1744 +0xd04
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1108 +0x7aa7
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:146 +0x184
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3372 +0x579
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x75d
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2227 +0x7e4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2232 +0x2452
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).makeExecPlan()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2274 +0x125
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1744 +0xd04
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1108 +0x7aa7
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:146 +0x184
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3372 +0x579
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x75d
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2227 +0x7e4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2232 +0x2452
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1108 +0x7aa7
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:146 +0x184
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3372 +0x579
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x75d
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2227 +0x7e4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2232 +0x2452
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1108 +0x7aa7
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:146 +0x184
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:3372 +0x579
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:145 +0x75d
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func1()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2227 +0x7e4
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2232 +0x2452
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2149 +0x3d5
  github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn()
      github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:953 +0x21d
  github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommands()
      github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:247 +0x572
  github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveImpl.func3()
      github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:1001 +0x21c
Previous read at 0x00c008ec9c8d by goroutine 536:
  encoding/base64.(*Encoding).Encode()
      GOROOT/src/encoding/base64/base64.go:174 +0x410
  encoding/json.encodeByteSlice()
      GOROOT/src/encoding/json/encode.go:791 +0x414
  encoding/json.structEncoder.encode()
      GOROOT/src/encoding/json/encode.go:706 +0x2bd
  encoding/json.structEncoder.encode-fm()
      <autogenerated>:1 +0xe4
  encoding/json.ptrEncoder.encode()
      GOROOT/src/encoding/json/encode.go:878 +0x3d1
  encoding/json.ptrEncoder.encode-fm()
      <autogenerated>:1 +0x86
  encoding/json.structEncoder.encode()
      GOROOT/src/encoding/json/encode.go:706 +0x2bd
  encoding/json.structEncoder.encode-fm()
      <autogenerated>:1 +0xe4
  encoding/json.ptrEncoder.encode()
      GOROOT/src/encoding/json/encode.go:878 +0x3d1
  encoding/json.ptrEncoder.encode-fm()
      <autogenerated>:1 +0x86
  encoding/json.(*encodeState).reflectValue()
      GOROOT/src/encoding/json/encode.go:323 +0x83
  encoding/json.interfaceEncoder()
      GOROOT/src/encoding/json/encode.go:660 +0x164
  encoding/json.structEncoder.encode()
      GOROOT/src/encoding/json/encode.go:706 +0x2bd
  encoding/json.structEncoder.encode-fm()
      <autogenerated>:1 +0xe4
  encoding/json.arrayEncoder.encode()
      GOROOT/src/encoding/json/encode.go:849 +0x112
  encoding/json.arrayEncoder.encode-fm()
      <autogenerated>:1 +0x86
  encoding/json.sliceEncoder.encode()
      GOROOT/src/encoding/json/encode.go:822 +0x4dd
  encoding/json.sliceEncoder.encode-fm()
      <autogenerated>:1 +0x86
  encoding/json.structEncoder.encode()
      GOROOT/src/encoding/json/encode.go:706 +0x2bd
  encoding/json.structEncoder.encode-fm()
      <autogenerated>:1 +0xe4
  encoding/json.ptrEncoder.encode()
      GOROOT/src/encoding/json/encode.go:878 +0x3d1
  encoding/json.ptrEncoder.encode-fm()
      <autogenerated>:1 +0x86
  encoding/json.(*encodeState).reflectValue()
      GOROOT/src/encoding/json/encode.go:323 +0x83
  encoding/json.(*encodeState).marshal()
      GOROOT/src/encoding/json/encode.go:295 +0x1af
  encoding/json.(*Encoder).Encode()
      GOROOT/src/encoding/json/stream.go:209 +0x16f
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.GRPCTransportFactory.func1()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport_race.go:120 +0x344
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 9158 (running) created at:
  github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).serveImpl()
      github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:998 +0xa4f
  github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn()
      github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:814 +0xe24
  github.com/cockroachdb/cockroach/pkg/server.(*systemServerWrapper).serveConn()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_sql.go:164 +0x1d7
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).sqlMux()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_sql.go:84 +0x5c9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).sqlMux-fm()
      <autogenerated>:1 +0xc4
  github.com/cockroachdb/cockroach/pkg/server.startServeSQL.func1.1()
      github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1918 +0x26b
  github.com/cockroachdb/cockroach/pkg/util/netutil.(*TCPServer).ServeWith.func1()
      github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:191 +0x1c6
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 536 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1a4
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.GRPCTransportFactory()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport_race.go:98 +0x133
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2312 +0xdba
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1925 +0x9ab
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1493 +0x1157
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*RangeIterator).Seek()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/range_iter.go:208 +0x75a
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1487 +0x29a
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1109 +0xa4f
  github.com/cockroachdb/cockroach/pkg/kv.lookupRangeFwdScan()
      github.com/cockroachdb/cockroach/pkg/kv/range_lookup.go:331 +0x861
  github.com/cockroachdb/cockroach/pkg/kv.RangeLookup()
      github.com/cockroachdb/cockroach/pkg/kv/range_lookup.go:206 +0x349
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeLookup()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:825 +0x12a
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache.(*RangeCache).performRangeLookup()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache/range_cache.go:1041 +0x3f0
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache.tryLookupImpl.func1()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache/range_cache.go:929 +0xb9
  github.com/cockroachdb/cockroach/pkg/util/timeutil.RunWithTimeout()
      github.com/cockroachdb/cockroach/pkg/util/timeutil/timeout.go:29 +0x12d
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache.tryLookupImpl()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache/range_cache.go:926 +0x206
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache.(*RangeCache).tryLookup.func3()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangecache/range_cache.go:824 +0xcd
  github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).doCall.func1()
      github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:389 +0x4b
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:319 +0x185
  github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).doCall()
      github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:388 +0x2c4
  github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight.(*Group).DoChan.func1()
      github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight/singleflight.go:358 +0xc6