Closed cockroach-teamcity closed 9 months ago
WARNING: DATA RACE
Read at 0x00c0074de000 by goroutine 256918:
github.com/cockroachdb/cockroach/pkg/settings.(*Values).TestingCopyForVirtualCluster()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/values.go:275 +0x12d
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).makeSharedProcessTenantConfig()
github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:170 +0x26c
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).newTenantServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:78 +0x14b
github.com/cockroachdb/cockroach/pkg/server.(*serverController).newServerForOrchestrator()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:262 +0x1bc
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5.2()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:371 +0x11a
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:395 +0x9aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Previous write at 0x00c0074de000 by goroutine 254383:
sync/atomic.SwapInt64()
src/runtime/race_amd64.s:262 +0xb
sync/atomic.SwapInt64()
<autogenerated>:1 +0x15
github.com/cockroachdb/cockroach/pkg/settings.(*Values).setInt64()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/values.go:204 +0x44
github.com/cockroachdb/cockroach/pkg/settings.(*IntSetting).set()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/int.go:101 +0xe5
github.com/cockroachdb/cockroach/pkg/settings.(*IntSetting).setToDefault()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/int.go:113 +0xd2
github.com/cockroachdb/cockroach/pkg/settings.(*ByteSizeSetting).setToDefault()
<autogenerated>:1 +0x4f
github.com/cockroachdb/cockroach/pkg/settings.updater.ResetRemaining()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/updater.go:252 +0x1fa
github.com/cockroachdb/cockroach/pkg/settings.(*updater).ResetRemaining()
<autogenerated>:1 +0x64
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start.func1()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:190 +0x14a
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start.func4()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:256 +0xbc
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).handleUpdate()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:385 +0x197
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:357 +0x10c4
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 256918 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 254383 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x57a5
github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:806 +0x104
github.com/cockroachdb/cockroach/pkg/server.(*testServer).Start()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:882 +0xda
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).Start()
github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:15 +0x73
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:290 +0x281
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:276 +0x151
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnly()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:305 +0x86
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServer()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:323 +0x6f
github.com/cockroachdb/cockroach/pkg/sql/tests.TestTruncateWithConcurrentMutations.func1()
github.com/cockroachdb/cockroach/pkg/sql/tests/truncate_test.go:94 +0x384
github.com/cockroachdb/cockroach/pkg/sql/tests.TestTruncateWithConcurrentMutations.func2()
github.com/cockroachdb/cockroach/pkg/sql/tests/truncate_test.go:364 +0xbc
testing.tRunner()
GOROOT/src/testing/testing.go:1595 +0x261
testing.(*T).Run.func1()
GOROOT/src/testing/testing.go:1648 +0x44
==================
I240109 18:59:02.865209 1 (gostd) testmain.go:191 [-] 1 Test //pkg/sql/tests:tests_test exited with error code 66
Seems likely to be due to 20132d33e74191972566cf80459ef507e095c404.
This is a race between the rangefeed watching settings changes on the system tenant and shared-process tenant being initialized on that same node. Hm, I'm not sure how to fix it. One idea is to create a deep copy for the shared-process args struct as:
diff --git a/pkg/server/testserver.go b/pkg/server/testserver.go
index 15a24bfed19..dfb1b283e67 100644
--- a/pkg/server/testserver.go
+++ b/pkg/server/testserver.go
@@ -651,8 +651,9 @@ func (ts *testServer) getSharedProcessDefaultTenantArgs() base.TestSharedProcess
TenantID: serverutils.TestTenantID(),
Knobs: ts.params.Knobs,
UseDatabase: ts.params.UseDatabase,
- Settings: ts.params.Settings,
}
+ args.Settings = &cluster.Settings{}
+ args.Settings.SV.TestingCopyForServer(&ts.params.Settings.SV, ts.params.Settings.SV.Opaque())
// See comment above on separate process tenant regarding the testing knobs.
args.Knobs.Server = &TestingKnobs{}
if ts.params.Knobs.Server != nil {
this should reduce the likelihood of the race, but wouldn't completely eliminate it.
@dt @stevendanna thoughts?
sql/tests.TestTruncateWithConcurrentMutations failed with artifacts on master @ 12ce879a77d139a51632c0b3b7799137d67e6524:
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 299948 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:806 +0x104
github.com/cockroachdb/cockroach/pkg/server.(*testServer).Start()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:882 +0xda
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).Start()
github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:15 +0x73
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:290 +0x281
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:276 +0x151
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnly()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:305 +0x86
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServer()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:323 +0x6f
github.com/cockroachdb/cockroach/pkg/sql/tests.TestTruncateWithConcurrentMutations.func1()
github.com/cockroachdb/cockroach/pkg/sql/tests/truncate_test.go:94 +0x384
github.com/cockroachdb/cockroach/pkg/sql/tests.TestTruncateWithConcurrentMutations.func2()
github.com/cockroachdb/cockroach/pkg/sql/tests/truncate_test.go:364 +0xbc
testing.tRunner()
GOROOT/src/testing/testing.go:1595 +0x261
testing.(*T).Run.func1()
GOROOT/src/testing/testing.go:1648 +0x44
==================
Parameters:
TAGS=bazel,gss
stress=true
sql/tests.TestTruncateWithConcurrentMutations failed with artifacts on master @ e2ad8d72175a985b11fefe1cb6eacede0654a36c:
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 327728 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus()
github.com/cockroachdb/cockroach/pkg/server/node.go:1200 +0x128
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:1990 +0x4fe4
github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
github.com/cockroachdb/cockroach/pkg/server.(*testServer).Start()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:883 +0xda
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).Start()
github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:15 +0x73
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:290 +0x281
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:276 +0x151
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnly()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:305 +0x86
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServer()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:323 +0x6f
github.com/cockroachdb/cockroach/pkg/sql/tests.TestTruncateWithConcurrentMutations.func1()
github.com/cockroachdb/cockroach/pkg/sql/tests/truncate_test.go:94 +0x384
github.com/cockroachdb/cockroach/pkg/sql/tests.TestTruncateWithConcurrentMutations.func2()
github.com/cockroachdb/cockroach/pkg/sql/tests/truncate_test.go:364 +0xbc
testing.tRunner()
GOROOT/src/testing/testing.go:1595 +0x261
testing.(*T).Run.func1()
GOROOT/src/testing/testing.go:1648 +0x44
==================
=== RUN TestTruncateWithConcurrentMutations/drop_column_which_owns_sequence
=== RUN TestTruncateWithConcurrentMutations/drop_column_which_uses_sequence
test_server_shim.go:168: automatically injected an external process virtual cluster under test; see comment at top of test_server_shim.go for details.
Parameters:
TAGS=bazel,gss
stress=true
sql/tests.TestTruncateWithConcurrentMutations failed with artifacts on master @ 66e264f9db0ab639a829997f66b8fd927c820a06:
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 243361 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
github.com/cockroachdb/cockroach/pkg/server.(*testServer).Start()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:883 +0xda
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).Start()
github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:15 +0x73
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:290 +0x281
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:276 +0x151
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnly()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:305 +0x86
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServer()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:323 +0x6f
github.com/cockroachdb/cockroach/pkg/sql/tests.TestTruncateWithConcurrentMutations.func1()
github.com/cockroachdb/cockroach/pkg/sql/tests/truncate_test.go:94 +0x384
github.com/cockroachdb/cockroach/pkg/sql/tests.TestTruncateWithConcurrentMutations.func2()
github.com/cockroachdb/cockroach/pkg/sql/tests/truncate_test.go:364 +0xbc
testing.tRunner()
GOROOT/src/testing/testing.go:1595 +0x261
testing.(*T).Run.func1()
GOROOT/src/testing/testing.go:1648 +0x44
==================
Parameters:
TAGS=bazel,gss
stress=true
sql/tests.TestTruncateWithConcurrentMutations failed with artifacts on master @ 9b10f083f8889cdc1bc549d9428f06fd784ba081:
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 304500 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
github.com/cockroachdb/cockroach/pkg/server.(*testServer).Start()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:883 +0xda
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).Start()
github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:15 +0x73
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:290 +0x281
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:276 +0x151
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnly()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:305 +0x86
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServer()
github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:323 +0x6f
github.com/cockroachdb/cockroach/pkg/sql/tests.TestTruncateWithConcurrentMutations.func1()
github.com/cockroachdb/cockroach/pkg/sql/tests/truncate_test.go:94 +0x384
github.com/cockroachdb/cockroach/pkg/sql/tests.TestTruncateWithConcurrentMutations.func2()
github.com/cockroachdb/cockroach/pkg/sql/tests/truncate_test.go:364 +0xbc
testing.tRunner()
GOROOT/src/testing/testing.go:1595 +0x261
testing.(*T).Run.func1()
GOROOT/src/testing/testing.go:1648 +0x44
==================
=== RUN TestTruncateWithConcurrentMutations/alter_primary_key
test_server_shim.go:168: automatically injected a shared process virtual cluster under test; see comment at top of test_server_shim.go for details.
=== RUN TestTruncateWithConcurrentMutations/drop_column_with_user-defined_type
test_server_shim.go:168: automatically injected an external process virtual cluster under test; see comment at top of test_server_shim.go for details.
=== RUN TestTruncateWithConcurrentMutations/drop_index
test_server_shim.go:168: automatically injected an external process virtual cluster under test; see comment at top of test_server_shim.go for details.
Parameters:
TAGS=bazel,gss
stress=true
sql/tests.TestTruncateWithConcurrentMutations failed with artifacts on master @ 01da063a52f59f5626f7be1a22fe202f32b90423:
Parameters:
TAGS=bazel,gss
stress=true
Help
See also: How To Investigate a Go Test Failure (internal)
/cc @cockroachdb/sql-queriesThis test on roachdash | Improve this report!
Jira issue: CRDB-35249