Closed cockroach-teamcity closed 9 months ago
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ 67664ad7a73b36145e4c48c14794636ef225bb9a:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 22788 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 20446 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
testing.tRunner()
GOROOT/src/testing/testing.go:1595 +0x261
testing.(*T).Run.func1()
GOROOT/src/testing/testing.go:1648 +0x44
==================
Parameters:
TAGS=bazel,gss
stress=true
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ e2ad8d72175a985b11fefe1cb6eacede0654a36c:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 22792 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 20459 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
testing.tRunner()
GOROOT/src/testing/testing.go:1595 +0x261
testing.(*T).Run.func1()
GOROOT/src/testing/testing.go:1648 +0x44
==================
Parameters:
TAGS=bazel,gss
stress=true
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ ee3495831f15ed85eeb8604c96e48b4f112b4f22:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 21394 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 19052 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
testing.tRunner()
GOROOT/src/testing/testing.go:1595 +0x261
testing.(*T).Run.func1()
GOROOT/src/testing/testing.go:1648 +0x44
==================
Parameters:
TAGS=bazel,gss
stress=true
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ 66e264f9db0ab639a829997f66b8fd927c820a06:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x568
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Previous read at 0x00c009e940f0 by goroutine 18605:
github.com/cockroachdb/cockroach/pkg/settings.(*Values).TestingCopyForVirtualCluster()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/values.go:275 +0x12d
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).makeSharedProcessTenantConfig()
github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:170 +0x26c
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).newTenantServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:78 +0x14b
github.com/cockroachdb/cockroach/pkg/server.(*serverController).newServerForOrchestrator()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:262 +0x1bc
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5.2()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:371 +0x11a
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:395 +0x9aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 18608 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x6e8
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:245 +0x65f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:335 +0xbb7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 18605 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
==================
Parameters:
TAGS=bazel,gss
stress=true
This is failing due to a race being detected. Here's the full test failure output:
=== RUN TestInsightsIntegrationForContention
test_log_scope.go:170: test logs captured to: /artifacts/tmp/_tmp/f75d8af2c2b6af04e0405b19fd4d23f3/logTestInsightsIntegrationForContention4073779800
test_log_scope.go:81: use -show-logs to present logs inline
test_server_shim.go:168: automatically injected a shared process virtual cluster under test; see comment at top of test_server_shim.go for details.
==================
WARNING: DATA RACE
Write at 0x00c009e940f0 by goroutine 18608:
sync/atomic.SwapInt64()
src/runtime/race_amd64.s:262 +0xb
sync/atomic.SwapInt64()
<autogenerated>:1 +0x15
github.com/cockroachdb/cockroach/pkg/settings.(*Values).setInt64()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/values.go:204 +0x44
github.com/cockroachdb/cockroach/pkg/settings.(*BoolSetting).set()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/bool.go:94 +0x538
github.com/cockroachdb/cockroach/pkg/settings.updater.setInternal()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/updater.go:180 +0x503
github.com/cockroachdb/cockroach/pkg/settings.updater.SetFromStorage()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/updater.go:270 +0x1d0
github.com/cockroachdb/cockroach/pkg/settings.(*updater).SetFromStorage()
<autogenerated>:1 +0xf5
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).setLocked()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:466 +0x749
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).applyValueFromSystemSettingsOrDefaultLocked()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:540 +0x18f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).maybeSet()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:407 +0x353
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).handleKV()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:374 +0x8c8
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start.func3()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:253 +0x48
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run.func3()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:269 +0x8f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).runInitialScan.func1()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/scanner.go:55 +0x2de
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).scanSpan.func1()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:185 +0x6c5
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec()
github.com/cockroachdb/cockroach/pkg/kv/txn.go:1032 +0x8f
github.com/cockroachdb/cockroach/pkg/kv.runTxn()
github.com/cockroachdb/cockroach/pkg/kv/db.go:1055 +0x64
github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl()
github.com/cockroachdb/cockroach/pkg/kv/db.go:1018 +0xec
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).scanSpan()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:167 +0x2dd
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).Scan()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:103 +0x884
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).runInitialScan()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/scanner.go:63 +0x3f8
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:279 +0x32f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x568
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Previous read at 0x00c009e940f0 by goroutine 18605:
github.com/cockroachdb/cockroach/pkg/settings.(*Values).TestingCopyForVirtualCluster()
github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/values.go:275 +0x12d
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).makeSharedProcessTenantConfig()
github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:170 +0x26c
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).newTenantServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:78 +0x14b
github.com/cockroachdb/cockroach/pkg/server.(*serverController).newServerForOrchestrator()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:262 +0x1bc
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5.2()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:371 +0x11a
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:395 +0x9aa
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 18608 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x6e8
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:245 +0x65f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:335 +0xbb7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 18605 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
==================
I240115 18:24:26.167982 1 (gostd) testmain.go:97 [-] 1 Test //pkg/sql/sqlstats/insights/integration:integration_test exited with error code 66
ERROR: exit status 66
The race complains about concurrent read/write access to valuesContainer.intVals
without mutex protection: https://github.com/cockroachdb/cockroach/blob/20132d33e74191972566cf80459ef507e095c404/pkg/settings/values.go#L81
One of the goroutines appears to be the server controller orchestrator:
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:395 +0x9aa
The other appears to be a rangefeed:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x568
I'm seeing the same race failure here: https://github.com/cockroachdb/cockroach/issues/117669
Also noticing this here: https://github.com/cockroachdb/cockroach/issues/117534
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ 515663b99f79f96e3e8fbdef0cd7789cc38612fa:
Goroutine 19915 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 17824 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus()
github.com/cockroachdb/cockroach/pkg/server/node.go:1200 +0x128
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:1990 +0x4fe4
github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
testing.tRunner()
GOROOT/src/testing/testing.go:1595 +0x261
testing.(*T).Run.func1()
GOROOT/src/testing/testing.go:1648 +0x44
==================
Parameters:
TAGS=bazel,gss
stress=true
On KV we're seeing this race in #117763, , https://github.com/cockroachdb/cockroach/issues/117611, and https://github.com/cockroachdb/cockroach/issues/117726.
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ 9b10f083f8889cdc1bc549d9428f06fd784ba081:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 17284 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
Goroutine 15230 (running) created at:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
testing.tRunner()
GOROOT/src/testing/testing.go:1595 +0x261
testing.(*T).Run.func1()
GOROOT/src/testing/testing.go:1648 +0x44
==================
Parameters:
TAGS=bazel,gss
stress=true
pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ 695f7efa3fca5e13668baa418dc4f4450ac9677b:
Parameters:
TAGS=bazel,gss
stress=true
Help
See also: How To Investigate a Go Test Failure (internal)
/cc @cockroachdb/cluster-observabilityThis test on roachdash | Improve this report!
Jira issue: CRDB-35307