search

cockroachdb / cockroach

CockroachDB — the cloud native, distributed SQL database designed for high availability, effortless scale, and control over data placement.
https://www.cockroachlabs.com
Other
30.06k stars 3.8k forks source link

pkg/sql/sqlstats/insights/integration/integration_test: TestInsightsIntegrationForContention failed #117688

Closed cockroach-teamcity closed 9 months ago

cockroach-teamcity commented 9 months ago

pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ 695f7efa3fca5e13668baa418dc4f4450ac9677b:

      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 16466 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
  github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
      github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
  github.com/cockroachdb/cockroach/pkg/util/startup.RunIdempotentWithRetry()
      github.com/cockroachdb/cockroach/pkg/util/startup/retry.go:120 +0xa7
  github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus()
      github.com/cockroachdb/cockroach/pkg/server/node.go:1200 +0x128
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/server.go:1990 +0x4fe4
  github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/testserver.go:806 +0x104
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
      github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
  pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
      pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
  testing.tRunner()
      GOROOT/src/testing/testing.go:1595 +0x261
  testing.(*T).Run.func1()
      GOROOT/src/testing/testing.go:1648 +0x44
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/cluster-observability

This test on roachdash | Improve this report!

Jira issue: CRDB-35307

cockroach-teamcity commented 9 months ago

pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ 67664ad7a73b36145e4c48c14794636ef225bb9a:

      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 22788 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 20446 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
  github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
      github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
  github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
      github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
  pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
      pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
  testing.tRunner()
      GOROOT/src/testing/testing.go:1595 +0x261
  testing.(*T).Run.func1()
      GOROOT/src/testing/testing.go:1648 +0x44
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!

cockroach-teamcity commented 9 months ago

pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ e2ad8d72175a985b11fefe1cb6eacede0654a36c:

      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 22792 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 20459 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
  github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
      github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
  github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
      github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
  pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
      pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
  testing.tRunner()
      GOROOT/src/testing/testing.go:1595 +0x261
  testing.(*T).Run.func1()
      GOROOT/src/testing/testing.go:1648 +0x44
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!

cockroach-teamcity commented 9 months ago

pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ ee3495831f15ed85eeb8604c96e48b4f112b4f22:

      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 21394 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 19052 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
  github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
      github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
  github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
      github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
  pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
      pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
  testing.tRunner()
      GOROOT/src/testing/testing.go:1595 +0x261
  testing.(*T).Run.func1()
      GOROOT/src/testing/testing.go:1648 +0x44
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!

cockroach-teamcity commented 9 months ago

pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ 66e264f9db0ab639a829997f66b8fd927c820a06:

      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x568
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Previous read at 0x00c009e940f0 by goroutine 18605:
  github.com/cockroachdb/cockroach/pkg/settings.(*Values).TestingCopyForVirtualCluster()
      github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/values.go:275 +0x12d
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).makeSharedProcessTenantConfig()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:170 +0x26c
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).newTenantServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:78 +0x14b
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).newServerForOrchestrator()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:262 +0x1bc
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5.2()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:371 +0x11a
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:395 +0x9aa
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 18608 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x6e8
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:245 +0x65f
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:335 +0xbb7
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 18605 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!

abarganier commented 9 months ago

This is failing due to a race being detected. Here's the full test failure output:

=== RUN   TestInsightsIntegrationForContention
    test_log_scope.go:170: test logs captured to: /artifacts/tmp/_tmp/f75d8af2c2b6af04e0405b19fd4d23f3/logTestInsightsIntegrationForContention4073779800
    test_log_scope.go:81: use -show-logs to present logs inline
    test_server_shim.go:168: automatically injected a shared process virtual cluster under test; see comment at top of test_server_shim.go for details.
==================
WARNING: DATA RACE
Write at 0x00c009e940f0 by goroutine 18608:
  sync/atomic.SwapInt64()
      src/runtime/race_amd64.s:262 +0xb
  sync/atomic.SwapInt64()
      <autogenerated>:1 +0x15
  github.com/cockroachdb/cockroach/pkg/settings.(*Values).setInt64()
      github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/values.go:204 +0x44
  github.com/cockroachdb/cockroach/pkg/settings.(*BoolSetting).set()
      github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/bool.go:94 +0x538
  github.com/cockroachdb/cockroach/pkg/settings.updater.setInternal()
      github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/updater.go:180 +0x503
  github.com/cockroachdb/cockroach/pkg/settings.updater.SetFromStorage()
      github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/updater.go:270 +0x1d0
  github.com/cockroachdb/cockroach/pkg/settings.(*updater).SetFromStorage()
      <autogenerated>:1 +0xf5
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).setLocked()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:466 +0x749
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).applyValueFromSystemSettingsOrDefaultLocked()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:540 +0x18f
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).maybeSet()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:407 +0x353
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).handleKV()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:374 +0x8c8
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start.func3()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:253 +0x48
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run.func3()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:269 +0x8f
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).runInitialScan.func1()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/scanner.go:55 +0x2de
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).scanSpan.func1()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:185 +0x6c5
  github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec()
      github.com/cockroachdb/cockroach/pkg/kv/txn.go:1032 +0x8f
  github.com/cockroachdb/cockroach/pkg/kv.runTxn()
      github.com/cockroachdb/cockroach/pkg/kv/db.go:1055 +0x64
  github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl()
      github.com/cockroachdb/cockroach/pkg/kv/db.go:1018 +0xec
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).scanSpan()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:167 +0x2dd
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).Scan()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:103 +0x884
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).runInitialScan()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/scanner.go:63 +0x3f8
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:279 +0x32f
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x568
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Previous read at 0x00c009e940f0 by goroutine 18605:
  github.com/cockroachdb/cockroach/pkg/settings.(*Values).TestingCopyForVirtualCluster()
      github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/values.go:275 +0x12d
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).makeSharedProcessTenantConfig()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:170 +0x26c
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).newTenantServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:78 +0x14b
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).newServerForOrchestrator()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:262 +0x1bc
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5.2()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:371 +0x11a
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:395 +0x9aa
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 18608 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x6e8
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:245 +0x65f
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:335 +0xbb7
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 18605 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
==================
I240115 18:24:26.167982 1 (gostd) testmain.go:97  [-] 1  Test //pkg/sql/sqlstats/insights/integration:integration_test exited with error code 66

ERROR: exit status 66
abarganier commented 9 months ago

The race complains about concurrent read/write access to valuesContainer.intVals without mutex protection: https://github.com/cockroachdb/cockroach/blob/20132d33e74191972566cf80459ef507e095c404/pkg/settings/values.go#L81

One of the goroutines appears to be the server controller orchestrator: 

  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:395 +0x9aa

The other appears to be a rangefeed:

  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x568
abarganier commented 9 months ago

I'm seeing the same race failure here: https://github.com/cockroachdb/cockroach/issues/117669

abarganier commented 9 months ago

Also noticing this here: https://github.com/cockroachdb/cockroach/issues/117534

cockroach-teamcity commented 9 months ago

pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ 515663b99f79f96e3e8fbdef0cd7789cc38612fa:

Goroutine 19915 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 17824 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
  github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
      github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
  github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus()
      github.com/cockroachdb/cockroach/pkg/server/node.go:1200 +0x128
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/server.go:1990 +0x4fe4
  github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
      github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
  pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
      pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
  testing.tRunner()
      GOROOT/src/testing/testing.go:1595 +0x261
  testing.(*T).Run.func1()
      GOROOT/src/testing/testing.go:1648 +0x44
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!

miraradeva commented 9 months ago

On KV we're seeing this race in #117763, , https://github.com/cockroachdb/cockroach/issues/117611, and https://github.com/cockroachdb/cockroach/issues/117726.

cockroach-teamcity commented 9 months ago

pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention failed with artifacts on master @ 9b10f083f8889cdc1bc549d9428f06fd784ba081:

      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 17284 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 15230 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
  github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
      github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
  github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/testserver.go:807 +0x104
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).PreStart()
      github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:20 +0x73
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.(*TestCluster).Start()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:417 +0x57e
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:238 +0xa4
  github.com/cockroachdb/cockroach/pkg/testutils/testcluster.StartTestCluster()
      github.com/cockroachdb/cockroach/pkg/testutils/testcluster/testcluster.go:237 +0x84
  pkg/sql/sqlstats/insights/integration/integration_test.TestInsightsIntegrationForContention()
      pkg/sql/sqlstats/insights/integration/integration_test/pkg/sql/sqlstats/insights/integration/insights_test.go:700 +0x169
  testing.tRunner()
      GOROOT/src/testing/testing.go:1595 +0x261
  testing.(*T).Run.func1()
      GOROOT/src/testing/testing.go:1648 +0x44
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!