search

cockroachdb / cockroach

CockroachDB — the cloud native, distributed SQL database designed for high availability, effortless scale, and control over data placement.
https://www.cockroachlabs.com
Other
30.06k stars 3.8k forks source link

kv/kvclient/rangefeed: TestRangeFeedIntegration failed #117669

Closed cockroach-teamcity closed 9 months ago

cockroach-teamcity commented 9 months ago

kv/kvclient/rangefeed.TestRangeFeedIntegration failed with artifacts on master @ d9c7307281096f1f1ac8bfe2613d253a2aa35fd6:

Goroutine 8670 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 5891 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x1ca
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:198 +0x4f
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start()
      github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:263 +0xf18
  github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart()
      github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1454 +0x339
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/server.go:2052 +0x5785
  github.com/cockroachdb/cockroach/pkg/server.(*testServer).PreStart()
      github.com/cockroachdb/cockroach/pkg/server/testserver.go:806 +0x104
  github.com/cockroachdb/cockroach/pkg/server.(*testServer).Start()
      github.com/cockroachdb/cockroach/pkg/server/testserver.go:882 +0xda
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.(*wrap).Start()
      github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/testutils/serverutils/ts_control_forwarder_generated.go:15 +0x73
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
      github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:290 +0x281
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnlyE()
      github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:276 +0x151
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServerOnly()
      github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:305 +0x86
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServer()
      github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:323 +0x6f
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed_test.TestRangeFeedIntegration()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed_test/pkg/kv/kvclient/rangefeed/rangefeed_external_test.go:67 +0x164
  testing.tRunner()
      GOROOT/src/testing/testing.go:1595 +0x261
  testing.(*T).Run.func1()
      GOROOT/src/testing/testing.go:1648 +0x44
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/replication

This test on roachdash | Improve this report!

Jira issue: CRDB-35295

cockroach-teamcity commented 9 months ago

kv/kvclient/rangefeed.TestRangeFeedIntegration failed with artifacts on master @ ee3495831f15ed85eeb8604c96e48b4f112b4f22:

      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:185 +0x6c5
  github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec()
      github.com/cockroachdb/cockroach/pkg/kv/txn.go:1032 +0x8f
  github.com/cockroachdb/cockroach/pkg/kv.runTxn()
      github.com/cockroachdb/cockroach/pkg/kv/db.go:1055 +0x64
  github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl()
      github.com/cockroachdb/cockroach/pkg/kv/db.go:1018 +0xec
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).scanSpan()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:167 +0x2dd
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).Scan()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:103 +0x884
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).runInitialScan()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/scanner.go:63 +0x3f8
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:279 +0x32f
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x568
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 6774 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 6777 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x6e8
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:245 +0x65f
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:335 +0xbb7
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!

cockroach-teamcity commented 9 months ago

kv/kvclient/rangefeed.TestRangeFeedIntegration failed with artifacts on master @ 3ccf718d2c31b0206911a36f39d83f2bd9a15903:

      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x568
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Previous read at 0x00c0017b60f0 by goroutine 5716:
  github.com/cockroachdb/cockroach/pkg/settings.(*Values).TestingCopyForVirtualCluster()
      github.com/cockroachdb/cockroach/pkg/settings/pkg/settings/values.go:275 +0x12d
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).makeSharedProcessTenantConfig()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:170 +0x26c
  github.com/cockroachdb/cockroach/pkg/server.(*topLevelServer).newTenantServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_new_server.go:78 +0x14b
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).newServerForOrchestrator()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:262 +0x1bc
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5.2()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:371 +0x11a
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer.func5()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:395 +0x9aa
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 5719 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0x6e8
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:245 +0x65f
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:335 +0xbb7
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1()
      github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:205 +0x2f7
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278

Goroutine 5716 (running) created at:
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:475 +0x638
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:346 +0xee4
  github.com/cockroachdb/cockroach/pkg/server.(*channelOrchestrator).startControlledServer()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_channel_orchestrator.go:296 +0x9e9
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).createServerEntryLocked()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:173 +0x2b2
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).startMissingServers()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:134 +0x2f1
  github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1()
      github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:64 +0x1c9
  github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
      github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:484 +0x278
==================

Parameters:

See also: How To Investigate a Go Test Failure (internal)

This test on roachdash | Improve this report!

miraradeva commented 9 months ago

Adding label P1 to match the original instance of this data race in https://github.com/cockroachdb/cockroach/issues/117688.