search

tikv / pd

Placement driver for TiKV
Apache License 2.0
1.04k stars 718 forks source link

data race at ConcurrentRunner #8386

Closed HuSharp closed 3 weeks ago

HuSharp commented 1 month ago

Bug Report

What did you do?

Run PD in a large cluster(by simulator)

What did you expect to see?

No data race

What did you see instead?

==================
WARNING: DATA RACE
Write at 0x00c002e81a48 by goroutine 639:
  github.com/tikv/pd/server/cluster.(*RaftCluster).InitCluster()
      /data/nvme0n1/husharp/proj/pd/server/cluster/cluster.go:299 +0x1184
  github.com/tikv/pd/server/cluster.(*RaftCluster).Start()
      /data/nvme0n1/husharp/proj/pd/server/cluster/cluster.go:313 +0x1dd
  github.com/tikv/pd/server.(*Server).createRaftCluster()
      /data/nvme0n1/husharp/proj/pd/server/server.go:807 +0x8a
  github.com/tikv/pd/server.(*Server).campaignLeader()
      /data/nvme0n1/husharp/proj/pd/server/server.go:1787 +0x187d
  github.com/tikv/pd/server.(*Server).leaderLoop()
      /data/nvme0n1/husharp/proj/pd/server/server.go:1707 +0x1564
  github.com/tikv/pd/server.(*Server).startServerLoop.func1()
      /data/nvme0n1/husharp/proj/pd/server/server.go:664 +0x33

Previous read at 0x00c002e81a48 by goroutine 41370597:
  github.com/tikv/pd/server/cluster.(*RaftCluster).GetRegionStats()
      <autogenerated>:1 +0x3b
  github.com/tikv/pd/pkg/cluster.Collect()
      /data/nvme0n1/husharp/proj/pd/pkg/cluster/cluster.go:81 +0xe1
  github.com/tikv/pd/server/cluster.(*RaftCluster).processRegionHeartbeat.func1()
      /data/nvme0n1/husharp/proj/pd/server/cluster/cluster.go:1066 +0xcb
  github.com/tikv/pd/pkg/ratelimit.(*ConcurrentRunner).run()
      /data/nvme0n1/husharp/proj/pd/pkg/ratelimit/runner.go:149 +0x7a
  github.com/tikv/pd/pkg/ratelimit.(*ConcurrentRunner).Start.func1.2()
      /data/nvme0n1/husharp/proj/pd/pkg/ratelimit/runner.go:121 +0x4f

Goroutine 639 (running) created at:
  github.com/tikv/pd/server.(*Server).startServerLoop()
      /data/nvme0n1/husharp/proj/pd/server/server.go:664 +0x15c
  github.com/tikv/pd/server.(*Server).Run()
      /data/nvme0n1/husharp/proj/pd/server/server.go:636 +0x1ad
  fmt.(*ss).doScanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:1230 +0x3ee
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  github.com/syndtr/goleveldb/leveldb.(*DB).checkAndCleanFiles()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db_util.go:52 +0x23e
  github.com/syndtr/goleveldb/leveldb.openDB()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:130 +0x884
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x224
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:657 +0x16e
  fmt.(*ss).doScanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:1230 +0x3ee
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  github.com/syndtr/goleveldb/leveldb.(*DB).recoverJournal()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:476 +0xa6
  github.com/syndtr/goleveldb/leveldb.openDB()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:125 +0x86d
  github.com/syndtr/goleveldb/leveldb.Open()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:197 +0x330
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  github.com/syndtr/goleveldb/leveldb.(*session).recover.func1()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/session.go:134 +0xbd
  runtime.deferreturn()
      /data/nvme0n1/husharp/go/go1.21.0/src/runtime/panic.go:477 +0x30
  github.com/syndtr/goleveldb/leveldb.Open()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:183 +0xf6
  github.com/syndtr/goleveldb/leveldb.OpenFile()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:219 +0x84
  github.com/tikv/pd/pkg/storage/kv.NewLevelDBKV()
      /data/nvme0n1/husharp/proj/pd/pkg/storage/kv/levedb_kv.go:34 +0x4d
  github.com/tikv/pd/pkg/storage.NewHotRegionsStorage()
      /data/nvme0n1/husharp/proj/pd/pkg/storage/hot_region_storage.go:123 +0x7d
  github.com/tikv/pd/server.(*Server).startServer()
      /data/nvme0n1/husharp/proj/pd/server/server.go:514 +0x2a65
  fmt.(*ss).doScanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:1230 +0x3ee
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  github.com/syndtr/goleveldb/leveldb.(*DB).checkAndCleanFiles()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db_util.go:52 +0x23e
  github.com/syndtr/goleveldb/leveldb.openDB()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:130 +0x884
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x224
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:657 +0x16e
  fmt.(*ss).doScanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:1230 +0x3ee
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  github.com/syndtr/goleveldb/leveldb.(*DB).recoverJournal()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:476 +0xa6
  github.com/syndtr/goleveldb/leveldb.openDB()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:125 +0x86d
  github.com/syndtr/goleveldb/leveldb.Open()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:197 +0x330
  fmt.Fscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:143 +0xdc
  fmt.Sscanf()
      /data/nvme0n1/husharp/go/go1.21.0/src/fmt/scan.go:114 +0x164
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0x1a
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x3f8
  github.com/syndtr/goleveldb/leveldb.(*session).recover.func1()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/session.go:134 +0xbd
  runtime.deferreturn()
      /data/nvme0n1/husharp/go/go1.21.0/src/runtime/panic.go:477 +0x30
  github.com/syndtr/goleveldb/leveldb.Open()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:183 +0xf6
  github.com/syndtr/goleveldb/leveldb.OpenFile()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:219 +0x84
  github.com/tikv/pd/pkg/storage/kv.NewLevelDBKV()
      /data/nvme0n1/husharp/proj/pd/pkg/storage/kv/levedb_kv.go:34 +0x4d
  github.com/tikv/pd/pkg/storage.newLevelDBBackend()
      /data/nvme0n1/husharp/proj/pd/pkg/storage/leveldb_backend.go:62 +0x64
  github.com/tikv/pd/pkg/storage.NewRegionStorageWithLevelDBBackend()
      /data/nvme0n1/husharp/proj/pd/pkg/storage/storage.go:67 +0x64
  github.com/tikv/pd/server.(*Server).startServer()
      /data/nvme0n1/husharp/proj/pd/server/server.go:464 +0xd87
  github.com/tikv/pd/server.(*Server).Run()
      /data/nvme0n1/husharp/proj/pd/server/server.go:627 +0x144
  main.start()
      /data/nvme0n1/husharp/proj/pd/cmd/pd-server/main.go:285 +0x13b1
  main.createServerWrapper()
      /data/nvme0n1/husharp/proj/pd/cmd/pd-server/main.go:191 +0xd5
  github.com/spf13/cobra.(*Command).execute()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/spf13/cobra@v1.8.0/command.go:987 +0xf9d
  github.com/spf13/cobra.(*Command).ExecuteC()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/spf13/cobra@v1.8.0/command.go:1115 +0x604
  github.com/spf13/cobra.(*Command).Execute()
      /data/nvme0n1/husharp/go/go1.21.0/pkg/mod/github.com/spf13/cobra@v1.8.0/command.go:1039 +0x224
  main.main()
      /data/nvme0n1/husharp/proj/pd/cmd/pd-server/main.go:71 +0x225

Goroutine 41370597 (finished) created at:
  github.com/tikv/pd/pkg/ratelimit.(*ConcurrentRunner).Start.func1()
      /data/nvme0n1/husharp/proj/pd/pkg/ratelimit/runner.go:121 +0x36b
==================

What version of PD are you using (pd-server -V)?

master

HuSharp commented 1 month ago

detail.log

HuSharp commented 1 month ago

The PD leader just dropped out and was immediately elected by the same node.

[2024/07/09 13:27:24.331 +08:00] [INFO] [runner.go:129] ["stopping async task runner"] [name=heartbeat-async]
[2024/07/09 13:27:24.332 +08:00] [INFO] [runner.go:129] ["stopping async task runner"] [name=misc-async]
[2024/07/09 13:27:24.332 +08:00] [INFO] [runner.go:129] ["stopping async task runner"] [name=log-async]
[2024/07/09 13:27:24.400 +08:00] [INFO] [cluster.go:728] ["update store stats background jobs has been stopped"]
[2024/07/09 13:27:24.400 +08:00] [INFO] [cluster.go:769] ["raft cluster is stopped"]
[2024/07/09 13:27:24.400 +08:00] [INFO] [tso.go:436] ["reset the timestamp in memory"] []
[2024/07/09 13:27:24.402 +08:00] [INFO] [server.go:1712] ["start to campaign PD leader"] [campaign-leader-name=pd-1]
[2024/07/09 13:27:24.405 +08:00] [INFO] [lease.go:66] ["lease granted"] [lease-id=3236276779607555392] [lease-timeout=3] [purpose="leader election"]
[2024/07/09 13:27:24.408 +08:00] [INFO] [leadership.go:182] ["check campaign resp"] [resp="{\"header\":{\"cluster_id\":18106971309470696072,\"member_id\":1394662113889169036,\"revision\":4134,\"raft_term\":2},\"succeeded\":true,\"responses\":[{\"Response\":{\"ResponsePut\":{\"header\":{\"revision\":4134}}}}]}"]
[2024/07/09 13:27:24.408 +08:00] [INFO] [leadership.go:191] ["write leaderData to leaderPath ok"] [leader-key=/pd/7389480197880276194/leader] [purpose="leader election"]
[2024/07/09 13:27:24.408 +08:00] [INFO] [server.go:1738] ["campaign PD leader ok"] [campaign-leader-name=pd-1]