search

ssvlabs / ssv

Secret-Shared-Validator(SSV) for ethereum staking
https://ssv.network
GNU General Public License v3.0
183 stars 95 forks source link

p2p: races (discovered with CI unit-test) #1855

Open iurii-ssv opened 4 days ago

iurii-ssv commented 4 days ago

Noticed CI pipeline failing - https://github.com/ssvlabs/ssv/actions/runs/11890850478/job/33130362086 - there are some races related to p2p & networking (upon cursory overview the root cause seems to be in our implementation itself, and not the test code, but obviously requires further investigation),

the relevant part describing those races:

==================
WARNING: DATA RACE
Read at 0x00c01772aa18 by goroutine 2247:
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).connectionsAtLimit()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:330 +0x9e
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).connectionsAtLimit-fm()
      <autogenerated>:1 +0x1f
  github.com/ssvlabs/ssv/network/peers/connections.(*connGater).InterceptAccept()
      /home/runner/work/ssv/ssv/network/peers/connections/conn_gater.go:85 +0x1d9
  github.com/libp2p/go-libp2p/p2p/net/upgrader.(*listener).handleIncoming()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/upgrader/listener.go:88 +0xbf7
  github.com/libp2p/go-libp2p/p2p/net/upgrader.(*upgrader).UpgradeListener.gowrap1()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/upgrader/upgrader.go:119 +0x33

Previous write at 0x00c01772aa18 by goroutine 1897:
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).setupPeerServices()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:190 +0x10a4
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).SetupServices()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:155 +0xc4
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).Setup()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:74 +0x45e
  github.com/ssvlabs/ssv/network/p2p.(*LocalNet).NewTestP2pNetwork()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:232 +0x18dd
  github.com/ssvlabs/ssv/network/p2p.NewLocalNet.func1()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:261 +0x1d7
  github.com/ssvlabs/ssv/network/testing.NewLocalTestnet()
      /home/runner/work/ssv/ssv/network/testing/local.go:22 +0x214
  github.com/ssvlabs/ssv/network/p2p.NewLocalNet()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:259 +0x277
  github.com/ssvlabs/ssv/network/p2p.CreateAndStartLocalNet.func1()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:76 +0xcf
  github.com/ssvlabs/ssv/network/p2p.CreateAndStartLocalNet()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:112 +0x20a
  github.com/ssvlabs/ssv/network/p2p.createNetworkAndSubscribe()
      /home/runner/work/ssv/ssv/network/p2p/p2p_test.go:260 +0xca
  github.com/ssvlabs/ssv/network/p2p.CreateVirtualNet()
      /home/runner/work/ssv/ssv/network/p2p/p2p_validation_test.go:338 +0x2b0
  github.com/ssvlabs/ssv/network/p2p.TestP2pNetwork_MessageValidation()
      /home/runner/work/ssv/ssv/network/p2p/p2p_validation_test.go:137 +0x5f1
  testing.tRunner()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1689 +0x21e
  testing.(*T).Run.gowrap1()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1742 +0x44

Goroutine 2247 (running) created at:
  github.com/libp2p/go-libp2p/p2p/net/upgrader.(*upgrader).UpgradeListener()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/upgrader/upgrader.go:119 +0x44a
  github.com/libp2p/go-libp2p/p2p/transport/tcp.(*TcpTransport).Listen()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/transport/tcp/tcp.go:255 +0x187
  github.com/libp2p/go-libp2p/p2p/net/swarm.(*Swarm).AddListenAddr()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/swarm/swarm_listen.go:116 +0x8a
  github.com/libp2p/go-libp2p/p2p/net/swarm.(*Swarm).Listen()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/swarm/swarm_listen.go:53 +0x304
  github.com/libp2p/go-libp2p/config.(*Config).NewNode.func3.1()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/config/config.go:487 +0x6b
  go.uber.org/fx/internal/lifecycle.(*Lifecycle).runStartHook()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/internal/lifecycle/lifecycle.go:256 +0x30a
  go.uber.org/fx/internal/lifecycle.(*Lifecycle).Start()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/internal/lifecycle/lifecycle.go:216 +0x6b3
  go.uber.org/fx.(*App).start-fm.(*App).start.func1()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:704 +0x70
  go.uber.org/fx.(*App).withRollback()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:686 +0x63
  go.uber.org/fx.(*App).start()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:703 +0x84
  go.uber.org/fx.(*App).start-fm()
      <autogenerated>:1 +0x1f
  go.uber.org/fx.withTimeout.func1()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:803 +0xe1

Goroutine 1897 (running) created at:
  testing.(*T).Run()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1742 +0x825
  testing.runTests.func1()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:2161 +0x85
  testing.tRunner()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1689 +0x21e
  testing.runTests()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:2159 +0x8be
  testing.(*M).Run()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:2027 +0xf17
  main.main()
      _testmain.go:85 +0x2e4
==================
==================
WARNING: DATA RACE
Read at 0x00c0238ea528 by goroutine 2247:
  github.com/ssvlabs/ssv/network/peers.(*peersIndex).AtLimit()
      /home/runner/work/ssv/ssv/network/peers/peers_index.go:100 +0x37
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).connectionsAtLimit()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:333 +0xdd
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).connectionsAtLimit-fm()
      <autogenerated>:1 +0x1f
  github.com/ssvlabs/ssv/network/peers/connections.(*connGater).InterceptAccept()
      /home/runner/work/ssv/ssv/network/peers/connections/conn_gater.go:85 +0x1d9
  github.com/libp2p/go-libp2p/p2p/net/upgrader.(*listener).handleIncoming()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/upgrader/listener.go:88 +0xbf7
  github.com/libp2p/go-libp2p/p2p/net/upgrader.(*upgrader).UpgradeListener.gowrap1()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/upgrader/upgrader.go:119 +0x33

Previous write at 0x00c0238ea528 by goroutine 1897:
  github.com/ssvlabs/ssv/network/peers.NewPeersIndex()
      /home/runner/work/ssv/ssv/network/peers/peers_index.go:52 +0xfb4
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).setupPeerServices()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:190 +0x7fe
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).SetupServices()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:155 +0xc4
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).Setup()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:74 +0x45e
  github.com/ssvlabs/ssv/network/p2p.(*LocalNet).NewTestP2pNetwork()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:232 +0x18dd
  github.com/ssvlabs/ssv/network/p2p.NewLocalNet.func1()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:261 +0x1d7
  github.com/ssvlabs/ssv/network/testing.NewLocalTestnet()
      /home/runner/work/ssv/ssv/network/testing/local.go:22 +0x214
  github.com/ssvlabs/ssv/network/p2p.NewLocalNet()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:259 +0x277
  github.com/ssvlabs/ssv/network/p2p.CreateAndStartLocalNet.func1()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:76 +0xcf
  github.com/ssvlabs/ssv/network/p2p.CreateAndStartLocalNet()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:112 +0x20a
  github.com/ssvlabs/ssv/network/p2p.createNetworkAndSubscribe()
      /home/runner/work/ssv/ssv/network/p2p/p2p_test.go:260 +0xca
  github.com/ssvlabs/ssv/network/p2p.CreateVirtualNet()
      /home/runner/work/ssv/ssv/network/p2p/p2p_validation_test.go:338 +0x2b0
  github.com/ssvlabs/ssv/network/p2p.TestP2pNetwork_MessageValidation()
      /home/runner/work/ssv/ssv/network/p2p/p2p_validation_test.go:137 +0x5f1
  testing.tRunner()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1689 +0x21e
  testing.(*T).Run.gowrap1()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1742 +0x44

Goroutine 2247 (running) created at:
  github.com/libp2p/go-libp2p/p2p/net/upgrader.(*upgrader).UpgradeListener()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/upgrader/upgrader.go:119 +0x44a
  github.com/libp2p/go-libp2p/p2p/transport/tcp.(*TcpTransport).Listen()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/transport/tcp/tcp.go:255 +0x187
  github.com/libp2p/go-libp2p/p2p/net/swarm.(*Swarm).AddListenAddr()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/swarm/swarm_listen.go:116 +0x8a
  github.com/libp2p/go-libp2p/p2p/net/swarm.(*Swarm).Listen()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/swarm/swarm_listen.go:53 +0x304
  github.com/libp2p/go-libp2p/config.(*Config).NewNode.func3.1()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/config/config.go:487 +0x6b
  go.uber.org/fx/internal/lifecycle.(*Lifecycle).runStartHook()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/internal/lifecycle/lifecycle.go:256 +0x30a
  go.uber.org/fx/internal/lifecycle.(*Lifecycle).Start()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/internal/lifecycle/lifecycle.go:216 +0x6b3
  go.uber.org/fx.(*App).start-fm.(*App).start.func1()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:704 +0x70
  go.uber.org/fx.(*App).withRollback()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:686 +0x63
  go.uber.org/fx.(*App).start()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:703 +0x84
  go.uber.org/fx.(*App).start-fm()
      <autogenerated>:1 +0x1f
  go.uber.org/fx.withTimeout.func1()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:803 +0xe1

Goroutine 1897 (running) created at:
  testing.(*T).Run()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1742 +0x825
  testing.runTests.func1()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:2161 +0x85
  testing.tRunner()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1689 +0x21e
  testing.runTests()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:2159 +0x8be
  testing.(*M).Run()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:2027 +0xf17
  main.main()
      _testmain.go:85 +0x2e4
==================
==================
WARNING: DATA RACE
Read at 0x00c0238ea4d8 by goroutine 2247:
  github.com/ssvlabs/ssv/network/peers.(*peersIndex).AtLimit()
      /home/runner/work/ssv/ssv/network/peers/peers_index.go:101 +0x64
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).connectionsAtLimit()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:333 +0xdd
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).connectionsAtLimit-fm()
      <autogenerated>:1 +0x1f
  github.com/ssvlabs/ssv/network/peers/connections.(*connGater).InterceptAccept()
      /home/runner/work/ssv/ssv/network/peers/connections/conn_gater.go:85 +0x1d9
  github.com/libp2p/go-libp2p/p2p/net/upgrader.(*listener).handleIncoming()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/upgrader/listener.go:88 +0xbf7
  github.com/libp2p/go-libp2p/p2p/net/upgrader.(*upgrader).UpgradeListener.gowrap1()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/upgrader/upgrader.go:119 +0x33

Previous write at 0x00c0238ea4d8 by goroutine 1897:
  github.com/ssvlabs/ssv/network/peers.NewPeersIndex()
      /home/runner/work/ssv/ssv/network/peers/peers_index.go:46 +0xda8
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).setupPeerServices()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:190 +0x7fe
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).SetupServices()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:155 +0xc4
  github.com/ssvlabs/ssv/network/p2p.(*p2pNetwork).Setup()
      /home/runner/work/ssv/ssv/network/p2p/p2p_setup.go:74 +0x45e
  github.com/ssvlabs/ssv/network/p2p.(*LocalNet).NewTestP2pNetwork()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:232 +0x18dd
  github.com/ssvlabs/ssv/network/p2p.NewLocalNet.func1()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:261 +0x1d7
  github.com/ssvlabs/ssv/network/testing.NewLocalTestnet()
      /home/runner/work/ssv/ssv/network/testing/local.go:22 +0x214
  github.com/ssvlabs/ssv/network/p2p.NewLocalNet()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:259 +0x277
  github.com/ssvlabs/ssv/network/p2p.CreateAndStartLocalNet.func1()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:76 +0xcf
  github.com/ssvlabs/ssv/network/p2p.CreateAndStartLocalNet()
      /home/runner/work/ssv/ssv/network/p2p/test_utils.go:112 +0x20a
  github.com/ssvlabs/ssv/network/p2p.createNetworkAndSubscribe()
      /home/runner/work/ssv/ssv/network/p2p/p2p_test.go:260 +0xca
  github.com/ssvlabs/ssv/network/p2p.CreateVirtualNet()
      /home/runner/work/ssv/ssv/network/p2p/p2p_validation_test.go:338 +0x2b0
  github.com/ssvlabs/ssv/network/p2p.TestP2pNetwork_MessageValidation()
      /home/runner/work/ssv/ssv/network/p2p/p2p_validation_test.go:137 +0x5f1
  testing.tRunner()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1689 +0x21e
  testing.(*T).Run.gowrap1()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1742 +0x44

Goroutine 2247 (running) created at:
  github.com/libp2p/go-libp2p/p2p/net/upgrader.(*upgrader).UpgradeListener()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/upgrader/upgrader.go:119 +0x44a
  github.com/libp2p/go-libp2p/p2p/transport/tcp.(*TcpTransport).Listen()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/transport/tcp/tcp.go:255 +0x187
  github.com/libp2p/go-libp2p/p2p/net/swarm.(*Swarm).AddListenAddr()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/swarm/swarm_listen.go:116 +0x8a
  github.com/libp2p/go-libp2p/p2p/net/swarm.(*Swarm).Listen()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/p2p/net/swarm/swarm_listen.go:53 +0x304
  github.com/libp2p/go-libp2p/config.(*Config).NewNode.func3.1()
      /home/runner/go/pkg/mod/github.com/libp2p/go-libp2p@v0.36.3/config/config.go:487 +0x6b
  go.uber.org/fx/internal/lifecycle.(*Lifecycle).runStartHook()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/internal/lifecycle/lifecycle.go:256 +0x30a
  go.uber.org/fx/internal/lifecycle.(*Lifecycle).Start()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/internal/lifecycle/lifecycle.go:216 +0x6b3
  go.uber.org/fx.(*App).start-fm.(*App).start.func1()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:704 +0x70
  go.uber.org/fx.(*App).withRollback()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:686 +0x63
  go.uber.org/fx.(*App).start()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:703 +0x84
  go.uber.org/fx.(*App).start-fm()
      <autogenerated>:1 +0x1f
  go.uber.org/fx.withTimeout.func1()
      /home/runner/go/pkg/mod/go.uber.org/fx@v1.22.1/app.go:803 +0xe1

Goroutine 1897 (running) created at:
  testing.(*T).Run()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1742 +0x825
  testing.runTests.func1()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:2161 +0x85
  testing.tRunner()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:1689 +0x21e
  testing.runTests()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:2159 +0x8be
  testing.(*M).Run()
      /opt/hostedtoolcache/go/1.22.9/x64/src/testing/testing.go:2027 +0xf17
  main.main()
      _testmain.go:85 +0x2e4
==================
nkryuchkov commented 2 days ago

duplicate of https://github.com/ssvlabs/ssv/issues/1732, although this one looks like it's written better