apple / swift-distributed-actors

Peer-to-peer cluster implementation for Swift Distributed Actors
https://apple.github.io/swift-distributed-actors/
Apache License 2.0
591 stars 55 forks source link

Revive clustered swim test: test_swim_shouldNotifyClusterAboutUnreachableNode_whenUnreachableDiscoveredByOtherNode #757

Open ktoso opened 4 years ago

ktoso commented 4 years ago
    func test_swim_shouldNotifyClusterAboutUnreachableNode_whenUnreachableDiscoveredByOtherNode() throws {
        let first = self.setUpFirst { settings in
            // purposefully too large timeouts, we want the first node to be informed by the third node
            // about the second node being unreachable/dead, and ensure that the first node also signals an
            // unreachability event to the cluster upon such discovery.
            settings.cluster.swim.lifeguard.suspicionTimeoutMax = .seconds(100)
            settings.cluster.swim.lifeguard.suspicionTimeoutMin = .seconds(10)
            settings.cluster.swim.pingTimeout = .seconds(3)
        }
        let second = self.setUpSecond()
        let secondNode = second.cluster.uniqueNode
        let third = self.setUpNode("third") { settings in
            settings.cluster.swim.lifeguard.suspicionTimeoutMin = .nanoseconds(2)
            settings.cluster.swim.lifeguard.suspicionTimeoutMax = .nanoseconds(2)
            settings.cluster.swim.pingTimeout = .milliseconds(300)
        }

        first.cluster.join(node: second.cluster.uniqueNode.node)
        third.cluster.join(node: second.cluster.uniqueNode.node)
        try assertAssociated(first, withExactly: [second.cluster.uniqueNode, third.cluster.uniqueNode])
        try assertAssociated(second, withExactly: [first.cluster.uniqueNode, third.cluster.uniqueNode])
        try assertAssociated(third, withExactly: [first.cluster.uniqueNode, second.cluster.uniqueNode])

        let firstTestKit = self.testKit(first)
        let p1 = firstTestKit.spawnTestProbe(expecting: Cluster.Event.self)
        first.cluster.events.subscribe(p1.ref)

        let thirdTestKit = self.testKit(third)
        let p3 = thirdTestKit.spawnTestProbe(expecting: Cluster.Event.self)
        third.cluster.events.subscribe(p3.ref)

        try self.expectReachabilityInSnapshot(firstTestKit, node: secondNode, expect: .reachable)
        try self.expectReachabilityInSnapshot(thirdTestKit, node: secondNode, expect: .reachable)

        // kill the second node
        second.shutdown()

        try self.expectReachabilityEvent(thirdTestKit, p3, node: secondNode, expect: .unreachable)
        try self.expectReachabilityEvent(firstTestKit, p1, node: secondNode, expect: .unreachable)

        // we also expect the snapshot to include the right reachability information now
        try self.expectReachabilityInSnapshot(firstTestKit, node: secondNode, expect: .unreachable)
        try self.expectReachabilityInSnapshot(thirdTestKit, node: secondNode, expect: .unreachable)
    }
ktoso commented 4 years ago

Note that it was known flaky, so perhaps re designing it while we're here would be good https://github.com/apple/swift-distributed-actors/issues/586