sonic-net / sonic-buildimage

Scripts which perform an installable binary image build for SONiC
Other
742 stars 1.43k forks source link

[dualtor] orchagent test_downstream_standby_mux_toggle_active fails #8087

Open theasianpianist opened 3 years ago

theasianpianist commented 3 years ago
conn_graph_facts = {'device_conn': {'str2-7050cx3-acs-06': {'Ethernet0': {'peerdevice': u'str2-7260cx3-acs-fan-10', 'peerport': u'Etherne...ard', 'HwSku': u'Arista-7050CX3-32S-C32', 'ManagementGw': u'10.3.146.1', 'ManagementIp': u'10.3.147.24/23', ...}}, ...}
ptfadapter = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
ptfhost = <tests.common.devices.ptf.PTFHost object at 0x7fd679d0f090>
rand_selected_dut = <MultiAsicSonicHost> str2-7050cx3-acs-07
rand_unselected_dut = <MultiAsicSonicHost> str2-7050cx3-acs-06
tbinfo = {'auto_recover': 'True', 'comment': 'hellogemini', 'conf-name': 'vms17-dual-t0-7050-1', 'duts': ['str2-7050cx3-acs-06', 'str2-7050cx3-acs-07'], ...}
tunnel_traffic_monitor = <class 'tests.common.dualtor.tunnel_traffic_utils.TunnelTrafficMonitor'>
vmhost = <tests.common.devices.vmhost.VMHost object at 0x7fd67abba790>
toggle_all_simulator_ports = <function _toggle at 0x7fd673a65950>
tor_mux_intfs = ['Ethernet4', 'Ethernet8', 'Ethernet12', 'Ethernet16', 'Ethernet20', 'Ethernet24', ...]

    def test_downstream_standby_mux_toggle_active(
        conn_graph_facts, ptfadapter, ptfhost,
        rand_selected_dut, rand_unselected_dut, tbinfo,
        tunnel_traffic_monitor, vmhost, toggle_all_simulator_ports,
        tor_mux_intfs
        ):
        # set rand_selected_dut as standby and rand_unselected_dut to active tor
        test_params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo)
        server_ipv4 = test_params["target_server_ip"]
        random_dst_ip = "1.1.1.2"
        pkt, exp_pkt = build_packet_to_server(rand_selected_dut, ptfadapter, random_dst_ip)
        ptf_t1_intf = random.choice(get_t1_ptf_ports(rand_selected_dut, tbinfo))

        def monitor_tunnel_and_server_traffic(torhost, expect_tunnel_traffic=True, expect_server_traffic=True):
            tunnel_monitor = tunnel_traffic_monitor(rand_selected_dut, existing=True)
            server_traffic_monitor = ServerTrafficMonitor(
                torhost, ptfhost, vmhost, tbinfo, test_params["selected_port"],
                conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo)
            )
            tunnel_monitor.existing = expect_tunnel_traffic
            server_traffic_monitor.existing = expect_server_traffic
            with tunnel_monitor, server_traffic_monitor:
                testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)

        logger.info("Stage 1: Verify Standby Forwarding")
        logger.info("Step 1.1: Add route to a nexthop which is a standby Neighbor")
        set_mux_state(rand_selected_dut, tbinfo, 'standby', tor_mux_intfs, toggle_all_simulator_ports)
        add_nexthop_routes(rand_selected_dut, random_dst_ip, nexthops=[server_ipv4])
        logger.info("Step 1.2: Verify traffic to this route dst is forwarded to Active ToR and equally distributed")
        check_tunnel_balance(**test_params)
>       monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=False, expect_tunnel_traffic=True)

conn_graph_facts = {'device_conn': {'str2-7050cx3-acs-06': {'Ethernet0': {'peerdevice': u'str2-7260cx3-acs-fan-10', 'peerport': u'Etherne...ard', 'HwSku': u'Arista-7050CX3-32S-C32', 'ManagementGw': u'10.3.146.1', 'ManagementIp': u'10.3.147.24/23', ...}}, ...}
exp_pkt    = <ptf.mask.Mask instance at 0x7fd64c2e62d0>
monitor_tunnel_and_server_traffic = <function monitor_tunnel_and_server_traffic at 0x7fd65f27e6d0>
pkt        = <Ether  dst=d4:af:f7:4d:af:18 src=7a:ad:6e:91:e4:00 type=0x800 |<IP  ihl=None ...16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@A' |>>>
ptf_t1_intf = 'eth29'
ptfadapter = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest>
ptfhost    = <tests.common.devices.ptf.PTFHost object at 0x7fd679d0f090>
rand_selected_dut = <MultiAsicSonicHost> str2-7050cx3-acs-07
rand_unselected_dut = <MultiAsicSonicHost> str2-7050cx3-acs-06
random_dst_ip = '1.1.1.2'
server_ipv4 = '192.168.0.16'
tbinfo     = {'auto_recover': 'True', 'comment': 'hellogemini', 'conf-name': 'vms17-dual-t0-7050-1', 'duts': ['str2-7050cx3-acs-06', 'str2-7050cx3-acs-07'], ...}
test_params = {'active_tor_ip': u'10.1.0.32', 'ptf_portchannel_indices': {'PortChannel0001': [29], 'PortChannel0002': [31], 'PortCha...04': [35]}, 'ptfhost': <tests.common.devices.ptf.PTFHost object at 0x7fd679d0f090>, 'selected_port': 'Ethernet60', ...}
toggle_all_simulator_ports = <function _toggle at 0x7fd673a65950>
tor_mux_intfs = ['Ethernet4', 'Ethernet8', 'Ethernet12', 'Ethernet16', 'Ethernet20', 'Ethernet24', ...]
tunnel_traffic_monitor = <class 'tests.common.dualtor.tunnel_traffic_utils.TunnelTrafficMonitor'>
vmhost     = <tests.common.devices.vmhost.VMHost object at 0x7fd67abba790>

dualtor/test_orchagent_standby_tor_downstream.py:254: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
dualtor/test_orchagent_standby_tor_downstream.py:246: in monitor_tunnel_and_server_traffic
    testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10)
common/dualtor/tunnel_traffic_utils.py:199: in __exit__
    queue_check_res = self._check_queue(self, self.rec_pkt)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tests.common.dualtor.tunnel_traffic_utils.TunnelTrafficMonitor object at 0x7fd673b6e3d0>
packet = <Ether  dst=52:54:00:82:35:55 src=d4:af:f7:4d:af:18 type=0x800 |<IP  version=4...6\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@A' |>>>>

    @staticmethod
    def _check_queue(self, packet):
        """Check queue for encap packet."""

        def _disassemble_ip_tos(tos):
            return tos >> 2, tos & 0x3

        outer_tos, inner_tos = packet[IP].tos, packet[IP].payload[IP].tos
        outer_dscp, outer_ecn = _disassemble_ip_tos(outer_tos)
        inner_dscp, inner_ecn = _disassemble_ip_tos(inner_tos)
        logging.info("Outer packet DSCP: {0:06b}, inner packet DSCP: {1:06b}".format(outer_dscp, inner_dscp))
        check_res = []
        if outer_dscp != inner_dscp:
            check_res.append("outer packet DSCP not same as inner packet DSCP")
        exp_queue = derive_queue_id_from_dscp(outer_dscp)

        time.sleep(10)
        queue_counter = self.standby_tor.shell('show queue counters | grep "UC"')['stdout']
        logging.debug('queue_counter:\n{}'.format(queue_counter))

        """
        regex search will look for following pattern in queue_counter outpute
        ----------------------------------------------------------------------------_---
        Port           TxQ    Counter/pkts     Counter/bytes     Drop/pkts    Drop/bytes
        -----------  -----  --------------  ---------------  -----------  --------------
        Ethernet124    UC1              10             1000            0             0
        """
        result = re.search(r'\S+\s+UC\d\s+10+\s+\S+\s+\S+\s+\S+', queue_counter)

        rec_queue = 0
        if result != None:
            output = result.group(0)
            output_list = output.split()
            rec_queue = int(output_list[1][2])

        if rec_queue != exp_queue:
>           pytest.fail("the expected Queue : {} not matching with received Queue : {}".format(exp_queue, rec_queue))
E           Failed: the expected Queue : 1 not matching with received Queue : 0

_disassemble_ip_tos = <function _disassemble_ip_tos at 0x7fd65b8ac950>
check_res  = []
exp_queue  = 1
inner_dscp = 1
inner_ecn  = 0
inner_tos  = 4
outer_dscp = 1
outer_ecn  = 0
outer_tos  = 4
packet     = <Ether  dst=52:54:00:82:35:55 src=d4:af:f7:4d:af:18 type=0x800 |<IP  version=4...6\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@A' |>>>>
queue_counter = u'Ethernet0    UC0               0                0            0             0...Ethernet132    UC9               0                0            0             0'
rec_queue  = 0
result     = None
self       = <tests.common.dualtor.tunnel_traffic_utils.TunnelTrafficMonitor object at 0x7fd673b6e3d0>

common/dualtor/tunnel_traffic_utils.py:136: Failed
theasianpianist commented 3 years ago

Test should continuously check queue counters for up to 60 sec.

theasianpianist commented 3 years ago

https://github.com/Azure/sonic-mgmt/blob/d2eaa907a99e5d4f85b54e47cbe70dbb81bb0c89/tests/common/dualtor/tunnel_traffic_utils.py#L116

kevinskwang commented 3 years ago

Merge the PR3761 to fix this issue.