Open theasianpianist opened 3 years ago
conn_graph_facts = {'device_conn': {'str2-7050cx3-acs-06': {'Ethernet0': {'peerdevice': u'str2-7260cx3-acs-fan-10', 'peerport': u'Etherne...ard', 'HwSku': u'Arista-7050CX3-32S-C32', 'ManagementGw': u'10.3.146.1', 'ManagementIp': u'10.3.147.24/23', ...}}, ...} ptfadapter = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest> ptfhost = <tests.common.devices.ptf.PTFHost object at 0x7fd679d0f090> rand_selected_dut = <MultiAsicSonicHost> str2-7050cx3-acs-07 rand_unselected_dut = <MultiAsicSonicHost> str2-7050cx3-acs-06 tbinfo = {'auto_recover': 'True', 'comment': 'hellogemini', 'conf-name': 'vms17-dual-t0-7050-1', 'duts': ['str2-7050cx3-acs-06', 'str2-7050cx3-acs-07'], ...} tunnel_traffic_monitor = <class 'tests.common.dualtor.tunnel_traffic_utils.TunnelTrafficMonitor'> vmhost = <tests.common.devices.vmhost.VMHost object at 0x7fd67abba790> toggle_all_simulator_ports = <function _toggle at 0x7fd673a65950> tor_mux_intfs = ['Ethernet4', 'Ethernet8', 'Ethernet12', 'Ethernet16', 'Ethernet20', 'Ethernet24', ...] def test_downstream_standby_mux_toggle_active( conn_graph_facts, ptfadapter, ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo, tunnel_traffic_monitor, vmhost, toggle_all_simulator_ports, tor_mux_intfs ): # set rand_selected_dut as standby and rand_unselected_dut to active tor test_params = dualtor_info(ptfhost, rand_selected_dut, rand_unselected_dut, tbinfo) server_ipv4 = test_params["target_server_ip"] random_dst_ip = "1.1.1.2" pkt, exp_pkt = build_packet_to_server(rand_selected_dut, ptfadapter, random_dst_ip) ptf_t1_intf = random.choice(get_t1_ptf_ports(rand_selected_dut, tbinfo)) def monitor_tunnel_and_server_traffic(torhost, expect_tunnel_traffic=True, expect_server_traffic=True): tunnel_monitor = tunnel_traffic_monitor(rand_selected_dut, existing=True) server_traffic_monitor = ServerTrafficMonitor( torhost, ptfhost, vmhost, tbinfo, test_params["selected_port"], conn_graph_facts, exp_pkt, existing=False, is_mocked=is_mocked_dualtor(tbinfo) ) tunnel_monitor.existing = expect_tunnel_traffic server_traffic_monitor.existing = expect_server_traffic with tunnel_monitor, server_traffic_monitor: testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) logger.info("Stage 1: Verify Standby Forwarding") logger.info("Step 1.1: Add route to a nexthop which is a standby Neighbor") set_mux_state(rand_selected_dut, tbinfo, 'standby', tor_mux_intfs, toggle_all_simulator_ports) add_nexthop_routes(rand_selected_dut, random_dst_ip, nexthops=[server_ipv4]) logger.info("Step 1.2: Verify traffic to this route dst is forwarded to Active ToR and equally distributed") check_tunnel_balance(**test_params) > monitor_tunnel_and_server_traffic(rand_selected_dut, expect_server_traffic=False, expect_tunnel_traffic=True) conn_graph_facts = {'device_conn': {'str2-7050cx3-acs-06': {'Ethernet0': {'peerdevice': u'str2-7260cx3-acs-fan-10', 'peerport': u'Etherne...ard', 'HwSku': u'Arista-7050CX3-32S-C32', 'ManagementGw': u'10.3.146.1', 'ManagementIp': u'10.3.147.24/23', ...}}, ...} exp_pkt = <ptf.mask.Mask instance at 0x7fd64c2e62d0> monitor_tunnel_and_server_traffic = <function monitor_tunnel_and_server_traffic at 0x7fd65f27e6d0> pkt = <Ether dst=d4:af:f7:4d:af:18 src=7a:ad:6e:91:e4:00 type=0x800 |<IP ihl=None ...16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@A' |>>> ptf_t1_intf = 'eth29' ptfadapter = <tests.common.plugins.ptfadapter.ptfadapter.PtfTestAdapter testMethod=runTest> ptfhost = <tests.common.devices.ptf.PTFHost object at 0x7fd679d0f090> rand_selected_dut = <MultiAsicSonicHost> str2-7050cx3-acs-07 rand_unselected_dut = <MultiAsicSonicHost> str2-7050cx3-acs-06 random_dst_ip = '1.1.1.2' server_ipv4 = '192.168.0.16' tbinfo = {'auto_recover': 'True', 'comment': 'hellogemini', 'conf-name': 'vms17-dual-t0-7050-1', 'duts': ['str2-7050cx3-acs-06', 'str2-7050cx3-acs-07'], ...} test_params = {'active_tor_ip': u'10.1.0.32', 'ptf_portchannel_indices': {'PortChannel0001': [29], 'PortChannel0002': [31], 'PortCha...04': [35]}, 'ptfhost': <tests.common.devices.ptf.PTFHost object at 0x7fd679d0f090>, 'selected_port': 'Ethernet60', ...} toggle_all_simulator_ports = <function _toggle at 0x7fd673a65950> tor_mux_intfs = ['Ethernet4', 'Ethernet8', 'Ethernet12', 'Ethernet16', 'Ethernet20', 'Ethernet24', ...] tunnel_traffic_monitor = <class 'tests.common.dualtor.tunnel_traffic_utils.TunnelTrafficMonitor'> vmhost = <tests.common.devices.vmhost.VMHost object at 0x7fd67abba790> dualtor/test_orchagent_standby_tor_downstream.py:254: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ dualtor/test_orchagent_standby_tor_downstream.py:246: in monitor_tunnel_and_server_traffic testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), pkt, count=10) common/dualtor/tunnel_traffic_utils.py:199: in __exit__ queue_check_res = self._check_queue(self, self.rec_pkt) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <tests.common.dualtor.tunnel_traffic_utils.TunnelTrafficMonitor object at 0x7fd673b6e3d0> packet = <Ether dst=52:54:00:82:35:55 src=d4:af:f7:4d:af:18 type=0x800 |<IP version=4...6\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@A' |>>>> @staticmethod def _check_queue(self, packet): """Check queue for encap packet.""" def _disassemble_ip_tos(tos): return tos >> 2, tos & 0x3 outer_tos, inner_tos = packet[IP].tos, packet[IP].payload[IP].tos outer_dscp, outer_ecn = _disassemble_ip_tos(outer_tos) inner_dscp, inner_ecn = _disassemble_ip_tos(inner_tos) logging.info("Outer packet DSCP: {0:06b}, inner packet DSCP: {1:06b}".format(outer_dscp, inner_dscp)) check_res = [] if outer_dscp != inner_dscp: check_res.append("outer packet DSCP not same as inner packet DSCP") exp_queue = derive_queue_id_from_dscp(outer_dscp) time.sleep(10) queue_counter = self.standby_tor.shell('show queue counters | grep "UC"')['stdout'] logging.debug('queue_counter:\n{}'.format(queue_counter)) """ regex search will look for following pattern in queue_counter outpute ----------------------------------------------------------------------------_--- Port TxQ Counter/pkts Counter/bytes Drop/pkts Drop/bytes ----------- ----- -------------- --------------- ----------- -------------- Ethernet124 UC1 10 1000 0 0 """ result = re.search(r'\S+\s+UC\d\s+10+\s+\S+\s+\S+\s+\S+', queue_counter) rec_queue = 0 if result != None: output = result.group(0) output_list = output.split() rec_queue = int(output_list[1][2]) if rec_queue != exp_queue: > pytest.fail("the expected Queue : {} not matching with received Queue : {}".format(exp_queue, rec_queue)) E Failed: the expected Queue : 1 not matching with received Queue : 0 _disassemble_ip_tos = <function _disassemble_ip_tos at 0x7fd65b8ac950> check_res = [] exp_queue = 1 inner_dscp = 1 inner_ecn = 0 inner_tos = 4 outer_dscp = 1 outer_ecn = 0 outer_tos = 4 packet = <Ether dst=52:54:00:82:35:55 src=d4:af:f7:4d:af:18 type=0x800 |<IP version=4...6\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@A' |>>>> queue_counter = u'Ethernet0 UC0 0 0 0 0...Ethernet132 UC9 0 0 0 0' rec_queue = 0 result = None self = <tests.common.dualtor.tunnel_traffic_utils.TunnelTrafficMonitor object at 0x7fd673b6e3d0> common/dualtor/tunnel_traffic_utils.py:136: Failed
Test should continuously check queue counters for up to 60 sec.
https://github.com/Azure/sonic-mgmt/blob/d2eaa907a99e5d4f85b54e47cbe70dbb81bb0c89/tests/common/dualtor/tunnel_traffic_utils.py#L116
Merge the PR3761 to fix this issue.