search

sonic-net / sonic-mgmt

Configuration management examples for SONiC
Other
196 stars 723 forks source link

[Arista] lldp/test_lldp.py test failure #6572

Closed yaqiangz closed 2 years ago

yaqiangz commented 2 years ago

Description Test failures seen in 720dt. Run tests by ./run_tests.sh -n [] -s [] -i [] -m individual -t m0,any cmd, and test_lldp.py failed. But run test by ./run_tests.sh -i [] -n[] -u -m individual -e --skip_sanity -e --disable_loganalyzer -c lldp/test_lldp.py can pass the test case Error logs:

2022-10-18T16:48:56.3301408Z ___________________ test_lldp_neighbor[bjw-can-720dt-1-None] ___________________
2022-10-18T16:48:56.3306077Z 
2022-10-18T16:48:56.3315596Z duthosts = [<MultiAsicSonicHost bjw-can-720dt-1>]
2022-10-18T16:48:56.3326450Z enum_rand_one_per_hwsku_frontend_hostname = 'bjw-can-720dt-1'
2022-10-18T16:48:56.3337796Z localhost = <tests.common.devices.local.Localhost object at 0x7fe2f7fde250>
2022-10-18T16:48:56.3350738Z eos = {'bgp_gr_timer': 700, 'ceos_image_mount_dir': '/data/ceos', 'snmp_location': 'str', 'snmp_rocommunity': 'public'}
2022-10-18T16:48:56.3361758Z collect_techsupport_all_duts = None
2022-10-18T16:48:56.3373875Z loganalyzer = {'bjw-can-720dt-1': <tests.common.plugins.loganalyzer.loganalyzer.LogAnalyzer instance at 0x7fe2ed4d38c0>}
2022-10-18T16:48:56.3385597Z enum_frontend_asic_index = None
2022-10-18T16:48:56.3397271Z tbinfo = {'auto_recover': 'True', 'comment': 'Beijing lab', 'conf-name': 'bjw-vms6-t0-720dt-1', 'duts': ['bjw-can-720dt-1'], ...}
2022-10-18T16:48:56.3402504Z 
2022-10-18T16:48:56.3412765Z     def test_lldp_neighbor(duthosts, enum_rand_one_per_hwsku_frontend_hostname, localhost, eos,
2022-10-18T16:48:56.3419723Z                            collect_techsupport_all_duts, loganalyzer, enum_frontend_asic_index, tbinfo):
2022-10-18T16:48:56.3423572Z         """ verify LLDP information on neighbors """
2022-10-18T16:48:56.3427441Z         duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
2022-10-18T16:48:56.3436254Z     
2022-10-18T16:48:56.3439349Z         if loganalyzer:
2022-10-18T16:48:56.3442947Z             loganalyzer[enum_rand_one_per_hwsku_frontend_hostname].ignore_regex.extend([
2022-10-18T16:48:56.3447090Z                 ".*ERR syncd#syncd: :- check_fdb_event_notification_data.*",
2022-10-18T16:48:56.3451022Z                 ".*ERR syncd#syncd: :- process_on_fdb_event: invalid OIDs in fdb \
2022-10-18T16:48:56.3455296Z                     notifications, NOT translating and NOT storing in ASIC DB.*",
2022-10-18T16:48:56.3459539Z                 ".*ERR syncd#syncd: :- process_on_fdb_event: FDB notification was \
2022-10-18T16:48:56.3463406Z                     not sent since it contain invalid OIDs, bug.*",
2022-10-18T16:48:56.3467035Z             ])
2022-10-18T16:48:56.3469890Z     
2022-10-18T16:48:56.3473589Z         res = duthost.shell("docker exec -i lldp lldpcli show chassis | grep \"SysDescr:\" | sed -e 's/^\\s*SysDescr:\\s*//g'")
2022-10-18T16:48:56.3477377Z         dut_system_description = res['stdout']
2022-10-18T16:48:56.3481566Z         lldpctl_facts = duthost.lldpctl_facts(asic_instance_id=enum_frontend_asic_index, skip_interface_pattern_list=["eth0", "Ethernet-BP", "Ethernet-IB"])['ansible_facts']
2022-10-18T16:48:56.3486440Z         config_facts = duthost.asic_instance(enum_frontend_asic_index).config_facts(host=duthost.hostname, source="running")['ansible_facts']
2022-10-18T16:48:56.3490363Z         if not lldpctl_facts['lldpctl'].items():
2022-10-18T16:48:56.3494144Z             pytest.fail("No LLDP neighbors received (lldpctl_facts are empty)")
2022-10-18T16:48:56.3498011Z         # We use the MAC of mgmt port to generate chassis ID as LLDPD dose.
2022-10-18T16:48:56.3501952Z         # To be compatible with PR #3331, we keep using router MAC on T2 devices
2022-10-18T16:48:56.3505600Z         switch_mac = ""
2022-10-18T16:48:56.3508819Z         if tbinfo["topo"]["type"] != "t2":
2022-10-18T16:48:56.3512400Z             mgmt_alias = duthost.get_extended_minigraph_facts(tbinfo)["minigraph_mgmt_interface"]["alias"]
2022-10-18T16:48:56.3516054Z             switch_mac = duthost.get_dut_iface_mac(mgmt_alias)
2022-10-18T16:48:56.3519440Z         elif tbinfo["topo"]["type"] == "t2":
2022-10-18T16:48:56.3523198Z             switch_mac = config_facts['DEVICE_METADATA']['localhost']['mac'].lower()
2022-10-18T16:48:56.3526596Z         else:
2022-10-18T16:48:56.3529766Z             switch_mac = duthost.facts['router_mac']
2022-10-18T16:48:56.3532985Z     
2022-10-18T16:48:56.3536692Z         nei_meta = config_facts.get('DEVICE_NEIGHBOR_METADATA', {})
2022-10-18T16:48:56.3540208Z     
2022-10-18T16:48:56.3543513Z         for k, v in lldpctl_facts['lldpctl'].items():
2022-10-18T16:48:56.3546919Z             try:
2022-10-18T16:48:56.3550007Z                 hostip = v['chassis']['mgmt-ip']
2022-10-18T16:48:56.3553076Z             except:
2022-10-18T16:48:56.3556683Z                 logger.info("Neighbor device {} does not sent management IP via lldp".format(v['chassis']['name']))
2022-10-18T16:48:56.3560429Z                 hostip = nei_meta[v['chassis']['name']]['mgmt_addr']
2022-10-18T16:48:56.3563630Z     
2022-10-18T16:48:56.3567263Z >           nei_lldp_facts = localhost.lldp_facts(host=hostip, version='v2c', community=eos['snmp_rocommunity'])['ansible_facts']
2022-10-18T16:48:56.3568919Z 
2022-10-18T16:48:56.3571948Z collect_techsupport_all_duts = None
2022-10-18T16:48:56.3577010Z config_facts = {'ACL_TABLE': {'DATAACL': {'policy_desc': u'DATAACL', 'ports': [u'PortChannel101', u'PortChannel103', u'PortChannel105...te_limit_interval': u'600', 'state': u'enabled'}, ...}, 'BGP_DEVICE_GLOBAL': {'STATE': {'tsa_enabled': u'false'}}, ...}
2022-10-18T16:48:56.3582581Z dut_system_description = u'SONiC Software Version: SONiC.20220531.08 - HwSku: Arista-720DT-48S - Distribution: Debian 11.5 - Kernel: 5.10.0-12-2-amd64'
2022-10-18T16:48:56.3586842Z duthost    = <MultiAsicSonicHost bjw-can-720dt-1>
2022-10-18T16:48:56.3590297Z duthosts   = [<MultiAsicSonicHost bjw-can-720dt-1>]
2022-10-18T16:48:56.3593490Z enum_frontend_asic_index = None
2022-10-18T16:48:56.3597022Z enum_rand_one_per_hwsku_frontend_hostname = 'bjw-can-720dt-1'
2022-10-18T16:48:56.3601290Z eos        = {'bgp_gr_timer': 700, 'ceos_image_mount_dir': '/data/ceos', 'snmp_location': 'str', 'snmp_rocommunity': 'public'}
2022-10-18T16:48:56.3605166Z hostip     = u'172.16.190.36'
2022-10-18T16:48:56.3608380Z k          = 'Ethernet50'
2022-10-18T16:48:56.3616860Z lldpctl_facts = {'lldpctl': {'Ethernet48': {'age': u'0 day, 00:22:45', 'chassis': {'Bridge': {'enabled': u'on'}, 'Router': {'enabled':..., ...}, 'port': {'aggregation': u'1000001', 'ifname': u'Ethernet1', 'mfs': u'9236', 'ttl': u'120'}, 'rid': u'4', ...}}}
2022-10-18T16:48:56.3621483Z localhost  = <tests.common.devices.local.Localhost object at 0x7fe2f7fde250>
2022-10-18T16:48:56.3625985Z loganalyzer = {'bjw-can-720dt-1': <tests.common.plugins.loganalyzer.loganalyzer.LogAnalyzer instance at 0x7fe2ed4d38c0>}
2022-10-18T16:48:56.3629636Z mgmt_alias = u'eth0'
2022-10-18T16:48:56.3634049Z nei_meta   = {'ARISTA01M1': {'hwsku': u'Arista-VM', 'lo_addr': u'None', 'mgmt_addr': u'172.16.190.34', 'type': u'MgmtLeafRouter'}, ...STA02MX': {'hwsku': u'Arista-VM', 'lo_addr': u'None', 'mgmt_addr': u'172.16.190.33', 'type': u'BmcMgmtToRRouter'}, ...}
2022-10-18T16:48:56.3639180Z res        = {'stderr_lines': [], u'cmd': u'docker exec -i lldp lldpcli show chassis | grep...48S - Distribution: Debian 11.5 - Kernel: 5.10.0-12-2-amd64'], 'failed': False}
2022-10-18T16:48:56.3643050Z switch_mac = u'2c:dd:e9:fc:df:08'
2022-10-18T16:48:56.3646865Z tbinfo     = {'auto_recover': 'True', 'comment': 'Beijing lab', 'conf-name': 'bjw-vms6-t0-720dt-1', 'duts': ['bjw-can-720dt-1'], ...}
2022-10-18T16:48:56.3651903Z v          = {'age': u'0 day, 01:46:05', 'chassis': {'Bridge': {'enabled': u'on'}, 'Router': {'enabled': u'on'}, 'mac': u'42:d3:a6:...6', ...}, 'port': {'aggregation': u'1000001', 'ifname': u'Ethernet1', 'mfs': u'9236', 'ttl': u'120'}, 'rid': u'3', ...}
2022-10-18T16:48:56.3654627Z 
2022-10-18T16:48:56.3657642Z lldp/test_lldp.py:75: 
2022-10-18T16:48:56.3661019Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
2022-10-18T16:48:56.3662474Z 
2022-10-18T16:48:56.3665760Z self = <tests.common.devices.local.Localhost object at 0x7fe2f7fde250>
2022-10-18T16:48:56.3669045Z module_args = ()
2022-10-18T16:48:56.3672555Z complex_args = {'community': 'public', 'host': u'172.16.190.36', 'version': 'v2c'}
2022-10-18T16:48:56.3676000Z previous_frame = <frame object at 0x55645df052f0>
2022-10-18T16:48:56.3679571Z filename = '/azp/agent/_work/1/s/tests/lldp/test_lldp.py', line_number = 75
2022-10-18T16:48:56.3683085Z function_name = 'test_lldp_neighbor'
2022-10-18T16:48:56.3686988Z lines = ["        nei_lldp_facts = localhost.lldp_facts(host=hostip, version='v2c', community=eos['snmp_rocommunity'])['ansible_facts']\n"]
2022-10-18T16:48:56.3690774Z index = 0, verbose = True, module_ignore_errors = False, module_async = False
2022-10-18T16:48:56.3692198Z 
2022-10-18T16:48:56.3695432Z     def _run(self, *module_args, **complex_args):
2022-10-18T16:48:56.3698716Z     
2022-10-18T16:48:56.3701828Z         previous_frame = inspect.currentframe().f_back
2022-10-18T16:48:56.3705708Z         filename, line_number, function_name, lines, index = inspect.getframeinfo(previous_frame)
2022-10-18T16:48:56.3708971Z     
2022-10-18T16:48:56.3712220Z         verbose = complex_args.pop('verbose', True)
2022-10-18T16:48:56.3715335Z     
2022-10-18T16:48:56.3718063Z         if verbose:
2022-10-18T16:48:56.3721226Z             logging.debug("{}::{}#{}: [{}] AnsibleModule::{}, args={}, kwargs={}"\
2022-10-18T16:48:56.3724754Z                 .format(filename, function_name, line_number, self.hostname,
2022-10-18T16:48:56.3728414Z                         self.module_name, json.dumps(module_args), json.dumps(complex_args)))
2022-10-18T16:48:56.3731678Z         else:
2022-10-18T16:48:56.3735003Z             logging.debug("{}::{}#{}: [{}] AnsibleModule::{} executing..."\
2022-10-18T16:48:56.3738890Z                 .format(filename, function_name, line_number, self.hostname, self.module_name))
2022-10-18T16:48:56.3742536Z     
2022-10-18T16:48:56.3746183Z         module_ignore_errors = complex_args.pop('module_ignore_errors', False)
2022-10-18T16:48:56.3749947Z         module_async = complex_args.pop('module_async', False)
2022-10-18T16:48:56.3753332Z     
2022-10-18T16:48:56.3756156Z         if module_async:
2022-10-18T16:48:56.3759043Z             def run_module(module_args, complex_args):
2022-10-18T16:48:56.3762401Z                 return self.module(*module_args, **complex_args)[self.hostname]
2022-10-18T16:48:56.3765589Z             pool = ThreadPool()
2022-10-18T16:48:56.3768669Z             result = pool.apply_async(run_module, (module_args, complex_args))
2022-10-18T16:48:56.3771872Z             return pool, result
2022-10-18T16:48:56.3774944Z     
2022-10-18T16:48:56.3778099Z         res = self.module(*module_args, **complex_args)[self.hostname]
2022-10-18T16:48:56.3781456Z     
2022-10-18T16:48:56.3792254Z         if verbose:
2022-10-18T16:48:56.3795269Z             logging.debug("{}::{}#{}: [{}] AnsibleModule::{} Result => {}"\
2022-10-18T16:48:56.3798947Z                 .format(filename, function_name, line_number, self.hostname, self.module_name, json.dumps(res)))
2022-10-18T16:48:56.3802274Z         else:
2022-10-18T16:48:56.3805173Z             logging.debug("{}::{}#{}: [{}] AnsibleModule::{} done, is_failed={}, rc={}"\
2022-10-18T16:48:56.3808650Z                 .format(filename, function_name, line_number, self.hostname, self.module_name, \
2022-10-18T16:48:56.3812041Z                         res.is_failed, res.get('rc', None)))
2022-10-18T16:48:56.3815201Z     
2022-10-18T16:48:56.3818353Z         if (res.is_failed or 'exception' in res) and not module_ignore_errors:
2022-10-18T16:48:56.3822027Z >           raise RunAnsibleModuleFail("run module {} failed".format(self.module_name), res)
2022-10-18T16:48:56.3825901Z E           RunAnsibleModuleFail: run module lldp_facts failed, Ansible Results =>
2022-10-18T16:48:56.3828852Z E           {
2022-10-18T16:48:56.3831366Z E               "changed": false, 
2022-10-18T16:48:56.3834039Z E               "failed": true, 
2022-10-18T16:48:56.3836617Z E               "invocation": {
2022-10-18T16:48:56.3839168Z E                   "module_args": {
2022-10-18T16:48:56.3841881Z E                       "authkey": null, 
2022-10-18T16:48:56.3844734Z E                       "community": "public", 
2022-10-18T16:48:56.3847602Z E                       "host": "172.16.190.36", 
2022-10-18T16:48:56.3850572Z E                       "integrity": null, 
2022-10-18T16:48:56.3853603Z E                       "level": null, 
2022-10-18T16:48:56.3856580Z E                       "privacy": null, 
2022-10-18T16:48:56.3859576Z E                       "privkey": null, 
2022-10-18T16:48:56.3862588Z E                       "removeplaceholder": null, 
2022-10-18T16:48:56.3865786Z E                       "username": null, 
2022-10-18T16:48:56.3868605Z E                       "version": "v2c"
2022-10-18T16:48:56.3871346Z E                   }
2022-10-18T16:48:56.3873993Z E               }, 
2022-10-18T16:48:56.3876798Z E               "msg": "No SNMP response received before timeout"
2022-10-18T16:48:56.3879775Z E           }
2022-10-18T16:48:56.3880706Z 
2022-10-18T16:48:56.3884020Z complex_args = {'community': 'public', 'host': u'172.16.190.36', 'version': 'v2c'}
2022-10-18T16:48:56.3887644Z filename   = '/azp/agent/_work/1/s/tests/lldp/test_lldp.py'
2022-10-18T16:48:56.3890879Z function_name = 'test_lldp_neighbor'
2022-10-18T16:48:56.3893934Z index      = 0
2022-10-18T16:48:56.3896733Z line_number = 75
2022-10-18T16:48:56.3900539Z lines      = ["        nei_lldp_facts = localhost.lldp_facts(host=hostip, version='v2c', community=eos['snmp_rocommunity'])['ansible_facts']\n"]
2022-10-18T16:48:56.3904123Z module_args = ()
2022-10-18T16:48:56.3906943Z module_async = False
2022-10-18T16:48:56.3909671Z module_ignore_errors = False
2022-10-18T16:48:56.3912458Z previous_frame = <frame object at 0x55645df052f0>
2022-10-18T16:48:56.3916469Z res        = {u'invocation': {u'module_args': {u'username': None, u'privacy': None, u'level... '_ansible_no_log': False, u'msg': u'No SNMP response received before timeout'}
2022-10-18T16:48:56.3920234Z self       = <tests.common.devices.local.Localhost object at 0x7fe2f7fde250>
2022-10-18T16:48:56.3923300Z verbose    = True
2022-10-18T16:48:56.3924239Z 
2022-10-18T16:48:56.3926972Z common/devices/base.py:89: RunAnsibleModuleFail
2022-10-18T16:48:56.3930458Z ---- generated xml file: /azp/agent/_work/1/s/tests/logs/lldp/test_lldp.xml ----
2022-10-18T16:48:56.3934267Z =========================== short test summary info ============================
2022-10-18T16:48:56.3938259Z FAILED lldp/test_lldp.py::test_lldp_neighbor[bjw-can-720dt-1-None] - RunAnsib...
2022-10-18T16:48:56.3945506Z ===================== 1 failed, 1 passed in 91.96 seconds ======================

Additional information you deem important:

**Output of `show version`:**

```
(paste your output here)
```

**Attach debug file `sudo generate_dump`:**

```
(paste your output here)
```
Blueve commented 2 years ago

Might be caused by our testbed connectivity issue. The sonic-mgmt docker is not able to connect cEOS container

Blueve commented 2 years ago

Fixed the sonic-mgmt container <-> cEOS connectivity issue. It was testbed network issue