search

atlanticwave-sdx / sdx-end-to-end-tests

SDX End to End tests
0 stars 0 forks source link

Flake end to end tests results for topology #2

Closed italovalcy closed 1 month ago

italovalcy commented 2 months ago

Hi,

Latest results for end-to-end tests are kind of flake for the topology checks:

============================= test session starts ==============================
platform linux -- Python 3.11.2, pytest-7.2.1, pluggy-1.0.0+repack
rootdir: /builds/amlight/sdx-end-to-end-tests/sdx-end-to-end-tests
plugins: unordered-0.6.1
collected 7 items

tests/test_01_topology.py .Fx                                            [ 42%]
tests/test_05_l2vpn.py ....                                              [100%]

=================================== FAILURES ===================================
_________ TestE2ETopology.test_015_check_topology_follows_model_2_0_0 __________

self = <test_01_topology.TestE2ETopology object at 0x7f087de8a4d0>

    def test_015_check_topology_follows_model_2_0_0(self):
        expected_topos = self.net.get_converted_topologies()
        for idx, oxp in enumerate(["ampath", "sax", "tenet"]):
            response = requests.get(f"http://{oxp}:8181/api/kytos/sdx/topology/2.0.0")
            topo = response.json()
            for node in topo["nodes"]:
                node["ports"] = unordered(node["ports"])
            for attr in ["name", "id", "model_version", "nodes", "links", "services"]:
                assert attr in topo, str(topo)
>               assert unordered(topo[attr]) == expected_topos[idx][attr], f"fount {attr}={topo[attr]}"
E               AssertionError: fount nodes=[{'name': 'Ampath1', 'id': 'urn:sdx:node:ampath.net:Ampath1', 'location': {'address': 'Miami', 'latitude': 25.77, 'longitude': -80.19, 'iso3166_2_lvl4': 'US-FL', 'private': []}, 'ports': [{'id': 'urn:sdx:port:ampath.net:Ampath1:1', 'name': 'Ampath1-eth1', 'node': 'urn:sdx:node:ampath.net:Ampath1', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath1/1_Ampath2/1', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath1:40', 'name': 'Ampath1-eth40', 'node': 'urn:sdx:node:ampath.net:Ampath1', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:port:sax.net:Sax01:40', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath1:50', 'name': 'Ampath1-eth50', 'node': 'urn:sdx:node:ampath.net:Ampath1', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': '', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath1:2', 'name': 'Ampath1-eth2', 'node': 'urn:sdx:node:ampath.net:Ampath1', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath1/2_Ampath3/2', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], 'status': 'up', 'state': 'enabled'}, {'name': 'Ampath2', 'id': 'urn:sdx:node:ampath.net:Ampath2', 'location': {'address': 'BocaRaton', 'latitude': 26.38, 'longitude': -80.11, 'iso3166_2_lvl4': 'US-FL', 'private': []}, 'ports': [{'id': 'urn:sdx:port:ampath.net:Ampath2:1', 'name': 'Ampath2-eth1', 'node': 'urn:sdx:node:ampath.net:Ampath2', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath1/1_Ampath2/1', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath2:50', 'name': 'Ampath2-eth50', 'node': 'urn:sdx:node:ampath.net:Ampath2', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': '', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath2:40', 'name': 'Ampath2-eth40', 'node': 'urn:sdx:node:ampath.net:Ampath2', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:port:sax.net:Sax02:40', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath2:3', 'name': 'Ampath2-eth3', 'node': 'urn:sdx:node:ampath.net:Ampath2', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath2/3_Ampath3/3', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], 'status': 'up', 'state': 'enabled'}, {'name': 'Ampath3', 'id': 'urn:sdx:node:ampath.net:Ampath3', 'location': {'address': 'Jacksonville', 'latitude': 30.27, 'longitude': -81.68, 'iso3166_2_lvl4': 'US-FL', 'private': []}, 'ports': [{'id': 'urn:sdx:port:ampath.net:Ampath3:50', 'name': 'Ampath3-eth50', 'node': 'urn:sdx:node:ampath.net:Ampath3', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': '', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath3:2', 'name': 'Ampath3-eth2', 'node': 'urn:sdx:node:ampath.net:Ampath3', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath1/2_Ampath3/2', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath3:3', 'name': 'Ampath3-eth3', 'node': 'urn:sdx:node:ampath.net:Ampath3', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath2/3_Ampath3/3', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], 'status': 'up', 'state': 'enabled'}]
E               assert [{'name': 'Ampath1', 'id': 'urn:sdx:node:ampath.net:Ampath1', 'location': {'address': 'Miami', 'latitude': 25.77, 'lon..., 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], 'status': 'up', 'state': 'enabled'}] == [{'id': 'urn:sdx:node:ampath.net:Ampath1', 'location': {'address': 'Miami', 'iso3166_2_lvl4': 'US-FL', 'latitude': 25....ni': '', ...}, {'id': 'urn:sdx:port:ampath.net:Ampath3:3', 'mtu': 1500, 'name': 'Ampath3-eth3', 'nni': '', ...}], ...}]
E                 Extra items in the left sequence:
E                 {'id': 'urn:sdx:node:ampath.net:Ampath1', 'location': {'address': 'Miami', 'iso3166_2_lvl4': 'US-FL', 'latitude': 25.7...:ampath.net:Ampath1/2_Ampath3/2', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], ...}
E                 {'id': 'urn:sdx:node:ampath.net:Ampath2', 'location': {'address': 'BocaRaton', 'iso3166_2_lvl4': 'US-FL', 'latitude': ...:ampath.net:Ampath2/3_Ampath3/3', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], ...}
E                 {'id': 'urn:sdx:node:ampath.net:Ampath3', 'location': {'address': 'Jacksonville', 'iso3166_2_lvl4': 'US-FL', 'latitude...:ampath.net:Ampath2/3_Ampath3/3', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], ...}
E                 Extra items in the right sequence:
E                 {'id': 'urn:sdx:node:ampath.net:Ampath1', 'location': {'address': 'Miami', 'iso3166_2_lvl4': 'US-FL', 'latitude': 25.7...nni': '', ...}, {'id': 'urn:sdx:port:ampath.net:Ampath1:2', 'mtu': 1500, 'name': 'Ampath1-eth2', 'nni': '', ...}], ...}
E                 {'id': 'urn:sdx:node:ampath.net:Ampath2', 'location': {'address': 'BocaRaton', 'iso3166_2_lvl4': 'US-FL', 'latitude': ...ax02:40', ...}, {'id': 'urn:sdx:port:ampath.net:Ampath2:3', 'mtu': 1500, 'name': 'Ampath2-eth3', 'nni': '', ...}], ...}
E                 {'id': 'urn:sdx:node:ampath.net:Ampath3', 'location': {'address': 'Jacksonville', 'iso3166_2_lvl4': 'US-FL', 'latitude...nni': '', ...}, {'id': 'urn:sdx:port:ampath.net:Ampath3:3', 'mtu': 1500, 'name': 'Ampath3-eth3', 'nni': '', ...}], ...}

tests/test_01_topology.py:59: AssertionError
------------------------------- start/stop times -------------------------------
tests/test_01_topology.py::TestE2ETopology::test_015_check_topology_follows_model_2_0_0: 2024-09-03,21:17:55.744009 - 2024-09-03,21:17:55.999923
=========================== short test summary info ============================
FAILED tests/test_01_topology.py::TestE2ETopology::test_015_check_topology_follows_model_2_0_0 - AssertionError: fount nodes=[{'name': 'Ampath1', 'id': 'urn:sdx:node:ampath.net:Ampath1', 'location': {'address': 'Miami', 'latitude': 25.77, 'longitude': -80.19, 'iso3166_2_lvl4': 'US-FL', 'private': []}, 'ports': [{'id': 'urn:sdx:port:ampath.net:Ampath1:1', 'name': 'Ampath1-eth1', 'node': 'urn:sdx:node:ampath.net:Ampath1', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath1/1_Ampath2/1', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath1:40', 'name': 'Ampath1-eth40', 'node': 'urn:sdx:node:ampath.net:Ampath1', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:port:sax.net:Sax01:40', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath1:50', 'name': 'Ampath1-eth50', 'node': 'urn:sdx:node:ampath.net:Ampath1', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': '', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath1:2', 'name': 'Ampath1-eth2', 'node': 'urn:sdx:node:ampath.net:Ampath1', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath1/2_Ampath3/2', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], 'status': 'up', 'state': 'enabled'}, {'name': 'Ampath2', 'id': 'urn:sdx:node:ampath.net:Ampath2', 'location': {'address': 'BocaRaton', 'latitude': 26.38, 'longitude': -80.11, 'iso3166_2_lvl4': 'US-FL', 'private': []}, 'ports': [{'id': 'urn:sdx:port:ampath.net:Ampath2:1', 'name': 'Ampath2-eth1', 'node': 'urn:sdx:node:ampath.net:Ampath2', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath1/1_Ampath2/1', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath2:50', 'name': 'Ampath2-eth50', 'node': 'urn:sdx:node:ampath.net:Ampath2', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': '', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath2:40', 'name': 'Ampath2-eth40', 'node': 'urn:sdx:node:ampath.net:Ampath2', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:port:sax.net:Sax02:40', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath2:3', 'name': 'Ampath2-eth3', 'node': 'urn:sdx:node:ampath.net:Ampath2', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath2/3_Ampath3/3', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], 'status': 'up', 'state': 'enabled'}, {'name': 'Ampath3', 'id': 'urn:sdx:node:ampath.net:Ampath3', 'location': {'address': 'Jacksonville', 'latitude': 30.27, 'longitude': -81.68, 'iso3166_2_lvl4': 'US-FL', 'private': []}, 'ports': [{'id': 'urn:sdx:port:ampath.net:Ampath3:50', 'name': 'Ampath3-eth50', 'node': 'urn:sdx:node:ampath.net:Ampath3', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': '', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath3:2', 'name': 'Ampath3-eth2', 'node': 'urn:sdx:node:ampath.net:Ampath3', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath1/2_Ampath3/2', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}, {'id': 'urn:sdx:port:ampath.net:Ampath3:3', 'name': 'Ampath3-eth3', 'node': 'urn:sdx:node:ampath.net:Ampath3', 'type': '10GE', 'status': 'up', 'state': 'enabled', 'mtu': 1500, 'nni': 'urn:sdx:link:ampath.net:Ampath2/3_Ampath3/3', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], 'status': 'up', 'state': 'enabled'}]
assert [{'name': 'Ampath1', 'id': 'urn:sdx:node:ampath.net:Ampath1', 'location': {'address': 'Miami', 'latitude': 25.77, 'lon..., 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], 'status': 'up', 'state': 'enabled'}] == [{'id': 'urn:sdx:node:ampath.net:Ampath1', 'location': {'address': 'Miami', 'iso3166_2_lvl4': 'US-FL', 'latitude': 25....ni': '', ...}, {'id': 'urn:sdx:port:ampath.net:Ampath3:3', 'mtu': 1500, 'name': 'Ampath3-eth3', 'nni': '', ...}], ...}]
  Extra items in the left sequence:
  {'id': 'urn:sdx:node:ampath.net:Ampath1', 'location': {'address': 'Miami', 'iso3166_2_lvl4': 'US-FL', 'latitude': 25.7...:ampath.net:Ampath1/2_Ampath3/2', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], ...}
  {'id': 'urn:sdx:node:ampath.net:Ampath2', 'location': {'address': 'BocaRaton', 'iso3166_2_lvl4': 'US-FL', 'latitude': ...:ampath.net:Ampath2/3_Ampath3/3', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], ...}
  {'id': 'urn:sdx:node:ampath.net:Ampath3', 'location': {'address': 'Jacksonville', 'iso3166_2_lvl4': 'US-FL', 'latitude...:ampath.net:Ampath2/3_Ampath3/3', 'services': {'l2vpn-ptp': {'vlan_range': [[1, 4095]]}}, 'private': ['status']}], ...}
  Extra items in the right sequence:
  {'id': 'urn:sdx:node:ampath.net:Ampath1', 'location': {'address': 'Miami', 'iso3166_2_lvl4': 'US-FL', 'latitude': 25.7...nni': '', ...}, {'id': 'urn:sdx:port:ampath.net:Ampath1:2', 'mtu': 1500, 'name': 'Ampath1-eth2', 'nni': '', ...}], ...}
  {'id': 'urn:sdx:node:ampath.net:Ampath2', 'location': {'address': 'BocaRaton', 'iso3166_2_lvl4': 'US-FL', 'latitude': ...ax02:40', ...}, {'id': 'urn:sdx:port:ampath.net:Ampath2:3', 'mtu': 1500, 'name': 'Ampath2-eth3', 'nni': '', ...}], ...}
  {'id': 'urn:sdx:node:ampath.net:Ampath3', 'location': {'address': 'Jacksonville', 'iso3166_2_lvl4': 'US-FL', 'latitude...nni': '', ...}, {'id': 'urn:sdx:port:ampath.net:Ampath3:3', 'mtu': 1500, 'name': 'Ampath3-eth3', 'nni': '', ...}], ...}
============== 1 failed, 5 passed, 1 xfailed in 230.36s (0:03:50) ==============

We have to double check and fix this error.

italovalcy commented 1 month ago

This issue was already fixed and the reason was basically due to the order of the ports in a link.