Flaky tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py::test_legacy_and_dynamic_sidecar_run

pcrespov commented 1 year ago

Main failure is

FAILED tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py::test_legacy_and_dynamic_sidecar_run[2.0.4] - AttributeError: 'NoneType' object has no attribute 'extract'2866

DETAILS

=================================== FAILURES =================================== __ test_legacy_and_dynamic_sidecar_run[2.0.4] __

dy_static_file_server_project = ProjectAtDB(uuid=UUID('d256acb9-4820-4a07-b3bc-d909faec3b7c'), name='Debbie Trujillo', description='Behavior style rep...ngState.NOT_STARTED: 'NOT_STARTED'>), boot_options=None)}, id=1, project_type='STANDARD', prj_owner=1, published=False) user_dict = {'created_at': datetime.datetime(2023, 1, 13, 9, 50, 13, 517015), 'created_ip': None, 'email': 'tammy76@example.com', 'expires_at': None, ...} services_endpoint = {'catalog': URL('http://10.1.0.51:8005'), 'director': URL('http://10.1.0.51:30000'), 'storage': URL('http://10.1.0.51:30005')} director_v2_client = <httpx.AsyncClient object at 0x7f73eae3ba30> ensure_services_stopped = None, mock_projects_networks_repository = None mock_dynamic_sidecar_client = None service_resources = {'container': ImageResources(image='simcore/services/dynamic/jupyter-math:2.0.5', resources={'CPU': ResourceValue(limit=0.1, reservation=0.1), 'RAM': ResourceValue(limit=2147483648, reservation=2147483648)})}

@pytest.mark.flaky(max_runs=3)
async def test_legacy_and_dynamic_sidecar_run(
    dy_static_file_server_project: ProjectAtDB,
    user_dict: dict[str, Any],
    services_endpoint: dict[str, URL],
    director_v2_client: httpx.AsyncClient,
    ensure_services_stopped: None,
    mock_projects_networks_repository: None,
    mock_dynamic_sidecar_client: None,
    service_resources: ServiceResourcesDict,
):
    """
    The test will start 3 dynamic services in the same project and check
    that the legacy and the 2 new dynamic-sidecar boot properly.

    Creates a project containing the following services:
    - dy-static-file-server (legacy)
    - dy-static-file-server-dynamic-sidecar  (sidecared w/ std config)
    - dy-static-file-server-dynamic-sidecar-compose (sidecared w/ docker-compose)
    """
    # FIXME: ANE can you instead parametrize this test?
    # why do we need to run all these services at the same time? it would be simpler one by one

  await asyncio.gather(
*( assert_start_service( director_v2_client=director_v2_client,

context

user_id=user_dict["id"], project_id=str(dy_static_file_server_project.uuid),

service

service_key=node.key, service_version=node.version, service_uuid=node_id,

extra config (legacy)

basepath=f"/x/{node_id}" if is_legacy(node) else None, catalog_url=services_endpoint["catalog"], ) for node_id, node in dy_static_file_server_project.workbench.items() ) )

tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py:263:

tests/integration/02/utils.py:289: in assert_start_service result = await director_v2_client.post( ../../.venv/lib/python3.9/site-packages/httpx/_client.py:1842: in post return await self.request( ../../.venv/lib/python3.9/site-packages/httpx/_client.py:1527: in request return await self.send(request, auth=auth, follow_redirects=follow_redirects) ../../.venv/lib/python3.9/site-packages/httpx/_client.py:1614: in send response = await self._send_handling_auth( ../../.venv/lib/python3.9/site-packages/httpx/_client.py:1642: in _send_handling_auth response = await self._send_handling_redirects( ../../.venv/lib/python3.9/site-packages/httpx/_client.py:1679: in _send_handling_redirects response = await self._send_single_request(request) ../../.venv/lib/python3.9/site-packages/httpx/_client.py:1716: in _send_single_request response = await transport.handle_async_request(request) ../../.venv/lib/python3.9/site-packages/httpx/_transports/asgi.py:152: in handle_async_request await self.app(scope, receive, send) ../../.venv/lib/python3.9/site-packages/fastapi/applications.py:270: in call await super().call(scope, receive, send) ../../.venv/lib/python3.9/site-packages/starlette/applications.py:124: in call await self.middleware_stack(scope, receive, send) ../../.venv/lib/python3.9/site-packages/starlette/middleware/errors.py:184: in call raise exc ../../.venv/lib/python3.9/site-packages/starlette/middleware/errors.py:162: in call await self.app(scope, receive, _send) ../../.venv/lib/python3.9/site-packages/starlette/middleware/base.py:72: in call response = await self.dispatch_func(request, call_next) ../../.venv/lib/python3.9/site-packages/fastapi_contrib/tracing/middlewares.py:52: in dispatch span = self.before_request(request, tracer)

request = <starlette.requests.Request object at 0x7f73eb6b0550>, tracer = None

@staticmethod
def before_request(request: Request, tracer):
    """
    Gather various info about the request and start new span with the data.
    """

  span_context = tracer.extract(
format=Format.HTTP_HEADERS, carrier=request.headers ) E AttributeError: 'NoneType' object has no attribute 'extract'

../../.venv/lib/python3.9/site-packages/fastapi_contrib/tracing/middlewares.py:24: AttributeError

pcrespov commented 1 year ago

Again https://github.com/ITISFoundation/osparc-simcore/actions/runs/3930582379/jobs/6721596222#step:12:2120

pcrespov commented 1 year ago

Again: https://github.com/ITISFoundation/osparc-simcore/actions/runs/4009651562/jobs/6885406866#step:12:8693

GitHK commented 1 year ago

So we have the same issue here, if you look at the PR and discussion. Please read the entire discussion from here https://github.com/ITISFoundation/osparc-simcore/pull/3647 , those are my best guesses.

pcrespov commented 1 year ago

https://github.com/ITISFoundation/osparc-simcore/actions/runs/4106353467/jobs/7085246233#step:12:51

pcrespov commented 1 year ago

Any extra guess on thisone? I think solutions to your original guesses (as stated some comments ago) did not really have an effect in here.

https://github.com/ITISFoundation/osparc-simcore/actions/runs/4894977821/jobs/8740589191?pr=4196#step:12:2538