[scylla-5.4] test_mutation_schema_change_restart fails due to "error system:98, posix_listen failed for address 127.214.90.4:9180: Address already in use"

[tchaikov]

==================================== ERRORS ====================================
____________ ERROR at setup of test_mutation_schema_change_restart _____________

event_loop = <_UnixSelectorEventLoop running=False closed=False debug=False>
request = <SubRequest 'manager' for <Function test_mutation_schema_change_restart>>
kwargs = {'manager_internal': <test.pylib.manager_client.ManagerClient object at 0x7fea28e16210>}
func = <function manager at 0x7fea28fb7600>
setup = <function _wrap_asyncgen_fixture.<locals>._asyncgen_fixture_wrapper.<locals>.setup at 0x7fea28dff1a0>
finalizer = <function _wrap_asyncgen_fixture.<locals>._asyncgen_fixture_wrapper.<locals>.finalizer at 0x7fea28dff240>

    @functools.wraps(fixture)
    def _asyncgen_fixture_wrapper(
        event_loop: asyncio.AbstractEventLoop, request: SubRequest, **kwargs: Any
    ):
        func = _perhaps_rebind_fixture_func(
            fixture, request.instance, fixturedef.unittest
        )
        gen_obj = func(**_add_kwargs(func, kwargs, event_loop, request))

        async def setup():
            res = await gen_obj.__anext__()
            return res

        def finalizer() -> None:
            """Yield again, to finalize."""

            async def async_finalizer() -> None:
                try:
                    await gen_obj.__anext__()
                except StopAsyncIteration:
                    pass
                else:
                    msg = "Async generator fixture didn't stop."
                    msg += "Yield only once."
                    raise ValueError(msg)

            event_loop.run_until_complete(async_finalizer())

>       result = event_loop.run_until_complete(setup())

/usr/lib/python3.11/site-packages/pytest_asyncio/plugin.py:299: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib64/python3.11/asyncio/base_events.py:653: in run_until_complete
    return future.result()
/usr/lib/python3.11/site-packages/pytest_asyncio/plugin.py:281: in setup
    res = await gen_obj.__anext__()
test/topology/conftest.py:162: in manager
    await manager_internal.before_test(test_case_name)
test/pylib/manager_client.py:89: in before_test
    cluster_str = await self.client.put_json(f"/cluster/before-test/{test_case_name}", timeout=600,
test/pylib/rest_client.py:104: in put_json
    ret = await self._fetch("PUT", resource_uri, response_type = response_type, host = host,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <test.pylib.rest_client.UnixRESTClient object at 0x7fea28e16190>
method = 'PUT'
resource = '/cluster/before-test/test_mutation_schema_change_restart'
response_type = 'json', host = None, port = None, params = None, json = None
timeout = 600

    async def _fetch(self, method: str, resource: str, response_type: Optional[str] = None,
                     host: Optional[str] = None, port: Optional[int] = None,
                     params: Optional[Mapping[str, str]] = None,
                     json: Optional[Mapping] = None, timeout: Optional[float] = None) -> Any:
        # Can raise exception. See https://docs.aiohttp.org/en/latest/web_exceptions.html
        assert method in ["GET", "POST", "PUT", "DELETE"], f"Invalid HTTP request method {method}"
        assert response_type is None or response_type in ["text", "json"], \
                f"Invalid response type requested {response_type} (expected 'text' or 'json')"
        # Build the URI
        port = port if port else self.default_port if hasattr(self, "default_port") else None
        port_str = f":{port}" if port else ""
        assert host is not None or hasattr(self, "default_host"), "_fetch: missing host for " \
                "{method} {resource}"
        host_str = host if host is not None else self.default_host
        uri = self.uri_scheme + "://" + host_str + port_str + resource
        logging.debug(f"RESTClient fetching {method} {uri}")

        client_timeout = ClientTimeout(total = timeout if timeout is not None else 300)
        async with request(method, uri,
                           connector = self.connector if hasattr(self, "connector") else None,
                           params = params, json = json, timeout = client_timeout) as resp:
            if resp.status != 200:
                text = await resp.text()
>               raise HTTPError(uri, resp.status, params, json, text)
E               test.pylib.rest_client.HTTPError: HTTP error 500, uri: http+unix://api/cluster/before-test/test_mutation_schema_change_restart, params: None, json: None, body:
E               Exception when starting cluster ScyllaCluster(name: 861df200-e6ab-11ee-97dc-a8a159ead570, running: ScyllaServer(304, 127.214.90.34, 666a3ead-1362-44c3-8851-581b7856ac84), ScyllaServer(307, 127.214.90.40, dad06103-355a-4af0-8151-6df8a01c1798), stopped: ):
E               failed to start the node, server_id 311, IP 127.214.90.4, workdir scylla-311, host_id <missing>, cql [not connected]
E               Check the log files:
E               /scylladir/testlog/x86_64/test.py.debug-release-dev.log
E               /scylladir/testlog/x86_64/dev/scylla-311.log

scylla-304.log scylla-307.log scylla-311.log topology.test_mutation_schema_change.1.log

spotted at https://jenkins.scylladb.com/job/scylla-5.4/job/scylla-ci/54

[denesb]

5.4 is EOL, closing.