NB RPC token was not retrieved successfully within the time limit error in test_nodes_restart[True] and test_nodereplacement_proactive_with_io_running

[pintojoy]

Check timeout for NB RPC token

self = <ocs_ci.ocs.resources.mcg.MCG object at 0x7fb3fca48a30>, timeout = 300 sleep = 30

def retrieve_nb_token(self, timeout=300, sleep=30): """ Try to retrieve a NB RPC token and decode its JSON

"""

def internal_retrieval_logic():
    try:
        rpc_response = self.send_rpc_query(
            "auth_api",
            "create_auth",
            params={
                "role": "admin",
                "system": "noobaa",
                "email": self.noobaa_user,
                "password": self.noobaa_password,
            },
        )
        return rpc_response.json().get("reply").get("token")

    except json.JSONDecodeError:
        logger.warning(
            "RPC did not respond with a JSON. Response: \n" + str(rpc_response)
        )
        logger.warning(
            "Failed to retrieve token, NooBaa might be unhealthy. Retrying"
        )
        return None

try:

  for token in TimeoutSampler(timeout, sleep, internal_retrieval_logic):

ocs_ci/ocs/resources/mcg.py:182:

self = <ocs_ci.utility.utils.TimeoutSampler object at 0x7fb4236cc3d0>

def iter(self): if self.start_time is None: self.start_time = time.time() while True: self.last_sample_time = time.time() if self.timeout <= (self.last_sample_time - self.start_time): raise self.timeout_exc_cls(self.timeout_exc_args) try: yield self.func(self.func_args, **self.func_kwargs) except Exception as ex: msg = f"Exception raised during iteration: {ex}" log.exception(msg) if self.timeout <= (time.time() - self.start_time):

      raise self.timeout_exc_cls(*self.timeout_exc_args)

E ocs_ci.ocs.exceptions.TimeoutExpiredError: Timed out after 300s running internal_retrieval_logic()

ocs_ci/utility/utils.py:1452: TimeoutExpiredError

During handling of the above exception, another exception occurred:

request = <SubRequest 'mcg_obj' for <Function test_nodes_restart[True]>>

@pytest.fixture() def mcg_obj(request):

return mcg_obj_fixture(request)

tests/conftest.py:2505:

tests/conftest.py:2524: in mcg_obj_fixture mcg_obj = MCG(args, *kwargs) ocs_ci/ocs/resources/mcg.py:134: in init self.update_s3_creds() ocs_ci/ocs/resources/mcg.py:1023: in update_s3_creds self.noobaa_token = self.retrieve_nb_token()

self = <ocs_ci.ocs.resources.mcg.MCG object at 0x7fb3fca48a30>, timeout = 300 sleep = 30

def retrieve_nb_token(self, timeout=300, sleep=30): """ Try to retrieve a NB RPC token and decode its JSON

"""

def internal_retrieval_logic():
    try:
        rpc_response = self.send_rpc_query(
            "auth_api",
            "create_auth",
            params={
                "role": "admin",
                "system": "noobaa",
                "email": self.noobaa_user,
                "password": self.noobaa_password,
            },
        )
        return rpc_response.json().get("reply").get("token")

    except json.JSONDecodeError:
        logger.warning(
            "RPC did not respond with a JSON. Response: \n" + str(rpc_response)
        )
        logger.warning(
            "Failed to retrieve token, NooBaa might be unhealthy. Retrying"
        )
        return None

try:
    for token in TimeoutSampler(timeout, sleep, internal_retrieval_logic):
        if token:
            return token
except TimeoutExpiredError:
    logger.error(
        "NB RPC token was not retrieved successfully within the time limit."
    )

  assert False, (
        "NB RPC token was not retrieved successfully " "within the time limit."
    )

E AssertionError: NB RPC token was not retrieved successfully within the time limit.

ocs_ci/ocs/resources/mcg.py:189: AssertionError

2024-08-09 17:41:03

[pintojoy]

Same issue is found in https://url.corp.redhat.com/24dde2d