tests.system.aiplatform.test_experiments.TestExperiments: test_delete_experiment failed

This test failed!
To configure my behavior, see the Flaky Bot documentation.
If I'm commenting on this issue too often, add the flakybot: quiet label and I will stop commenting.
commit: 837d12e2683ecf23c413a1dc87bc0a9fcbfbf360 buildURL: Build Status, Sponge status: failed
Test output
args = (name: "projects/580378083368/locations/us-central1/tensorboards/6041271036861743104/experiments/tmpvrtxsdk-e2e--2bd5e115-94fa-4aeb-8784-a087b6a7928e/runs/run-1"
,)
kwargs = {'metadata': [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/tensorboards/604127103686174...928e/runs/run-1'), ('x-goog-api-client', 'model-builder/1.23.0 gl-python/3.8.13 grpc/1.47.0 gax/1.32.0 gapic/1.23.0')]}

    @six.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
>           return callable_(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 
request = name: "projects/580378083368/locations/us-central1/tensorboards/6041271036861743104/experiments/tmpvrtxsdk-e2e--2bd5e115-94fa-4aeb-8784-a087b6a7928e/runs/run-1"

timeout = None
metadata = [('x-goog-request-params', 'name=projects/580378083368/locations/us-central1/tensorboards/6041271036861743104/experime...7928e/runs/run-1'), ('x-goog-api-client', 'model-builder/1.23.0 gl-python/3.8.13 grpc/1.47.0 gax/1.32.0 gapic/1.23.0')]
credentials = None, wait_for_ready = None, compression = None

    def __call__(self,
                 request,
                 timeout=None,
                 metadata=None,
                 credentials=None,
                 wait_for_ready=None,
                 compression=None):
        state, call, = self._blocking(request, timeout, metadata, credentials,
                                      wait_for_ready, compression)
>       return _end_unary_response_blocking(state, call, False, None)

.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

state = 
call = 
with_call = False, deadline = None

    def _end_unary_response_blocking(state, call, with_call, deadline):
        if state.code is grpc.StatusCode.OK:
            if with_call:
                rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
                return state.response, rendezvous
            else:
                return state.response
        else:
>           raise _InactiveRpcError(state)
E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
E               status = StatusCode.UNAVAILABLE
E               details = "502:Bad Gateway"
E               debug_error_string = "{"created":"@1679002637.552977655","description":"Error received from peer ipv4:74.125.195.95:443","file":"src/core/lib/surface/call.cc","file_line":966,"grpc_message":"502:Bad Gateway","grpc_status":14}"
E           >

.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError

The above exception was the direct cause of the following exception:

self = 

    def test_delete_experiment(self):
        experiment = aiplatform.Experiment(
            experiment_name=self._experiment_name,
            project=e2e_base._PROJECT,
            location=e2e_base._LOCATION,
        )
>       experiment.delete(delete_backing_tensorboard_runs=True)

tests/system/aiplatform/test_experiments.py:464: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/aiplatform/metadata/experiment_resources.py:357: in delete
    experiment_run.delete(
google/cloud/aiplatform/metadata/experiment_run_resource.py:1301: in delete
    self._backing_tensorboard_run.resource.delete()
google/cloud/aiplatform/base.py:810: in wrapper
    return method(*args, **kwargs)
google/cloud/aiplatform/base.py:1265: in delete
    lro = getattr(self.api_client, self._delete_method)(name=self.resource_name)
google/cloud/aiplatform_v1/services/tensorboard_service/client.py:2641: in delete_tensorboard_run
    response = rpc(
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in __call__
    return wrapped_func(*args, **kwargs)
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable
    six.raise_from(exceptions.from_grpc_error(exc), exc)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

value = None
from_value = <_InactiveRpcError of RPC that terminated with:
    status = StatusCode.UNAVAILABLE
    details = "502:Bad Gateway"
    debug_e...195.95:443","file":"src/core/lib/surface/call.cc","file_line":966,"grpc_message":"502:Bad Gateway","grpc_status":14}"
>

>   ???
E   google.api_core.exceptions.ServiceUnavailable: 503 502:Bad Gateway

:3: ServiceUnavailable
googleapis / python-aiplatform

tests.system.aiplatform.test_experiments.TestExperiments: test_delete_experiment failed #2024
