tests.system.aiplatform.test_matching_engine_index.TestMatchingEngine: test_create_get_list_matching_engine_index failed

flaky-bot[bot] commented 1 year ago

Note: #2305 was also for this test, but it was closed more than 10 days ago. So, I didn't mark it flaky.

commit: 0fa47aea08b635159e9c6dc64f7d330f047dfb5d buildURL: Build Status, Sponge status: failed

Test output

args = (parent: "projects/ucaip-sample-tests/locations/us-central1"
index_endpoint {
  display_name: "public_endpoint_name"
 ...ption: "my public endpoint"
  labels {
    key: "my_key"
    value: "my_value"
  }
  public_endpoint_enabled: true
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/ucaip-sample-tests/locations/us-central1'), ('x-goog-api-client', 'model-builder/1.32.0 gl-python/3.11.1 grpc/1.58.0 gax/2.11.1 gapic/1.32.0')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
>           return callable_(*args, **kwargs)

.nox/system-3-11/lib/python3.11/site-packages/google/api_core/grpc_helpers.py:72: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.nox/system-3-11/lib/python3.11/site-packages/grpc/_channel.py:1161: in __call__
    return _end_unary_response_blocking(state, call, False, None)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

state = 
call = 
with_call = False, deadline = None

    def _end_unary_response_blocking(
        state: _RPCState,
        call: cygrpc.SegregatedCall,
        with_call: bool,
        deadline: Optional[float],
    ) -> Union[ResponseType, Tuple[ResponseType, grpc.Call]]:
        if state.code is grpc.StatusCode.OK:
            if with_call:
                rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
                return state.response, rendezvous
            else:
                return state.response
        else:
>           raise _InactiveRpcError(state)  # pytype: disable=not-instantiable
E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
E               status = StatusCode.FAILED_PRECONDITION
E               details = "Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15."
E               debug_error_string = "UNKNOWN:Error received from peer ipv4:74.125.197.95:443 {created_time:"2023-09-13T10:40:17.014736581+00:00", grpc_status:9, grpc_message:"Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15."}"
E           >

.nox/system-3-11/lib/python3.11/site-packages/grpc/_channel.py:1004: _InactiveRpcError

The above exception was the direct cause of the following exception:

self = 
shared_state = {'index': ...dex object at 0x7ff89492d950> 
resource name: projects/580378083368/locations/us-central1/indexes/4011431834498891776]}

    def test_create_get_list_matching_engine_index(self, shared_state):
        aiplatform.init(
            project=e2e_base._PROJECT,
            location=e2e_base._LOCATION,
        )

        # Create an index
        index = aiplatform.MatchingEngineIndex.create_tree_ah_index(
            display_name=_TEST_INDEX_DISPLAY_NAME,
            contents_delta_uri=_TEST_CONTENTS_DELTA_URI,
            dimensions=_TEST_INDEX_CONFIG_DIMENSIONS,
            approximate_neighbors_count=_TEST_INDEX_APPROXIMATE_NEIGHBORS_COUNT,
            distance_measure_type=_TEST_INDEX_DISTANCE_MEASURE_TYPE,
            leaf_node_embedding_count=_TEST_LEAF_NODE_EMBEDDING_COUNT,
            leaf_nodes_to_search_percent=_TEST_LEAF_NODES_TO_SEARCH_PERCENT,
            description=_TEST_INDEX_DESCRIPTION,
            labels=_TEST_LABELS,
        )

        shared_state["resources"] = [index]
        shared_state["index"] = index
        shared_state["index_name"] = index.resource_name

        # Verify that the retrieved index is the same
        get_index = aiplatform.MatchingEngineIndex(index_name=index.resource_name)
        assert index.resource_name == get_index.resource_name

        # Create index and check that it is listed
        list_indexes = aiplatform.MatchingEngineIndex.list()
        assert get_index.resource_name in [
            index.resource_name for index in list_indexes
        ]

        # Update the index metadata
        updated_index = get_index.update_metadata(
            display_name=_TEST_DISPLAY_NAME_UPDATE,
            description=_TEST_DESCRIPTION_UPDATE,
            labels=_TEST_LABELS_UPDATE,
        )

        assert updated_index.name == get_index.name
        # TODO: Reinstate assertions once b/220005272 is fixed.
        # assert updated_index.display_name == _TEST_DISPLAY_NAME_UPDATE
        # assert updated_index.description == _TEST_DESCRIPTION_UPDATE
        # assert updated_index.labels == _TEST_LABELS_UPDATE

        # Update the index embeddings
        updated_index = get_index.update_embeddings(
            contents_delta_uri=_TEST_CONTENTS_DELTA_URI_UPDATE,
            is_complete_overwrite=_TEST_IS_COMPLETE_OVERWRITE,
        )

        assert updated_index.name == get_index.name

        # Create endpoint and check that it is listed
        my_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(
            display_name=_TEST_INDEX_ENDPOINT_DISPLAY_NAME,
            description=_TEST_INDEX_ENDPOINT_DESCRIPTION,
            network=e2e_base._VPC_NETWORK_URI,
            labels=_TEST_LABELS,
        )
        assert my_index_endpoint.resource_name in [
            index_endpoint.resource_name
            for index_endpoint in aiplatform.MatchingEngineIndexEndpoint.list()
        ]

        assert my_index_endpoint.labels == _TEST_LABELS
        assert my_index_endpoint.display_name == _TEST_INDEX_ENDPOINT_DISPLAY_NAME
        assert my_index_endpoint.description == _TEST_INDEX_ENDPOINT_DESCRIPTION

        # Create endpoint and check that it is listed
>       public_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(
            display_name=_TEST_PUBLIC_INDEX_ENDPOINT_DISPLAY_NAME,
            description=_TEST_PUBLIC_INDEX_ENDPOINT_DESCRIPTION,
            public_endpoint_enabled=True,
            labels=_TEST_LABELS,
        )

tests/system/aiplatform/test_matching_engine_index.py:249: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/aiplatform/matching_engine/matching_engine_index_endpoint.py:227: in create
    return cls._create(
google/cloud/aiplatform/base.py:819: in wrapper
    return method(*args, **kwargs)
google/cloud/aiplatform/matching_engine/matching_engine_index_endpoint.py:329: in _create
    create_lro = api_client.create_index_endpoint(
google/cloud/aiplatform_v1/services/index_endpoint_service/client.py:594: in create_index_endpoint
    response = rpc(
.nox/system-3-11/lib/python3.11/site-packages/google/api_core/gapic_v1/method.py:113: in __call__
    return wrapped_func(*args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = (parent: "projects/ucaip-sample-tests/locations/us-central1"
index_endpoint {
  display_name: "public_endpoint_name"
 ...ption: "my public endpoint"
  labels {
    key: "my_key"
    value: "my_value"
  }
  public_endpoint_enabled: true
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/ucaip-sample-tests/locations/us-central1'), ('x-goog-api-client', 'model-builder/1.32.0 gl-python/3.11.1 grpc/1.58.0 gax/2.11.1 gapic/1.32.0')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
            return callable_(*args, **kwargs)
        except grpc.RpcError as exc:
>           raise exceptions.from_grpc_error(exc) from exc
E           google.api_core.exceptions.FailedPrecondition: 400 Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15.

.nox/system-3-11/lib/python3.11/site-packages/google/api_core/grpc_helpers.py:74: FailedPrecondition

flaky-bot[bot] commented 11 months ago

commit: 50f857f47dc854851ca5812840c79f260d61c868 buildURL: Build Status, Sponge status: failed

Test output

args = (parent: "projects/ucaip-sample-tests/locations/us-central1"
index_endpoint {
  display_name: "public_endpoint_name"
 ...ption: "my public endpoint"
  labels {
    key: "my_key"
    value: "my_value"
  }
  public_endpoint_enabled: true
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/ucaip-sample-tests/locations/us-central1'), ('x-goog-api-clie...or_method+google.cloud.aiplatform.matching_engine.matching_engine_index_endpoint.MatchingEngineIndexEndpoint.create')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
>           return callable_(*args, **kwargs)

.nox/system-3-10/lib/python3.10/site-packages/google/api_core/grpc_helpers.py:65: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.nox/system-3-10/lib/python3.10/site-packages/grpc/_channel.py:946: in __call__
    return _end_unary_response_blocking(state, call, False, None)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

state = 
call = 
with_call = False, deadline = None

    def _end_unary_response_blocking(state, call, with_call, deadline):
        if state.code is grpc.StatusCode.OK:
            if with_call:
                rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
                return state.response, rendezvous
            else:
                return state.response
        else:
>           raise _InactiveRpcError(state)
E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
E               status = StatusCode.FAILED_PRECONDITION
E               details = "Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15."
E               debug_error_string = "UNKNOWN:Error received from peer ipv4:173.194.203.95:443 {created_time:"2023-10-26T18:27:52.400636454+00:00", grpc_status:9, grpc_message:"Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15."}"
E           >

.nox/system-3-10/lib/python3.10/site-packages/grpc/_channel.py:849: _InactiveRpcError

The above exception was the direct cause of the following exception:

self = 
shared_state = {'index': ...dex object at 0x7f90884d6920> 
resource name: projects/580378083368/locations/us-central1/indexes/5520010166319185920]}

    def test_create_get_list_matching_engine_index(self, shared_state):
        aiplatform.init(
            project=e2e_base._PROJECT,
            location=e2e_base._LOCATION,
        )

        # Create an index
        index = aiplatform.MatchingEngineIndex.create_tree_ah_index(
            display_name=_TEST_INDEX_DISPLAY_NAME,
            contents_delta_uri=_TEST_CONTENTS_DELTA_URI,
            dimensions=_TEST_INDEX_CONFIG_DIMENSIONS,
            approximate_neighbors_count=_TEST_INDEX_APPROXIMATE_NEIGHBORS_COUNT,
            distance_measure_type=_TEST_INDEX_DISTANCE_MEASURE_TYPE,
            leaf_node_embedding_count=_TEST_LEAF_NODE_EMBEDDING_COUNT,
            leaf_nodes_to_search_percent=_TEST_LEAF_NODES_TO_SEARCH_PERCENT,
            description=_TEST_INDEX_DESCRIPTION,
            labels=_TEST_LABELS,
        )

        shared_state["resources"] = [index]
        shared_state["index"] = index
        shared_state["index_name"] = index.resource_name

        # Verify that the retrieved index is the same
        get_index = aiplatform.MatchingEngineIndex(index_name=index.resource_name)
        assert index.resource_name == get_index.resource_name

        # Create index and check that it is listed
        list_indexes = aiplatform.MatchingEngineIndex.list()
        assert get_index.resource_name in [
            index.resource_name for index in list_indexes
        ]

        # Update the index metadata
        updated_index = get_index.update_metadata(
            display_name=_TEST_DISPLAY_NAME_UPDATE,
            description=_TEST_DESCRIPTION_UPDATE,
            labels=_TEST_LABELS_UPDATE,
        )

        assert updated_index.name == get_index.name
        # TODO: Reinstate assertions once b/220005272 is fixed.
        # assert updated_index.display_name == _TEST_DISPLAY_NAME_UPDATE
        # assert updated_index.description == _TEST_DESCRIPTION_UPDATE
        # assert updated_index.labels == _TEST_LABELS_UPDATE

        # Update the index embeddings
        updated_index = get_index.update_embeddings(
            contents_delta_uri=_TEST_CONTENTS_DELTA_URI_UPDATE,
            is_complete_overwrite=_TEST_IS_COMPLETE_OVERWRITE,
        )

        assert updated_index.name == get_index.name

        # Create endpoint and check that it is listed
        my_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(
            display_name=_TEST_INDEX_ENDPOINT_DISPLAY_NAME,
            description=_TEST_INDEX_ENDPOINT_DESCRIPTION,
            network=e2e_base._VPC_NETWORK_URI,
            labels=_TEST_LABELS,
        )
        assert my_index_endpoint.resource_name in [
            index_endpoint.resource_name
            for index_endpoint in aiplatform.MatchingEngineIndexEndpoint.list()
        ]

        assert my_index_endpoint.labels == _TEST_LABELS
        assert my_index_endpoint.display_name == _TEST_INDEX_ENDPOINT_DISPLAY_NAME
        assert my_index_endpoint.description == _TEST_INDEX_ENDPOINT_DESCRIPTION

        # Create endpoint and check that it is listed
>       public_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(
            display_name=_TEST_PUBLIC_INDEX_ENDPOINT_DISPLAY_NAME,
            description=_TEST_PUBLIC_INDEX_ENDPOINT_DESCRIPTION,
            public_endpoint_enabled=True,
            labels=_TEST_LABELS,
        )

tests/system/aiplatform/test_matching_engine_index.py:249: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/aiplatform/matching_engine/matching_engine_index_endpoint.py:227: in create
    return cls._create(
google/cloud/aiplatform/base.py:817: in wrapper
    return method(*args, **kwargs)
google/cloud/aiplatform/matching_engine/matching_engine_index_endpoint.py:329: in _create
    create_lro = api_client.create_index_endpoint(
google/cloud/aiplatform_v1/services/index_endpoint_service/client.py:594: in create_index_endpoint
    response = rpc(
.nox/system-3-10/lib/python3.10/site-packages/google/api_core/gapic_v1/method.py:113: in __call__
    return wrapped_func(*args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = (parent: "projects/ucaip-sample-tests/locations/us-central1"
index_endpoint {
  display_name: "public_endpoint_name"
 ...ption: "my public endpoint"
  labels {
    key: "my_key"
    value: "my_value"
  }
  public_endpoint_enabled: true
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/ucaip-sample-tests/locations/us-central1'), ('x-goog-api-clie...or_method+google.cloud.aiplatform.matching_engine.matching_engine_index_endpoint.MatchingEngineIndexEndpoint.create')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
            return callable_(*args, **kwargs)
        except grpc.RpcError as exc:
>           raise exceptions.from_grpc_error(exc) from exc
E           google.api_core.exceptions.FailedPrecondition: 400 Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15.

.nox/system-3-10/lib/python3.10/site-packages/google/api_core/grpc_helpers.py:67: FailedPrecondition

flaky-bot[bot] commented 11 months ago

commit: 0b825c9f4615d119859dc3c3ecaf558614d5e148 buildURL: Build Status, Sponge status: failed

Test output

args = (parent: "projects/ucaip-sample-tests/locations/us-central1"
index_endpoint {
  display_name: "public_endpoint_name"
 ...ption: "my public endpoint"
  labels {
    key: "my_key"
    value: "my_value"
  }
  public_endpoint_enabled: true
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/ucaip-sample-tests/locations/us-central1'), ('x-goog-api-clie...or_method+google.cloud.aiplatform.matching_engine.matching_engine_index_endpoint.MatchingEngineIndexEndpoint.create')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
>           return callable_(*args, **kwargs)

.nox/system-3-10/lib/python3.10/site-packages/google/api_core/grpc_helpers.py:65: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.nox/system-3-10/lib/python3.10/site-packages/grpc/_channel.py:946: in __call__
    return _end_unary_response_blocking(state, call, False, None)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

state = 
call = 
with_call = False, deadline = None

    def _end_unary_response_blocking(state, call, with_call, deadline):
        if state.code is grpc.StatusCode.OK:
            if with_call:
                rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
                return state.response, rendezvous
            else:
                return state.response
        else:
>           raise _InactiveRpcError(state)
E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
E               status = StatusCode.FAILED_PRECONDITION
E               details = "Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15."
E               debug_error_string = "UNKNOWN:Error received from peer ipv4:142.250.99.95:443 {created_time:"2023-10-27T03:50:30.645516471+00:00", grpc_status:9, grpc_message:"Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15."}"
E           >

.nox/system-3-10/lib/python3.10/site-packages/grpc/_channel.py:849: _InactiveRpcError

The above exception was the direct cause of the following exception:

self = 
shared_state = {'index': ...dex object at 0x7fc3be7029b0> 
resource name: projects/580378083368/locations/us-central1/indexes/5839765739862491136]}

    def test_create_get_list_matching_engine_index(self, shared_state):
        aiplatform.init(
            project=e2e_base._PROJECT,
            location=e2e_base._LOCATION,
        )

        # Create an index
        index = aiplatform.MatchingEngineIndex.create_tree_ah_index(
            display_name=_TEST_INDEX_DISPLAY_NAME,
            contents_delta_uri=_TEST_CONTENTS_DELTA_URI,
            dimensions=_TEST_INDEX_CONFIG_DIMENSIONS,
            approximate_neighbors_count=_TEST_INDEX_APPROXIMATE_NEIGHBORS_COUNT,
            distance_measure_type=_TEST_INDEX_DISTANCE_MEASURE_TYPE,
            leaf_node_embedding_count=_TEST_LEAF_NODE_EMBEDDING_COUNT,
            leaf_nodes_to_search_percent=_TEST_LEAF_NODES_TO_SEARCH_PERCENT,
            description=_TEST_INDEX_DESCRIPTION,
            labels=_TEST_LABELS,
        )

        shared_state["resources"] = [index]
        shared_state["index"] = index
        shared_state["index_name"] = index.resource_name

        # Verify that the retrieved index is the same
        get_index = aiplatform.MatchingEngineIndex(index_name=index.resource_name)
        assert index.resource_name == get_index.resource_name

        # Create index and check that it is listed
        list_indexes = aiplatform.MatchingEngineIndex.list()
        assert get_index.resource_name in [
            index.resource_name for index in list_indexes
        ]

        # Update the index metadata
        updated_index = get_index.update_metadata(
            display_name=_TEST_DISPLAY_NAME_UPDATE,
            description=_TEST_DESCRIPTION_UPDATE,
            labels=_TEST_LABELS_UPDATE,
        )

        assert updated_index.name == get_index.name
        # TODO: Reinstate assertions once b/220005272 is fixed.
        # assert updated_index.display_name == _TEST_DISPLAY_NAME_UPDATE
        # assert updated_index.description == _TEST_DESCRIPTION_UPDATE
        # assert updated_index.labels == _TEST_LABELS_UPDATE

        # Update the index embeddings
        updated_index = get_index.update_embeddings(
            contents_delta_uri=_TEST_CONTENTS_DELTA_URI_UPDATE,
            is_complete_overwrite=_TEST_IS_COMPLETE_OVERWRITE,
        )

        assert updated_index.name == get_index.name

        # Create endpoint and check that it is listed
        my_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(
            display_name=_TEST_INDEX_ENDPOINT_DISPLAY_NAME,
            description=_TEST_INDEX_ENDPOINT_DESCRIPTION,
            network=e2e_base._VPC_NETWORK_URI,
            labels=_TEST_LABELS,
        )
        assert my_index_endpoint.resource_name in [
            index_endpoint.resource_name
            for index_endpoint in aiplatform.MatchingEngineIndexEndpoint.list()
        ]

        assert my_index_endpoint.labels == _TEST_LABELS
        assert my_index_endpoint.display_name == _TEST_INDEX_ENDPOINT_DISPLAY_NAME
        assert my_index_endpoint.description == _TEST_INDEX_ENDPOINT_DESCRIPTION

        # Create endpoint and check that it is listed
>       public_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(
            display_name=_TEST_PUBLIC_INDEX_ENDPOINT_DISPLAY_NAME,
            description=_TEST_PUBLIC_INDEX_ENDPOINT_DESCRIPTION,
            public_endpoint_enabled=True,
            labels=_TEST_LABELS,
        )

tests/system/aiplatform/test_matching_engine_index.py:249: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/aiplatform/matching_engine/matching_engine_index_endpoint.py:227: in create
    return cls._create(
google/cloud/aiplatform/base.py:817: in wrapper
    return method(*args, **kwargs)
google/cloud/aiplatform/matching_engine/matching_engine_index_endpoint.py:329: in _create
    create_lro = api_client.create_index_endpoint(
google/cloud/aiplatform_v1/services/index_endpoint_service/client.py:594: in create_index_endpoint
    response = rpc(
.nox/system-3-10/lib/python3.10/site-packages/google/api_core/gapic_v1/method.py:113: in __call__
    return wrapped_func(*args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = (parent: "projects/ucaip-sample-tests/locations/us-central1"
index_endpoint {
  display_name: "public_endpoint_name"
 ...ption: "my public endpoint"
  labels {
    key: "my_key"
    value: "my_value"
  }
  public_endpoint_enabled: true
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/ucaip-sample-tests/locations/us-central1'), ('x-goog-api-clie...or_method+google.cloud.aiplatform.matching_engine.matching_engine_index_endpoint.MatchingEngineIndexEndpoint.create')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
            return callable_(*args, **kwargs)
        except grpc.RpcError as exc:
>           raise exceptions.from_grpc_error(exc) from exc
E           google.api_core.exceptions.FailedPrecondition: 400 Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15.

.nox/system-3-10/lib/python3.10/site-packages/google/api_core/grpc_helpers.py:67: FailedPrecondition

nayaknishant commented 11 months ago

Deleted all the index endpoints from our test project account.

Matching Engine Index Endpoints may not be properly deleted here.

Closing for now, will check if MatchingEngine tests are still failing in next run.

flaky-bot[bot] commented 11 months ago

Looks like this issue is flaky. :worried:

I'm going to leave this open and stop commenting.

A human should fix and close this.

commit: 1def3f6213e4468ddf0c3b55a05fa028f12e6949 buildURL: Build Status, Sponge status: failed

Test output

args = (parent: "projects/ucaip-sample-tests/locations/us-central1"
index_endpoint {
  display_name: "public_endpoint_name"
 ...ption: "my public endpoint"
  labels {
    key: "my_key"
    value: "my_value"
  }
  public_endpoint_enabled: true
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/ucaip-sample-tests/locations/us-central1'), ('x-goog-api-clie...or_method+google.cloud.aiplatform.matching_engine.matching_engine_index_endpoint.MatchingEngineIndexEndpoint.create')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
>           return callable_(*args, **kwargs)

.nox/system-3-10/lib/python3.10/site-packages/google/api_core/grpc_helpers.py:65: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.nox/system-3-10/lib/python3.10/site-packages/grpc/_channel.py:946: in __call__
    return _end_unary_response_blocking(state, call, False, None)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

state = 
call = 
with_call = False, deadline = None

    def _end_unary_response_blocking(state, call, with_call, deadline):
        if state.code is grpc.StatusCode.OK:
            if with_call:
                rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
                return state.response, rendezvous
            else:
                return state.response
        else:
>           raise _InactiveRpcError(state)
E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
E               status = StatusCode.FAILED_PRECONDITION
E               details = "Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15."
E               debug_error_string = "UNKNOWN:Error received from peer ipv4:74.125.142.95:443 {created_time:"2023-10-27T18:25:06.196047817+00:00", grpc_status:9, grpc_message:"Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15."}"
E           >

.nox/system-3-10/lib/python3.10/site-packages/grpc/_channel.py:849: _InactiveRpcError

The above exception was the direct cause of the following exception:

self = 
shared_state = {'index': ...dex object at 0x7f9deb8a1840> 
resource name: projects/580378083368/locations/us-central1/indexes/2002698857342828544]}

    def test_create_get_list_matching_engine_index(self, shared_state):
        aiplatform.init(
            project=e2e_base._PROJECT,
            location=e2e_base._LOCATION,
        )

        # Create an index
        index = aiplatform.MatchingEngineIndex.create_tree_ah_index(
            display_name=_TEST_INDEX_DISPLAY_NAME,
            contents_delta_uri=_TEST_CONTENTS_DELTA_URI,
            dimensions=_TEST_INDEX_CONFIG_DIMENSIONS,
            approximate_neighbors_count=_TEST_INDEX_APPROXIMATE_NEIGHBORS_COUNT,
            distance_measure_type=_TEST_INDEX_DISTANCE_MEASURE_TYPE,
            leaf_node_embedding_count=_TEST_LEAF_NODE_EMBEDDING_COUNT,
            leaf_nodes_to_search_percent=_TEST_LEAF_NODES_TO_SEARCH_PERCENT,
            description=_TEST_INDEX_DESCRIPTION,
            labels=_TEST_LABELS,
        )

        shared_state["resources"] = [index]
        shared_state["index"] = index
        shared_state["index_name"] = index.resource_name

        # Verify that the retrieved index is the same
        get_index = aiplatform.MatchingEngineIndex(index_name=index.resource_name)
        assert index.resource_name == get_index.resource_name

        # Create index and check that it is listed
        list_indexes = aiplatform.MatchingEngineIndex.list()
        assert get_index.resource_name in [
            index.resource_name for index in list_indexes
        ]

        # Update the index metadata
        updated_index = get_index.update_metadata(
            display_name=_TEST_DISPLAY_NAME_UPDATE,
            description=_TEST_DESCRIPTION_UPDATE,
            labels=_TEST_LABELS_UPDATE,
        )

        assert updated_index.name == get_index.name
        # TODO: Reinstate assertions once b/220005272 is fixed.
        # assert updated_index.display_name == _TEST_DISPLAY_NAME_UPDATE
        # assert updated_index.description == _TEST_DESCRIPTION_UPDATE
        # assert updated_index.labels == _TEST_LABELS_UPDATE

        # Update the index embeddings
        updated_index = get_index.update_embeddings(
            contents_delta_uri=_TEST_CONTENTS_DELTA_URI_UPDATE,
            is_complete_overwrite=_TEST_IS_COMPLETE_OVERWRITE,
        )

        assert updated_index.name == get_index.name

        # Create endpoint and check that it is listed
        my_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(
            display_name=_TEST_INDEX_ENDPOINT_DISPLAY_NAME,
            description=_TEST_INDEX_ENDPOINT_DESCRIPTION,
            network=e2e_base._VPC_NETWORK_URI,
            labels=_TEST_LABELS,
        )
        assert my_index_endpoint.resource_name in [
            index_endpoint.resource_name
            for index_endpoint in aiplatform.MatchingEngineIndexEndpoint.list()
        ]

        assert my_index_endpoint.labels == _TEST_LABELS
        assert my_index_endpoint.display_name == _TEST_INDEX_ENDPOINT_DISPLAY_NAME
        assert my_index_endpoint.description == _TEST_INDEX_ENDPOINT_DESCRIPTION

        # Create endpoint and check that it is listed
>       public_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(
            display_name=_TEST_PUBLIC_INDEX_ENDPOINT_DISPLAY_NAME,
            description=_TEST_PUBLIC_INDEX_ENDPOINT_DESCRIPTION,
            public_endpoint_enabled=True,
            labels=_TEST_LABELS,
        )

tests/system/aiplatform/test_matching_engine_index.py:249: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/aiplatform/matching_engine/matching_engine_index_endpoint.py:227: in create
    return cls._create(
google/cloud/aiplatform/base.py:817: in wrapper
    return method(*args, **kwargs)
google/cloud/aiplatform/matching_engine/matching_engine_index_endpoint.py:329: in _create
    create_lro = api_client.create_index_endpoint(
google/cloud/aiplatform_v1/services/index_endpoint_service/client.py:594: in create_index_endpoint
    response = rpc(
.nox/system-3-10/lib/python3.10/site-packages/google/api_core/gapic_v1/method.py:113: in __call__
    return wrapped_func(*args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = (parent: "projects/ucaip-sample-tests/locations/us-central1"
index_endpoint {
  display_name: "public_endpoint_name"
 ...ption: "my public endpoint"
  labels {
    key: "my_key"
    value: "my_value"
  }
  public_endpoint_enabled: true
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'parent=projects/ucaip-sample-tests/locations/us-central1'), ('x-goog-api-clie...or_method+google.cloud.aiplatform.matching_engine.matching_engine_index_endpoint.MatchingEngineIndexEndpoint.create')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
            return callable_(*args, **kwargs)
        except grpc.RpcError as exc:
>           raise exceptions.from_grpc_error(exc) from exc
E           google.api_core.exceptions.FailedPrecondition: 400 Exceeds maximum number of PublicEndpoint(s) allowed for a single Project, which is 15.

.nox/system-3-10/lib/python3.10/site-packages/google/api_core/grpc_helpers.py:67: FailedPrecondition

flaky-bot[bot] commented 11 months ago

Oops! Looks like this issue is still flaky. It failed again. :grimacing: