tests.system.aiplatform.test_model_monitoring.TestModelDeploymentMonitoring: test_mdm_two_models_invalid_configs_xai failed

[flaky-bot]

Note: #1551 was also for this test, but it was closed more than 10 days ago. So, I didn't mark it flaky.

---

commit: 6bafca5
buildURL: Build Status, Sponge
status: failed

Test output

args = (name: "projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320"
,)
kwargs = {'metadata': [('x-goog-request-params', 'name=projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320'), ('x-goog-api-client', 'model-builder/1.16.1 gl-python/3.8.12 grpc/1.47.0 gax/1.32.0 gapic/1.16.1')]}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f259391a4c0>

request = name: "projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320"
timeout = None

metadata = [('x-goog-request-params', 'name=projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320'), ('x-goog-api-client', 'model-builder/1.16.1 gl-python/3.8.12 grpc/1.47.0 gax/1.32.0 gapic/1.16.1')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7f2593b356d0>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f2592a2e7c0>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.NOT_FOUND

E           	details = "Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found."

E           	debug_error_string = "{"created":"@1662004158.146285600","description":"Error received from peer ipv4:74.125.142.95:443","file":"src/core/lib/surface/call.cc","file_line":966,"grpc_message":"Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.","grpc_status":5}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_model_monitoring.TestModelDeploymentMonitoring object at 0x7f25982c91f0>
def test_mdm_two_models_invalid_configs_xai(self):
    [deployed_model1, deployed_model2] = list(


      map(lambda x: x.id, self.endpoint.list_models())


    )

tests/system/aiplatform/test_model_monitoring.py:277:

google/cloud/aiplatform/models.py:1650: in list_models

self._sync_gca_resource()

google/cloud/aiplatform/base.py:642: in _sync_gca_resource

self._gca_resource = self._get_gca_resource(resource_name=self.resource_name)

google/cloud/aiplatform/base.py:672: in resource_name

self._assert_gca_resource_is_available()

google/cloud/aiplatform/models.py:224: in _assert_gca_resource_is_available

self._sync_gca_resource_if_skipped()

google/cloud/aiplatform/models.py:216: in _sync_gca_resource_if_skipped

self._gca_resource = self._get_gca_resource(

google/cloud/aiplatform/base.py:635: in _get_gca_resource

return getattr(self.api_client, self._getter_method)(

google/cloud/aiplatform_v1/services/endpoint_service/client.py:732: in get_endpoint

response = rpc(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.NOT_FOUND

details = "Endpoint projects/precise-t...point projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.","grpc_status":5}"



???

E   google.api_core.exceptions.NotFound: 404 Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.

:3: NotFound

[rosiezou]

fixed with #1633

[flaky-bot]

Looks like this issue is flaky. 😟

I'm going to leave this open and stop commenting.

A human should fix and close this.

---

commit: b8a6586
buildURL: Build Status, Sponge
status: failed

Test output

args = (name: "projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320"
,)
kwargs = {'metadata': [('x-goog-request-params', 'name=projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320'), ('x-goog-api-client', 'model-builder/1.16.1 gl-python/3.8.13 grpc/1.47.0 gax/1.32.0 gapic/1.16.1')]}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f58eed6e040>

request = name: "projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320"
timeout = None

metadata = [('x-goog-request-params', 'name=projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320'), ('x-goog-api-client', 'model-builder/1.16.1 gl-python/3.8.13 grpc/1.47.0 gax/1.32.0 gapic/1.16.1')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7f58ef07b190>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f58ef08ef40>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.NOT_FOUND

E           	details = "Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found."

E           	debug_error_string = "{"created":"@1662078894.600087007","description":"Error received from peer ipv4:74.125.197.95:443","file":"src/core/lib/surface/call.cc","file_line":966,"grpc_message":"Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.","grpc_status":5}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_model_monitoring.TestModelDeploymentMonitoring object at 0x7f58efa44f10>
def test_mdm_two_models_invalid_configs_xai(self):
    [deployed_model1, deployed_model2] = list(


      map(lambda x: x.id, self.endpoint.list_models())


    )

tests/system/aiplatform/test_model_monitoring.py:277:

google/cloud/aiplatform/models.py:1650: in list_models

self._sync_gca_resource()

google/cloud/aiplatform/base.py:642: in _sync_gca_resource

self._gca_resource = self._get_gca_resource(resource_name=self.resource_name)

google/cloud/aiplatform/base.py:672: in resource_name

self._assert_gca_resource_is_available()

google/cloud/aiplatform/models.py:224: in _assert_gca_resource_is_available

self._sync_gca_resource_if_skipped()

google/cloud/aiplatform/models.py:216: in _sync_gca_resource_if_skipped

self._gca_resource = self._get_gca_resource(

google/cloud/aiplatform/base.py:635: in _get_gca_resource

return getattr(self.api_client, self._getter_method)(

google/cloud/aiplatform_v1/services/endpoint_service/client.py:732: in get_endpoint

response = rpc(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.NOT_FOUND

details = "Endpoint projects/precise-t...point projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.","grpc_status":5}"



???

E   google.api_core.exceptions.NotFound: 404 Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.

:3: NotFound