tests.system.aiplatform.test_model_monitoring.TestModelDeploymentMonitoring: test_mdm_invalid_config_incorrect_model_id failed

[flaky-bot]

This test failed!

To configure my behavior, see the Flaky Bot documentation.

If I'm commenting on this issue too often, add the flakybot: quiet label and
I will stop commenting.

---

commit: 7e6b60f
buildURL: Build Status, Sponge
status: failed

Test output

self = 
shared_state = {'resources': [ 
resource name: projects/580378083368/locations/us-central1/models/1440918784293470208]}
def test_mdm_invalid_config_incorrect_model_id(self, shared_state):


  [temp_endpoint, model] = self.temp_endpoint(shared_state)


tests/system/aiplatform/test_model_monitoring.py:323:

tests/system/aiplatform/test_model_monitoring.py:147: in temp_endpoint

endpoint = model.deploy(machine_type="n1-standard-2")

google/cloud/aiplatform/models.py:3201: in deploy

return self._deploy(

google/cloud/aiplatform/base.py:793: in wrapper

return method(*args, **kwargs)

google/cloud/aiplatform/models.py:3376: in _deploy

endpoint._deploy_call(

google/cloud/aiplatform/models.py:1233: in _deploy_call

operation_future.result()

self = <google.api_core.operation.Operation object at 0x7f6160674790>

timeout = None, retry = <google.api_core.retry.Retry object at 0x7f617b5694c0>
def result(self, timeout=None, retry=DEFAULT_RETRY):
    """Get the result of the operation, blocking if necessary.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.

    Returns:
        google.protobuf.Message: The Operation's result.

    Raises:
        google.api_core.GoogleAPICallError: If the operation errors or if
            the timeout is reached before the operation completes.
    """
    kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
    self._blocking_poll(timeout=timeout, **kwargs)

    if self._exception is not None:
        # pylint: disable=raising-bad-type
        # Pylint doesn't recognize that this is valid in this case.


      raise self._exception


E           google.api_core.exceptions.ResourceExhausted: 429 The following quotas are exceeded: DeployedCustomModelsPerProjectPerRegion
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:135: ResourceExhausted

[flaky-bot]

Looks like this issue is flaky. 😟

I'm going to leave this open and stop commenting.

A human should fix and close this.

---

When run at the same commit (7e6b60f), this test passed in one build (Build Status, Sponge) and failed in another build (Build Status, Sponge).

[rosiezou]

Closing this issue for now since #1578 provides a temporary fix.

[flaky-bot]

Oops! Looks like this issue is still flaky. It failed again. 😬

I reopened the issue, but a human will need to close it again.

---

commit: 9e19a40
buildURL: Build Status, Sponge
status: failed

Test output

args = (endpoint: "projects/580378083368/locations/us-central1/endpoints/7581941914073563136"
instances {
  struct_value {
  ...    key: "user_pseudo_id"
      value {
        string_value: "104B0770BAE16E8B53DF330C95881893"
      }
    }
  }
}
,)
kwargs = {'metadata': [('x-goog-request-params', 'endpoint=projects/580378083368/locations/us-central1/endpoints/7581941914073563136'), ('x-goog-api-client', 'model-builder/1.16.1 gl-python/3.8.12 grpc/1.47.0 gax/1.32.0 gapic/1.16.1')]}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7fa5debcef40>

request = endpoint: "projects/580378083368/locations/us-central1/endpoints/7581941914073563136"

instances {

struct_value {

...      key: "user_pseudo_id"

value {

string_value: "104B0770BAE16E8B53DF330C95881893"

}

}

}

}
timeout = None

metadata = [('x-goog-request-params', 'endpoint=projects/580378083368/locations/us-central1/endpoints/7581941914073563136'), ('x-goog-api-client', 'model-builder/1.16.1 gl-python/3.8.12 grpc/1.47.0 gax/1.32.0 gapic/1.16.1')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7fa5debc6700>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7fa5def15340>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.UNAVAILABLE

E           	details = "Authentication backend unavailable."

E           	debug_error_string = "{"created":"@1661418779.371570712","description":"Error received from peer ipv4:74.125.197.95:443","file":"src/core/lib/surface/call.cc","file_line":966,"grpc_message":"Authentication backend unavailable.","grpc_status":14}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_model_monitoring.TestModelDeploymentMonitoring object at 0x7fa5dfb10520>

shared_state = {'resources': [<google.cloud.aiplatform.models.Model object at 0x7fa5debcad60>

resource name: projects/580378083368/locations/us-central1/models/7692548385781317632]}
def test_mdm_invalid_config_incorrect_model_id(self, shared_state):


  [temp_endpoint, model] = self.temp_endpoint(shared_state)


tests/system/aiplatform/test_model_monitoring.py:339:

tests/system/aiplatform/test_model_monitoring.py:153: in temp_endpoint

predict_response = endpoint.predict(instances=[_DEFAULT_INPUT])

google/cloud/aiplatform/models.py:1514: in predict

prediction_response = self._prediction_client.predict(

google/cloud/aiplatform_v1/services/prediction_service/client.py:582: in predict

response = rpc(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.UNAVAILABLE

details = "Authentication backend un...src/core/lib/surface/call.cc","file_line":966,"grpc_message":"Authentication backend unavailable.","grpc_status":14}"



???

E   google.api_core.exceptions.ServiceUnavailable: 503 Authentication backend unavailable.

:3: ServiceUnavailable

[flaky-bot]

Oops! Looks like this issue is still flaky. It failed again. 😬

I reopened the issue, but a human will need to close it again.

---

commit: 6bafca5
buildURL: Build Status, Sponge
status: failed

Test output

args = (name: "projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320"
,)
kwargs = {'metadata': [('x-goog-request-params', 'name=projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320'), ('x-goog-api-client', 'model-builder/1.16.1 gl-python/3.8.12 grpc/1.47.0 gax/1.32.0 gapic/1.16.1')]}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f2593bf3370>

request = name: "projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320"
timeout = None

metadata = [('x-goog-request-params', 'name=projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320'), ('x-goog-api-client', 'model-builder/1.16.1 gl-python/3.8.12 grpc/1.47.0 gax/1.32.0 gapic/1.16.1')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7f2592a09a60>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f25929dc240>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.NOT_FOUND

E           	details = "Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found."

E           	debug_error_string = "{"created":"@1662004157.130951352","description":"Error received from peer ipv4:74.125.142.95:443","file":"src/core/lib/surface/call.cc","file_line":966,"grpc_message":"Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.","grpc_status":5}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_model_monitoring.TestModelDeploymentMonitoring object at 0x7f25982c9520>
def test_mdm_invalid_config_incorrect_model_id(self):
    with pytest.raises(ValueError) as e:


      aiplatform.ModelDeploymentMonitoringJob.create(


            display_name=self._make_display_name(key=JOB_NAME),
            logging_sampling_strategy=sampling_strategy,
            schedule_config=schedule_config,
            alert_config=alert_config,
            objective_configs=objective_config,
            create_request_timeout=3600,
            project=e2e_base._PROJECT,
            location=e2e_base._LOCATION,
            endpoint=self.endpoint,
            predict_instance_schema_uri="",
            analysis_instance_schema_uri="",
            deployed_model_ids=[""],
        )

tests/system/aiplatform/test_model_monitoring.py:238:

google/cloud/aiplatform/jobs.py:2330: in create

mdm_objective_config_seq = cls._parse_configs(

google/cloud/aiplatform/jobs.py:2107: in _parse_configs

for model in endpoint.list_models():

google/cloud/aiplatform/models.py:1650: in list_models

self._sync_gca_resource()

google/cloud/aiplatform/base.py:642: in _sync_gca_resource

self._gca_resource = self._get_gca_resource(resource_name=self.resource_name)

google/cloud/aiplatform/base.py:672: in resource_name

self._assert_gca_resource_is_available()

google/cloud/aiplatform/models.py:224: in _assert_gca_resource_is_available

self._sync_gca_resource_if_skipped()

google/cloud/aiplatform/models.py:216: in _sync_gca_resource_if_skipped

self._gca_resource = self._get_gca_resource(

google/cloud/aiplatform/base.py:635: in _get_gca_resource

return getattr(self.api_client, self._getter_method)(

google/cloud/aiplatform_v1/services/endpoint_service/client.py:732: in get_endpoint

response = rpc(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.NOT_FOUND

details = "Endpoint projects/precise-t...point projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.","grpc_status":5}"



???

E   google.api_core.exceptions.NotFound: 404 Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.

:3: NotFound

[rosiezou]

fixed with #1633

[flaky-bot]

Oops! Looks like this issue is still flaky. It failed again. 😬

I reopened the issue, but a human will need to close it again.

---

commit: b8a6586
buildURL: Build Status, Sponge
status: failed

Test output

args = (name: "projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320"
,)
kwargs = {'metadata': [('x-goog-request-params', 'name=projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320'), ('x-goog-api-client', 'model-builder/1.16.1 gl-python/3.8.13 grpc/1.47.0 gax/1.32.0 gapic/1.16.1')]}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f58ef0cf400>

request = name: "projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320"
timeout = None

metadata = [('x-goog-request-params', 'name=projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320'), ('x-goog-api-client', 'model-builder/1.16.1 gl-python/3.8.13 grpc/1.47.0 gax/1.32.0 gapic/1.16.1')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7f58ee3f6220>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f58eeeaf2c0>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.NOT_FOUND

E           	details = "Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found."

E           	debug_error_string = "{"created":"@1662078893.269782695","description":"Error received from peer ipv4:74.125.197.95:443","file":"src/core/lib/surface/call.cc","file_line":966,"grpc_message":"Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.","grpc_status":5}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
self = <tests.system.aiplatform.test_model_monitoring.TestModelDeploymentMonitoring object at 0x7f58efb53d60>
def test_mdm_invalid_config_incorrect_model_id(self):
    with pytest.raises(ValueError) as e:


      aiplatform.ModelDeploymentMonitoringJob.create(


            display_name=self._make_display_name(key=JOB_NAME),
            logging_sampling_strategy=sampling_strategy,
            schedule_config=schedule_config,
            alert_config=alert_config,
            objective_configs=objective_config,
            create_request_timeout=3600,
            project=e2e_base._PROJECT,
            location=e2e_base._LOCATION,
            endpoint=self.endpoint,
            predict_instance_schema_uri="",
            analysis_instance_schema_uri="",
            deployed_model_ids=[""],
        )

tests/system/aiplatform/test_model_monitoring.py:238:

google/cloud/aiplatform/jobs.py:2330: in create

mdm_objective_config_seq = cls._parse_configs(

google/cloud/aiplatform/jobs.py:2107: in _parse_configs

for model in endpoint.list_models():

google/cloud/aiplatform/models.py:1650: in list_models

self._sync_gca_resource()

google/cloud/aiplatform/base.py:642: in _sync_gca_resource

self._gca_resource = self._get_gca_resource(resource_name=self.resource_name)

google/cloud/aiplatform/base.py:672: in resource_name

self._assert_gca_resource_is_available()

google/cloud/aiplatform/models.py:224: in _assert_gca_resource_is_available

self._sync_gca_resource_if_skipped()

google/cloud/aiplatform/models.py:216: in _sync_gca_resource_if_skipped

self._gca_resource = self._get_gca_resource(

google/cloud/aiplatform/base.py:635: in _get_gca_resource

return getattr(self.api_client, self._getter_method)(

google/cloud/aiplatform_v1/services/endpoint_service/client.py:732: in get_endpoint

response = rpc(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189: in retry_target

return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.NOT_FOUND

details = "Endpoint projects/precise-t...point projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.","grpc_status":5}"



???

E   google.api_core.exceptions.NotFound: 404 Endpoint projects/precise-truck-742/locations/us-central1/endpoints/8289570005524152320 is not found.

:3: NotFound