MLEngine API commands

component_sdk/python/kfp_component/google/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

component_sdk/python/kfp_component/google/common/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._utils import normalize_name, dump_file, check_resource_changed

component_sdk/python/kfp_component/google/common/_utils.py

@@ -0,0 +1,86 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+
+def normalize_name(name,
+              valid_first_char_pattern='a-zA-Z',
+              valid_char_pattern='0-9a-zA-Z_',
+              invalid_char_placeholder='_',
+              prefix_placeholder='x_'):
+        """Normalize a name to a valid resource name.
+
+        Uses ``valid_first_char_pattern`` and ``valid_char_pattern`` regex pattern
+        to find invalid characters from ``name`` and replaces them with 
+        ``invalid_char_placeholder`` or prefix the name with ``prefix_placeholder``.
+
+        Args:
+             name: The name to be normalized.
+             valid_first_char_pattern: The regex pattern for the first character.
+             valid_char_pattern: The regex pattern for all the characters in the name.
+             invalid_char_placeholder: The placeholder to replace invalid characters.
+             prefix_placeholder: The placeholder to prefix the name if the first char 
+                is invalid.
+        
+        Returns:
+            The normalized name. Unchanged if all characters are valid.
+        """
+        if not name:
+            return name
+        normalized_name = re.sub('[^{}]+'.format(valid_char_pattern), 
+            invalid_char_placeholder, name)
+        if not re.match('[{}]'.format(valid_first_char_pattern), 
+            normalized_name[0]):
+            normalized_name = prefix_placeholder + normalized_name
+        if name != normalized_name:
+            logging.info('Normalize name from "{}" to "{}".'.format(
+                name, normalized_name))
+        return normalized_name
+
+def dump_file(path, content):
+    """Dumps string into local file.
+
+    Args:
+        path: the local path to the file.
+        content: the string content to dump.
+    """
+    with open(path, 'w') as f:
+            f.write(content)
+
+def check_resource_changed(requested_resource, 
+    existing_resource, property_names):
+    """Check if a resource has been changed.
+
+    The function checks requested resource with existing resource
+    by comparing specified property names. Check fails if any property
+    name in the list is in ``requested_resource`` but its value is
+    different with the value in ``existing_resource``.
+
+    Args:
+        requested_resource: the user requested resource paylod.
+        existing_resource: the existing resource payload from data storage.
+        property_names: a list of property names.
+
+    Return:
+        True if ``requested_resource`` has been changed.
+    """
+    for property_name in property_names:
+        if not property_name in requested_resource:
+            continue
+        existing_value = existing_resource.get(property_name, None)
+        if requested_resource[property_name] != existing_value:
+            return True
+    return False
+

component_sdk/python/kfp_component/google/ml_engine/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._create_job import create_job
+from ._create_model import create_model
+from ._create_version import create_version
+from ._delete_version import delete_version

component_sdk/python/kfp_component/google/ml_engine/_client.py

@@ -0,0 +1,208 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+
+import googleapiclient.discovery as discovery
+from googleapiclient import errors
+
+class MLEngineClient:
+    """ Client for calling MLEngine APIs.
+    """
+    def __init__(self):
+        self._ml_client = discovery.build('ml', 'v1')
+
+    def create_job(self, project_id, job):
+        """Create a new job.
+
+        Args:
+            project_id: the ID of the parent project.
+            job: the payload of the job.
+
+        Returns:
+            The created job.
+        """
+        return self._ml_client.projects().jobs().create(
+            parent = 'projects/{}'.format(project_id),
+            body = job
+        ).execute()
+
+    def cancel_job(self, project_id, job_id):
+        """Cancel the specified job.
+
+        Args:
+            project_id: the parent project ID of the job.
+            job_id: the ID of the job.
+        """
+        job_name = 'projects/{}/jobs/{}'.format(project_id, job_id)
+        self._ml_client.projects().jobs().cancel(
+            name = job_name,
+            body = {
+                'name': job_name
+            },
+        ).execute()
+
+    def get_job(self, project_id, job_id):
+        """Gets the job by ID.
+
+        Args:
+            project_id: the ID of the parent project.
+            job_id: the ID of the job to retrieve.
+        Returns:
+            The retrieved job payload.
+        """
+        job_name = 'projects/{}/jobs/{}'.format(project_id, job_id)
+        return self._ml_client.projects().jobs().get(
+            name=job_name).execute()
+
+    def create_model(self, project_id, model):
+        """Creates a new model.
+
+        Args:
+            project_id: the ID of the parent project.
+            model: the payload of the model.
+        Returns:
+            The created model.
+        """
+        return self._ml_client.projects().models().create(
+            parent = 'projects/{}'.format(project_id),
+            body = model
+        ).execute()
+
+    def get_model(self, project_id, model_name):
+        """Gets a model.
+
+        Args:
+            project_id: the ID of the parent project.
+            model_name: the name of the model.
+        Returns:
+            The retrieved model.
+        """
+        return self._ml_client.projects().models().get(
+            name = 'projects/{}/models/{}'.format(
+                project_id, model_name)
+        ).execute()
+
+    def create_version(self, project_id, model_name, version):
+        """Creates a new version.
+
+        Args:
+            project_id: the ID of the parent project.
+            model_name: the name of the parent model.
+            version: the payload of the version.
+
+        Returns:
+            The created version.
+        """
+        return self._ml_client.projects().models().versions().create(
+            parent = 'projects/{}/models/{}'.format(project_id, model_name),
+            body = version
+        ).execute()
+
+    def get_version(self, project_id, model_name, version_name):
+        """Gets a version.
+
+        Args:
+            project_id: the ID of the parent project.
+            model_name: the name of the parent model.
+            version_name: the name of the version.
+
+        Returns:
+            The retrieved version. None if the version is not found.
+        """
+        try:
+            return self._ml_client.projects().models().versions().get(
+                name = 'projects/{}/models/{}/version/{}'.format(
+                    project_id, model_name, version_name)
+            ).execute()
+        except errors.HttpError as e:
+            if e.resp.status == 404:
+                return None
+            raise
+
+    def delete_version(self, project_id, model_name, version_name):
+        """Deletes a version.
+
+        Args:
+            project_id: the ID of the parent project.
+            model_name: the name of the parent model.
+            version_name: the name of the version.
+
+        Returns:
+            The delete operation. None if the version is not found.
+        """
+        try:
+            return self._ml_client.projects().models().versions().delete(
+                name = 'projects/{}/models/{}/version/{}'.format(
+                    project_id, model_name, version_name)
+            ).execute()
+        except errors.HttpError as e:
+            if e.resp.status == 404:
+                logging.info('The version has already been deleted.')
+                return None
+            raise
+
+    def get_operation(self, operation_name):
+        """Gets an operation.
+
+        Args:
+            operation_name: the name of the operation.
+
+        Returns:
+            The retrieved operation.
+        """
+        return self._ml_client.projects().operations().get(
+            name = operation_name
+        ).execute()
+
+    def wait_for_operation_done(self, operation_name, wait_interval):
+        """Waits for an operation to be done.
+
+        Args:
+            operation_name: the name of the operation.
+            wait_interval: the wait interview between pulling job
+                status.
+
+        Returns:
+            The completed operation.
+        """
+        operation = None
+        while True:
+            operation = self._ml_client.projects().operations().get(
+                name = operation_name
+            ).execute()
+            done = operation.get('done', False)
+            if done:
+                break
+            logging.info('Operation {} is not done. Wait for {}s.'.format(operation_name, wait_interval))
+            time.sleep(wait_interval)
+        error = operation.get('error', None)
+        if error:
+            raise RuntimeError('Failed to complete operation {}: {} {}'.format(
+                operation_name,
+                error.get('code', 'Unknown code'),
+                error.get('message', 'Unknown message'),
+            ))
+        return operation
+
+    def cancel_operation(self, operation_name):
+        """Cancels an operation.
+
+        Args:
+            operation_name: the name of the operation.
+        """
+        self._ml_client.projects().operations().cancel(
+            name = operation_name
+        ).execute()