From 18f8fb16200408bf259eb536e5bf78ae9d120ef2 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Sun, 22 Sep 2019 17:22:59 -0700 Subject: [PATCH 1/2] SDK - Moved the _container_builder from kfp.compiler to kfp.containers This only moves the files. The imports remain the same for now. --- sdk/python/kfp/compiler/__init__.py | 2 +- sdk/python/kfp/containers/_build_image_api.py | 3 +-- .../kfp/{compiler => containers}/_component_builder.py | 0 .../kfp/{compiler => containers}/_container_builder.py | 2 +- sdk/python/kfp/{compiler => containers}/_gcs_helper.py | 0 sdk/python/tests/compiler/component_builder_test.py | 10 +++++----- sdk/python/tests/compiler/container_builder_test.py | 4 ++-- test/sample-test/sample_test_launcher.py | 2 +- 8 files changed, 11 insertions(+), 12 deletions(-) rename sdk/python/kfp/{compiler => containers}/_component_builder.py (100%) rename sdk/python/kfp/{compiler => containers}/_container_builder.py (99%) rename sdk/python/kfp/{compiler => containers}/_gcs_helper.py (100%) diff --git a/sdk/python/kfp/compiler/__init__.py b/sdk/python/kfp/compiler/__init__.py index f397748ebaa..716aaf61217 100644 --- a/sdk/python/kfp/compiler/__init__.py +++ b/sdk/python/kfp/compiler/__init__.py @@ -14,4 +14,4 @@ from .compiler import Compiler -from ._component_builder import build_python_component, build_docker_image, VersionedDependency \ No newline at end of file +from ..containers._component_builder import build_python_component, build_docker_image, VersionedDependency \ No newline at end of file diff --git a/sdk/python/kfp/containers/_build_image_api.py b/sdk/python/kfp/containers/_build_image_api.py index af5bbc187c0..af91b561ed7 100644 --- a/sdk/python/kfp/containers/_build_image_api.py +++ b/sdk/python/kfp/containers/_build_image_api.py @@ -25,7 +25,7 @@ import requests -from ..compiler._container_builder import ContainerBuilder +from ._container_builder import ContainerBuilder default_base_image = 'gcr.io/deeplearning-platform-release/tf-cpu.1-14' @@ -40,7 +40,6 @@ def _get_default_image_builder(): global _default_image_builder if _default_image_builder is None: - from ..compiler._container_builder import ContainerBuilder _default_image_builder = ContainerBuilder() return _default_image_builder diff --git a/sdk/python/kfp/compiler/_component_builder.py b/sdk/python/kfp/containers/_component_builder.py similarity index 100% rename from sdk/python/kfp/compiler/_component_builder.py rename to sdk/python/kfp/containers/_component_builder.py diff --git a/sdk/python/kfp/compiler/_container_builder.py b/sdk/python/kfp/containers/_container_builder.py similarity index 99% rename from sdk/python/kfp/compiler/_container_builder.py rename to sdk/python/kfp/containers/_container_builder.py index 599af2d905b..b7def6bbcdc 100644 --- a/sdk/python/kfp/compiler/_container_builder.py +++ b/sdk/python/kfp/containers/_container_builder.py @@ -181,7 +181,7 @@ def build(self, local_dir, docker_filename : str = 'Dockerfile', target_image=No docker_filename=docker_filename, target_image=target_image) logging.info('Start a kaniko job for build.') - from ._k8s_helper import K8sHelper + from ..compiler._k8s_helper import K8sHelper k8s_helper = K8sHelper() result_pod_obj = k8s_helper.run_job(kaniko_spec, timeout) logging.info('Kaniko job complete.') diff --git a/sdk/python/kfp/compiler/_gcs_helper.py b/sdk/python/kfp/containers/_gcs_helper.py similarity index 100% rename from sdk/python/kfp/compiler/_gcs_helper.py rename to sdk/python/kfp/containers/_gcs_helper.py diff --git a/sdk/python/tests/compiler/component_builder_test.py b/sdk/python/tests/compiler/component_builder_test.py index 8d928755e9d..583e3dc89a9 100644 --- a/sdk/python/tests/compiler/component_builder_test.py +++ b/sdk/python/tests/compiler/component_builder_test.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.compiler._component_builder import _generate_dockerfile -from kfp.compiler._component_builder import _dependency_to_requirements -from kfp.compiler._component_builder import VersionedDependency -from kfp.compiler._component_builder import DependencyHelper +from kfp.containers._component_builder import _generate_dockerfile +from kfp.containers._component_builder import _dependency_to_requirements +from kfp.containers._component_builder import VersionedDependency +from kfp.containers._component_builder import DependencyHelper import os import unittest @@ -113,7 +113,7 @@ def test_add_python_package(self): class TestGenerator(unittest.TestCase): def test_generate_dockerfile(self): """ Test generate dockerfile """ - from kfp.compiler._component_builder import _generate_dockerfile + from kfp.containers._component_builder import _generate_dockerfile # prepare test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') diff --git a/sdk/python/tests/compiler/container_builder_test.py b/sdk/python/tests/compiler/container_builder_test.py index 3bf7509bb49..e03a80c1ca1 100644 --- a/sdk/python/tests/compiler/container_builder_test.py +++ b/sdk/python/tests/compiler/container_builder_test.py @@ -18,12 +18,12 @@ import yaml import tempfile import mock -from kfp.compiler._component_builder import ContainerBuilder +from kfp.containers._component_builder import ContainerBuilder GCS_BASE = 'gs://kfp-testing/' GCR_IMAGE_TAG = 'gcr.io/kfp-testing/image' -@mock.patch('kfp.compiler._gcs_helper.GCSHelper') +@mock.patch('kfp.containers._gcs_helper.GCSHelper') class TestContainerBuild(unittest.TestCase): def test_wrap_dir_in_tarball(self, mock_gcshelper): diff --git a/test/sample-test/sample_test_launcher.py b/test/sample-test/sample_test_launcher.py index 94cf43c8d63..aa24c50f0a6 100644 --- a/test/sample-test/sample_test_launcher.py +++ b/test/sample-test/sample_test_launcher.py @@ -27,7 +27,7 @@ from constants import PAPERMILL_ERR_MSG, BASE_DIR, TEST_DIR, SCHEMA_CONFIG, CONFIG_DIR, DEFAULT_CONFIG from check_notebook_results import NoteBookChecker -from kfp.compiler._gcs_helper import GCSHelper +from kfp.containers._gcs_helper import GCSHelper from run_sample_test import PySampleChecker From a60b3abfebf37ca5e09ea01fbc2efa053aef4ae1 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Wed, 25 Sep 2019 14:50:51 -0700 Subject: [PATCH 2/2] Simplified the imports. --- sdk/python/tests/compiler/component_builder_test.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sdk/python/tests/compiler/component_builder_test.py b/sdk/python/tests/compiler/component_builder_test.py index 583e3dc89a9..031cdfb95e8 100644 --- a/sdk/python/tests/compiler/component_builder_test.py +++ b/sdk/python/tests/compiler/component_builder_test.py @@ -12,10 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp.containers._component_builder import _generate_dockerfile -from kfp.containers._component_builder import _dependency_to_requirements -from kfp.containers._component_builder import VersionedDependency -from kfp.containers._component_builder import DependencyHelper +from kfp.containers._component_builder import _generate_dockerfile, _dependency_to_requirements, VersionedDependency, DependencyHelper import os import unittest @@ -113,8 +110,6 @@ def test_add_python_package(self): class TestGenerator(unittest.TestCase): def test_generate_dockerfile(self): """ Test generate dockerfile """ - from kfp.containers._component_builder import _generate_dockerfile - # prepare test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') target_dockerfile = os.path.join(test_data_dir, 'component.temp.dockerfile')