From a48f442e7975b19199dc4c9a0d807938c810cec5 Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Thu, 31 Jan 2019 22:20:51 -0800 Subject: [PATCH 1/4] Add launcher module --- .../python/kfp_component/__init__.py | 4 +- .../python/kfp_component/launcher/__init__.py | 24 ++++++++++ .../python/kfp_component/launcher/__main__.py | 26 +++++++++++ .../python/kfp_component/launcher/launcher.py | 45 +++++++++++++++++++ component_sdk/python/requirements.txt | 3 +- component_sdk/python/run_test.sh | 2 +- .../python/tests/launcher/__init__.py | 13 ++++++ component_sdk/python/tests/launcher/echo.py | 2 + .../python/tests/launcher/test_launcher.py | 31 +++++++++++++ 9 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 component_sdk/python/kfp_component/launcher/__init__.py create mode 100644 component_sdk/python/kfp_component/launcher/__main__.py create mode 100644 component_sdk/python/kfp_component/launcher/launcher.py create mode 100644 component_sdk/python/tests/launcher/__init__.py create mode 100644 component_sdk/python/tests/launcher/echo.py create mode 100644 component_sdk/python/tests/launcher/test_launcher.py diff --git a/component_sdk/python/kfp_component/__init__.py b/component_sdk/python/kfp_component/__init__.py index c2fc82ab83f..689556d9213 100644 --- a/component_sdk/python/kfp_component/__init__.py +++ b/component_sdk/python/kfp_component/__init__.py @@ -10,4 +10,6 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. + +from . import launcher, core diff --git a/component_sdk/python/kfp_component/launcher/__init__.py b/component_sdk/python/kfp_component/launcher/__init__.py new file mode 100644 index 00000000000..8a939155c4c --- /dev/null +++ b/component_sdk/python/kfp_component/launcher/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Entrypoint module to launch python module or file dynamically. + +This module makes it easier to build kfp component with python code +by defining a dynamic entrypoint and generate command line arg parser +by python-fire module. It can be used as an entrypoint in the +container spec to run arbitary python module or code in the local +image. +""" + +from .launcher import launch \ No newline at end of file diff --git a/component_sdk/python/kfp_component/launcher/__main__.py b/component_sdk/python/kfp_component/launcher/__main__.py new file mode 100644 index 00000000000..0f9fce37fdb --- /dev/null +++ b/component_sdk/python/kfp_component/launcher/__main__.py @@ -0,0 +1,26 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fire +import importlib +import sys +import logging +from .launcher import launch + +def main(): + logging.basicConfig(level=logging.INFO) + fire.Fire(launch) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/component_sdk/python/kfp_component/launcher/launcher.py b/component_sdk/python/kfp_component/launcher/launcher.py new file mode 100644 index 00000000000..43ab722b70a --- /dev/null +++ b/component_sdk/python/kfp_component/launcher/launcher.py @@ -0,0 +1,45 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fire +import importlib +import sys +import logging + +def launch(file_or_module, *args): + """Launches a python file or module as a command entrypoint. + + Args: + file_or_module: it is either a file path to python file + a module path. + args: the args passed to the entrypoint function. + + Returns: + The return value from the launched function. + """ + try: + module = importlib.import_module(file_or_module) + except Exception: + try: + if sys.version_info.major > 2: + spec = importlib.util.spec_from_file_location('module', file_or_module) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + else: + import imp + module = imp.load_source('module', file_or_module) + except Exception: + logging.error('Failed to find the module or file: {}'.format(file_or_module)) + sys.exit(1) + return fire.Fire(module, command=args) \ No newline at end of file diff --git a/component_sdk/python/requirements.txt b/component_sdk/python/requirements.txt index 70cfe557632..2738caa50d7 100644 --- a/component_sdk/python/requirements.txt +++ b/component_sdk/python/requirements.txt @@ -1 +1,2 @@ -kubernetes \ No newline at end of file +kubernetes +fire \ No newline at end of file diff --git a/component_sdk/python/run_test.sh b/component_sdk/python/run_test.sh index ce89389b7d2..598ac42ebc7 100755 --- a/component_sdk/python/run_test.sh +++ b/component_sdk/python/run_test.sh @@ -1,2 +1,2 @@ pip install -U tox virtualenv -tox \ No newline at end of file +tox "$@" \ No newline at end of file diff --git a/component_sdk/python/tests/launcher/__init__.py b/component_sdk/python/tests/launcher/__init__.py new file mode 100644 index 00000000000..c2fc82ab83f --- /dev/null +++ b/component_sdk/python/tests/launcher/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/component_sdk/python/tests/launcher/echo.py b/component_sdk/python/tests/launcher/echo.py new file mode 100644 index 00000000000..75ccce9e8b3 --- /dev/null +++ b/component_sdk/python/tests/launcher/echo.py @@ -0,0 +1,2 @@ +def echo(message): + return message \ No newline at end of file diff --git a/component_sdk/python/tests/launcher/test_launcher.py b/component_sdk/python/tests/launcher/test_launcher.py new file mode 100644 index 00000000000..abccc4d4714 --- /dev/null +++ b/component_sdk/python/tests/launcher/test_launcher.py @@ -0,0 +1,31 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unittest +import math +import os + +from kfp_component.launcher import launch + +TEST_PY_FILE = os.path.join(os.path.dirname(__file__), 'echo.py') + +class LauncherTest(unittest.TestCase): + + def test_launch_module_succeed(self): + self.assertEqual(math.pi, launch('math', 'pi')) + + def test_launch_py_file_succeed(self): + self.assertEqual('hello', + launch(TEST_PY_FILE, 'echo', 'hello')) \ No newline at end of file From 742242f741f04003f6b1a5f32303c4bca00a7b4b Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Sun, 3 Feb 2019 22:32:32 -0800 Subject: [PATCH 2/4] Add argparse to wrap file launcher to avoid dup output prints. --- .../python/kfp_component/launcher/__main__.py | 10 +++++++++- .../python/kfp_component/launcher/launcher.py | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/component_sdk/python/kfp_component/launcher/__main__.py b/component_sdk/python/kfp_component/launcher/__main__.py index 0f9fce37fdb..7d9937935c9 100644 --- a/component_sdk/python/kfp_component/launcher/__main__.py +++ b/component_sdk/python/kfp_component/launcher/__main__.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import fire import importlib import sys @@ -20,7 +21,14 @@ def main(): logging.basicConfig(level=logging.INFO) - fire.Fire(launch) + parser = argparse.ArgumentParser( + prog='launcher', + description='Launch a python module or file.') + parser.add_argument('file_or_module', type=str, + help='Either a python file path or a module name.') + parser.add_argument('args', nargs=argparse.REMAINDER) + args = parser.parse_args() + launch(args.file_or_module, args.args) if __name__ == '__main__': main() \ No newline at end of file diff --git a/component_sdk/python/kfp_component/launcher/launcher.py b/component_sdk/python/kfp_component/launcher/launcher.py index 43ab722b70a..362e0ebfed1 100644 --- a/component_sdk/python/kfp_component/launcher/launcher.py +++ b/component_sdk/python/kfp_component/launcher/launcher.py @@ -17,7 +17,7 @@ import sys import logging -def launch(file_or_module, *args): +def launch(file_or_module, args): """Launches a python file or module as a command entrypoint. Args: @@ -42,4 +42,4 @@ def launch(file_or_module, *args): except Exception: logging.error('Failed to find the module or file: {}'.format(file_or_module)) sys.exit(1) - return fire.Fire(module, command=args) \ No newline at end of file + return fire.Fire(module, command=args, name=module.__name__) \ No newline at end of file From 48af0a5464a569bb629760f291a15c9ad9173338 Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Wed, 6 Feb 2019 09:26:46 -0800 Subject: [PATCH 3/4] Fix PR comments --- component_sdk/python/requirements.txt | 4 ++-- component_sdk/python/tests/launcher/echo.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/component_sdk/python/requirements.txt b/component_sdk/python/requirements.txt index 2738caa50d7..187c82d92aa 100644 --- a/component_sdk/python/requirements.txt +++ b/component_sdk/python/requirements.txt @@ -1,2 +1,2 @@ -kubernetes -fire \ No newline at end of file +kubernetes == 8.0.1 +fire == 0.1.3 diff --git a/component_sdk/python/tests/launcher/echo.py b/component_sdk/python/tests/launcher/echo.py index 75ccce9e8b3..911348cd572 100644 --- a/component_sdk/python/tests/launcher/echo.py +++ b/component_sdk/python/tests/launcher/echo.py @@ -1,2 +1,16 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + def echo(message): return message \ No newline at end of file From f09e025e20433a1bea7810aec592020d046da55b Mon Sep 17 00:00:00 2001 From: Hongye Sun Date: Wed, 6 Feb 2019 09:28:15 -0800 Subject: [PATCH 4/4] fix test issue --- component_sdk/python/tests/launcher/test_launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component_sdk/python/tests/launcher/test_launcher.py b/component_sdk/python/tests/launcher/test_launcher.py index abccc4d4714..977a5c6c040 100644 --- a/component_sdk/python/tests/launcher/test_launcher.py +++ b/component_sdk/python/tests/launcher/test_launcher.py @@ -28,4 +28,4 @@ def test_launch_module_succeed(self): def test_launch_py_file_succeed(self): self.assertEqual('hello', - launch(TEST_PY_FILE, 'echo', 'hello')) \ No newline at end of file + launch(TEST_PY_FILE, ['echo', 'hello'])) \ No newline at end of file