Add Launcher module in component SDK (#769)

* Add launcher module

* Add argparse to wrap file launcher to avoid dup output prints.

* Fix PR comments

* fix test issue

component_sdk/python/kfp_component/__init__.py

@@ -10,4 +10,6 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.
+
+from . import launcher, core

component_sdk/python/kfp_component/launcher/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Entrypoint module to launch python module or file dynamically.
+
+This module makes it easier to build kfp component with python code 
+by defining a dynamic entrypoint and generate command line arg parser
+by python-fire module. It can be used as an entrypoint in the 
+container spec to run arbitary python module or code in the local 
+image.
+"""
+
+from .launcher import launch

component_sdk/python/kfp_component/launcher/__main__.py

@@ -0,0 +1,34 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import fire
+import importlib
+import sys
+import logging
+from .launcher import launch
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    parser = argparse.ArgumentParser(
+        prog='launcher',
+        description='Launch a python module or file.')
+    parser.add_argument('file_or_module', type=str,
+        help='Either a python file path or a module name.')
+    parser.add_argument('args', nargs=argparse.REMAINDER)
+    args = parser.parse_args()
+    launch(args.file_or_module, args.args)
+
+if __name__ == '__main__':
+    main()

component_sdk/python/kfp_component/launcher/launcher.py

@@ -0,0 +1,45 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import fire
+import importlib
+import sys
+import logging
+
+def launch(file_or_module, args):
+    """Launches a python file or module as a command entrypoint.
+
+    Args:
+        file_or_module: it is either a file path to python file
+            a module path.
+        args: the args passed to the entrypoint function.
+
+    Returns:
+        The return value from the launched function.
+    """
+    try:
+        module = importlib.import_module(file_or_module)
+    except Exception:
+        try:
+            if sys.version_info.major > 2:
+                spec = importlib.util.spec_from_file_location('module', file_or_module)
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)
+            else:
+                import imp
+                module = imp.load_source('module', file_or_module)
+        except Exception:
+            logging.error('Failed to find the module or file: {}'.format(file_or_module))
+            sys.exit(1)
+    return fire.Fire(module, command=args, name=module.__name__)

component_sdk/python/requirements.txt

@@ -1 +1,2 @@
-kubernetes
+kubernetes == 8.0.1
+fire == 0.1.3

component_sdk/python/run_test.sh

@@ -1,2 +1,2 @@
 pip install -U tox virtualenv
-tox
+tox "$@"

component_sdk/python/tests/launcher/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

component_sdk/python/tests/launcher/echo.py

@@ -0,0 +1,16 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def echo(message):
+    return message

component_sdk/python/tests/launcher/test_launcher.py

@@ -0,0 +1,31 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import unittest
+import math
+import os
+
+from kfp_component.launcher import launch
+
+TEST_PY_FILE = os.path.join(os.path.dirname(__file__), 'echo.py')
+
+class LauncherTest(unittest.TestCase):
+
+    def test_launch_module_succeed(self):
+        self.assertEqual(math.pi, launch('math', 'pi'))
+
+    def test_launch_py_file_succeed(self):
+        self.assertEqual('hello', 
+            launch(TEST_PY_FILE, ['echo', 'hello']))