refactor entrypoint

Signed-off-by: weiwee <wbwmat@gmail.com>

python/fate/components/__main__.py

@@ -18,23 +18,10 @@
 execute with python -m fate.components --execution_id xxx --config xxx
 """
 
-
 if __name__ == "__main__":
     import click
-    from fate.components.entrypoint import (
-        cleanup_cli,
-        desc_cli,
-        execute_cli,
-        list_cli,
-        task_schema_cli,
-    )
+    from fate.components.entrypoint.cli.component.__main__ import component
 
     cli = click.Group()
-    component = click.Group(name="component")
-    component.add_command(execute_cli.execute)
-    component.add_command(cleanup_cli.cleanup)
-    component.add_command(desc_cli.desc)
-    component.add_command(list_cli.list)
-    component.add_command(task_schema_cli.task_schema)
     cli.add_command(component)
     cli(prog_name="python -m fate.components")

python/fate/components/core/spec/task.py

@@ -13,7 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 import os
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Union
 
 import pydantic
 

python/fate/components/entrypoint/cli/component/__main__.py

@@ -0,0 +1,18 @@
+import click
+from fate.components.entrypoint.cli.component import (
+    cleanup_cli,
+    desc_cli,
+    execute_cli,
+    list_cli,
+    task_schema_cli,
+)
+
+component = click.Group(name="component")
+component.add_command(execute_cli.execute)
+component.add_command(cleanup_cli.cleanup)
+component.add_command(desc_cli.desc)
+component.add_command(list_cli.list)
+component.add_command(task_schema_cli.task_schema)
+
+if __name__ == "__main__":
+    component(prog_name="python -m fate.components.entrypoint.cli.component")

python/fate/components/entrypoint/cleanup_cli.py → python/fate/components/entrypoint/cli/component/cleanup_cli.py

@@ -2,29 +2,38 @@
 
 
 @click.command()
-@click.option("--process-tag", required=True, help="unique id to identify this execution process")
+@click.option("--process-tag", required=False, help="unique id to identify this execution process")
 @click.option("--config", required=False, type=click.File(), help="config path")
-def cleanup(process_tag, config):
-    import logging
+@click.option("--env-name", required=False, type=str, help="env name for config")
+def cleanup(process_tag, config, env_name):
+    """cleanup"""
     import traceback
 
     from fate.arch import Context
     from fate.components.core import load_computing, load_federation
     from fate.components.core.spec.task import TaskCleanupConfigSpec
+    from fate.components.entrypoint.utils import (
+        load_config_from_env,
+        load_config_from_file,
+    )
 
-    logger = logging.getLogger(__name__)
-    config = TaskCleanupConfigSpec.parse_obj(config)
+    configs = {}
+    configs = load_config_from_env(configs, env_name)
+    load_config_from_file(configs, config)
+    config = TaskCleanupConfigSpec.parse_obj(configs)
 
     try:
+        print("start cleanup")
         computing = load_computing(config.computing)
         federation = load_federation(config.federation, computing)
         ctx = Context(
             computing=computing,
             federation=federation,
         )
         ctx.destroy()
+        print("cleanup done")
     except Exception as e:
-        traceback.print_exception(e)
+        traceback.print_exc()
         raise e
 
 

python/fate/components/entrypoint/execute_cli.py → python/fate/components/entrypoint/cli/component/execute_cli.py

@@ -36,12 +36,24 @@
     help="path for execution meta generated by component when execution finished",
 )
 def execute(process_tag, config, config_entrypoint, properties, env_prefix, env_name, execution_final_meta_path):
-    "execute component_desc"
-    if config is None and config_entrypoint is None and not properties and env_name is None:
-        raise click.UsageError("at least one of config, config-entrypoint, properties, env-name should be provided")
+    """
+    execute component
+    """
     import logging
 
     from fate.components.core.spec.task import TaskConfigSpec
+    from fate.components.entrypoint.utils import (
+        load_config_from_entrypoint,
+        load_config_from_env,
+        load_config_from_file,
+        load_config_from_properties,
+        load_properties,
+        load_properties_from_env,
+    )
+
+    "execute component_desc"
+    if config is None and config_entrypoint is None and not properties and env_name is None:
+        raise click.UsageError("at least one of config, config-entrypoint, properties, env-name should be provided")
 
     # parse properties
     properties_items = {}
@@ -141,91 +153,5 @@ def execute_component_from_config(config: "TaskConfigSpec", output_path):
         raise e
 
 
-def load_properties(properties) -> dict:
-    properties_dict = {}
-    for property_item in properties:
-        k, v = property_item.split("=")
-        k = k.strip()
-        v = v.strip()
-        properties_dict[k] = v
-    return properties_dict
-
-
-def load_properties_from_env(env_filter_prefix):
-    import os
-
-    properties_dict = {}
-    if env_filter_prefix:
-        env_prefix_size = len(env_filter_prefix)
-        for k, v in os.environ.items():
-            if k.startswith(env_filter_prefix):
-                property_key = k[env_prefix_size:]
-                if property_key:
-                    properties_dict[property_key] = v
-    return properties_dict
-
-
-def load_config_from_properties(configs, properties_dict):
-    for k, v in properties_dict.items():
-        lens_and_setter = configs, None
-
-        def _setter(d, k):
-            def _set(v):
-                d[k] = v
-
-            return _set
-
-        for s in k.split("."):
-            lens, _ = lens_and_setter
-            if not s.endswith("]"):
-                print("in", lens)
-                if lens.get(s) is None:
-                    lens[s] = {}
-                lens_and_setter = lens[s], _setter(lens, s)
-            else:
-                name, index = s.rstrip("]").split("[")
-                index = int(index)
-                if lens.get(name) is None:
-                    lens[name] = []
-                lens = lens[name]
-                if (short_size := index + 1 - len(lens)) > 0:
-                    lens.extend([None] * short_size)
-                    lens[index] = {}
-                lens_and_setter = lens[index], _setter(lens, index)
-        _, setter = lens_and_setter
-        if setter is not None:
-            setter(v)
-
-
-def load_config_from_file(configs, config_file):
-    from ruamel import yaml
-
-    if config_file is not None:
-        configs.update(yaml.safe_load(config_file))
-    return configs
-
-
-def load_config_from_entrypoint(configs, config_entrypoint):
-    import requests
-
-    if config_entrypoint is not None:
-        try:
-            resp = requests.get(config_entrypoint).json()
-            configs.update(resp["config"])
-        except:
-            pass
-    return configs
-
-
-def load_config_from_env(configs, env_name):
-    import os
-
-    from ruamel import yaml
-
-    if env_name is not None and os.environ.get(env_name):
-        configs.update(yaml.safe_load(os.environ[env_name]))
-    return configs
-
-
 if __name__ == "__main__":
-    main()
+    execute()

python/fate/components/entrypoint/utils.py

@@ -0,0 +1,84 @@
+def load_properties(properties) -> dict:
+    properties_dict = {}
+    for property_item in properties:
+        k, v = property_item.split("=")
+        k = k.strip()
+        v = v.strip()
+        properties_dict[k] = v
+    return properties_dict
+
+
+def load_properties_from_env(env_filter_prefix):
+    import os
+
+    properties_dict = {}
+    if env_filter_prefix:
+        env_prefix_size = len(env_filter_prefix)
+        for k, v in os.environ.items():
+            if k.startswith(env_filter_prefix):
+                property_key = k[env_prefix_size:]
+                if property_key:
+                    properties_dict[property_key] = v
+    return properties_dict
+
+
+def load_config_from_properties(configs, properties_dict):
+    for k, v in properties_dict.items():
+        lens_and_setter = configs, None
+
+        def _setter(d, k):
+            def _set(v):
+                d[k] = v
+
+            return _set
+
+        for s in k.split("."):
+            lens, _ = lens_and_setter
+            if not s.endswith("]"):
+                print("in", lens)
+                if lens.get(s) is None:
+                    lens[s] = {}
+                lens_and_setter = lens[s], _setter(lens, s)
+            else:
+                name, index = s.rstrip("]").split("[")
+                index = int(index)
+                if lens.get(name) is None:
+                    lens[name] = []
+                lens = lens[name]
+                if (short_size := index + 1 - len(lens)) > 0:
+                    lens.extend([None] * short_size)
+                    lens[index] = {}
+                lens_and_setter = lens[index], _setter(lens, index)
+        _, setter = lens_and_setter
+        if setter is not None:
+            setter(v)
+
+
+def load_config_from_file(configs, config_file):
+    from ruamel import yaml
+
+    if config_file is not None:
+        configs.update(yaml.safe_load(config_file))
+    return configs
+
+
+def load_config_from_entrypoint(configs, config_entrypoint):
+    import requests
+
+    if config_entrypoint is not None:
+        try:
+            resp = requests.get(config_entrypoint).json()
+            configs.update(resp["config"])
+        except:
+            pass
+    return configs
+
+
+def load_config_from_env(configs, env_name):
+    import os
+
+    from ruamel import yaml
+
+    if env_name is not None and os.environ.get(env_name):
+        configs.update(yaml.safe_load(os.environ[env_name]))
+    return configs