Remote Execution Support

mlonmcu/artifact.py

@@ -18,7 +18,7 @@
 #
 """Artifacts defintions internally used to refer to intermediate results."""
 
-from enum import Enum
+from enum import IntFlag, auto
 from pathlib import Path
 
 from mlonmcu.setup import utils
@@ -28,24 +28,25 @@
 # TODO: decide if inheritance based scheme would fit better
 
 
-class ArtifactFormat(Enum):  # TODO: ArtifactType, ArtifactKind?
+# class ArtifactFormat(Enum):  # TODO: ArtifactType, ArtifactKind?
+class ArtifactFormat(IntFlag):
     """Enumeration of artifact types."""
 
-    UNKNOWN = 0
-    SOURCE = 1
-    TEXT = 2
-    MLF = 3
-    MODEL = 4
-    IMAGE = 5
-    DATA = 6
-    NUMPY = 7
-    PARAMS = 8
-    JSON = 9  # TODO: how about YAML or more general: DICT?
-    PATH = 10  # NOT A DIRECTORY?
-    RAW = 11
-    BIN = 11
-    SHARED_OBJECT = 12  # Here: the parent tar archive
-    ARCHIVE = 13
+    UNKNOWN = auto()
+    SOURCE = auto()
+    TEXT = auto()
+    MLF = auto()
+    MODEL = auto()
+    IMAGE = auto()
+    DATA = auto()
+    NUMPY = auto()
+    PARAMS = auto()
+    JSON = auto()  # TODO: how about YAML or more general: DICT?
+    PATH = auto()  # NOT A DIRECTORY?
+    RAW = auto()
+    BIN = RAW
+    SHARED_OBJECT = auto()  # Here: the parent tar archive
+    ARCHIVE = auto()
 
 
 def lookup_artifacts(artifacts, name=None, fmt=None, flags=None, first_only=False):
@@ -100,12 +101,27 @@ def __init__(
         self.optional = optional
         self.validate()
 
+    def serialize(self):
+        return {
+            "name": self.name,
+            "content": self.content,
+            "path": str(self.path) if self.path else None,
+            "data": self.data,
+            "raw": self.raw,
+            "fmt": self.fmt.value,
+            "flags": list(self.flags),
+            "archive": self.archive,
+            "optional": self.optional,
+        }
+
+    # TODO: unserialize
+
     def __repr__(self):
         return f"Artifact({self.name}, fmt={self.fmt}, flags={self.flags})"
 
     @property
     def exported(self):
-        """Returns true if the artifact was writtem to disk."""
+        """Returns true if the artifact was written to disk."""
         return bool(self.path is not None)
 
     def validate(self):

mlonmcu/cli/build.py

@@ -22,7 +22,7 @@
 from mlonmcu.cli.common import kickoff_runs
 from mlonmcu.cli.load import handle as handle_load, add_load_options
 from mlonmcu.context.context import MlonMcuContext
-from mlonmcu.session.run import RunStage
+from mlonmcu.session.run import RunStage, RunInitializer
 from mlonmcu.platform.lookup import get_platforms_targets, get_platforms_backends
 from .helper.parse import (
     extract_backend_names,
@@ -73,9 +73,12 @@ def _handle(args, context, require_target=False):
     session = context.sessions[-1]
     new_runs = []
     for run in session.runs:
+        if isinstance(run, RunInitializer) and run.frozen:
+            new_runs.append(run)
+            continue
         for target_name in targets:
             for backend_name in backends:
-                new_run = run.copy()
+                new_run = run.copy(session=session)
                 if backend_name is not None:
                     platform_name = None
                     for platform in platforms:

mlonmcu/cli/common.py

@@ -22,6 +22,7 @@
 import logging
 import argparse
 
+from mlonmcu.config import str2bool
 from mlonmcu.platform import get_platforms
 from mlonmcu.session.postprocess import SUPPORTED_POSTPROCESSES
 from mlonmcu.feature.features import get_available_feature_names
@@ -93,6 +94,14 @@ def add_flow_options(parser):
         choices=get_available_feature_names(),
         help="Enabled features for target/framework/backend (choices: %(choices)s)",
     )
+    flow_parser.add_argument(
+        "--initializer",
+        type=str,
+        metavar="INITIALIZER",
+        nargs="+",
+        # action="append",
+        help="List of yml files for initializing runs",
+    )
     flow_parser.add_argument(
         "-c",
         "--config",
@@ -154,6 +163,11 @@ def add_gen_args(parser, number):
         action="store_true",
         help="Display progress bar (default: %(default)s)",
     )
+    flow_parser.add_argument(
+        "--noop",
+        action="store_true",
+        help="Skip processing of runs, just initialize (default: %(default)s)",
+    )
     flow_parser.add_argument(
         "--resume",
         action="store_true",
@@ -218,16 +232,18 @@ def kickoff_runs(args, until, context):
     assert len(context.sessions) > 0
     session = context.sessions[-1]
     # session.label = args.label
-    config = extract_config(args)
+    config, config_gen = extract_config(args)
     # TODO: move into context/session
     per_stage = True
     print_report = True
     if "runs_per_stage" in config:
-        per_stage = bool(config["runs_per_stage"])
+        value = config["runs_per_stage"]
+        per_stage = str2bool(value) if isinstance(value, str) else value
     elif "runs_per_stage" in context.environment.vars:
         per_stage = bool(context.environment.vars["runs_per_stage"])
     if "print_report" in config:
-        print_report = bool(config["print_report"])
+        value = config["print_report"]
+        print_report = str2bool(value) if isinstance(value, str) else value
     elif "print_report" in context.environment.vars:
         print_report = bool(context.environment.vars["print_report"])
     with session:
@@ -239,6 +255,7 @@ def kickoff_runs(args, until, context):
             progress=args.progress,
             context=context,
             export=True,
+            noop=args.noop,
         )
     if not success:
         logger.error("At least one error occured!")

mlonmcu/cli/compile.py

@@ -24,7 +24,7 @@
     add_build_options,
 )
 from mlonmcu.context.context import MlonMcuContext
-from mlonmcu.session.run import RunStage
+from mlonmcu.session.run import RunStage, RunInitializer
 from mlonmcu.platform.lookup import get_platforms_targets
 from .helper.parse import extract_target_names, extract_platform_names, extract_config_and_feature_names
 
@@ -54,13 +54,16 @@ def _handle(args, context):
     session = context.sessions[-1]
     new_runs = []
     for run in session.runs:
-        if run.target is None:
+        if isinstance(run, RunInitializer) and run.frozen:
+            new_runs.append(run)
+            continue
+        if not run.has_target():
             # assert run.compile_platform is None
             targets_ = targets
         else:
             targets_ = [None]
         for target_name in targets_:
-            new_run = run.copy()
+            new_run = run.copy(session=session)
             if target_name is not None:
                 platform_name = None
                 for platform in platforms:

mlonmcu/cli/load.py

@@ -18,6 +18,8 @@
 #
 """Command line subcommand for the load stage."""
 
+from pathlib import Path
+
 from mlonmcu.cli.common import (
     add_common_options,
     add_context_options,
@@ -30,7 +32,7 @@
 from mlonmcu.context.context import MlonMcuContext
 from mlonmcu.models import SUPPORTED_FRONTENDS
 from mlonmcu.models.lookup import apply_modelgroups
-from mlonmcu.session.run import RunStage
+from mlonmcu.session.run import RunStage, RunInitializer
 
 
 def add_load_options(parser):
@@ -62,11 +64,19 @@ def _handle(args, context):
     config = context.environment.vars
     new_config, features, gen_config, gen_features = extract_config_and_feature_names(args, context=context)
     config.update(new_config)
+    session = context.get_session(label=args.label, resume=args.resume, config=config)
+    initializers = args.initializer
+    if initializers is not None:
+        for initializer_file in initializers:
+            initializer_file = Path(initializer_file).resolve()
+            initializer = RunInitializer.from_file(initializer_file)
+            session.add_run(initializer, ignore_idx=True)
     frontends = extract_frontend_names(args, context=context)
     postprocesses = extract_postprocess_names(args, context=context)
-    session = context.get_session(label=args.label, resume=args.resume, config=config)
     models = apply_modelgroups(args.models, context=context)
     for model in models:
+        if model == "_":
+            continue
         for f in gen_features:
             for c in gen_config:
                 all_config = {**config, **c}

mlonmcu/context/context.py

@@ -28,7 +28,7 @@
 
 from mlonmcu.utils import ask_user
 from mlonmcu.logging import get_logger, set_log_file
-from mlonmcu.session.run import Run
+from mlonmcu.session.run import Run, ArchivedRun
 from mlonmcu.session.session import Session
 from mlonmcu.setup.cache import TaskCache
 import mlonmcu.setup.utils as utils
@@ -186,9 +186,9 @@
             run_directory = runs_directory / str(rid)
             # run_file = run_directory / "run.txt"
             # run = Run.from_file(run_file)  # TODO: actually implement run restore
-            run = Run()  # TODO: fix
-            run.archived = True
-            run.dir = run_directory
+            run = ArchivedRun.from_dir(run_directory)
+            # run.archived = True
+            # run.dir = run_directory
             runs.append(run)
         session = Session(idx=sid, archived=True, dir=session_directory)
         session.runs = runs
@@ -307,7 +307,12 @@
         self.cache = TaskCache()
         self.export_paths = set()
 
-    def create_session(self, label="", config=None):
+    def create_session(self, label="", config=None, custom_dir=None):
+        if custom_dir is not None:
+            logger.debug("Creating a new session with idx %s", idx)
+            session_dir = Path(custom_dir)
+            session = Session(idx=None, label=label, dir=session_dir, config=config)
+            return session
         try:
             lock = self.latest_session_link_lock.acquire(timeout=10)
         except filelock.Timeout as err:
@@ -574,3 +579,13 @@
             logger.debug("Releasing lock on context")
             self.deps_lock.release()
         return False
+
+    def get_read_only_context(self):
+        return MlonMcuContextMinimal(self)
+
+
+class MlonMcuContextMinimal:
+
+    def __init__(self, context: MlonMcuContext):
+        self.environment = context.environment
+        self.cache = context.cache

mlonmcu/environment/environment.py

@@ -18,7 +18,7 @@
 #
 import logging
 
 from .config import (
     DefaultsConfig,
     PathConfig,
     RepoConfig,
@@ -368,90 +368,14 @@
                 PathConfig("./models"),
             ],
         }
-        self.repos = {
-            "tensorflow": RepoConfig("https://github.com/tensorflow/tensorflow.git", ref="v2.5.2"),
-            "tflite_micro_compiler": RepoConfig(
-                "https://github.com/cpetig/tflite_micro_compiler.git", ref="master"
-            ),  # TODO: freeze ref?
-            "tvm": RepoConfig(
-                "https://github.com/tum-ei-eda/tvm.git", ref="tumeda"
-            ),  # TODO: use upstream repo with suitable commit?
-            "utvm_staticrt_codegen": RepoConfig(
-                "https://github.com/tum-ei-eda/utvm_staticrt_codegen.git", ref="master"
-            ),  # TODO: freeze ref?
-            "etiss": RepoConfig("https://github.com/tum-ei-eda/etiss.git", ref="master"),  # TODO: freeze ref?
-        }
-        self.frameworks = [
-            FrameworkConfig(
-                "tflm",
-                enabled=True,
-                backends=[
-                    BackendConfig("tflmc", enabled=True, features=[]),
-                    BackendConfig("tflmi", enabled=True, features=[]),
-                ],
-                features=[
-                    FrameworkFeatureConfig("muriscvnn", framework="tflm", supported=False),
-                ],
-            ),
-            FrameworkConfig(
-                "utvm",
-                enabled=True,
-                backends=[
-                    BackendConfig(
-                        "tvmaot",
-                        enabled=True,
-                        features=[
-                            BackendFeatureConfig("unpacked_api", backend="tvmaot", supported=True),
-                        ],
-                    ),
-                    BackendConfig("tvmrt", enabled=True, features=[]),
-                    BackendConfig("tvmcg", enabled=True, features=[]),
-                ],
-                features=[
-                    FrameworkFeatureConfig("memplan", framework="utvm", supported=False),
-                ],
-            ),
-        ]
-        self.frontends = [
-            FrontendConfig("saved_model", enabled=False),
-            FrontendConfig("ipynb", enabled=False),
-            FrontendConfig(
-                "tflite",
-                enabled=True,
-                features=[
-                    FrontendFeatureConfig("packing", frontend="tflite", supported=False),
-                ],
-            ),
-        ]
-        self.vars = {
-            "TEST": "abc",
-        }
+        self.repos = {}
+        self.frameworks = []
+        self.frontends = []
+        self.vars = {}
         self.flags = {}
-        self.platforms = [
-            PlatformConfig(
-                "mlif",
-                enabled=True,
-                features=[PlatformFeatureConfig("debug", platform="mlif", supported=True)],
-            )
-        ]
+        self.platforms = []
         self.toolchains = {}
-        self.targets = [
-            TargetConfig(
-                "etiss_pulpino",
-                features=[
-                    TargetFeatureConfig("debug", target="etiss_pulpino", supported=True),
-                    TargetFeatureConfig("attach", target="etiss_pulpino", supported=True),
-                    TargetFeatureConfig("trace", target="etiss_pulpino", supported=True),
-                ],
-            ),
-            TargetConfig(
-                "host_x86",
-                features=[
-                    TargetFeatureConfig("debug", target="host_x86", supported=True),
-                    TargetFeatureConfig("attach", target="host_x86", supported=True),
-                ],
-            ),
-        ]
+        self.targets = []
 
 
 class UserEnvironment(DefaultEnvironment):

mlonmcu/platform/espidf/espidf.py

@@ -125,7 +125,7 @@ def init_directory(self, path=None, context=None):
         if self.project_dir is not None:
             self.project_dir.mkdir(exist_ok=True)
             logger.debug("Project directory already initialized")
-            return
+            return self.project_dir
         dir_name = self.name
         if path is not None:
             self.project_dir = Path(path)
@@ -146,6 +146,7 @@ def init_directory(self, path=None, context=None):
                 self.project_dir = Path(self.tempdir.name) / dir_name
                 logger.debug("Temporary project directory: %s", self.project_dir)
         self.project_dir.mkdir(exist_ok=True)
+        return self.project_dir
 
     def get_supported_targets(self):
         text = self.invoke_idf_exe("--list-targets", live=self.print_outputs)