fix & feat: invalid filename & maro grass status

maro/cli/grass/executors/grass_azure_executor.py

@@ -14,12 +14,13 @@
 import yaml
 
 from maro.cli.grass.executors.grass_executor import GrassExecutor
-from maro.cli.grass.utils.copy import copy_files_to_node, copy_files_from_node, sync_mkdir
+from maro.cli.grass.utils.copy import copy_files_to_node, copy_files_from_node, sync_mkdir, copy_and_rename
 from maro.cli.grass.utils.hash import get_checksum
 from maro.cli.utils.details import load_cluster_details, save_cluster_details, load_job_details, save_job_details, \
     load_schedule_details, save_schedule_details
 from maro.cli.utils.executors.azure_executor import AzureExecutor
-from maro.cli.utils.naming import generate_cluster_id, generate_job_id, generate_component_id, generate_node_name
+from maro.cli.utils.naming import generate_cluster_id, generate_job_id, generate_component_id, generate_node_name, \
+    get_valid_file_name
 from maro.cli.utils.params import GlobalParams, GlobalPaths
 from maro.cli.utils.subprocess import SubProcess
 from maro.cli.utils.validation import validate_and_fill_dict
@@ -708,16 +709,17 @@ def push_image(self, image_name: str, image_path: str, remote_context_path: str,
 
         # Push image
         if image_name:
-            image_path = f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/images/{image_name}"
-            GrassAzureExecutor._save_image(
+            new_file_name = get_valid_file_name(image_name)
+            image_path = f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/images/{new_file_name}"
+            self._save_image(
                 image_name=image_name,
-                export_path=image_path
+                export_path=os.path.expanduser(image_path)
             )
             if self._check_checksum_validity(
                     local_file_path=os.path.expanduser(image_path),
                     remote_file_path=os.path.join(images_dir, image_name)
             ):
-                logger.info_green(f"The image '{image_name}' already exists")
+                logger.info_green(f"The image file '{new_file_name}' already exists")
                 return
             copy_files_to_node(
                 local_path=image_path,
@@ -729,15 +731,21 @@ def push_image(self, image_name: str, image_path: str, remote_context_path: str,
             logger.info_green(f"Image {image_name} is loaded")
         elif image_path:
             file_name = os.path.basename(image_path)
+            new_file_name = get_valid_file_name(file_name)
+            image_path = f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/images/{new_file_name}"
+            copy_and_rename(
+                source_path=os.path.expanduser(image_path),
+                target_dir=image_path
+            )
             if self._check_checksum_validity(
                     local_file_path=os.path.expanduser(image_path),
-                    remote_file_path=os.path.join(images_dir, file_name)
+                    remote_file_path=os.path.join(images_dir, new_file_name)
             ):
-                logger.info_green(f"The image file '{file_name}' already exists")
+                logger.info_green(f"The image file '{new_file_name}' already exists")
                 return
             copy_files_to_node(
                 local_path=image_path,
-                remote_dir=f"{GlobalPaths.MARO_CLUSTERS}/{self.cluster_name}/images/",
+                remote_dir=images_dir,
                 admin_username=admin_username, node_ip_address=master_public_ip_address
             )
             self.grass_executor.remote_update_image_files_details()
@@ -754,7 +762,7 @@ def push_image(self, image_name: str, image_path: str, remote_context_path: str,
     @staticmethod
     def _save_image(image_name: str, export_path: str):
         # Save image to specific folder
-        command = f"docker save {image_name} > {export_path}"
+        command = f"docker save '{image_name}' --output '{export_path}'"
         _ = SubProcess.run(command)
 
     def _batch_load_images(self):
@@ -1028,6 +1036,24 @@ def clean(self):
         # Remote clean
         self.grass_executor.remote_clean(parallels=GlobalParams.PARALLELS)
 
+    # maro grass status
+
+    def status(self, resource_name: str):
+        if resource_name == "master":
+            return_status = self.grass_executor.remote_get_master_details()
+        elif resource_name == "nodes":
+            return_status = self.grass_executor.remote_get_nodes_details()
+        else:
+            raise CliException(f"Resource {resource_name} is unsupported")
+
+        # Print status
+        logger.info(
+            json.dumps(
+                return_status,
+                indent=4, sort_keys=True
+            )
+        )
+
     # maro grass template
 
     @staticmethod

maro/cli/grass/executors/grass_executor.py

@@ -49,6 +49,13 @@ def remote_get_jobs_details(self):
         return_str = SubProcess.run(command)
         return json.loads(return_str)
 
+    def remote_get_master_details(self):
+        command = f"ssh -o StrictHostKeyChecking=no " \
+                  f"{self.admin_username}@{self.cluster_details['master']['public_ip_address']} " \
+                  f"'python3 {GlobalPaths.MARO_GRASS_LIB}/scripts/get_master_details.py {self.cluster_name}'"
+        return_str = SubProcess.run(command)
+        return json.loads(return_str)
+
     def remote_get_node_details(self, node_name: str):
         command = f"ssh -o StrictHostKeyChecking=no " \
                   f"{self.admin_username}@{self.cluster_details['master']['public_ip_address']} " \

maro/cli/grass/lib/scripts/load_images.py

@@ -3,6 +3,7 @@
 
 
 import argparse
+import os
 import subprocess
 import sys
 from multiprocessing.pool import ThreadPool
@@ -12,12 +13,12 @@
 from utils import load_cluster_details, get_node_details, set_node_details, get_master_details
 
 LOAD_IMAGE_COMMAND = '''\
-docker load -q -i {image_path}
+docker load -q -i "{image_path}"
 '''
 
 
-def load_image(unloaded_image: str):
-    command = LOAD_IMAGE_COMMAND.format(image_path=unloaded_image)
+def load_image(image_path: str):
+    command = LOAD_IMAGE_COMMAND.format(image_path=image_path)
     completed_process = subprocess.run(command,
                                        shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf8')
     if completed_process.returncode != 0:
@@ -69,7 +70,8 @@ def load_image(unloaded_image: str):
 
     # Parallel load
     with ThreadPool(args.parallels) as pool:
-        params = [[unloaded_image] for unloaded_image in unloaded_images]
+        params = [[os.path.expanduser(f"~/.maro/clusters/{args.cluster_name}/images/{unloaded_image}")]
+                  for unloaded_image in unloaded_images]
         pool.starmap(
             load_image,
             params

maro/cli/grass/lib/scripts/update_image_files_details.py

@@ -11,14 +11,15 @@
 from utils import load_cluster_details, get_master_details, set_master_details
 
 
-def get_current_image_files_details() -> dict:
-    current_images = glob.glob(os.path.expanduser(f"~/.maro/clusters/maro_grass_test/images/*"))
+def get_current_image_files_details(cluster_name: str) -> dict:
+    image_paths = glob.glob(os.path.expanduser(f"~/.maro/clusters/{cluster_name}/images/*"))
     image_files_details = {}
 
-    for current_image in current_images:
-        image_files_details[current_image] = {
-            'modify_time': os.path.getmtime(current_image),
-            'size': os.path.getsize(current_image)
+    for image_path in image_paths:
+        file_name = os.path.basename(image_path)
+        image_files_details[file_name] = {
+            'modify_time': os.path.getmtime(image_path),
+            'size': os.path.getsize(image_path)
         }
 
     return image_files_details
@@ -39,7 +40,7 @@ def get_current_image_files_details() -> dict:
                   charset="utf-8", decode_responses=True)
 
     # Get details
-    curr_image_files_details = get_current_image_files_details()
+    curr_image_files_details = get_current_image_files_details(cluster_name=args.cluster_name)
     with redis.lock("lock:master"):
         master_details = get_master_details(
             redis=redis,

maro/cli/grass/status.py

@@ -0,0 +1,18 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+
+from maro.cli.grass.executors.grass_azure_executor import GrassAzureExecutor
+from maro.cli.utils.checkers import check_details_validity
+from maro.cli.utils.details import load_cluster_details
+from maro.cli.utils.lock import lock
+
+
+@check_details_validity(mode='grass')
+@lock
+def status(cluster_name: str, resource_name: str, **kwargs):
+    cluster_details = load_cluster_details(cluster_name=cluster_name)
+
+    if cluster_details['cloud']['infra'] == 'azure':
+        executor = GrassAzureExecutor(cluster_name=cluster_name)
+        executor.status(resource_name=resource_name)

maro/cli/maro.py

@@ -388,6 +388,18 @@ def load_parser_grass(prev_parser: ArgumentParser, global_parser: ArgumentParser
         'cluster_name', help='Name of the cluster')
     parser_clean.set_defaults(func=clean)
 
+    # maro grass status
+    from maro.cli.grass.status import status
+    parser_status = subparsers.add_parser(
+        'status',
+        help='Get status of the cluster',
+        examples=CliExamples.MARO_GRASS_STATUS,
+        parents=[global_parser]
+    )
+    parser_status.add_argument('cluster_name', help='Name of the cluster')
+    parser_status.add_argument('resource_name', help='Name of the resource')
+    parser_status.set_defaults(func=status)
+
     # maro grass template
     from maro.cli.grass.template import template
     parser_clean = subparsers.add_parser(

maro/cli/utils/examples.py

@@ -109,6 +109,13 @@
     maro grass clean MyClusterName
 """
 
+MARO_GRASS_STATUS = """
+Examples:
+  Get status of the resource in the cluster
+    maro grass status MyClusterName master
+    maro grass status MyClusterName nodes
+"""
+
 MARO_GRASS_TEMPLATES = """
 Examples:
   Get deployment templates to target directory

maro/cli/utils/naming.py

@@ -3,9 +3,14 @@
 
 
 import hashlib
+import re
 import uuid
 
 
+def get_valid_file_name(file_name: str):
+    return re.sub(r'[\\/*?:"<>|]', "_", file_name)
+
+
 def generate_name_with_uuid(prefix: str, uuid_len: int = 16) -> str:
     postfix = uuid.uuid4().hex[:uuid_len]
     return f"{prefix}{postfix}"