Merge pull request #545 from VisLab/develop

Updated remodeling terminology to use operation instead of command

hed/tools/analysis/annotation_util.py

@@ -109,7 +109,7 @@ def generate_sidecar_entry(column_name, column_values=None):
             if column_value == "n/a":
                 continue
             value_label = re.sub(r'[^A-Za-z0-9-]+', '_', column_value)
-            levels[column_value] = f"Description for {column_value} of {column_name}"
+            levels[column_value] = f"Here describe column value {column_value} of column {column_name}"
             hed[column_value] = f"(Label/{name_label}, Label/{value_label})"
         sidecar_entry["Levels"] = levels
         sidecar_entry["HED"] = hed

hed/tools/remodeling/cli/run_remodel.py

@@ -8,22 +8,6 @@
 from hed.tools.remodeling.backup_manager import BackupManager
 
 
-# def check_commands(commands):
-#     """ Verify that the remodeling file is correct.
-#
-#     Args:
-#         args (object
-#
-#     """
-#
-#     command_list, errors = Dispatcher.parse_commands(commands)
-#     if errors:
-#         raise ValueError("UnableToFullyParseCommands",
-#                          f"Fatal command errors, cannot continue:\n{Dispatcher.errors_to_str(errors)}")
-#     return
-#
-
-
 def get_parser():
     parser = argparse.ArgumentParser(description="Converts event files based on a json file specifying operations.")
     parser.add_argument("data_dir", help="Full path of dataset root directory.")
@@ -46,7 +30,7 @@ def get_parser():
     return parser
 
 
-def parse_commands(arg_list=None):
+def parse_arguments(arg_list=None):
     parser = get_parser()
     args = parser.parse_args(arg_list)
     if '*' in args.file_suffix:
@@ -57,14 +41,14 @@ def parse_commands(arg_list=None):
     args.exclude_dirs = args.exclude_dirs + ['remodel']
     args.model_path = os.path.realpath(args.model_path)
     if args.verbose:
-        print(f"Data directory: {args.data_dir}\nCommand path: {args.json_remodel_path}")
+        print(f"Data directory: {args.data_dir}\nJSON remodel path: {args.json_remodel_path}")
     with open(args.model_path, 'r') as fp:
-        commands = json.load(fp)
-    command_list, errors = Dispatcher.parse_commands(commands)
+        operations = json.load(fp)
+    parsed_operations, errors = Dispatcher.parse_operations(operations)
     if errors:
-        raise ValueError("UnableToFullyParseCommands",
-                         f"Fatal command errors, cannot continue:\n{Dispatcher.errors_to_str(errors)}")
-    return args, commands
+        raise ValueError("UnableToFullyParseOperations",
+                         f"Fatal operation error, cannot continue:\n{Dispatcher.errors_to_str(errors)}")
+    return args, operations
 
 
 def run_bids_ops(dispatch, args):
@@ -107,15 +91,15 @@ def run_direct_ops(dispatch, args):
 
 
 def main(arg_list=None):
-    args, commands = parse_commands(arg_list)
+    args, operations = parse_arguments(arg_list)
     if not os.path.isdir(args.data_dir):
         raise ValueError("DataDirectoryDoesNotExist", f"The root data directory {args.data_dir} does not exist")
     backup_man = BackupManager(args.data_dir)
     if not backup_man.get_backup(args.backup_name):
         raise HedFileError("BackupDoesNotExist", f"Backup {args.backup_name} does not exist. "
                            f"Please run_remodel_backup first", "")
     backup_man.restore_backup(args.backup_name, verbose=args.verbose)
-    dispatch = Dispatcher(commands, data_root=args.data_dir, backup_name=args.backup_name)
+    dispatch = Dispatcher(operations, data_root=args.data_dir, backup_name=args.backup_name)
     if args.use_bids:
         run_bids_ops(dispatch, args)
     else:

hed/tools/remodeling/dispatcher.py

@@ -5,15 +5,16 @@
 import zipfile
 from hed.schema.hed_schema_io import load_schema_version
 from hed.tools.remodeling.backup_manager import BackupManager
-from hed.tools.remodeling.operations.operation_list import operations
+from hed.tools.remodeling.operations.operation_list import valid_operations
 from hed.errors.exceptions import HedFileError
 from hed.tools.util.io_util import generate_filename
 
 
 class Dispatcher:
     REMODELING_SUMMARY_PATH = 'derivatives/remodel/summaries'
 
-    def __init__(self, command_list, data_root=None, backup_name=BackupManager.DEFAULT_BACKUP_NAME, hed_versions=None):
+    def __init__(self, operation_list, data_root=None,
+                 backup_name=BackupManager.DEFAULT_BACKUP_NAME, hed_versions=None):
         self.data_root = data_root
         self.backup_name = backup_name
         self.backup_man = None
@@ -23,10 +24,10 @@ def __init__(self, command_list, data_root=None, backup_name=BackupManager.DEFAU
                 raise HedFileError("BackupDoesNotExist",
                                    f"Remodeler cannot be run with a dataset without first creating the "
                                    f"{self.backup_name} backup for {self.data_root}", "")
-        op_list, errors = self.parse_commands(command_list)
+        op_list, errors = self.parse_operations(operation_list)
         if errors:
-            these_errors = self.errors_to_str(errors, 'Dispatcher failed due to invalid commands')
-            raise ValueError("InvalidCommandList", f"{these_errors}")
+            these_errors = self.errors_to_str(errors, 'Dispatcher failed due to invalid operations')
+            raise ValueError("InvalidOperationList", f"{these_errors}")
         self.parsed_ops = op_list
         if hed_versions:
             self.hed_schema = load_schema_version(hed_versions)
@@ -95,7 +96,7 @@ def get_context_save_dir(self):
         raise HedFileError("NoDataRoot", f"Dispatcher must have a data root to produce directories", "")
 
     def run_operations(self, file_path, sidecar=None, verbose=False):
-        """ Run the dispatcher commands on a file.
+        """ Run the dispatcher operations on a file.
         Parameters:
             file_path (str):      Full path of the file to be remodeled.
             sidecar (Sidecar or file-like):   Only needed for HED operations.
@@ -130,32 +131,32 @@ def save_context(self, save_formats=['.json', '.txt'], verbose=True):
             context_item.save(summary_path, save_formats, verbose=verbose)
 
     @staticmethod
-    def parse_commands(command_list):
+    def parse_operations(operation_list):
         errors = []
-        commands = []
-        for index, item in enumerate(command_list):
+        operations = []
+        for index, item in enumerate(operation_list):
             try:
                 if not isinstance(item, dict):
-                    raise TypeError("InvalidCommandFormat",
-                                    f"Each commands must be a dictionary but command {str(item)} is {type(item)}")
-                if "command" not in item:
-                    raise KeyError("MissingCommand",
-                                   f"Command {str(item)} does not have a command key")
+                    raise TypeError("InvalidOperationFormat",
+                                    f"Each operations must be a dictionary but operation {str(item)} is {type(item)}")
+                if "operation" not in item:
+                    raise KeyError("MissingOperation",
+                                   f"operation {str(item)} does not have a operation key")
                 if "parameters" not in item:
                     raise KeyError("MissingParameters",
-                                   f"Command {str(item)} does not have a parameters key")
-                if item["command"] not in operations:
-                    raise KeyError("CommandCanNotBeDispatched",
-                                   f"Command {item['command']} must be added to operations_list"
+                                   f"Operation {str(item)} does not have a parameters key")
+                if item["operation"] not in valid_operations:
+                    raise KeyError("OperationCanNotBeDispatched",
+                                   f"Operation {item['operation']} must be added to operations_list"
                                    f"before it can be executed.")
-                new_command = operations[item["command"]](item["parameters"])
-                commands.append(new_command)
+                new_operation = valid_operations[item["operation"]](item["parameters"])
+                operations.append(new_operation)
             except Exception as ex:
                 errors.append({"index": index, "item": f"{item}", "error_type": type(ex),
                                "error_code": ex.args[0], "error_msg": ex.args[1]})
         if errors:
             return [], errors
-        return commands, []
+        return operations, []
 
     @staticmethod
     def prep_events(df):
@@ -172,7 +173,7 @@ def prep_events(df):
     def errors_to_str(messages, title="", sep='\n'):
         error_list = [0]*len(messages)
         for index, message in enumerate(messages):
-            error_list[index] = f"Command[{message.get('index', None)}] " + \
+            error_list[index] = f"Operation[{message.get('index', None)}] " + \
                                 f"has error:{message.get('error_type', None)}" + \
                                 f" with error code:{message.get('error_code', None)} " + \
                                 f"\n\terror msg:{message.get('error_msg', None)}"

hed/tools/remodeling/operations/base_op.py

@@ -2,11 +2,11 @@
 
 class BaseOp:
 
-    def __init__(self, command, required_params, optional_params):
-        if command:
-            self.command = command
+    def __init__(self, operation, required_params, optional_params):
+        if operation:
+            self.operation = operation
         else:
-            raise ValueError("OpMustHaveCommand", "BaseOp command is empty")
+            raise ValueError("OpMustHaveOperation", "BaseOp operation is empty")
         self.required_params = required_params
         self.optional_params = optional_params
 
@@ -15,14 +15,16 @@ def check_parameters(self, parameters):
         required = set(self.required_params.keys())
         required_missing = required.difference(set(parameters.keys()))
         if required_missing:
-            raise KeyError("MissingRequiredParameters", f"{self.command} requires parameters {list(required_missing)}")
+            raise KeyError("MissingRequiredParameters",
+                           f"{self.operation} requires parameters {list(required_missing)}")
         for param_name, param_value in parameters.items():
             if param_name in self.required_params:
                 param_type = self.required_params[param_name]
             elif param_name in self.optional_params:
                 param_type = self.optional_params[param_name]
             else:
-                raise KeyError("BadParameter", f"{param_name} not a required or optional parameter for {self.command}")
+                raise KeyError("BadParameter",
+                               f"{param_name} not a required or optional parameter for {self.operation}")
             if isinstance(param_type, list):
                 self._check_list_type(param_value, param_type)
             elif not isinstance(param_value, param_type):

hed/tools/remodeling/operations/factor_column_op.py

@@ -4,7 +4,7 @@
 # TODO: Does not handle optional return columns.
 
 PARAMS = {
-    "command": "factor_column",
+    "operation": "factor_column",
     "required_parameters": {
         "column_name": str,
         "factor_values": list,
@@ -30,7 +30,7 @@ class FactorColumnOp(BaseOp):
     """
 
     def __init__(self, parameters):
-        super().__init__(PARAMS["command"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
+        super().__init__(PARAMS["operation"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
         self.check_parameters(parameters)
         self.column_name = parameters['column_name']
         self.factor_values = parameters['factor_values']

hed/tools/remodeling/operations/factor_hed_tags_op.py

@@ -5,7 +5,7 @@
 from hed.tools.analysis.analysis_util import get_assembled_strings
 
 PARAMS = {
-    "command": "factor_hed_tags",
+    "operation": "factor_hed_tags",
     "required_parameters": {
         "filters": list,
         "queries": list,
@@ -30,7 +30,7 @@ class FactorHedTagsOp(BaseOp):
     """
 
     def __init__(self, parameters):
-        super().__init__(PARAMS["command"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
+        super().__init__(PARAMS["operation"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
         self.check_parameters(parameters)
         self.filters = parameters['filters']
         self.queries = parameters['queries']

hed/tools/remodeling/operations/factor_hed_type_op.py

@@ -9,7 +9,7 @@
 # TODO: restricted factor values are not implemented yet.
 
 PARAMS = {
-    "command": "factor_hed_type",
+    "operation": "factor_hed_type",
     "required_parameters": {
         "type_tag": str,
         "type_values": list,
@@ -32,7 +32,7 @@ class FactorHedTypeOp(BaseOp):
      """
 
     def __init__(self, parameters):
-        super().__init__(PARAMS["command"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
+        super().__init__(PARAMS["operation"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
         self.check_parameters(parameters)
         self.type_tag = parameters["type_tag"]
         self.type_values = parameters["type_values"]
@@ -41,7 +41,7 @@ def __init__(self, parameters):
         if self.factor_encoding not in HedTypeFactors.ALLOWED_ENCODINGS:
             raise ValueError("BadFactorEncoding",
                              f"{self.factor_encoding} is not in the allowed encodings: " +
-                             f"{str(HedTypeFactors.ALLOWED_ENDCODINGS)}")
+                             f"{str(HedTypeFactors.ALLOWED_ENCODINGS)}")
 
     def do_op(self, dispatcher, df, name, sidecar=None):
         """ Factor columns based on HED type.

hed/tools/remodeling/operations/filter_hed_op.py

@@ -5,7 +5,7 @@
 from hed.tools.analysis.analysis_util import get_assembled_strings
 
 PARAMS = {
-    "command": "factor_hed_tags",
+    "operation": "factor_hed_tags",
     "required_parameters": {
         "filters": list,
         "queries": list,
@@ -30,7 +30,7 @@ class FilterHedOp(BaseOp):
     """
 
     def __init__(self, parameters):
-        super().__init__(PARAMS["command"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
+        super().__init__(PARAMS["operation"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
         self.check_parameters(parameters)
         self.filters = parameters['filters']
         self.queries = parameters['queries']

hed/tools/remodeling/operations/merge_consecutive_op.py

@@ -2,7 +2,7 @@
 from hed.tools.remodeling.operations.base_op import BaseOp
 
 PARAMS = {
-    "command": "merge_consecutive",
+    "operation": "merge_consecutive",
     "required_parameters": {
         "column_name": str,
         "event_code": [str, int, float],
@@ -17,7 +17,7 @@
 class MergeConsecutiveOp(BaseOp):
 
     def __init__(self, parameters):
-        super().__init__(PARAMS["command"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
+        super().__init__(PARAMS["operation"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
         self.check_parameters(parameters)
         self.column_name = parameters["column_name"]
         self.event_code = parameters["event_code"]

hed/tools/remodeling/operations/number_groups_op.py

@@ -4,7 +4,7 @@
 import itertools
 
 PARAMS = {
-    "command": "number_groups",
+    "operation": "number_groups",
     "required_parameters": {
         "number_column_name": str,
         "source_column": str,
@@ -18,7 +18,7 @@
 class NumberGroupsOp(BaseOp):
 
     def __init__(self, parameters):
-        super().__init__(PARAMS["command"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
+        super().__init__(PARAMS["operation"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
         self.check_parameters(parameters)
         self.number_column_name = parameters['number_column_name']
         self.source_column = parameters['source_column']
@@ -39,7 +39,7 @@ def __init__(self, parameters):
                     param_type = self.start_stop_test[param_name]
                 else:
                     raise KeyError("BadParameter",
-                                   f"{param_name} not a required or optional parameter for {self.command}")
+                                   f"{param_name} not a required or optional parameter for {self.operation}")
                 # TODO: This has a syntax error
                 # if not isinstance(param_value, param_type):
                 #     raise TypeError("BadType" f"{param_value} has type {type(param_value)} not {param_type}")

hed/tools/remodeling/operations/number_rows_op.py

@@ -2,7 +2,7 @@
 from hed.tools.remodeling.operations.base_op import BaseOp
 
 PARAMS = {
-    "command": "number_rows",
+    "operation": "number_rows",
     "required_parameters": {
         "number_column_name": str
     },
@@ -13,7 +13,7 @@
 class NumberRowsOp(BaseOp):
 
     def __init__(self, parameters):
-        super().__init__(PARAMS["command"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
+        super().__init__(PARAMS["operation"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
         self.check_parameters(parameters)
         self.number_column_name = parameters['number_column_name']
         self.overwrite = parameters.get('overwrite', False)
@@ -30,7 +30,7 @@ def __init__(self, parameters):
                     param_type = self.match_value_params[param_name]
                 else:
                     raise KeyError("BadParameter",
-                                   f"{param_name} not a required or optional parameter for {self.command}")
+                                   f"{param_name} not a required or optional parameter for {self.operation}")
                 # TODO: this has a syntax error
                 # if not isinstance(param_value, param_type):
                 #     raise TypeError("BadType" f"{param_value} has type {type(param_value)} not {param_type}")

hed/tools/remodeling/operations/operation_list.py

@@ -14,7 +14,7 @@
 from hed.tools.remodeling.operations.summarize_column_values_op import SummarizeColumnValuesOp
 from hed.tools.remodeling.operations.summarize_hed_type_op import SummarizeHedTypeOp
 
-operations = {
+valid_operations = {
     'factor_column': FactorColumnOp,
     'factor_hed_tags': FactorHedTagsOp,
     'factor_hed_type': FactorHedTypeOp,

hed/tools/remodeling/operations/remap_columns_op.py

@@ -3,7 +3,7 @@
 from hed.tools.analysis.key_map import KeyMap
 
 PARAMS = {
-    "command": "remap_columns",
+    "operation": "remap_columns",
     "required_parameters": {
         "source_columns": list,
         "destination_columns": list,
@@ -21,7 +21,7 @@ class RemapColumnsOp(BaseOp):
 
     """
     def __init__(self, parameters):
-        super().__init__(PARAMS["command"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
+        super().__init__(PARAMS["operation"], PARAMS["required_parameters"], PARAMS["optional_parameters"])
         self.check_parameters(parameters)
         self.source_columns = parameters['source_columns']
         self.destination_columns = parameters['destination_columns']