Addition of pargs for passing parameters to custom parameter generation

maestrowf/maestro.py

@@ -44,7 +44,8 @@
 from maestrowf.datastructures.core import Study
 from maestrowf.datastructures.environment import Variable
 from maestrowf.utils import \
-    create_parentdir, csvtable_to_dict, make_safe_path, start_process
+    create_parentdir, create_dictionary, csvtable_to_dict, make_safe_path, \
+    start_process
 
 
 # Program Globals
@@ -88,12 +89,14 @@ def cancel_study(args):
     return 0
 
 
-def load_parameter_generator(path):
+def load_parameter_generator(path, kwargs):
     """
     Import and load custom parameter Python files.
 
     :param path: Path to a Python file containing the function
-    'get_custom_generator()'
+    'get_custom_generator'.
+    :param kwargs: Dictionary containing keyword arguments for the function
+    'get_custom_generator'.
     :returns: A populated ParameterGenerator instance.
     """
     path = os.path.abspath(path)
@@ -105,20 +108,20 @@ def load_parameter_generator(path):
         spec = importlib.util.spec_from_file_location("custom_gen", path)
         f = importlib.util.module_from_spec(spec)
         spec.loader.exec_module(f)
-        return f.get_custom_generator()
+        return f.get_custom_generator(**kwargs)
     except ImportError:
         try:
             # Python 3.3
             from importlib.machinery import SourceFileLoader
             LOGGER.debug("Using Python 3.4 SourceFileLoader...")
             f = SourceFileLoader("custom_gen", path).load_module()
-            return f.get_custom_generator()
+            return f.get_custom_generator(**kwargs)
         except ImportError:
             # Python 2
             import imp
             LOGGER.debug("Using Python 2 imp library...")
             f = imp.load_source("custom_gen", path)
-            return f.get_custom_generator()
+            return f.get_custom_generator(**kwargs)
     except Exception as e:
         LOGGER.exception(str(e))
         raise e
@@ -173,11 +176,20 @@ def run_study(args):
     # Now that we know outpath, set up logging.
     setup_logging(args, output_path, spec.name.replace(" ", "_").lower())
 
+    # Check for pargs without the matching pgen
+    if args.pargs and not args.pgen:
+        msg = "Cannot use the 'pargs' parameter without specifying a 'pgen'!"
+        LOGGER.exception(msg)
+        raise ArgumentError(msg)
+
     # Handle loading a custom ParameterGenerator if specified.
     if args.pgen:
+        # 'pgen_args' has a default of an empty list, which should translate
+        # to an empty dictionary.
+        kwargs = create_dictionary(args.pargs)
         # Copy the Python file used to generate parameters.
         shutil.copy(args.pgen, output_path)
-        parameters = load_parameter_generator(args.pgen)
+        parameters = load_parameter_generator(args.pgen, kwargs)
     else:
         parameters = spec.get_parameters()
 
@@ -315,6 +327,10 @@ def setup_argparser():
                      help="Path to a Python code file containing a function "
                      "that returns a custom filled ParameterGenerator"
                      "instance.")
+    run.add_argument("--pargs", type=str, action="append", default=[],
+                     help="A string that represents a single argument to pass "
+                     "a custom parameter generation function. Reuse '--parg' "
+                     "to pass multiple arguments. [Use with '--pgen']")
     run.add_argument("-o", "--out", type=str,
                      help="Output path to place study in. [NOTE: overrides "
                      "OUTPUT_PATH in the specified specification]")

maestrowf/utils.py

@@ -235,3 +235,26 @@ def ping_url(url):
     else:
         response.read()
         return
+
+
+def create_dictionary(list_keyvalues, token=":"):
+    """
+    Create a dictionary from a list of key-value pairs.
+
+    :param list_keyvalues: List of token separates key-values.
+    :param token: The token to split each key-value by.
+    :returns: A dictionary containing the key-value pairings in list_keyvalues.
+    """
+    _dict = {}
+    for item in list_keyvalues:
+        try:
+            key, value = [i.strip() for i in item.split(token, 1)]
+            _dict[key] = value
+        except ValueError:
+            msg = "'{}' is not capable of being split by the token '{}'. " \
+                  "Verify that all other parameters are formatted properly." \
+                  .format(item, token)
+            LOGGER.exception(msg)
+            raise ValueError(msg)
+
+    return _dict

samples/parameterization/lulesh_montecarlo_args.py

@@ -0,0 +1,42 @@
+"""An example file that produces a custom parameters for the LULESH example."""
+
+from random import randint
+
+from maestrowf.datastructures.core import ParameterGenerator
+
+
+def get_custom_generator(**kwargs):
+    """
+    Create a custom populated ParameterGenerator.
+
+    This function recreates the exact same parameter set as the sample LULESH
+    specifications. The point of this file is to present an example of how to
+    generate custom parameters.
+
+    :params kwargs: A dictionary of keyword arguments this function uses.
+    :returns: A ParameterGenerator populated with parameters.
+    """
+    p_gen = ParameterGenerator()
+    trials = int(kwargs.get("trials"))
+    size_min = int(kwargs.get("smin"))
+    size_max = int(kwargs.get("smax"))
+    iterations = int(kwargs.get("iter"))
+    params = {
+        "TRIAL": {
+            "values": [i for i in range(1, trials)],
+            "label": "TRIAL.%%"
+        },
+        "SIZE": {
+            "values": [randint(size_min, size_max) for i in range(1, trials)],
+            "label": "SIZE.%%"
+        },
+        "ITERATIONS": {
+            "values": [iterations for i in range(1, trials)],
+            "label": "ITERATIONS.%%"
+        }
+    }
+
+    for key, value in params.items():
+        p_gen.add_parameter(key, value["values"], value["label"])
+
+    return p_gen

setup.py

@@ -3,7 +3,7 @@
 setup(name='maestrowf',
       description='A tool and library for specifying and conducting general '
       'workflows.',
-      version='1.1.4dev1.1',
+      version='1.1.4dev1.2',
       author='Francesco Di Natale',
       author_email='dinatale3@llnl.gov',
       url='https://github.com/llnl/maestrowf',