Merge branch 'develop' into documentation/update2

docs/source/getting_started.rst

@@ -0,0 +1,38 @@
+Getting Started
+================
+
+Installing MaestroWF
+*********************
+
+MaestroWF can be installed via pip::
+
+    $ pip install maestrowf
+
+.. note:: Using a `virtualenv <https://virtualenv.pypa.io/en/stable/>`_ is recommended.
+
+Once installed run::
+
+    $ maestro -h
+    
+    usage: maestro [-h] [-l LOGPATH] [-d DEBUG_LVL] [-c] {cancel,run,status} ...
+
+    The Maestro Workflow Conductor for specifiying, launching, and managing general workflows.
+
+    positional arguments:
+      {cancel,run,status}
+        cancel              Cancel all running jobs.
+        run                 Launch a study based on a specification
+        status              Check the status of a running study.
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      -l LOGPATH, --logpath LOGPATH
+                            Alternate path to store program logging.
+      -d DEBUG_LVL, --debug_lvl DEBUG_LVL
+                            Level of logging messages to be output:
+                            5 - Critical
+                            4 - Error
+                            3 - Warning
+                            2 - Info (Default)
+                            1 - Debug
+      -c, --logstdout       Log to stdout in addition to a file. [Default: True]

docs/source/index.rst

@@ -10,6 +10,11 @@ Welcome to Maestro Workflow Conductor Documentation
    :maxdepth: 4
    :caption: Contents:
 
+   getting_started
+   quick_start
+   lulesh_breakdown
+   maestro_core
+
    modules
 
 Indices and tables

docs/source/lulesh_breakdown.rst

@@ -0,0 +1,4 @@
+LULESH Specification Breakdown
+===============================
+
+Stub

docs/source/maestro_core.rst

@@ -0,0 +1,4 @@
+Maestro Core Concepts
+======================
+
+Stub

maestrowf/datastructures/core/executiongraph.py

@@ -89,6 +89,7 @@ def generate_script(self, adapter, tmp_dir=""):
                     self.script, self.restart_script, self.to_be_scheduled)
 
     def execute(self, adapter):
+        self.mark_submitted()
         retcode, jobid = self._execute(adapter, self.script)
 
         if retcode == SubmissionCode.OK:
@@ -139,7 +140,7 @@ def mark_submitted(self):
             self._submit_time = datetime.now()
         else:
             logger.warning(
-                "Cannot set the submission time of '%s' because it has"
+                "Cannot set the submission time of '%s' because it has "
                 "already been set.", self.name
             )
 
@@ -580,7 +581,6 @@ def _execute_record(self, record, adapter, restart=False):
                 # Generate the script for execution on the fly.
                 record.setup_workspace()    # Generate the workspace.
                 record.generate_script(adapter, self._tmp_dir)
-                record.mark_submitted()
                 retcode = record.execute(adapter)
             # Otherwise, it's a restart.
             else:
@@ -766,7 +766,7 @@ def execute_ready_steps(self):
                                    "resubmit step '%s'.", name)
                     # We can just let the logic below handle submission with
                     # everything else.
-                    self.ready_steps.append(record)
+                    self.ready_steps.append(name)
 
                 elif status == State.FAILED:
                     logger.warning(
@@ -811,23 +811,23 @@ def execute_ready_steps(self):
 
                 logger.debug(
                     "Unfulfilled dependencies: %s",
-                    self._dependencies[record.name])
+                    self._dependencies[key])
 
                 s_completed = filter(
                     lambda x: x in self.completed_steps,
-                    self._dependencies[record.name])
-                self._dependencies[record.name] = \
-                    self._dependencies[record.name] - set(s_completed)
+                    self._dependencies[key])
+                self._dependencies[key] = \
+                    self._dependencies[key] - set(s_completed)
                 logger.debug(
                     "Completed dependencies: %s\n"
                     "Remaining dependencies: %s",
-                    s_completed, self._dependencies[record.name])
+                    s_completed, self._dependencies[key])
 
                 # If the gating dependencies set is empty, we can execute.
-                if not self._dependencies[record.name]:
+                if not self._dependencies[key]:
                     if key not in self.ready_steps:
                         logger.debug("All dependencies completed. Staging.")
-                        self.ready_steps.append(record)
+                        self.ready_steps.append(key)
                     else:
                         logger.debug("Already staged. Passing.")
                         continue
@@ -841,12 +841,18 @@ def execute_ready_steps(self):
         else:
             # Compute the number of available slots we have for execution.
             _available = self._submission_throttle - len(self.in_progress)
+            # Available slots should never be negative, but on the off chance
+            # we are in a slot deficit, then we will just say none are free.
             _available = max(0, _available)
+            # Now, we need to take the min of the length of the queue and the
+            # computed number of slots. We could have free slots, but have less
+            # in the queue.
+            _available = min(_available, len(self.ready_steps))
             logger.info("Found %d available slots...", _available)
 
         for i in range(0, _available):
             # Pop the record and execute using the helper method.
-            _record = self.ready_steps.popleft()
+            _record = self.values[self.ready_steps.popleft()]
 
             # If we get to this point and we've cancelled, cancel the record.
             if self.is_canceled:

maestrowf/datastructures/environment/variable.py

@@ -95,7 +95,8 @@ def _verify(self):
 
         :returns: True if Variable is valid, False otherwise.
         """
-        return bool(self.name) and bool(self.value)
+        _valid = bool(self.name) and self.value is not None
+        return _valid
 
     def __str__(self):
         """