Merge pull request #711 from douglasjacobsen/workspace-metadata

Add metadata tracking to workspace object

lib/ramble/ramble/namespace.py

@@ -62,3 +62,5 @@ class namespace:
 
     # For variants
     package_manager = "package_manager"
+
+    metadata = "metadata"

lib/ramble/ramble/pipeline.py

@@ -123,6 +123,9 @@ def _construct_workspace_hash(self):
             with open(os.path.join(self.workspace.root, self.workspace.hash_file_name), "w+") as f:
                 f.write(self.workspace.workspace_hash + "\n")
 
+            self.workspace.update_metadata("workspace_digest", self.workspace.workspace_hash)
+            self.workspace._write_metadata()
+
     def _prepare(self):
         """Perform preparation for pipeline execution"""
         pass
@@ -436,16 +439,9 @@ def _complete(self):
                 _upload_file(tar_path, remote_tar_path)
                 logger.all_msg(f"Archive Uploaded to {remote_tar_path}")
 
-                # Record upload URL to the filesystem
-                url_extension = ".url"
-                tar_url_path = tar_path + url_extension
-                with open(tar_url_path, "w") as f:
-                    f.write(remote_tar_path)
-
-                tar_url_path_latest = os.path.join(
-                    self.workspace.archive_dir, "archive.latest" + url_extension
-                )
-                self.create_simlink(tar_url_path, tar_url_path_latest)
+                # Record upload URL to workspace metadata
+                self.workspace.update_metadata("archive_url", remote_tar_path)
+                self.workspace._write_metadata()
 
 
 class MirrorPipeline(Pipeline):

lib/ramble/ramble/test/workspace_hashing/workspace_name_does_not_change_hash.py

@@ -14,6 +14,8 @@
 import ramble.config
 import ramble.software_environments
 from ramble.main import RambleCommand
+from ramble.namespace import namespace
+import spack.util.spack_yaml as syaml
 
 
 # everything here uses the mock_workspace_path
@@ -70,4 +72,14 @@ def test_workspace_name_does_not_change_hash(
             ws2._re_read()
             workspace("setup", "--dry-run", global_args=["-w", workspace2_name])
 
-            assert ws1.workspace_hash == ws2.workspace_hash
+            metadata_path = os.path.join(ws1.root, ramble.workspace.metadata_file_name)
+            with open(metadata_path) as f:
+                data = syaml.load(f)
+                ws1_hash = data[namespace.metadata]["workspace_digest"]
+
+            metadata_path = os.path.join(ws2.root, ramble.workspace.metadata_file_name)
+            with open(metadata_path) as f:
+                data = syaml.load(f)
+                ws2_hash = data[namespace.metadata]["workspace_digest"]
+
+            assert ws1_hash == ws2_hash

lib/ramble/ramble/workspace/__init__.py

@@ -34,6 +34,7 @@
     config_file_name,
     licenses_file,
     licenses_file_name,
+    metadata_file_name,
     workspace_software_path,
     auxiliary_software_dir_name,
     template_path,
@@ -73,6 +74,7 @@
     "config_file_name",
     "licenses_file",
     "licenses_file_name",
+    "metadata_file_name",
     "workspace_software_path",
     "auxiliary_software_dir_name",
     "template_path",

lib/ramble/ramble/workspace/workspace.py

@@ -106,6 +106,8 @@
 config_file_name = "ramble.yaml"
 licenses_file_name = "licenses.yaml"
 
+metadata_file_name = "workspace_metadata.yaml"
+
 workspace_all_experiments_file = "all_experiments"
 
 workspace_execution_template = "execute_experiment" + workspace_template_extension
@@ -419,6 +421,7 @@ def __init__(self, root, dry_run=False, read_default_template=True):
         self.input_mirror_cache = None
         self.software_mirror_cache = None
         self.software_environments = None
+        self.metadata = syaml.syaml_dict()
         self.hash_inventory = {"experiments": [], "versions": []}
 
         from ramble.main import get_version
@@ -526,6 +529,8 @@ def _read(self):
                 template_name = workspace_execution_template[0:-ext_len]
                 self._read_template(template_name, self._template_execute_script())
 
+            self._read_metadata()
+
     @classmethod
     def _template_execute_script(self):
         shell = ramble.config.get("config:shell")
@@ -612,6 +617,32 @@ def _read_config(self, section, f, raw_yaml=None):
         self._read_yaml(config, f, raw_yaml)
         self._check_deprecated(config["yaml"])
 
+    def _read_metadata(self):
+        """Read workspace metadata file
+
+        If a metadata file exists in the workspace root, read it in, and
+        populate this workspace's metadata object with its contents.
+        """
+        metadata_file_path = os.path.join(self.root, metadata_file_name)
+
+        if os.path.exists(metadata_file_path):
+            with open(metadata_file_path) as f:
+                self.metadata = syaml.load(f)
+        else:
+            self.metadata = syaml.syaml_dict()
+            self.metadata[namespace.metadata] = syaml.syaml_dict()
+
+    def _write_metadata(self):
+        """Write out workspace metadata file
+
+        Create, and populate the metadata file in the root of the workspace.
+        This file can be used to house cross-pipeline information.
+        """
+        metadata_file_path = os.path.join(self.root, metadata_file_name)
+
+        with open(metadata_file_path, "w+") as f:
+            syaml.dump(self.metadata, stream=f)
+
     def _check_deprecated(self, config):
         """
         Trap and warn (or error) on deprecated configuration settings
@@ -682,6 +713,8 @@ def write(self, software_dir=None, inputs_dir=None):
 
             self._write_templates()
 
+            self._write_metadata()
+
     def _write_config(self, section, force=False):
         """Update YAML config file for this workspace, based on
         changes and write it"""
@@ -702,6 +735,29 @@ def _write_templates(self):
             with open(template_path, "w+") as f:
                 f.write(conf["contents"])
 
+    def get_metadata(self, key):
+        """Get the value of a metadata key
+
+        Args:
+            key (str): Name of metadata key to retrieve
+
+        Returns:
+            (any): Value associated with key in metadata
+        """
+        if key in self.metadata[namespace.metadata]:
+            return self.metadata[namespace.metadata][key]
+        else:
+            return None
+
+    def update_metadata(self, key, value):
+        """Set the metadata key value
+
+        Args:
+            key (str): Key of metadata to set
+            value (any): Value to set in the metadata object
+        """
+        self.metadata[namespace.metadata][key] = value
+
     def clear(self):
         self.config_sections = {}
         self.application_configs = []