diff --git a/lib/ramble/ramble/pipeline.py b/lib/ramble/ramble/pipeline.py index c998fd298..1ab3aa896 100644 --- a/lib/ramble/ramble/pipeline.py +++ b/lib/ramble/ramble/pipeline.py @@ -68,8 +68,11 @@ def __init__(self, workspace, filters): self.workspace.software_environments = self._software_environments self._experiment_set = workspace.build_experiment_set() - def _construct_hash(self): - """Hash all of the experiments, construct workspace inventory""" + def _construct_experiment_hashes(self): + """Hash all of the experiments. + + Populate the workspace inventory information with experiment hash data. + """ for exp, app_inst, _ in self._experiment_set.all_experiments(): app_inst.populate_inventory( self.workspace, @@ -77,6 +80,12 @@ def _construct_hash(self): require_exist=self.require_inventory, ) + def _construct_workspace_hash(self): + """Construct workspace inventory + + Assumes experiment hashes are already constructed and populated into + the workspace. + """ workspace_inventory = os.path.join(self.workspace.root, self.workspace.inventory_file_name) workspace_hash_file = os.path.join(self.workspace.root, self.workspace.hash_file_name) @@ -280,7 +289,8 @@ def _prepare(self): " Make sure your workspace is setup with\n" " ramble workspace setup" ) - super()._construct_hash() + super()._construct_experiment_hashes() + super()._construct_workspace_hash() super()._prepare() def _complete(self): @@ -328,7 +338,8 @@ def __init__( ) def _prepare(self): - super()._construct_hash() + super()._construct_experiment_hashes() + super()._construct_workspace_hash() super()._prepare() date_str = self.workspace.date_string() @@ -488,6 +499,10 @@ def __init__(self, workspace, filters): self.action_string = "Setting up" def _prepare(self): + # Check if the selected phases require the inventory is successful + if "write_inventory" in self.filters.phases or "*" in self.filters.phases: + self.require_inventory = True + super()._prepare() experiment_file = open(self.workspace.all_experiments_path, "w+") shell = ramble.config.get("config:shell") @@ -495,13 +510,11 @@ def _prepare(self): experiment_file.write(f"#!{shell_path}\n") self.workspace.experiments_script = experiment_file - def _complete(self): - # Check if the selected phases require the inventory is successful - if "write_inventory" in self.filters.phases or "*" in self.filters.phases: - self.require_inventory = True + super()._construct_experiment_hashes() + def _complete(self): try: - super()._construct_hash() + super()._construct_workspace_hash() except FileNotFoundError as e: tty.warn("Unable to construct workspace hash due to missing file") tty.warn(e) diff --git a/lib/ramble/ramble/test/end_to_end/experiment_hashes.py b/lib/ramble/ramble/test/end_to_end/experiment_hashes.py new file mode 100644 index 000000000..697e030a8 --- /dev/null +++ b/lib/ramble/ramble/test/end_to_end/experiment_hashes.py @@ -0,0 +1,138 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os + +import ramble.workspace +import spack.util.spack_json as sjson +from ramble.main import RambleCommand +from ramble.application import ApplicationBase + + +workspace = RambleCommand("workspace") + + +def test_experiment_hashes(mutable_config, mutable_mock_workspace_path, request): + workspace_name = request.node.name + + ws1 = ramble.workspace.create(workspace_name) + + global_args = ["-w", workspace_name] + + workspace( + "generate-config", + "gromacs", + "--wf", + "water_bare", + "-e", + "unit_test", + "-v", + "n_nodes=1", + "-v", + "n_ranks=1", + "-p", + "spack", + global_args=global_args, + ) + + workspace("concretize", global_args=global_args) + workspace("setup", "--dry-run", global_args=global_args) + + experiment_inventory = os.path.join( + ws1.experiment_dir, + "gromacs", + "water_bare", + "unit_test", + ApplicationBase._inventory_file_name, + ) + + workspace_inventory = os.path.join(ws1.root, ramble.workspace.Workspace.inventory_file_name) + + # Test experiment inventory + assert os.path.isfile(experiment_inventory) + with open(experiment_inventory) as f: + data = sjson.load(f) + + assert "application_definition" in data + assert data["application_definition"] != "" + assert data["application_definition"] is not None + + # Test Attributes + expected_attrs = {"variables", "modifiers", "env_vars", "internals", "chained_experiments"} + assert "attributes" in data + for attr in data["attributes"]: + if attr["name"] in expected_attrs: + assert attr["digest"] != "" + assert attr["digest"] is not None + expected_attrs.remove(attr["name"]) + + assert len(expected_attrs) == 0 + + # Test Templates + expected_templates = {"execute_experiment"} + assert "templates" in data + for temp in data["templates"]: + if temp["name"] in expected_templates: + assert temp["digest"] != "" + assert temp["digest"] is not None + expected_templates.remove(temp["name"]) + + assert len(expected_templates) == 0 + + # Test software environments + expected_envs = {"software/gromacs"} + assert "software" in data + for env in data["software"]: + if env["name"] in expected_envs: + assert env["digest"] != "" + assert env["digest"] is not None + expected_envs.remove(env["name"]) + + assert len(expected_envs) == 0 + + # Test package manager + expected_pkgmans = {"spack"} + assert "package_manager" in data + for pkgman in data["package_manager"]: + if pkgman["name"] in expected_pkgmans: + assert pkgman["digest"] != "" + assert pkgman["digest"] is not None + assert pkgman["version"] != "" + assert pkgman["version"] is not None + expected_pkgmans.remove(pkgman["name"]) + + assert len(expected_pkgmans) == 0 + + # Test workspace inventory + assert os.path.isfile(workspace_inventory) + with open(workspace_inventory) as f: + data = sjson.load(f) + + # Test experiments + expected_experiments = {"gromacs.water_bare.unit_test"} + + assert "experiments" in data + for exp in data["experiments"]: + if exp["name"] in expected_experiments: + assert exp["digest"] != "" + assert exp["digest"] is not None + assert "contents" in exp + expected_experiments.remove(exp["name"]) + + assert len(expected_experiments) == 0 + + # Test versions + expected_versions = {"ramble"} + + assert "versions" in data + for ver in data["versions"]: + if ver["name"] in expected_versions: + assert ver["digest"] != "" + assert ver["digest"] is not None + expected_versions.remove(ver["name"]) + assert len(expected_versions) == 0 diff --git a/var/ramble/repos/builtin/package_managers/environment-modules/package_manager.py b/var/ramble/repos/builtin/package_managers/environment-modules/package_manager.py index 657ec8ba5..b425a8541 100644 --- a/var/ramble/repos/builtin/package_managers/environment-modules/package_manager.py +++ b/var/ramble/repos/builtin/package_managers/environment-modules/package_manager.py @@ -35,19 +35,43 @@ class EnvironmentModules(PackageManagerBase): run_before=["make_experiments"], ) + def _generate_loads_content(self, workspace): + if not hasattr(self, "_load_string"): + app_context = self.app_inst.expander.expand_var_name( + self.keywords.env_name + ) + + require_env = self.environment_required() + + software_envs = workspace.software_environments + software_env = software_envs.render_environment( + app_context, self.app_inst.expander, self, require=require_env + ) + + load_content = [] + + if software_env is not None: + for spec in software_envs.package_specs_for_environment( + software_env + ): + load_content.append(f"module load {spec}") + + self._load_string = "\n".join(load_content) + + return self._load_string + def populate_inventory( self, workspace, force_compute=False, require_exist=False ): - env_path = self.app_inst.expander.env_path - self.app_inst.hash_inventory["package_manager"].append( { "name": self.name, } ) - env_hash = ramble.util.hashing.hash_file( - os.path.join(env_path, "module_loads") + env_path = self.app_inst.expander.env_path + env_hash = ramble.util.hashing.hash_string( + self._generate_loads_content(workspace) ) self.app_inst.hash_inventory["software"].append( @@ -58,32 +82,16 @@ def populate_inventory( ) def _write_module_commands(self, workspace, app_inst=None): - - app_context = self.app_inst.expander.expand_var_name( - self.keywords.env_name - ) - - require_env = self.environment_required() - - software_envs = workspace.software_environments - software_env = software_envs.render_environment( - app_context, self.app_inst.expander, self, require=require_env - ) - env_path = self.app_inst.expander.env_path module_file_path = os.path.join(env_path, "module_loads") fs.mkdirp(env_path) - module_file = open(module_file_path, "w+") + loads_content = self._generate_loads_content(workspace) - if software_env is not None: - for spec in software_envs.package_specs_for_environment( - software_env - ): - module_file.write(f"module load {spec}\n") - module_file.close() + with open(module_file_path, "w+") as f: + f.write(loads_content) register_builtin("module_load", required=True)