From 530185e8c7a0ec305ad387113cfd53505af8ebb9 Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Mon, 3 Jun 2024 11:51:29 -0700 Subject: [PATCH 1/3] Apply `black` to Ramble's object files * Add in a separate config file for the object files (apps and mods,) to accommodate for the different line-length requirements between primary files and object files * Update style cmd to apply black separately for primary and object files --- lib/ramble/ramble/cmd/style.py | 34 +++++++++++++++++++++++++++------- pyproject_objects.toml | 8 ++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 pyproject_objects.toml diff --git a/lib/ramble/ramble/cmd/style.py b/lib/ramble/ramble/cmd/style.py index 14bb45678..14e2aa2ea 100644 --- a/lib/ramble/ramble/cmd/style.py +++ b/lib/ramble/ramble/cmd/style.py @@ -344,6 +344,11 @@ def filter_file(source, dest, output=False): sys.stdout.write(line) +def _split_file_list(file_list): + """Return a tuple of (primary_files, obj_files)""" + return [f for f in file_list if not is_object(f)], [f for f in file_list if is_object(f)] + + @tool("flake8") def run_flake8(flake8_cmd, file_list, args): temp = tempfile.mkdtemp() @@ -352,8 +357,7 @@ def run_flake8(flake8_cmd, file_list, args): print_tool_header("flake8", file_list) # run flake8 on the temporary tree, once for core, once for apps - application_file_list = [f for f in file_list if is_object(f)] - primary_file_list = [f for f in file_list if not is_object(f)] + primary_file_list, application_file_list = _split_file_list(file_list) # filter files into a temporary directory with exemptions added. # TODO: DRY this duplication @@ -424,13 +428,29 @@ def run_flake8(flake8_cmd, file_list, args): @tool("black") def run_black(black_cmd, file_list, args): print_tool_header("black", file_list) - black_args = ("--config", os.path.join(ramble.paths.prefix, "pyproject.toml")) + common_args = ("--config", os.path.join(ramble.paths.prefix, "pyproject.toml")) if not args.fix: - black_args += ("--check", "--diff") - black_args += tuple(file_list) + common_args += ("--check", "--diff") + primary_files, obj_files = _split_file_list(file_list) + output = "" + returncode = 0 + + # Operate on primary and object (apps and mods) files spearately with varying configs. + if primary_files: + output += black_cmd( + *(common_args + tuple(primary_files)), fail_on_error=False, output=str, error=str + ) + returncode |= black_cmd.returncode + + if obj_files: + output += black_cmd( + *(common_args + ("--config", "pyproject_objects.toml") + tuple(obj_files)), + fail_on_error=False, + output=str, + error=str, + ) + returncode |= black_cmd.returncode - output = black_cmd(*black_args, fail_on_error=False, output=str, error=str) - returncode = black_cmd.returncode print_output(output, args) print_tool_result("black", returncode) return returncode diff --git a/pyproject_objects.toml b/pyproject_objects.toml new file mode 100644 index 000000000..801973ccf --- /dev/null +++ b/pyproject_objects.toml @@ -0,0 +1,8 @@ +# Config file for Ramble's object files (apps and mods) + +[tool.black] +line-length = 79 +target-version = ["py36", "py37", "py38", "py39", "py310", "py311", "py312"] +include = ''' + \.pyi?$ +''' From 6d1b58b4d1b651427bbfb70ba985689856ea1bde Mon Sep 17 00:00:00 2001 From: Lin Guo Date: Mon, 3 Jun 2024 11:55:39 -0700 Subject: [PATCH 2/3] Format object files with `ramble style -af` --- .../basic-inherited/application.py | 19 +- .../applications/basic/application.py | 65 +- .../applications/expanded_foms/application.py | 34 +- .../applications/glob-patterns/application.py | 138 ++-- .../applications/input-test/application.py | 37 +- .../interleved-env-vars/application.py | 61 +- .../applications/maintained-1/application.py | 9 +- .../applications/maintained-2/application.py | 9 +- .../register-builtin/application.py | 51 +- .../shared-context/application.py | 52 +- .../success-function/application.py | 13 +- .../applications/tagged-1/application.py | 6 +- .../unmaintained-1/application.py | 7 +- .../applications/untagged-1/application.py | 4 +- .../workload-groups-inherited/application.py | 6 +- .../workload-groups/application.py | 34 +- .../applications/workload-tags/application.py | 8 +- .../applications/zlib-configs/application.py | 22 +- .../applications/zlib/application.py | 20 +- .../append-env-var-mod-paths/modifier.py | 10 +- .../append-env-var-mod-vars/modifier.py | 14 +- .../modifiers/glob-patterns-mod/modifier.py | 23 +- .../modifiers/maintained-1/modifier.py | 7 +- .../modifiers/maintained-2/modifier.py | 7 +- .../modifiers/mod-phase/modifier.py | 17 +- .../multiple-modes-no-default/modifier.py | 7 +- .../multiple-modes-with-default/modifier.py | 9 +- .../modifiers/no-docstring-mod/modifier.py | 4 +- .../modifiers/no-variable-mods/modifier.py | 5 +- .../modifiers/prepare-analysis/modifier.py | 20 +- .../prepend-env-var-mod-paths/modifier.py | 10 +- .../modifiers/set-env-var-mod/modifier.py | 22 +- .../modifiers/spack-failed-reqs/modifier.py | 32 +- .../modifiers/spack-mod/modifier.py | 63 +- .../modifiers/success-criteria/modifier.py | 19 +- .../modifiers/tagged-1/modifier.py | 5 +- .../modifiers/test-mod/modifier.py | 105 ++- .../modifiers/unmaintained-1/modifier.py | 5 +- .../modifiers/unset-env-var-mod/modifier.py | 7 +- .../modifiers/untagged-1/modifier.py | 3 +- .../applications/cloverleaf/application.py | 253 ++++--- .../builtin/applications/elk/application.py | 84 ++- .../applications/gromacs/application.py | 606 +++++++++++------ .../builtin/applications/hmmer/application.py | 130 ++-- .../applications/hostname/application.py | 63 +- .../builtin/applications/hpcc/application.py | 188 +++--- .../builtin/applications/hpcg/application.py | 197 +++--- .../builtin/applications/hpl/application.py | 627 ++++++++++------- .../applications/intel-hpl/application.py | 631 +++++++++++------- .../intel-mpi-benchmarks/application.py | 347 ++++++---- .../builtin/applications/ior/application.py | 230 ++++--- .../applications/iperf2/application.py | 141 ++-- .../applications/lammps/application.py | 441 ++++++++---- .../applications/lulesh/application.py | 161 +++-- .../applications/md-test/application.py | 98 ++- .../applications/minixyce/application.py | 354 ++++++---- .../builtin/applications/namd/application.py | 374 ++++++----- .../applications/nvbandwidth/application.py | 36 +- .../applications/openfoam/application.py | 592 ++++++++++------ .../osu-micro-benchmarks/application.py | 351 ++++++---- .../quantum-espresso/application.py | 351 ++++++---- .../builtin/applications/sleep/application.py | 130 ++-- .../applications/spack-stack/application.py | 193 +++--- .../applications/streamc/application.py | 215 +++--- .../ufs-weather-model/application.py | 318 +++++---- .../builtin/applications/wrfv3/application.py | 259 ++++--- .../builtin/applications/wrfv4/application.py | 276 +++++--- .../modifiers/execution-date/modifier.py | 57 +- .../modifiers/gcp-metadata/modifier.py | 80 ++- .../modifiers/install-ramble/modifier.py | 92 ++- .../modifiers/install-spack/modifier.py | 72 +- .../builtin/modifiers/intel-aps/modifier.py | 200 ++++-- .../repos/builtin/modifiers/lscpu/modifier.py | 112 ++-- .../modifiers/pre-exec-print/modifier.py | 28 +- 74 files changed, 5837 insertions(+), 3439 deletions(-) diff --git a/var/ramble/repos/builtin.mock/applications/basic-inherited/application.py b/var/ramble/repos/builtin.mock/applications/basic-inherited/application.py index a289a9f76..ce952ed47 100644 --- a/var/ramble/repos/builtin.mock/applications/basic-inherited/application.py +++ b/var/ramble/repos/builtin.mock/applications/basic-inherited/application.py @@ -14,11 +14,18 @@ class BasicInherited(BaseBasic): name = "basic-inherited" - input_file('inherited_input', url='file:///tmp/inherited_file.log', - description='Again, not a file', extension='.log') + input_file( + "inherited_input", + url="file:///tmp/inherited_file.log", + description="Again, not a file", + extension=".log", + ) - workload('test_wl3', executable='foo', input='inherited_input') + workload("test_wl3", executable="foo", input="inherited_input") - workload_variable('my_var', default='1.0', - description='Shadowed Example var', - workload='test_wl') + workload_variable( + "my_var", + default="1.0", + description="Shadowed Example var", + workload="test_wl", + ) diff --git a/var/ramble/repos/builtin.mock/applications/basic/application.py b/var/ramble/repos/builtin.mock/applications/basic/application.py index 399ea9d40..7752c05f9 100644 --- a/var/ramble/repos/builtin.mock/applications/basic/application.py +++ b/var/ramble/repos/builtin.mock/applications/basic/application.py @@ -12,30 +12,41 @@ class Basic(ExecutableApplication): name = "basic" - executable('foo', 'bar', use_mpi=False) - executable('bar', 'baz', use_mpi=True) - executable('echo', 'echo "0.25 seconds"', use_mpi=False) - - input_file('input', url='file:///tmp/test_file.log', - description='Not a file', extension='.log') - - workload('test_wl', executable='foo', input='input') - workload('test_wl2', executable='bar', input='input') - workload('working_wl', executable='echo') - - workload_variable('my_base_var', default='0.0', - description='Example var', - workload='test_wl') - - workload_variable('my_var', default='1.0', - description='Example var', - workload='test_wl') - - environment_variable('TEST_ENV', value="1", - description="test var", workload="test_wl") - - archive_pattern('{experiment_run_dir}/archive_test.*') - - figure_of_merit('test_fom', - fom_regex=r'(?P[0-9]+\.[0-9]+).*seconds.*', - group_name='test', units='s') + executable("foo", "bar", use_mpi=False) + executable("bar", "baz", use_mpi=True) + executable("echo", 'echo "0.25 seconds"', use_mpi=False) + + input_file( + "input", + url="file:///tmp/test_file.log", + description="Not a file", + extension=".log", + ) + + workload("test_wl", executable="foo", input="input") + workload("test_wl2", executable="bar", input="input") + workload("working_wl", executable="echo") + + workload_variable( + "my_base_var", + default="0.0", + description="Example var", + workload="test_wl", + ) + + workload_variable( + "my_var", default="1.0", description="Example var", workload="test_wl" + ) + + environment_variable( + "TEST_ENV", value="1", description="test var", workload="test_wl" + ) + + archive_pattern("{experiment_run_dir}/archive_test.*") + + figure_of_merit( + "test_fom", + fom_regex=r"(?P[0-9]+\.[0-9]+).*seconds.*", + group_name="test", + units="s", + ) diff --git a/var/ramble/repos/builtin.mock/applications/expanded_foms/application.py b/var/ramble/repos/builtin.mock/applications/expanded_foms/application.py index e75c1afea..2d28583bd 100644 --- a/var/ramble/repos/builtin.mock/applications/expanded_foms/application.py +++ b/var/ramble/repos/builtin.mock/applications/expanded_foms/application.py @@ -12,23 +12,29 @@ class ExpandedFoms(ExecutableApplication): name = "expanded-Foms" - executable('foo', template=['bar', 'echo "{my_var}"'], use_mpi=False) + executable("foo", template=["bar", 'echo "{my_var}"'], use_mpi=False) - input_file('input', url='file:///tmp/test_file.log', - description='Not a file', extension='.log') + input_file( + "input", + url="file:///tmp/test_file.log", + description="Not a file", + extension=".log", + ) - workload('test_wl', executables=['foo'], input='input') + workload("test_wl", executables=["foo"], input="input") - workload_variable('my_var', default='1.0', - description='Example var', - workload='test_wl') + workload_variable( + "my_var", default="1.0", description="Example var", workload="test_wl" + ) - archive_pattern('{experiment_run_dir}/archive_test.*') + archive_pattern("{experiment_run_dir}/archive_test.*") - figure_of_merit('test_fom {var}', - fom_regex=r'Collect FOM (?P\w+)\s=\s(?P[0-9]+\.[0-9]+) seconds', - log_file='{log_file}', - group_name='test', units='s') + figure_of_merit( + "test_fom {var}", + fom_regex=r"Collect FOM (?P\w+)\s=\s(?P[0-9]+\.[0-9]+) seconds", + log_file="{log_file}", + group_name="test", + units="s", + ) - success_criteria('Run', mode='string', - match=r'Collect', file='{log_file}') + success_criteria("Run", mode="string", match=r"Collect", file="{log_file}") diff --git a/var/ramble/repos/builtin.mock/applications/glob-patterns/application.py b/var/ramble/repos/builtin.mock/applications/glob-patterns/application.py index 9267c7532..ffb57b5e5 100644 --- a/var/ramble/repos/builtin.mock/applications/glob-patterns/application.py +++ b/var/ramble/repos/builtin.mock/applications/glob-patterns/application.py @@ -12,46 +12,98 @@ class GlobPatterns(ExecutableApplication): name = "glob-patterns" - executable('test', 'base test {test_var} {glob_var} {baz_var}', use_mpi=False) - executable('test-foo', 'test foo {test_var} {glob_var} {baz_var} {mod_var}', use_mpi=False) - executable('test-bar', 'test bar {test_var} {glob_var} {baz_var}', use_mpi=True) - executable('baz', 'baz {test_var} {glob_var} {baz_var}', use_mpi=True) - - input_file('input', url='file:///tmp/test_file.log', - description='Not a file', extension='.log') - input_file('input-foo', url='file:///tmp/test_foo_file.log', - description='Not a file', extension='.log') - input_file('input-bar', url='file:///tmp/test_bar_file.log', - description='Not a file', extension='.log') - input_file('baz', url='file:///tmp/baz_file.log', - description='Not a file', extension='.log') - - workload('test_one_exec', executables=['test'], inputs=['input']) - workload('test_three_exec', executables=['test*'], inputs=['input*']) - workload('one_baz_exec', executables=['baz'], inputs=['baz']) - - environment_variable('env_var_test', 'set', description='Test env var', - workloads=['test_one_exec']) - environment_variable('env_var_glob', 'set', description='Test env var', - workloads=['test*']) - environment_variable('env_var_baz', 'set', description='Test env var', - workloads=['one_baz_exec']) - environment_variable('env_var_mod', 'set', description='Env var to be modified', - workloads=['test_three_exec']) - - workload_variable('test_var', default='wl_var_test', - description='Example var', - workloads=['test_one_exec']) - workload_variable('glob_var', default='wl_var_glob', - description='Example var', - workloads=['test*']) - workload_variable('baz_var', default='wl_var_baz', - description='Example var', - workloads=['one_baz_exec']) - workload_variable('var_mod', default='wl_var_mod', - description='Variable to be modified', - workloads=['test_three_exec']) - - figure_of_merit('test_fom', - fom_regex=r'(?P[0-9]+\.[0-9]+).*seconds.*', - group_name='test', units='s') + executable( + "test", "base test {test_var} {glob_var} {baz_var}", use_mpi=False + ) + executable( + "test-foo", + "test foo {test_var} {glob_var} {baz_var} {mod_var}", + use_mpi=False, + ) + executable( + "test-bar", "test bar {test_var} {glob_var} {baz_var}", use_mpi=True + ) + executable("baz", "baz {test_var} {glob_var} {baz_var}", use_mpi=True) + + input_file( + "input", + url="file:///tmp/test_file.log", + description="Not a file", + extension=".log", + ) + input_file( + "input-foo", + url="file:///tmp/test_foo_file.log", + description="Not a file", + extension=".log", + ) + input_file( + "input-bar", + url="file:///tmp/test_bar_file.log", + description="Not a file", + extension=".log", + ) + input_file( + "baz", + url="file:///tmp/baz_file.log", + description="Not a file", + extension=".log", + ) + + workload("test_one_exec", executables=["test"], inputs=["input"]) + workload("test_three_exec", executables=["test*"], inputs=["input*"]) + workload("one_baz_exec", executables=["baz"], inputs=["baz"]) + + environment_variable( + "env_var_test", + "set", + description="Test env var", + workloads=["test_one_exec"], + ) + environment_variable( + "env_var_glob", "set", description="Test env var", workloads=["test*"] + ) + environment_variable( + "env_var_baz", + "set", + description="Test env var", + workloads=["one_baz_exec"], + ) + environment_variable( + "env_var_mod", + "set", + description="Env var to be modified", + workloads=["test_three_exec"], + ) + + workload_variable( + "test_var", + default="wl_var_test", + description="Example var", + workloads=["test_one_exec"], + ) + workload_variable( + "glob_var", + default="wl_var_glob", + description="Example var", + workloads=["test*"], + ) + workload_variable( + "baz_var", + default="wl_var_baz", + description="Example var", + workloads=["one_baz_exec"], + ) + workload_variable( + "var_mod", + default="wl_var_mod", + description="Variable to be modified", + workloads=["test_three_exec"], + ) + + figure_of_merit( + "test_fom", + fom_regex=r"(?P[0-9]+\.[0-9]+).*seconds.*", + group_name="test", + units="s", + ) diff --git a/var/ramble/repos/builtin.mock/applications/input-test/application.py b/var/ramble/repos/builtin.mock/applications/input-test/application.py index cd13549cc..09439b310 100644 --- a/var/ramble/repos/builtin.mock/applications/input-test/application.py +++ b/var/ramble/repos/builtin.mock/applications/input-test/application.py @@ -13,17 +13,30 @@ class InputTest(ExecutableApplication): name = "input-test" - executable('test', 'echo "repo test"', use_mpi=False) + executable("test", 'echo "repo test"', use_mpi=False) cwd = os.getcwd() - input_file('test-input1', url=f'file://{cwd}/input1.tar.gz', - description='Test input') - - input_file('test-input2', url=f'file://{cwd}/input2.tar.gz', - description='Test input') - - input_file('test-input3', url=f'file://{cwd}/input3.txt', - expand=False, - description='Test input') - - workload('test', executables=['test'], inputs=['test-input1', 'test-input2', 'test-input3']) + input_file( + "test-input1", + url=f"file://{cwd}/input1.tar.gz", + description="Test input", + ) + + input_file( + "test-input2", + url=f"file://{cwd}/input2.tar.gz", + description="Test input", + ) + + input_file( + "test-input3", + url=f"file://{cwd}/input3.txt", + expand=False, + description="Test input", + ) + + workload( + "test", + executables=["test"], + inputs=["test-input1", "test-input2", "test-input3"], + ) diff --git a/var/ramble/repos/builtin.mock/applications/interleved-env-vars/application.py b/var/ramble/repos/builtin.mock/applications/interleved-env-vars/application.py index 5767e563d..8add190ec 100644 --- a/var/ramble/repos/builtin.mock/applications/interleved-env-vars/application.py +++ b/var/ramble/repos/builtin.mock/applications/interleved-env-vars/application.py @@ -12,26 +12,41 @@ class InterlevedEnvVars(ExecutableApplication): name = "interleved-env-vars" - executable('foo', 'bar', use_mpi=False) - executable('bar', 'baz', use_mpi=True) - executable('baz', 'foo', use_mpi=True) - - input_file('input', url='file:///tmp/test_file.log', - description='Not a file', extension='.log') - - workload('test_wl', executables=['builtin::env_vars', 'foo'], input='input') - workload('test_wl2', executables=['bar', 'builtin::env_vars'], input='input') - workload('test_wl3', executables=['baz'], input='input') - - environment_variable('FROM_DIRECTIVE', 'set', description='Test env var', - workloads=['test_wl', 'test_wl2', 'test_wl3']) - - workload_variable('my_var', default='1.0', - description='Example var', - workload='test_wl') - - archive_pattern('{experiment_run_dir}/archive_test.*') - - figure_of_merit('test_fom', - fom_regex=r'(?P[0-9]+\.[0-9]+).*seconds.*', - group_name='test', units='s') + executable("foo", "bar", use_mpi=False) + executable("bar", "baz", use_mpi=True) + executable("baz", "foo", use_mpi=True) + + input_file( + "input", + url="file:///tmp/test_file.log", + description="Not a file", + extension=".log", + ) + + workload( + "test_wl", executables=["builtin::env_vars", "foo"], input="input" + ) + workload( + "test_wl2", executables=["bar", "builtin::env_vars"], input="input" + ) + workload("test_wl3", executables=["baz"], input="input") + + environment_variable( + "FROM_DIRECTIVE", + "set", + description="Test env var", + workloads=["test_wl", "test_wl2", "test_wl3"], + ) + + workload_variable( + "my_var", default="1.0", description="Example var", workload="test_wl" + ) + + archive_pattern("{experiment_run_dir}/archive_test.*") + + figure_of_merit( + "test_fom", + fom_regex=r"(?P[0-9]+\.[0-9]+).*seconds.*", + group_name="test", + units="s", + ) diff --git a/var/ramble/repos/builtin.mock/applications/maintained-1/application.py b/var/ramble/repos/builtin.mock/applications/maintained-1/application.py index bef19f933..177649324 100644 --- a/var/ramble/repos/builtin.mock/applications/maintained-1/application.py +++ b/var/ramble/repos/builtin.mock/applications/maintained-1/application.py @@ -12,8 +12,11 @@ class Maintained1(ExecutableApplication): name = "maintained-1" - maintainers('maintainer-1') + maintainers("maintainer-1") - executable('foo', 'bar', use_mpi=False) + executable("foo", "bar", use_mpi=False) - workload('test_wl', executable='foo',) + workload( + "test_wl", + executable="foo", + ) diff --git a/var/ramble/repos/builtin.mock/applications/maintained-2/application.py b/var/ramble/repos/builtin.mock/applications/maintained-2/application.py index 9d5307462..89873a868 100644 --- a/var/ramble/repos/builtin.mock/applications/maintained-2/application.py +++ b/var/ramble/repos/builtin.mock/applications/maintained-2/application.py @@ -12,8 +12,11 @@ class Maintained2(ExecutableApplication): name = "maintained-2" - maintainers('maintainer-1') + maintainers("maintainer-1") - executable('foo', 'bar', use_mpi=False) + executable("foo", "bar", use_mpi=False) - workload('test_wl', executable='foo',) + workload( + "test_wl", + executable="foo", + ) diff --git a/var/ramble/repos/builtin.mock/applications/register-builtin/application.py b/var/ramble/repos/builtin.mock/applications/register-builtin/application.py index da5bc5328..b757ae390 100644 --- a/var/ramble/repos/builtin.mock/applications/register-builtin/application.py +++ b/var/ramble/repos/builtin.mock/applications/register-builtin/application.py @@ -12,30 +12,37 @@ class RegisterBuiltin(ExecutableApplication): name = "register-builtin" - executable('foo', 'bar', use_mpi=False) - executable('bar', 'baz', use_mpi=True) - executable('baz', 'foo', use_mpi=True) - - input_file('input', url='file:///tmp/test_file.log', - description='Not a file', extension='.log') - - workload('test_wl', executables=['foo'], input='input') - workload('test_wl2', executables=['bar'], input='input') - workload('test_wl3', executables=['baz'], input='input') - - workload_variable('my_var', default='1.0', - description='Example var', - workload='test_wl') - - archive_pattern('{experiment_run_dir}/archive_test.*') - - figure_of_merit('test_fom', - fom_regex=r'(?P[0-9]+\.[0-9]+).*seconds.*', - group_name='test', units='s') + executable("foo", "bar", use_mpi=False) + executable("bar", "baz", use_mpi=True) + executable("baz", "foo", use_mpi=True) + + input_file( + "input", + url="file:///tmp/test_file.log", + description="Not a file", + extension=".log", + ) + + workload("test_wl", executables=["foo"], input="input") + workload("test_wl2", executables=["bar"], input="input") + workload("test_wl3", executables=["baz"], input="input") + + workload_variable( + "my_var", default="1.0", description="Example var", workload="test_wl" + ) + + archive_pattern("{experiment_run_dir}/archive_test.*") + + figure_of_merit( + "test_fom", + fom_regex=r"(?P[0-9]+\.[0-9]+).*seconds.*", + group_name="test", + units="s", + ) # Test disabling the typically included env_vars builtin. - register_builtin('env_vars', required=False) - register_builtin('test_builtin', required=True) + register_builtin("env_vars", required=False) + register_builtin("test_builtin", required=True) def test_builtin(self): return ['echo "foo"'] diff --git a/var/ramble/repos/builtin.mock/applications/shared-context/application.py b/var/ramble/repos/builtin.mock/applications/shared-context/application.py index 0134582d2..041b71daa 100644 --- a/var/ramble/repos/builtin.mock/applications/shared-context/application.py +++ b/var/ramble/repos/builtin.mock/applications/shared-context/application.py @@ -12,23 +12,35 @@ class SharedContext(ExecutableApplication): name = "shared-context" - executable('foo', 'bar', use_mpi=False) - executable('bar', 'baz', use_mpi=True) - - input_file('input', url='file:///tmp/test_file.log', - description='Not a file', extension='.log') - - workload('test_wl', executable='foo', input='input') - workload('test_wl2', executable='bar', input='input') - - workload_variable('my_var', default='1.0', - description='Example var', - workload='test_wl') - - archive_pattern('{experiment_run_dir}/archive_test.*') - - figure_of_merit('test_fom', - fom_regex=r'(?P[0-9]+\.[0-9]+).*seconds.*', - group_name='test', units='s', contexts=['test_shared_context']) - - figure_of_merit_context('test_shared_context', regex=r'(?P[0-9]+\.[0-9]+).*seconds.*', output_format='matched_shared_context') + executable("foo", "bar", use_mpi=False) + executable("bar", "baz", use_mpi=True) + + input_file( + "input", + url="file:///tmp/test_file.log", + description="Not a file", + extension=".log", + ) + + workload("test_wl", executable="foo", input="input") + workload("test_wl2", executable="bar", input="input") + + workload_variable( + "my_var", default="1.0", description="Example var", workload="test_wl" + ) + + archive_pattern("{experiment_run_dir}/archive_test.*") + + figure_of_merit( + "test_fom", + fom_regex=r"(?P[0-9]+\.[0-9]+).*seconds.*", + group_name="test", + units="s", + contexts=["test_shared_context"], + ) + + figure_of_merit_context( + "test_shared_context", + regex=r"(?P[0-9]+\.[0-9]+).*seconds.*", + output_format="matched_shared_context", + ) diff --git a/var/ramble/repos/builtin.mock/applications/success-function/application.py b/var/ramble/repos/builtin.mock/applications/success-function/application.py index 9f0fffb34..602d290e4 100644 --- a/var/ramble/repos/builtin.mock/applications/success-function/application.py +++ b/var/ramble/repos/builtin.mock/applications/success-function/application.py @@ -12,13 +12,16 @@ class SuccessFunction(ExecutableApplication): name = "success-function" - executable('foo', 'echo "0.9 seconds"', use_mpi=False) + executable("foo", 'echo "0.9 seconds"', use_mpi=False) - workload('test_wl', executable='foo') + workload("test_wl", executable="foo") - figure_of_merit('test_fom', - fom_regex=r'(?P[0-9]+\.[0-9]+).*seconds.*', - group_name='test', units='s') + figure_of_merit( + "test_fom", + fom_regex=r"(?P[0-9]+\.[0-9]+).*seconds.*", + group_name="test", + units="s", + ) def evaluate_success(self): """Always fail, to help testing success functions""" diff --git a/var/ramble/repos/builtin.mock/applications/tagged-1/application.py b/var/ramble/repos/builtin.mock/applications/tagged-1/application.py index 9a0a3d6fe..98cf21236 100644 --- a/var/ramble/repos/builtin.mock/applications/tagged-1/application.py +++ b/var/ramble/repos/builtin.mock/applications/tagged-1/application.py @@ -12,8 +12,8 @@ class Tagged1(ExecutableApplication): name = "tagged-1" - tags('tag-1') + tags("tag-1") - executable('foo', 'bar', use_mpi=False) + executable("foo", "bar", use_mpi=False) - workload('test_wl', executable='foo') + workload("test_wl", executable="foo") diff --git a/var/ramble/repos/builtin.mock/applications/unmaintained-1/application.py b/var/ramble/repos/builtin.mock/applications/unmaintained-1/application.py index 11e0cfdb2..b0590d0f8 100644 --- a/var/ramble/repos/builtin.mock/applications/unmaintained-1/application.py +++ b/var/ramble/repos/builtin.mock/applications/unmaintained-1/application.py @@ -12,6 +12,9 @@ class Unmaintained1(ExecutableApplication): name = "unmaintained-1" - executable('foo', 'bar', use_mpi=False) + executable("foo", "bar", use_mpi=False) - workload('test_wl', executable='foo',) + workload( + "test_wl", + executable="foo", + ) diff --git a/var/ramble/repos/builtin.mock/applications/untagged-1/application.py b/var/ramble/repos/builtin.mock/applications/untagged-1/application.py index c64725afe..4c322d241 100644 --- a/var/ramble/repos/builtin.mock/applications/untagged-1/application.py +++ b/var/ramble/repos/builtin.mock/applications/untagged-1/application.py @@ -12,6 +12,6 @@ class Untagged1(ExecutableApplication): name = "untagged-1" - executable('foo', 'bar', use_mpi=False) + executable("foo", "bar", use_mpi=False) - workload('test_wl', executable='foo') + workload("test_wl", executable="foo") diff --git a/var/ramble/repos/builtin.mock/applications/workload-groups-inherited/application.py b/var/ramble/repos/builtin.mock/applications/workload-groups-inherited/application.py index 0771ea808..976d70498 100644 --- a/var/ramble/repos/builtin.mock/applications/workload-groups-inherited/application.py +++ b/var/ramble/repos/builtin.mock/applications/workload-groups-inherited/application.py @@ -14,9 +14,7 @@ class WorkloadGroupsInherited(WorkloadGroups): name = "workload-groups-inherited" - workload('test_wl3', executable='baz') + workload("test_wl3", executable="baz") # Test populated group applies existing vars to new workload - workload_group('test_wlg', - workloads=['test_wl3'], - mode='append') + workload_group("test_wlg", workloads=["test_wl3"], mode="append") diff --git a/var/ramble/repos/builtin.mock/applications/workload-groups/application.py b/var/ramble/repos/builtin.mock/applications/workload-groups/application.py index 43cb03caa..a68152538 100644 --- a/var/ramble/repos/builtin.mock/applications/workload-groups/application.py +++ b/var/ramble/repos/builtin.mock/applications/workload-groups/application.py @@ -12,27 +12,31 @@ class WorkloadGroups(ExecutableApplication): name = "workload-groups" - executable('foo', 'echo "bar"', use_mpi=False) - executable('bar', 'echo "baz"', use_mpi=False) + executable("foo", 'echo "bar"', use_mpi=False) + executable("bar", 'echo "baz"', use_mpi=False) - workload('test_wl', executable='foo') - workload('test_wl2', executable='bar') + workload("test_wl", executable="foo") + workload("test_wl2", executable="bar") # Test empty group - workload_group('empty', - workloads=[]) + workload_group("empty", workloads=[]) # Test populated group - workload_group('test_wlg', - workloads=['test_wl', 'test_wl2']) + workload_group("test_wlg", workloads=["test_wl", "test_wl2"]) # Test workload_variable that uses a group - workload_variable('test_var', default='2.0', - description='Test workload vars and groups', - workload_group='test_wlg') + workload_variable( + "test_var", + default="2.0", + description="Test workload vars and groups", + workload_group="test_wlg", + ) # Test passing both groups an explicit workloads - workload_variable('test_var_mixed', default='3.0', - description='Test vars for workload and groups', - workload='test_wl', - workload_group='test_wlg') + workload_variable( + "test_var_mixed", + default="3.0", + description="Test vars for workload and groups", + workload="test_wl", + workload_group="test_wlg", + ) diff --git a/var/ramble/repos/builtin.mock/applications/workload-tags/application.py b/var/ramble/repos/builtin.mock/applications/workload-tags/application.py index af92fd377..7c536e71a 100644 --- a/var/ramble/repos/builtin.mock/applications/workload-tags/application.py +++ b/var/ramble/repos/builtin.mock/applications/workload-tags/application.py @@ -12,8 +12,8 @@ class WorkloadTags(ExecutableApplication): name = "workload-tags" - executable('foo', 'echo "bar"', use_mpi=False) - executable('bar', 'echo "baz"', use_mpi=False) + executable("foo", 'echo "bar"', use_mpi=False) + executable("bar", 'echo "baz"', use_mpi=False) - workload('test_wl', executable='foo', tags=['wl-tag1', 'wl-shared-tag']) - workload('test_wl2', executable='bar', tags=['wl-tag2', 'wl-shared-tag']) + workload("test_wl", executable="foo", tags=["wl-tag1", "wl-shared-tag"]) + workload("test_wl2", executable="bar", tags=["wl-tag2", "wl-shared-tag"]) diff --git a/var/ramble/repos/builtin.mock/applications/zlib-configs/application.py b/var/ramble/repos/builtin.mock/applications/zlib-configs/application.py index 4d0f11afd..4eac4cfc2 100644 --- a/var/ramble/repos/builtin.mock/applications/zlib-configs/application.py +++ b/var/ramble/repos/builtin.mock/applications/zlib-configs/application.py @@ -12,17 +12,21 @@ class ZlibConfigs(SpackApplication): name = "zlib-configs" - software_spec('zlib', spack_spec='zlib') + software_spec("zlib", spack_spec="zlib") - executable('list_lib', 'ls {zlib}/lib', use_mpi=False) + executable("list_lib", "ls {zlib}/lib", use_mpi=False) - workload('ensure_installed', executable='list_lib') + workload("ensure_installed", executable="list_lib") - package_manager_config('enable_debug', 'config:debug:true') + package_manager_config("enable_debug", "config:debug:true") - figure_of_merit('zlib_installed', - fom_regex=r'(?Plibz.so.*)', group_name='lib_name', - units='') + figure_of_merit( + "zlib_installed", + fom_regex=r"(?Plibz.so.*)", + group_name="lib_name", + units="", + ) - success_criteria('zlib_installed', mode='string', - match=r'libz.so', file='{log_file}') + success_criteria( + "zlib_installed", mode="string", match=r"libz.so", file="{log_file}" + ) diff --git a/var/ramble/repos/builtin.mock/applications/zlib/application.py b/var/ramble/repos/builtin.mock/applications/zlib/application.py index d14862b03..d00e1371a 100644 --- a/var/ramble/repos/builtin.mock/applications/zlib/application.py +++ b/var/ramble/repos/builtin.mock/applications/zlib/application.py @@ -12,15 +12,19 @@ class Zlib(SpackApplication): name = "zlib" - software_spec('zlib', spack_spec='zlib') + software_spec("zlib", spack_spec="zlib") - executable('list_lib', 'ls {zlib}/lib', use_mpi=False) + executable("list_lib", "ls {zlib}/lib", use_mpi=False) - workload('ensure_installed', executable='list_lib') + workload("ensure_installed", executable="list_lib") - figure_of_merit('zlib_installed', - fom_regex=r'(?Plibz.so.*)', group_name='lib_name', - units='') + figure_of_merit( + "zlib_installed", + fom_regex=r"(?Plibz.so.*)", + group_name="lib_name", + units="", + ) - success_criteria('zlib_installed', mode='string', - match=r'libz.so', file='{log_file}') + success_criteria( + "zlib_installed", mode="string", match=r"libz.so", file="{log_file}" + ) diff --git a/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-paths/modifier.py b/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-paths/modifier.py index c3f225027..36e9c9a75 100644 --- a/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-paths/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-paths/modifier.py @@ -12,11 +12,13 @@ class AppendEnvVarModPaths(BasicModifier): """Define a modifier with only an environment variable modification using the append method and a colon separator""" + name = "append-env-var-mod-paths" - tags('test') + tags("test") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") - env_var_modification('test_var', modification='test_val', method='append', - mode='test') + env_var_modification( + "test_var", modification="test_val", method="append", mode="test" + ) diff --git a/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-vars/modifier.py b/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-vars/modifier.py index a2cb0e96d..76240c440 100644 --- a/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-vars/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/append-env-var-mod-vars/modifier.py @@ -12,11 +12,17 @@ class AppendEnvVarModVars(BasicModifier): """Define a modifier with only an environment variable modification using the append method and a non-colon separator""" + name = "append-env-var-mod-vars" - tags('test') + tags("test") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") - env_var_modification('test_var', modification='test_val', method='append', - mode='test', separator=',') + env_var_modification( + "test_var", + modification="test_val", + method="append", + mode="test", + separator=",", + ) diff --git a/var/ramble/repos/builtin.mock/modifiers/glob-patterns-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/glob-patterns-mod/modifier.py index 0d62d4046..aa75a159a 100644 --- a/var/ramble/repos/builtin.mock/modifiers/glob-patterns-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/glob-patterns-mod/modifier.py @@ -14,17 +14,24 @@ class GlobPatternsMod(BasicModifier): This modifier tests globbing in the modifier language. """ + name = "glob-patterns-mod" - tags('test') + tags("test") - mode('base', description='This is a base mode with no modifications') - mode('test-glob', description='This test mode turns on mods using globbing') - default_mode('base') + mode("base", description="This is a base mode with no modifications") + mode( + "test-glob", description="This test mode turns on mods using globbing" + ) + default_mode("base") - variable_modification('var_mod', '{mod_var}', modes=['test*']) + variable_modification("var_mod", "{mod_var}", modes=["test*"]) - env_var_modification('env_var_mod', 'modded', modes=['test*']) + env_var_modification("env_var_mod", "modded", modes=["test*"]) - modifier_variable('mod_var', default='var_mod_modified', - description='This is a modifier variable', modes=['test*']) + modifier_variable( + "mod_var", + default="var_mod_modified", + description="This is a modifier variable", + modes=["test*"], + ) diff --git a/var/ramble/repos/builtin.mock/modifiers/maintained-1/modifier.py b/var/ramble/repos/builtin.mock/modifiers/maintained-1/modifier.py index 47124c7b8..706c57079 100644 --- a/var/ramble/repos/builtin.mock/modifiers/maintained-1/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/maintained-1/modifier.py @@ -11,10 +11,11 @@ class Maintained1(BasicModifier): """Define a maintained modifier""" + name = "maintained-1" - tags('test') + tags("test") - maintainers('maintainer-1') + maintainers("maintainer-1") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") diff --git a/var/ramble/repos/builtin.mock/modifiers/maintained-2/modifier.py b/var/ramble/repos/builtin.mock/modifiers/maintained-2/modifier.py index 66cd5a7cd..dad7898a8 100644 --- a/var/ramble/repos/builtin.mock/modifiers/maintained-2/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/maintained-2/modifier.py @@ -11,10 +11,11 @@ class Maintained2(BasicModifier): """Define a maintained modifier""" + name = "maintained-2" - tags('test') + tags("test") - maintainers('maintainer-1') + maintainers("maintainer-1") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") diff --git a/var/ramble/repos/builtin.mock/modifiers/mod-phase/modifier.py b/var/ramble/repos/builtin.mock/modifiers/mod-phase/modifier.py index 382c2e3f9..f96cee301 100644 --- a/var/ramble/repos/builtin.mock/modifiers/mod-phase/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/mod-phase/modifier.py @@ -12,18 +12,23 @@ class ModPhase(BasicModifier): """Define a modifier that defines a new phase with register_phase""" + name = "mod-phase" - tags('test') + tags("test") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") - register_phase('first_phase', pipeline='setup', run_before=['get_inputs']) + register_phase("first_phase", pipeline="setup", run_before=["get_inputs"]) def _first_phase(self, workspace, app_inst=None): - logger.all_msg('Inside a phase: first_phase') + logger.all_msg("Inside a phase: first_phase") - register_phase('after_make_experiments', pipeline='setup', run_after=['make_experiments']) + register_phase( + "after_make_experiments", + pipeline="setup", + run_after=["make_experiments"], + ) def _after_make_experiments(self, workspace, app_inst=None): - logger.all_msg('Inside a phase: after_make_experiments') + logger.all_msg("Inside a phase: after_make_experiments") diff --git a/var/ramble/repos/builtin.mock/modifiers/multiple-modes-no-default/modifier.py b/var/ramble/repos/builtin.mock/modifiers/multiple-modes-no-default/modifier.py index 95c3e05d4..c567dfd4e 100644 --- a/var/ramble/repos/builtin.mock/modifiers/multiple-modes-no-default/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/multiple-modes-no-default/modifier.py @@ -11,10 +11,11 @@ class MultipleModesNoDefault(BasicModifier): """Define modifier with multiple modes and no default mode""" + name = "multiple-modes-no-default" - tags('test') + tags("test") - mode('test_mode1', description='This is the first test mode') + mode("test_mode1", description="This is the first test mode") - mode('test_mode2', description='This is the second test mode') + mode("test_mode2", description="This is the second test mode") diff --git a/var/ramble/repos/builtin.mock/modifiers/multiple-modes-with-default/modifier.py b/var/ramble/repos/builtin.mock/modifiers/multiple-modes-with-default/modifier.py index 3e0be401d..daf742511 100644 --- a/var/ramble/repos/builtin.mock/modifiers/multiple-modes-with-default/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/multiple-modes-with-default/modifier.py @@ -11,12 +11,13 @@ class MultipleModesWithDefault(BasicModifier): """Define modifier with multiple modes and a default mode""" + name = "multiple-modes-with-default" - tags('test') + tags("test") - mode('test_mode1', description='This is the first test mode') + mode("test_mode1", description="This is the first test mode") - mode('test_mode2', description='This is the second test mode') + mode("test_mode2", description="This is the second test mode") - default_mode('test_mode2') + default_mode("test_mode2") diff --git a/var/ramble/repos/builtin.mock/modifiers/no-docstring-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/no-docstring-mod/modifier.py index a1b522a24..c06f9fd0c 100644 --- a/var/ramble/repos/builtin.mock/modifiers/no-docstring-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/no-docstring-mod/modifier.py @@ -12,6 +12,6 @@ class NoDocstringMod(BasicModifier): name = "no-docstring-mod" - tags('test') + tags("test") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") diff --git a/var/ramble/repos/builtin.mock/modifiers/no-variable-mods/modifier.py b/var/ramble/repos/builtin.mock/modifiers/no-variable-mods/modifier.py index a0aee9ffe..4ef90fa7f 100644 --- a/var/ramble/repos/builtin.mock/modifiers/no-variable-mods/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/no-variable-mods/modifier.py @@ -11,8 +11,9 @@ class NoVariableMods(BasicModifier): """Define modifier with no variable modifications""" + name = "no-variable-mods" - tags('test') + tags("test") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") diff --git a/var/ramble/repos/builtin.mock/modifiers/prepare-analysis/modifier.py b/var/ramble/repos/builtin.mock/modifiers/prepare-analysis/modifier.py index a58d16fe7..0979e5ff6 100644 --- a/var/ramble/repos/builtin.mock/modifiers/prepare-analysis/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/prepare-analysis/modifier.py @@ -11,17 +11,23 @@ class PrepareAnalysis(BasicModifier): """Define a modifier that defines a prepare_analysis hook""" + name = "prepare-analysis" - tags('test') + tags("test") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") - _log_file = '{experiment_run_dir}/.prepare_analysis_hook_data' + _log_file = "{experiment_run_dir}/.prepare_analysis_hook_data" - figure_of_merit('test-fom', fom_regex='Test fom = (?P.*)', log_file=_log_file, - units='', group_name='fom') + figure_of_merit( + "test-fom", + fom_regex="Test fom = (?P.*)", + log_file=_log_file, + units="", + group_name="fom", + ) def _prepare_analysis(self, workspace): - with open(self.expander.expand_var(self._log_file), 'w+') as f: - f.write('Test fom = This test worked') + with open(self.expander.expand_var(self._log_file), "w+") as f: + f.write("Test fom = This test worked") diff --git a/var/ramble/repos/builtin.mock/modifiers/prepend-env-var-mod-paths/modifier.py b/var/ramble/repos/builtin.mock/modifiers/prepend-env-var-mod-paths/modifier.py index 23e60f9f5..8f3542b45 100644 --- a/var/ramble/repos/builtin.mock/modifiers/prepend-env-var-mod-paths/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/prepend-env-var-mod-paths/modifier.py @@ -12,11 +12,13 @@ class PrependEnvVarModPaths(BasicModifier): """Define a modifier with only an environment variable modification using the prepend method and a colon separator""" + name = "prepend-env-var-mod-paths" - tags('test') + tags("test") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") - env_var_modification('test_var', modification='test_val', method='prepend', - mode='test') + env_var_modification( + "test_var", modification="test_val", method="prepend", mode="test" + ) diff --git a/var/ramble/repos/builtin.mock/modifiers/set-env-var-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/set-env-var-mod/modifier.py index 4db104a65..f3adfdd79 100644 --- a/var/ramble/repos/builtin.mock/modifiers/set-env-var-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/set-env-var-mod/modifier.py @@ -12,14 +12,24 @@ class SetEnvVarMod(BasicModifier): """Define a modifier with only an environment variable modification using the set method""" + name = "set-env-var-mod" - tags('test') + tags("test") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") - modifier_variable('mask_test', default='0x0', description='Test mask var', - modes=['test'], expandable=False) + modifier_variable( + "mask_test", + default="0x0", + description="Test mask var", + modes=["test"], + expandable=False, + ) - env_var_modification('test_var', modification='test_val', method='set', mode='test') - env_var_modification('mask_env_var', modification='{mask_test}', method='set', mode='test') + env_var_modification( + "test_var", modification="test_val", method="set", mode="test" + ) + env_var_modification( + "mask_env_var", modification="{mask_test}", method="set", mode="test" + ) diff --git a/var/ramble/repos/builtin.mock/modifiers/spack-failed-reqs/modifier.py b/var/ramble/repos/builtin.mock/modifiers/spack-failed-reqs/modifier.py index 8820fab1d..81a309f83 100644 --- a/var/ramble/repos/builtin.mock/modifiers/spack-failed-reqs/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/spack-failed-reqs/modifier.py @@ -11,22 +11,30 @@ class SpackFailedReqs(SpackModifier): """Define spack modifier requirements that will fail""" + name = "spack-mod" - tags('test') + tags("test") - mode('default', description='This is the default mode for the spack-failed-reqs modifier') + mode( + "default", + description="This is the default mode for the spack-failed-reqs modifier", + ) - define_compiler('mod_compiler', - spack_spec='mod_compiler@1.1 target=x86_64', - compiler_spec='mod_compiler@1.1') + define_compiler( + "mod_compiler", + spack_spec="mod_compiler@1.1 target=x86_64", + compiler_spec="mod_compiler@1.1", + ) - software_spec('mod_package1', - spack_spec='mod_package1@1.1', - compiler='mod_compiler') + software_spec( + "mod_package1", spack_spec="mod_package1@1.1", compiler="mod_compiler" + ) - software_spec('mod_package2', - spack_spec='mod_package2@1.1', - compiler='mod_compiler') + software_spec( + "mod_package2", spack_spec="mod_package2@1.1", compiler="mod_compiler" + ) - package_manager_requirement('list not-a-package', validation_type='not_empty', modes=['default']) + package_manager_requirement( + "list not-a-package", validation_type="not_empty", modes=["default"] + ) diff --git a/var/ramble/repos/builtin.mock/modifiers/spack-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/spack-mod/modifier.py index a0d3844e7..95a80eb56 100644 --- a/var/ramble/repos/builtin.mock/modifiers/spack-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/spack-mod/modifier.py @@ -11,29 +11,44 @@ class SpackMod(SpackModifier): """Define spack modifier with various software aspects""" - name = "spack-mod" - - tags('test') - - mode('default', description='This is the default mode for the spack-mod') - - package_manager_config('enable_debug', 'config:debug:true') - define_compiler('mod_compiler', - spack_spec='mod_compiler@1.1 target=x86_64', - compiler_spec='mod_compiler@1.1') - - software_spec('mod_package1', - spack_spec='mod_package1@1.1', - compiler='mod_compiler') - - software_spec('mod_package2', - spack_spec='mod_package2@1.1', - compiler='mod_compiler') + name = "spack-mod" - package_manager_requirement('list not-a-package', validation_type='empty', modes=['default']) - package_manager_requirement('list zlib', validation_type='not_empty', modes=['default']) - package_manager_requirement('info zlib', validation_type='contains_regex', modes=['default'], - regex=r'\s*Safe versions:\s*') - package_manager_requirement('info zlib', validation_type='does_not_contain_regex', - modes=['default'], regex=r'\s*Broken versions:\s*') + tags("test") + + mode("default", description="This is the default mode for the spack-mod") + + package_manager_config("enable_debug", "config:debug:true") + + define_compiler( + "mod_compiler", + spack_spec="mod_compiler@1.1 target=x86_64", + compiler_spec="mod_compiler@1.1", + ) + + software_spec( + "mod_package1", spack_spec="mod_package1@1.1", compiler="mod_compiler" + ) + + software_spec( + "mod_package2", spack_spec="mod_package2@1.1", compiler="mod_compiler" + ) + + package_manager_requirement( + "list not-a-package", validation_type="empty", modes=["default"] + ) + package_manager_requirement( + "list zlib", validation_type="not_empty", modes=["default"] + ) + package_manager_requirement( + "info zlib", + validation_type="contains_regex", + modes=["default"], + regex=r"\s*Safe versions:\s*", + ) + package_manager_requirement( + "info zlib", + validation_type="does_not_contain_regex", + modes=["default"], + regex=r"\s*Broken versions:\s*", + ) diff --git a/var/ramble/repos/builtin.mock/modifiers/success-criteria/modifier.py b/var/ramble/repos/builtin.mock/modifiers/success-criteria/modifier.py index 3454343af..2b5440fb4 100644 --- a/var/ramble/repos/builtin.mock/modifiers/success-criteria/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/success-criteria/modifier.py @@ -11,15 +11,26 @@ class SuccessCriteria(BasicModifier): """Define a modifier with a success criteria""" + name = "success-criteria" - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") - success_criteria('status', mode='string', match='.*Experiment status: SUCCESS', file='{log_file}') + success_criteria( + "status", + mode="string", + match=".*Experiment status: SUCCESS", + file="{log_file}", + ) - variable_modification('experiment_status', modification='Experiment status: SUCCESS', method='set', mode='test') + variable_modification( + "experiment_status", + modification="Experiment status: SUCCESS", + method="set", + mode="test", + ) - register_builtin('echo_status', required=True) + register_builtin("echo_status", required=True) def echo_status(self): return ['echo "Experiment status: {experiment_status}" >> {log_file}'] diff --git a/var/ramble/repos/builtin.mock/modifiers/tagged-1/modifier.py b/var/ramble/repos/builtin.mock/modifiers/tagged-1/modifier.py index 3c8706b2e..af16fdaa3 100644 --- a/var/ramble/repos/builtin.mock/modifiers/tagged-1/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/tagged-1/modifier.py @@ -11,8 +11,9 @@ class Tagged1(BasicModifier): """Define a tagged modifier""" + name = "tagged-1" - tags('tag-1') + tags("tag-1") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") diff --git a/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py index 44be66531..ec3109856 100644 --- a/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/test-mod/modifier.py @@ -14,41 +14,78 @@ class TestMod(BasicModifier): This modifier is just a test of various aspects of the modifier language. """ - name = "test-mod" - - tags('test') - - mode('test', description='This is a test mode') - default_mode('test') - - mode('app-scope', description='This is a test mode at the application scope') - - mode('wl-scope', description='This is a test mode at the workload scope') - - mode('exp-scope', description='This is a test mode at the experiment scope') - - variable_modification('mpi_command', 'echo "prefix_mpi_command" >> {log_file}; ', method='prepend', modes=['test']) - - variable_modification('analysis_log', '{experiment_run_dir}/test_analysis.log', method='set', modes=['test']) - software_spec('analysis_spec', spack_spec='analysis_pkg@1.1', compiler='gcc') - - fom_regex = r'(?Pfom_context)(?P.*)' - - figure_of_merit('test_mod_fom', fom_regex=fom_regex, group_name='fom', - units='', log_file='{analysis_log}', contexts=['test_mod_context']) - - figure_of_merit('shared_context_fom', fom_regex=fom_regex, group_name='fom', - units='', log_file='{analysis_log}', contexts=['test_shared_context']) - - figure_of_merit_context('test_mod_context', regex=fom_regex, output_format='{context}') - - figure_of_merit_context('test_shared_context', regex=fom_regex, output_format='matched_shared_context') - - register_builtin('test_builtin', required=True, injection_method='append') + name = "test-mod" - test_attr = 'test_value' + tags("test") + + mode("test", description="This is a test mode") + default_mode("test") + + mode( + "app-scope", description="This is a test mode at the application scope" + ) + + mode("wl-scope", description="This is a test mode at the workload scope") + + mode( + "exp-scope", description="This is a test mode at the experiment scope" + ) + + variable_modification( + "mpi_command", + 'echo "prefix_mpi_command" >> {log_file}; ', + method="prepend", + modes=["test"], + ) + + variable_modification( + "analysis_log", + "{experiment_run_dir}/test_analysis.log", + method="set", + modes=["test"], + ) + + software_spec( + "analysis_spec", spack_spec="analysis_pkg@1.1", compiler="gcc" + ) + + fom_regex = r"(?Pfom_context)(?P.*)" + + figure_of_merit( + "test_mod_fom", + fom_regex=fom_regex, + group_name="fom", + units="", + log_file="{analysis_log}", + contexts=["test_mod_context"], + ) + + figure_of_merit( + "shared_context_fom", + fom_regex=fom_regex, + group_name="fom", + units="", + log_file="{analysis_log}", + contexts=["test_shared_context"], + ) + + figure_of_merit_context( + "test_mod_context", regex=fom_regex, output_format="{context}" + ) + + figure_of_merit_context( + "test_shared_context", + regex=fom_regex, + output_format="matched_shared_context", + ) + + register_builtin("test_builtin", required=True, injection_method="append") + + test_attr = "test_value" def test_builtin(self): - return ['echo "fom_contextFOM_GOES_HERE" >> {analysis_log}', - f'echo "{self.test_attr}"' + ' >> {analysis_log}'] + return [ + 'echo "fom_contextFOM_GOES_HERE" >> {analysis_log}', + f'echo "{self.test_attr}"' + " >> {analysis_log}", + ] diff --git a/var/ramble/repos/builtin.mock/modifiers/unmaintained-1/modifier.py b/var/ramble/repos/builtin.mock/modifiers/unmaintained-1/modifier.py index 92be2893d..35aa72e15 100644 --- a/var/ramble/repos/builtin.mock/modifiers/unmaintained-1/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/unmaintained-1/modifier.py @@ -11,8 +11,9 @@ class Unmaintained1(BasicModifier): """Define an unmaintained modifier""" + name = "test-mod" - tags('test') + tags("test") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") diff --git a/var/ramble/repos/builtin.mock/modifiers/unset-env-var-mod/modifier.py b/var/ramble/repos/builtin.mock/modifiers/unset-env-var-mod/modifier.py index 0ddbd252e..a55f1e687 100644 --- a/var/ramble/repos/builtin.mock/modifiers/unset-env-var-mod/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/unset-env-var-mod/modifier.py @@ -12,10 +12,11 @@ class UnsetEnvVarMod(BasicModifier): """Define a modifier with only an environment variable modification using the unset method""" + name = "unset-env-var-mod" - tags('test') + tags("test") - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") - env_var_modification('test_var', method='unset', mode='test') + env_var_modification("test_var", method="unset", mode="test") diff --git a/var/ramble/repos/builtin.mock/modifiers/untagged-1/modifier.py b/var/ramble/repos/builtin.mock/modifiers/untagged-1/modifier.py index 64fd6e10d..e3f94ca0b 100644 --- a/var/ramble/repos/builtin.mock/modifiers/untagged-1/modifier.py +++ b/var/ramble/repos/builtin.mock/modifiers/untagged-1/modifier.py @@ -11,6 +11,7 @@ class Untagged1(BasicModifier): """Define an untagged modifier""" + name = "untagged-1" - mode('test', description='This is a test mode') + mode("test", description="This is a test mode") diff --git a/var/ramble/repos/builtin/applications/cloverleaf/application.py b/var/ramble/repos/builtin/applications/cloverleaf/application.py index 32b3491db..55fe6a8ed 100644 --- a/var/ramble/repos/builtin/applications/cloverleaf/application.py +++ b/var/ramble/repos/builtin/applications/cloverleaf/application.py @@ -13,106 +13,171 @@ class Cloverleaf(SpackApplication): - '''Define CLOVERLEAF application''' - name = 'cloverleaf' - - maintainers('rfbgo') - - tags('cfd', 'fluid', 'dynamics', 'euler', 'miniapp', 'minibenchmark', 'mini-benchmark') - - define_compiler('gcc12', spack_spec='gcc@12.2.0') - - software_spec('ompi414', spack_spec='openmpi@4.1.4 +legacylaunchers +cxx', - compiler='gcc12') - software_spec('cloverleaf', - spack_spec='cloverleaf@1.1 build=ref', - compiler='gcc12') - - required_package('cloverleaf') - - executable('execute', 'clover_leaf', use_mpi=True) - - executable('get_input', - template=['cp {cloverleaf}/doc/tests/clover_{workload_name}.in {experiment_run_dir}/clover.in'], - use_mpi=False) - - executable('get_default_input', template=['cp {cloverleaf}/doc/tests/clover.in {experiment_run_dir}/clover.in'], - use_mpi=False) - - workload('bm_short_c', executables=['get_input', 'execute']) - workload('bm_short', executables=['get_input', 'execute']) - workload('qa', executables=['get_input', 'execute']) - workload('sodbig', executables=['get_input', 'execute']) - workload('sodx', executables=['get_input', 'execute']) - workload('sodxy', executables=['get_input', 'execute']) - workload('sody', executables=['get_input', 'execute']) - workload('clover', executables=['get_default_input', 'execute']) + """Define CLOVERLEAF application""" + + name = "cloverleaf" + + maintainers("rfbgo") + + tags( + "cfd", + "fluid", + "dynamics", + "euler", + "miniapp", + "minibenchmark", + "mini-benchmark", + ) + + define_compiler("gcc12", spack_spec="gcc@12.2.0") + + software_spec( + "ompi414", + spack_spec="openmpi@4.1.4 +legacylaunchers +cxx", + compiler="gcc12", + ) + software_spec( + "cloverleaf", spack_spec="cloverleaf@1.1 build=ref", compiler="gcc12" + ) + + required_package("cloverleaf") + + executable("execute", "clover_leaf", use_mpi=True) + + executable( + "get_input", + template=[ + "cp {cloverleaf}/doc/tests/clover_{workload_name}.in {experiment_run_dir}/clover.in" + ], + use_mpi=False, + ) + + executable( + "get_default_input", + template=[ + "cp {cloverleaf}/doc/tests/clover.in {experiment_run_dir}/clover.in" + ], + use_mpi=False, + ) + + workload("bm_short_c", executables=["get_input", "execute"]) + workload("bm_short", executables=["get_input", "execute"]) + workload("qa", executables=["get_input", "execute"]) + workload("sodbig", executables=["get_input", "execute"]) + workload("sodx", executables=["get_input", "execute"]) + workload("sodxy", executables=["get_input", "execute"]) + workload("sody", executables=["get_input", "execute"]) + workload("clover", executables=["get_default_input", "execute"]) # 2 through 8K for k in range(1, 14): - workload('bm' + str(2**k), executables=['get_input', 'execute']) + workload("bm" + str(2**k), executables=["get_input", "execute"]) # 2 through 8K for k in range(1, 14): - workload('bm' + str(2**k) + '_short', executables=['get_input', 'execute']) + workload( + "bm" + str(2**k) + "_short", executables=["get_input", "execute"] + ) # 1 through 16K for k in range(15): - workload('bm' + str(2**k) + 's_short', executables=['get_input', 'execute']) - - log_str = os.path.join(Expander.expansion_str('experiment_run_dir'), 'clover.out') - - floating_point_regex = r'[\+\-]*[0-9]*\.*[0-9]+E*[\+\-]*[0-9]*' - - step_count_regex = r'\s*Step\s+(?P[0-9]+).*\s+timestep\s+(?P' + floating_point_regex + r')' - - wall_clock_regex = r'\s*Wall clock\s+(?P[0-9]+\.[0-9]+)' - - figure_of_merit('Timestep', log_file=log_str, - fom_regex=step_count_regex, - group_name='timestep', - units='s', contexts=['step'] - ) - - figure_of_merit('Wall Clock', log_file=log_str, - fom_regex=wall_clock_regex, - group_name='wall_clock', - units='s', contexts=['step'] - ) - - figure_of_merit_context('step', - regex=step_count_regex, - output_format='Step {step}') - - step_summary_regex = (r'\s*step:\s+(?P[0-9]+)\s+' + - r'(?P' + floating_point_regex + r')\s+' + - r'(?P' + floating_point_regex + r')\s+' + - r'(?P' + floating_point_regex + r')\s+' + - r'(?P' + floating_point_regex + r')\s+' + - r'(?P' + floating_point_regex + r')\s+' + - r'(?P' + floating_point_regex + r')\s+' + - r'(?P' + floating_point_regex + r')') - - figure_of_merit('Total step count', log_file=log_str, - fom_regex=step_summary_regex, - group_name='step', - units='' - ) - - figure_of_merit('Final Kinetic Energy', log_file=log_str, - fom_regex=step_summary_regex, - group_name='kinetic_energy', - units='Joules' - ) - - figure_of_merit('Total Elapsed Time', log_file=log_str, - fom_regex=wall_clock_regex, - group_name='wall_clock', - units='s' - ) - - figure_of_merit('First step overhead', log_file=log_str, - fom_regex=(r'\s*First step overhead\s+(?P' + floating_point_regex + r')'), - group_name='overhead', - units='s' - ) + workload( + "bm" + str(2**k) + "s_short", executables=["get_input", "execute"] + ) + + log_str = os.path.join( + Expander.expansion_str("experiment_run_dir"), "clover.out" + ) + + floating_point_regex = r"[\+\-]*[0-9]*\.*[0-9]+E*[\+\-]*[0-9]*" + + step_count_regex = ( + r"\s*Step\s+(?P[0-9]+).*\s+timestep\s+(?P" + + floating_point_regex + + r")" + ) + + wall_clock_regex = r"\s*Wall clock\s+(?P[0-9]+\.[0-9]+)" + + figure_of_merit( + "Timestep", + log_file=log_str, + fom_regex=step_count_regex, + group_name="timestep", + units="s", + contexts=["step"], + ) + + figure_of_merit( + "Wall Clock", + log_file=log_str, + fom_regex=wall_clock_regex, + group_name="wall_clock", + units="s", + contexts=["step"], + ) + + figure_of_merit_context( + "step", regex=step_count_regex, output_format="Step {step}" + ) + + step_summary_regex = ( + r"\s*step:\s+(?P[0-9]+)\s+" + + r"(?P" + + floating_point_regex + + r")\s+" + + r"(?P" + + floating_point_regex + + r")\s+" + + r"(?P" + + floating_point_regex + + r")\s+" + + r"(?P" + + floating_point_regex + + r")\s+" + + r"(?P" + + floating_point_regex + + r")\s+" + + r"(?P" + + floating_point_regex + + r")\s+" + + r"(?P" + + floating_point_regex + + r")" + ) + + figure_of_merit( + "Total step count", + log_file=log_str, + fom_regex=step_summary_regex, + group_name="step", + units="", + ) + + figure_of_merit( + "Final Kinetic Energy", + log_file=log_str, + fom_regex=step_summary_regex, + group_name="kinetic_energy", + units="Joules", + ) + + figure_of_merit( + "Total Elapsed Time", + log_file=log_str, + fom_regex=wall_clock_regex, + group_name="wall_clock", + units="s", + ) + + figure_of_merit( + "First step overhead", + log_file=log_str, + fom_regex=( + r"\s*First step overhead\s+(?P" + + floating_point_regex + + r")" + ), + group_name="overhead", + units="s", + ) diff --git a/var/ramble/repos/builtin/applications/elk/application.py b/var/ramble/repos/builtin/applications/elk/application.py index c2c2f6dae..b90721e45 100644 --- a/var/ramble/repos/builtin/applications/elk/application.py +++ b/var/ramble/repos/builtin/applications/elk/application.py @@ -10,7 +10,7 @@ class Elk(ExecutableApplication): - ''' + """ An all-electron full-potential linearised augmented-plane wave (LAPW) code with many advanced features. Written originally at Karl-Franzens-Universität Graz as a milestone of the EXCITING EU Research @@ -25,40 +25,70 @@ class Elk(ExecutableApplication): user to manage both the package install (eg via yum), and the environment through explicit paths. By default, yum installs to /usr/lib64/openmpi/bin/elk_openmpi - ''' + """ - name = 'elk' + name = "elk" - tags('LAPW', 'density-functional-theory') + tags("LAPW", "density-functional-theory") - input_file('examples', - url='https://master.dl.sourceforge.net/project/elk/elk-4.3.6.tgz', - sha256='efd2893a55143ac045656d2acd1407becf773408a116c90771ed3ee9fede35c9', - description='Main installer which contains various example input decks and species' - ) + input_file( + "examples", + url="https://master.dl.sourceforge.net/project/elk/elk-4.3.6.tgz", + sha256="efd2893a55143ac045656d2acd1407becf773408a116c90771ed3ee9fede35c9", + description="Main installer which contains various example input decks and species", + ) - executable('execute', '{install_prefix}/elk_openmpi', use_mpi=True) - executable('copy', 'cp {input_path}/elk.in {experiment_run_dir}/.', use_mpi=False) - executable('update_input', "sed -i 's|../../../species/|{examples}/species/|g' elk.in", use_mpi=False) + executable("execute", "{install_prefix}/elk_openmpi", use_mpi=True) + executable( + "copy", "cp {input_path}/elk.in {experiment_run_dir}/.", use_mpi=False + ) + executable( + "update_input", + "sed -i 's|../../../species/|{examples}/species/|g' elk.in", + use_mpi=False, + ) - workload('Cu', executables=['copy', 'update_input', 'execute'], input='examples') + workload( + "Cu", executables=["copy", "update_input", "execute"], input="examples" + ) - workload_variable('install_prefix', default='/usr/lib64/openmpi/bin', - description='Install path to main executable', - workloads=['Cu']) + workload_variable( + "install_prefix", + default="/usr/lib64/openmpi/bin", + description="Install path to main executable", + workloads=["Cu"], + ) - workload_variable('input_path', default='{examples}/examples/basic/Cu', - description='Path to input deck', - workloads=['Cu']) + workload_variable( + "input_path", + default="{examples}/examples/basic/Cu", + description="Path to input deck", + workloads=["Cu"], + ) - success_criteria('prints_done', mode='string', match=r'.*Elk code stopped.*', file='{experiment_run_dir}/{experiment_name}.out') + success_criteria( + "prints_done", + mode="string", + match=r".*Elk code stopped.*", + file="{experiment_run_dir}/{experiment_name}.out", + ) - output_file = 'INFO.OUT' - metrics = ['Timings (CPU seconds)', 'initialisation', 'Hamiltonian and overlap matrix set up', 'first-variational eigenvalue equation', 'charge density calculation', 'potential calculation', 'total'] + output_file = "INFO.OUT" + metrics = [ + "Timings (CPU seconds)", + "initialisation", + "Hamiltonian and overlap matrix set up", + "first-variational eigenvalue equation", + "charge density calculation", + "potential calculation", + "total", + ] for metric in metrics: - figure_of_merit(metric, - log_file=output_file, - fom_regex=rf'\s*(?P{metric})\s+:\s+(?P[0-9]+\.[0-9]*).*', - group_name='value', units='s' - ) + figure_of_merit( + metric, + log_file=output_file, + fom_regex=rf"\s*(?P{metric})\s+:\s+(?P[0-9]+\.[0-9]*).*", + group_name="value", + units="s", + ) diff --git a/var/ramble/repos/builtin/applications/gromacs/application.py b/var/ramble/repos/builtin/applications/gromacs/application.py index eeb957161..30364993d 100644 --- a/var/ramble/repos/builtin/applications/gromacs/application.py +++ b/var/ramble/repos/builtin/applications/gromacs/application.py @@ -12,220 +12,400 @@ class Gromacs(SpackApplication): - '''Define a Gromacs application''' - name = 'gromacs' - - maintainers('douglasjacobsen') - - tags('molecular-dynamics') - - define_compiler('gcc9', spack_spec='gcc@9.3.0') - software_spec('impi2018', spack_spec='intel-mpi@2018.4.274') - software_spec('gromacs', spack_spec='gromacs@2020.5', compiler='gcc9') - - executable('pre-process', '{grompp} ' + - '-f {input_path}/{type}.mdp ' + - '-c {input_path}/conf.gro ' + - '-p {input_path}/topol.top ' + - '-o exp_input.tpr', use_mpi=False, output_capture=OUTPUT_CAPTURE.ALL) - executable('execute-gen', '{mdrun} {notunepme} -dlb {dlb} ' + - '{verbose} -resetstep {resetstep} -noconfout -nsteps {nsteps} ' + - '-s exp_input.tpr', use_mpi=True, output_capture=OUTPUT_CAPTURE.ALL) - executable('execute', '{mdrun} {notunepme} -dlb {dlb} ' + - '{verbose} -resetstep {resetstep} -noconfout -nsteps {nsteps} ' + - '-s {input_path}', use_mpi=True, output_capture=OUTPUT_CAPTURE.ALL) - - input_file('water_gmx50_bare', url='https://ftp.gromacs.org/pub/benchmarks/water_GMX50_bare.tar.gz', - sha256='2219c10acb97787f80f6638132bad3ff2ca1e68600eef1bc8b89d9560e74c66a', - description='') - input_file('water_bare_hbonds', url='https://ftp.gromacs.org/pub/benchmarks/water_bare_hbonds.tar.gz', - sha256='b2e09d30f5c6b00ecf1c13ea6fa715ad132747863ef89f983f6c09a872cf2776', - description='') - input_file('lignocellulose', - url='https://repository.prace-ri.eu/ueabs/GROMACS/1.2/GROMACS_TestCaseB.tar.gz', - sha256='8a12db0232465e1d47c6a4eb89f615cdbbdc8fc360a86088b131331bd462f35c', - description='A model of cellulose and lignocellulosic biomass in an aqueous ' + - 'solution. This system of 3.3M atoms is inhomogeneous, at ' + - 'least with GROMACS 4.5. This system uses reaction-field' + - 'electrostatics instead of PME and therefore should scale well.') - input_file('HECBioSim', - url='https://github.com/victorusu/GROMACS_Benchmark_Suite/archive/refs/tags/1.0.0.tar.gz', - sha256='9cb2ad61ec2a422fc33578047e7cb2fd2c37ae9a75a6162d662fa2b711e9737f', - description='https://www.hecbiosim.ac.uk/access-hpc/benchmarks') - - input_file('BenchPEP', url='https://www.mpinat.mpg.de/benchPEP.zip', - sha256='f11745201dbb9e6a29a39cb016ee8123f6b0f519b250c94660f0a9623e497b22', - description='12M Atoms, Peptides in Water, 2fs time step, all bonds constrained. https://www.mpinat.mpg.de/grubmueller/bench') - - input_file('BenchPEP_h', url='https://www.mpinat.mpg.de/benchPEP-h.zip', - sha256='3ca8902fd9a6cf005b266f83b57217397b4ba4af987b97dc01e04185bd098bce', - description='12M Atoms, Peptides in Water, 2fs time step, h-bonds constrained. https://www.mpinat.mpg.de/grubmueller/bench') - - input_file('BenchMEM', url='https://www.mpinat.mpg.de/benchMEM.zip', - sha256='3c1c8cd4f274d532f48c4668e1490d389486850d6b3b258dfad4581aa11380a4', - description='82k atoms, protein in membrane surrounded by water, 2 fs time step. https://www.mpinat.mpg.de/grubmueller/bench') - - input_file('BenchRIB', url='https://www.mpinat.mpg.de/benchRIB.zip', - sha256='39acb014a79ed9a9ff2ad6294a2c09f9b85ea6986dfc204a3639814503eeb60a', - description='2 M atoms, ribosome in water, 4 fs time step. https://www.mpinat.mpg.de/grubmueller/bench') - - input_file('JCP_benchmarks', - url='https://zenodo.org/record/3893789/files/GROMACS_heterogeneous_parallelization_benchmark_info_and_systems_JCP.tar.gz?download=1', - sha256='82449291f44f4d5b7e5c192d688b57b7c2a2e267fe8b12e7a15b5d68f96c7b20', - description='GROMACS_heterogeneous_parallelization_benchmark_info_and_systems_JCP') - - workload('water_gmx50', executables=['pre-process', 'execute-gen'], - input='water_gmx50_bare') - workload('water_bare', executables=['pre-process', 'execute-gen'], - input='water_bare_hbonds') - workload('lignocellulose', executables=['execute'], - input='lignocellulose') - workload('hecbiosim', executables=['execute'], - input='HECBioSim') - workload('benchpep', executables=['execute'], - input='BenchPEP') - workload('benchpep_h', executables=['execute'], - input='BenchPEP_h') - workload('benchmem', executables=['execute'], - input='BenchMEM') - workload('benchrib', executables=['execute'], - input='BenchRIB') - workload('stmv_rf', executables=['pre-process', 'execute-gen'], - input='JCP_benchmarks') - workload('stmv_pme', executables=['pre-process', 'execute-gen'], - input='JCP_benchmarks') - workload('rnase_cubic', executables=['pre-process', 'execute-gen'], - input='JCP_benchmarks') - workload('ion_channel', executables=['pre-process', 'execute-gen'], - input='JCP_benchmarks') - workload('adh_dodec', executables=['pre-process', 'execute-gen'], - input='JCP_benchmarks') + """Define a Gromacs application""" + + name = "gromacs" + + maintainers("douglasjacobsen") + + tags("molecular-dynamics") + + define_compiler("gcc9", spack_spec="gcc@9.3.0") + software_spec("impi2018", spack_spec="intel-mpi@2018.4.274") + software_spec("gromacs", spack_spec="gromacs@2020.5", compiler="gcc9") + + executable( + "pre-process", + "{grompp} " + + "-f {input_path}/{type}.mdp " + + "-c {input_path}/conf.gro " + + "-p {input_path}/topol.top " + + "-o exp_input.tpr", + use_mpi=False, + output_capture=OUTPUT_CAPTURE.ALL, + ) + executable( + "execute-gen", + "{mdrun} {notunepme} -dlb {dlb} " + + "{verbose} -resetstep {resetstep} -noconfout -nsteps {nsteps} " + + "-s exp_input.tpr", + use_mpi=True, + output_capture=OUTPUT_CAPTURE.ALL, + ) + executable( + "execute", + "{mdrun} {notunepme} -dlb {dlb} " + + "{verbose} -resetstep {resetstep} -noconfout -nsteps {nsteps} " + + "-s {input_path}", + use_mpi=True, + output_capture=OUTPUT_CAPTURE.ALL, + ) + + input_file( + "water_gmx50_bare", + url="https://ftp.gromacs.org/pub/benchmarks/water_GMX50_bare.tar.gz", + sha256="2219c10acb97787f80f6638132bad3ff2ca1e68600eef1bc8b89d9560e74c66a", + description="", + ) + input_file( + "water_bare_hbonds", + url="https://ftp.gromacs.org/pub/benchmarks/water_bare_hbonds.tar.gz", + sha256="b2e09d30f5c6b00ecf1c13ea6fa715ad132747863ef89f983f6c09a872cf2776", + description="", + ) + input_file( + "lignocellulose", + url="https://repository.prace-ri.eu/ueabs/GROMACS/1.2/GROMACS_TestCaseB.tar.gz", + sha256="8a12db0232465e1d47c6a4eb89f615cdbbdc8fc360a86088b131331bd462f35c", + description="A model of cellulose and lignocellulosic biomass in an aqueous " + + "solution. This system of 3.3M atoms is inhomogeneous, at " + + "least with GROMACS 4.5. This system uses reaction-field" + + "electrostatics instead of PME and therefore should scale well.", + ) + input_file( + "HECBioSim", + url="https://github.com/victorusu/GROMACS_Benchmark_Suite/archive/refs/tags/1.0.0.tar.gz", + sha256="9cb2ad61ec2a422fc33578047e7cb2fd2c37ae9a75a6162d662fa2b711e9737f", + description="https://www.hecbiosim.ac.uk/access-hpc/benchmarks", + ) + + input_file( + "BenchPEP", + url="https://www.mpinat.mpg.de/benchPEP.zip", + sha256="f11745201dbb9e6a29a39cb016ee8123f6b0f519b250c94660f0a9623e497b22", + description="12M Atoms, Peptides in Water, 2fs time step, all bonds constrained. https://www.mpinat.mpg.de/grubmueller/bench", + ) + + input_file( + "BenchPEP_h", + url="https://www.mpinat.mpg.de/benchPEP-h.zip", + sha256="3ca8902fd9a6cf005b266f83b57217397b4ba4af987b97dc01e04185bd098bce", + description="12M Atoms, Peptides in Water, 2fs time step, h-bonds constrained. https://www.mpinat.mpg.de/grubmueller/bench", + ) + + input_file( + "BenchMEM", + url="https://www.mpinat.mpg.de/benchMEM.zip", + sha256="3c1c8cd4f274d532f48c4668e1490d389486850d6b3b258dfad4581aa11380a4", + description="82k atoms, protein in membrane surrounded by water, 2 fs time step. https://www.mpinat.mpg.de/grubmueller/bench", + ) + + input_file( + "BenchRIB", + url="https://www.mpinat.mpg.de/benchRIB.zip", + sha256="39acb014a79ed9a9ff2ad6294a2c09f9b85ea6986dfc204a3639814503eeb60a", + description="2 M atoms, ribosome in water, 4 fs time step. https://www.mpinat.mpg.de/grubmueller/bench", + ) + + input_file( + "JCP_benchmarks", + url="https://zenodo.org/record/3893789/files/GROMACS_heterogeneous_parallelization_benchmark_info_and_systems_JCP.tar.gz?download=1", + sha256="82449291f44f4d5b7e5c192d688b57b7c2a2e267fe8b12e7a15b5d68f96c7b20", + description="GROMACS_heterogeneous_parallelization_benchmark_info_and_systems_JCP", + ) + + workload( + "water_gmx50", + executables=["pre-process", "execute-gen"], + input="water_gmx50_bare", + ) + workload( + "water_bare", + executables=["pre-process", "execute-gen"], + input="water_bare_hbonds", + ) + workload("lignocellulose", executables=["execute"], input="lignocellulose") + workload("hecbiosim", executables=["execute"], input="HECBioSim") + workload("benchpep", executables=["execute"], input="BenchPEP") + workload("benchpep_h", executables=["execute"], input="BenchPEP_h") + workload("benchmem", executables=["execute"], input="BenchMEM") + workload("benchrib", executables=["execute"], input="BenchRIB") + workload( + "stmv_rf", + executables=["pre-process", "execute-gen"], + input="JCP_benchmarks", + ) + workload( + "stmv_pme", + executables=["pre-process", "execute-gen"], + input="JCP_benchmarks", + ) + workload( + "rnase_cubic", + executables=["pre-process", "execute-gen"], + input="JCP_benchmarks", + ) + workload( + "ion_channel", + executables=["pre-process", "execute-gen"], + input="JCP_benchmarks", + ) + workload( + "adh_dodec", + executables=["pre-process", "execute-gen"], + input="JCP_benchmarks", + ) all_workloads = [ - 'water_gmx50', - 'water_bare', - 'lignocellulose', - 'hecbiosim', - 'benchpep', - 'benchpep_h', - 'benchmem', - 'benchrib', - 'stmv_rf', - 'stmv_pme', - 'rnase_cubic', - 'ion_channel', - 'adh_dodec', + "water_gmx50", + "water_bare", + "lignocellulose", + "hecbiosim", + "benchpep", + "benchpep_h", + "benchmem", + "benchrib", + "stmv_rf", + "stmv_pme", + "rnase_cubic", + "ion_channel", + "adh_dodec", ] - workload_variable('gmx', default='gmx_mpi', - description='Name of the gromacs binary', - workloads=all_workloads) - workload_variable('grompp', default='{gmx} grompp', - description='How to run grompp', - workloads=all_workloads) - workload_variable('mdrun', default='{gmx} mdrun', - description='How to run mdrun', - workloads=all_workloads) - workload_variable('nsteps', default=str(20000), - description='Simulation steps', - workloads=all_workloads) - workload_variable('resetstep', default='{str(int(0.9*{nsteps}))}', - description='Reset performance counters at this step', - workloads=all_workloads) - workload_variable('verbose', default="", values=['', '-v'], - description='Set to empty string to run without verbose mode', - workloads=all_workloads) - workload_variable('notunepme', default='-notunepme', values=['', '-notunepme'], - description='Whether to set -notunepme for mdrun', - workloads=all_workloads) - workload_variable('dlb', default='yes', values=['yes', 'no'], - description='Whether to use dynamic load balancing for mdrun', - workloads=all_workloads) - - workload_variable('size', default='1536', - values=['0000.65', '0000.96', '0001.5', - '0003', '0006', '0012', '0024', - '0048', '0096', '0192', '0384', - '0768', '1536', '3072'], - description='Workload size', - workloads=['water_gmx50', 'water_bare'], - expandable=False) - workload_variable('type', default='pme', - description='Workload type.', - values=['pme', 'rf'], - workloads=['water_gmx50', 'water_bare']) - workload_variable('input_path', default='{water_gmx50_bare}/{size}', - description='Input path for water GMX50', - workload='water_gmx50') - workload_variable('input_path', default='{water_bare_hbonds}/{size}', - description='Input path for water bare hbonds', - workload='water_bare') - workload_variable('input_path', default='{lignocellulose}/lignocellulose-rf.tpr', - description='Input path for lignocellulose', - workload='lignocellulose') - workload_variable('type', default='Crambin', - description='Workload type. Valid values are ''Crambin'', ''Glutamine-Binding-Protein'', ''hEGFRDimer'', ''hEGFRDimerPair'', ''hEGFRDimerSmallerPL'', ''hEGFRtetramerPair''', - workload='hecbiosim') - workload_variable('input_path', default='{HECBioSim}/HECBioSim/{type}/benchmark.tpr', - description='Input path for hecbiosim', - workload='hecbiosim') - workload_variable('input_path', default='{BenchPEP}/benchPEP.tpr', - description='Input path for Bench PEP workload', - workload='benchpep') - workload_variable('input_path', default='{BenchMEM}/benchMEM.tpr', - description='Input path for Bench MEM workload', - workload='benchmem') - workload_variable('input_path', default='{BenchRIB}/benchRIB.tpr', - description='Input path for Bench RIB workload', - workload='benchrib') - workload_variable('input_path', default='{BenchPEP_h}/benchPEP-h.tpr', - description='Input path for Bench PEP-h workload', - workload='benchpep_h') - workload_variable('type', default='rf_nvt', - description='Workload type for JCP_benchmarks', - workload='stmv_rf') - workload_variable('type', default='pme_nvt', - description='Workload type for JCP_benchmarks', - workload='stmv_pme') - workload_variable('type', default='grompp', - description='Workload type for JCP_benchmarks', - workloads=['ion_channel', 'rnase_cubic']) - workload_variable('input_path', default='{JCP_benchmarks}/stmv', - description='Input path for JCP_benchmark {workload_name}', - workloads=['stmv_rf', 'stmv_pme']) - workload_variable('input_path', default='{JCP_benchmarks}/{workload_name}', - description='Input path for JCP_benchmark {workload_name}', - workloads=['ion_channel', 'rnase_cubic']) - workload_variable('input_path', default='{JCP_benchmarks}/{workload_name}', - description='Input path for JCP_benchmark {workload_name}', - workloads=['adh_dodec']) - workload_variable('type', default='pme_verlet', - description='Workload type for JCP_benchmarks', - workloads=['adh_dodec']) - - log_str = os.path.join(Expander.expansion_str('experiment_run_dir'), - 'md.log') - - figure_of_merit('Core Time', log_file=log_str, - fom_regex=r'\s+Time:\s+(?P[0-9]+\.[0-9]+).*', - group_name='core_time', units='s') - - figure_of_merit('Wall Time', log_file=log_str, - fom_regex=r'\s+Time:\s+[0-9]+\.[0-9]+\s+' + - r'(?P[0-9]+\.[0-9]+).*', - group_name='wall_time', units='s') - - figure_of_merit('Percent Core Time', log_file=log_str, - fom_regex=r'\s+Time:\s+[0-9]+\.[0-9]+\s+[0-9]+\.[0-9]+\s+' + - r'(?P[0-9]+\.[0-9]+).*', - group_name='perc_core_time', units='%') - - figure_of_merit('Nanosecs per day', log_file=log_str, - fom_regex=r'Performance:\s+' + - r'(?P[0-9]+\.[0-9]+).*', - group_name='ns_per_day', units='ns/day') - - figure_of_merit('Hours per nanosec', log_file=log_str, - fom_regex=r'Performance:\s+[0-9]+\.[0-9]+\s+' + - r'(?P[0-9]+\.[0-9]+).*', - group_name='hours_per_ns', units='hours/ns') + workload_variable( + "gmx", + default="gmx_mpi", + description="Name of the gromacs binary", + workloads=all_workloads, + ) + workload_variable( + "grompp", + default="{gmx} grompp", + description="How to run grompp", + workloads=all_workloads, + ) + workload_variable( + "mdrun", + default="{gmx} mdrun", + description="How to run mdrun", + workloads=all_workloads, + ) + workload_variable( + "nsteps", + default=str(20000), + description="Simulation steps", + workloads=all_workloads, + ) + workload_variable( + "resetstep", + default="{str(int(0.9*{nsteps}))}", + description="Reset performance counters at this step", + workloads=all_workloads, + ) + workload_variable( + "verbose", + default="", + values=["", "-v"], + description="Set to empty string to run without verbose mode", + workloads=all_workloads, + ) + workload_variable( + "notunepme", + default="-notunepme", + values=["", "-notunepme"], + description="Whether to set -notunepme for mdrun", + workloads=all_workloads, + ) + workload_variable( + "dlb", + default="yes", + values=["yes", "no"], + description="Whether to use dynamic load balancing for mdrun", + workloads=all_workloads, + ) + + workload_variable( + "size", + default="1536", + values=[ + "0000.65", + "0000.96", + "0001.5", + "0003", + "0006", + "0012", + "0024", + "0048", + "0096", + "0192", + "0384", + "0768", + "1536", + "3072", + ], + description="Workload size", + workloads=["water_gmx50", "water_bare"], + expandable=False, + ) + workload_variable( + "type", + default="pme", + description="Workload type.", + values=["pme", "rf"], + workloads=["water_gmx50", "water_bare"], + ) + workload_variable( + "input_path", + default="{water_gmx50_bare}/{size}", + description="Input path for water GMX50", + workload="water_gmx50", + ) + workload_variable( + "input_path", + default="{water_bare_hbonds}/{size}", + description="Input path for water bare hbonds", + workload="water_bare", + ) + workload_variable( + "input_path", + default="{lignocellulose}/lignocellulose-rf.tpr", + description="Input path for lignocellulose", + workload="lignocellulose", + ) + workload_variable( + "type", + default="Crambin", + description="Workload type. Valid values are " + "Crambin" + ", " + "Glutamine-Binding-Protein" + ", " + "hEGFRDimer" + ", " + "hEGFRDimerPair" + ", " + "hEGFRDimerSmallerPL" + ", " + "hEGFRtetramerPair" + "", + workload="hecbiosim", + ) + workload_variable( + "input_path", + default="{HECBioSim}/HECBioSim/{type}/benchmark.tpr", + description="Input path for hecbiosim", + workload="hecbiosim", + ) + workload_variable( + "input_path", + default="{BenchPEP}/benchPEP.tpr", + description="Input path for Bench PEP workload", + workload="benchpep", + ) + workload_variable( + "input_path", + default="{BenchMEM}/benchMEM.tpr", + description="Input path for Bench MEM workload", + workload="benchmem", + ) + workload_variable( + "input_path", + default="{BenchRIB}/benchRIB.tpr", + description="Input path for Bench RIB workload", + workload="benchrib", + ) + workload_variable( + "input_path", + default="{BenchPEP_h}/benchPEP-h.tpr", + description="Input path for Bench PEP-h workload", + workload="benchpep_h", + ) + workload_variable( + "type", + default="rf_nvt", + description="Workload type for JCP_benchmarks", + workload="stmv_rf", + ) + workload_variable( + "type", + default="pme_nvt", + description="Workload type for JCP_benchmarks", + workload="stmv_pme", + ) + workload_variable( + "type", + default="grompp", + description="Workload type for JCP_benchmarks", + workloads=["ion_channel", "rnase_cubic"], + ) + workload_variable( + "input_path", + default="{JCP_benchmarks}/stmv", + description="Input path for JCP_benchmark {workload_name}", + workloads=["stmv_rf", "stmv_pme"], + ) + workload_variable( + "input_path", + default="{JCP_benchmarks}/{workload_name}", + description="Input path for JCP_benchmark {workload_name}", + workloads=["ion_channel", "rnase_cubic"], + ) + workload_variable( + "input_path", + default="{JCP_benchmarks}/{workload_name}", + description="Input path for JCP_benchmark {workload_name}", + workloads=["adh_dodec"], + ) + workload_variable( + "type", + default="pme_verlet", + description="Workload type for JCP_benchmarks", + workloads=["adh_dodec"], + ) + + log_str = os.path.join( + Expander.expansion_str("experiment_run_dir"), "md.log" + ) + + figure_of_merit( + "Core Time", + log_file=log_str, + fom_regex=r"\s+Time:\s+(?P[0-9]+\.[0-9]+).*", + group_name="core_time", + units="s", + ) + + figure_of_merit( + "Wall Time", + log_file=log_str, + fom_regex=r"\s+Time:\s+[0-9]+\.[0-9]+\s+" + + r"(?P[0-9]+\.[0-9]+).*", + group_name="wall_time", + units="s", + ) + + figure_of_merit( + "Percent Core Time", + log_file=log_str, + fom_regex=r"\s+Time:\s+[0-9]+\.[0-9]+\s+[0-9]+\.[0-9]+\s+" + + r"(?P[0-9]+\.[0-9]+).*", + group_name="perc_core_time", + units="%", + ) + + figure_of_merit( + "Nanosecs per day", + log_file=log_str, + fom_regex=r"Performance:\s+" + r"(?P[0-9]+\.[0-9]+).*", + group_name="ns_per_day", + units="ns/day", + ) + + figure_of_merit( + "Hours per nanosec", + log_file=log_str, + fom_regex=r"Performance:\s+[0-9]+\.[0-9]+\s+" + + r"(?P[0-9]+\.[0-9]+).*", + group_name="hours_per_ns", + units="hours/ns", + ) diff --git a/var/ramble/repos/builtin/applications/hmmer/application.py b/var/ramble/repos/builtin/applications/hmmer/application.py index e4ed969bd..ad56c5222 100644 --- a/var/ramble/repos/builtin/applications/hmmer/application.py +++ b/var/ramble/repos/builtin/applications/hmmer/application.py @@ -12,7 +12,7 @@ class Hmmer(SpackApplication): - '''HMMER is used for searching sequence databases for sequence homologs, + """HMMER is used for searching sequence databases for sequence homologs, and for making sequence alignments. It implements methods using probabilistic models called profile hidden Markov models (profile HMMs). @@ -20,64 +20,88 @@ class Hmmer(SpackApplication): families), Rfam (for non-coded RNA families), Dfam (for repetitive DNA based), etc. - Homepage: www.hmmer.org''' + Homepage: www.hmmer.org""" - name = 'hmmer' + name = "hmmer" - maintainers('rfbgo') + maintainers("rfbgo") - tags('molecular-dynamics', 'hidden-markov-models', 'bio-molecule') + tags("molecular-dynamics", "hidden-markov-models", "bio-molecule") - define_compiler('gcc9', spack_spec='gcc@9.3.0') + define_compiler("gcc9", spack_spec="gcc@9.3.0") - software_spec('impi_2018', spack_spec='intel-mpi@2018.4.274') + software_spec("impi_2018", spack_spec="intel-mpi@2018.4.274") - software_spec('hmmer', spack_spec='hmmer@3.3.2', compiler='gcc9') + software_spec("hmmer", spack_spec="hmmer@3.3.2", compiler="gcc9") # This would ideally not use the current_release, as the package will need to be manually updated per release # Here current_release == 'Pfam36.0' - input_file('Pfam_A', - url='http://ftp.ebi.ac.uk/pub/databases/Pfam/current_release/Pfam-A.hmm.gz', - sha256='8779125bd2ed533886c559a5808903800f5c262798e5b086d276c2b829566b3a', - description='The Pfam database is a large collection of protein families, ' + - 'each represented by multiple sequence alignments and hidden Markov models (HMMs).') - - input_file('uniprot_sprot_fasta', - url='https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot.fasta.gz', - sha256='b0cacbf62fbf02f410dfd5e7963099762e53ed9f6fb123ba655583a9f7f0adab', - description='Uniprot Swiss Prot fasta search input') - - executable('execute', - 'hmmsearch --mpi -o hmmsearch.out {database_path} {input_path}', - use_mpi=True) - - executable('tail_hmmsearch_out', - 'tail -100 hmmsearch.out', - use_mpi=False) - - workload('fasta_pfam', executables=['execute', 'tail_hmmsearch_out'], - inputs=['Pfam_A', 'uniprot_sprot_fasta']) - - workload_variable('database_path', default='{Pfam_A}/Pfam-A.hmm', - description='Database path for Pfam-A', - workloads=['fasta_pfam']) - - workload_variable('input_path', default='{uniprot_sprot_fasta}/uniprot_sprot.fasta', - description='Input path for uniprot_sprot.fasta', - workloads=['fasta_pfam']) - - hmmsearch_out = os.path.join(Expander.expansion_str('experiment_run_dir'), - 'hmmsearch.out') - - out_file = os.path.join(Expander.expansion_str('experiment_run_dir'), - Expander.expansion_str('experiment_name') + '.out') - - figure_of_merit('Elapsed time', - fom_regex=r'# CPU.*Elapsed:\s+(?P[0-9]+:[0-9]+:[0-9]+\.*[0-9]*)\s*$', - group_name='elapsed_time', log_file=out_file, units='hms') - - figure_of_merit('Million dynamic programming cells per second', - fom_regex=r'^#\s*Mc/sec:\s+(?P[0-9]+)\s*', - group_name='mc_per_sec', log_file=out_file, units='Mc/s') - - success_criteria('ok', mode='string', match=r'^\[ok\]$', file=out_file) + input_file( + "Pfam_A", + url="http://ftp.ebi.ac.uk/pub/databases/Pfam/current_release/Pfam-A.hmm.gz", + sha256="8779125bd2ed533886c559a5808903800f5c262798e5b086d276c2b829566b3a", + description="The Pfam database is a large collection of protein families, " + + "each represented by multiple sequence alignments and hidden Markov models (HMMs).", + ) + + input_file( + "uniprot_sprot_fasta", + url="https://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot.fasta.gz", + sha256="b0cacbf62fbf02f410dfd5e7963099762e53ed9f6fb123ba655583a9f7f0adab", + description="Uniprot Swiss Prot fasta search input", + ) + + executable( + "execute", + "hmmsearch --mpi -o hmmsearch.out {database_path} {input_path}", + use_mpi=True, + ) + + executable("tail_hmmsearch_out", "tail -100 hmmsearch.out", use_mpi=False) + + workload( + "fasta_pfam", + executables=["execute", "tail_hmmsearch_out"], + inputs=["Pfam_A", "uniprot_sprot_fasta"], + ) + + workload_variable( + "database_path", + default="{Pfam_A}/Pfam-A.hmm", + description="Database path for Pfam-A", + workloads=["fasta_pfam"], + ) + + workload_variable( + "input_path", + default="{uniprot_sprot_fasta}/uniprot_sprot.fasta", + description="Input path for uniprot_sprot.fasta", + workloads=["fasta_pfam"], + ) + + hmmsearch_out = os.path.join( + Expander.expansion_str("experiment_run_dir"), "hmmsearch.out" + ) + + out_file = os.path.join( + Expander.expansion_str("experiment_run_dir"), + Expander.expansion_str("experiment_name") + ".out", + ) + + figure_of_merit( + "Elapsed time", + fom_regex=r"# CPU.*Elapsed:\s+(?P[0-9]+:[0-9]+:[0-9]+\.*[0-9]*)\s*$", + group_name="elapsed_time", + log_file=out_file, + units="hms", + ) + + figure_of_merit( + "Million dynamic programming cells per second", + fom_regex=r"^#\s*Mc/sec:\s+(?P[0-9]+)\s*", + group_name="mc_per_sec", + log_file=out_file, + units="Mc/s", + ) + + success_criteria("ok", mode="string", match=r"^\[ok\]$", file=out_file) diff --git a/var/ramble/repos/builtin/applications/hostname/application.py b/var/ramble/repos/builtin/applications/hostname/application.py index 1b708b9a7..39c61e42c 100644 --- a/var/ramble/repos/builtin/applications/hostname/application.py +++ b/var/ramble/repos/builtin/applications/hostname/application.py @@ -1,4 +1,3 @@ - # Copyright 2022-2024 The Ramble Authors # # Licensed under the Apache License, Version 2.0 [0-9]+\.[0-9]+)user.*', - group_name='user_time', units='s') + figure_of_merit( + "user time from file", + log_file=time_file, + fom_regex=r"(?P[0-9]+\.[0-9]+)user.*", + group_name="user_time", + units="s", + ) - figure_of_merit('user time', - fom_regex=r'(?P[0-9]+\.[0-9]+)user.*', - group_name='user_time', units='s') + figure_of_merit( + "user time", + fom_regex=r"(?P[0-9]+\.[0-9]+)user.*", + group_name="user_time", + units="s", + ) - figure_of_merit('possible hostname', - fom_regex=r'(?P\S+)\s*', - group_name='hostname', units='') + figure_of_merit( + "possible hostname", + fom_regex=r"(?P\S+)\s*", + group_name="hostname", + units="", + ) - success_criteria('wrote_anything', mode='string', match=r'.*') + success_criteria("wrote_anything", mode="string", match=r".*") diff --git a/var/ramble/repos/builtin/applications/hpcc/application.py b/var/ramble/repos/builtin/applications/hpcc/application.py index 2fba0c057..5138a2755 100644 --- a/var/ramble/repos/builtin/applications/hpcc/application.py +++ b/var/ramble/repos/builtin/applications/hpcc/application.py @@ -11,7 +11,7 @@ class Hpcc(SpackApplication): - '''Define the HPCC application + """Define the HPCC application HPCC is a collection of multiple benchmarks, which include: - HPL @@ -21,89 +21,121 @@ class Hpcc(SpackApplication): - MPIRandomAccess - FFT - LatencyBandwidth - ''' - name = 'hpcc' - - maintainers('rfbgo') - - tags('benchmark-app', 'mini-app', 'benchmark', 'DGEMM') - - define_compiler('gcc9', spack_spec='gcc@9.3.0') - - software_spec('impi2018', - spack_spec='intel-mpi@2018.4.274') - - software_spec('hpcc', - spack_spec='hpcc@1.5.0', - compiler='gcc9') - - required_package('hpcc') - - input_file('hpccinf', url='{config_file}', expand=False, - sha256='fe9e5f4118c1b40980e162dc3c52d224fd6287e9706b95bb40ae7dfc96b38622', - description='Input/Config file for HPCC benchmark') - - executable('copy-config', template='cp -R {workload_input_dir}/* {experiment_run_dir}/.', use_mpi=False) - - executable('execute', 'hpcc', use_mpi=True) - - workload('standard', executables=['copy-config', 'execute'], input='hpccinf') - - workload_variable('config_file', default='https://raw.githubusercontent.com/icl-utk-edu/hpcc/1.5.0/_hpccinf.txt', - description='Default config file', - workloads=['standard']) - - workload_variable('out_file', default='{experiment_run_dir}/hpccoutf.txt', - description='Output file for results', - workloads=['standard']) - - output_sections = ['HPL', 'LatencyBandwidth', 'MPIFFT', 'MPIRandomAccess_LCG', - 'MPIRandomAccess', 'PTRANS', 'SingleDGEMM', 'SingleFFT', - 'SingleRandomAccess_LCG', 'SingleRandomAccess', 'SingleSTREAM', - 'StarDGEMM', 'StarFFT', 'StarRandomAccess_LCG', 'StarRandomAccess', - 'StarSTREAM', 'Summary'] + """ + + name = "hpcc" + + maintainers("rfbgo") + + tags("benchmark-app", "mini-app", "benchmark", "DGEMM") + + define_compiler("gcc9", spack_spec="gcc@9.3.0") + + software_spec("impi2018", spack_spec="intel-mpi@2018.4.274") + + software_spec("hpcc", spack_spec="hpcc@1.5.0", compiler="gcc9") + + required_package("hpcc") + + input_file( + "hpccinf", + url="{config_file}", + expand=False, + sha256="fe9e5f4118c1b40980e162dc3c52d224fd6287e9706b95bb40ae7dfc96b38622", + description="Input/Config file for HPCC benchmark", + ) + + executable( + "copy-config", + template="cp -R {workload_input_dir}/* {experiment_run_dir}/.", + use_mpi=False, + ) + + executable("execute", "hpcc", use_mpi=True) + + workload( + "standard", executables=["copy-config", "execute"], input="hpccinf" + ) + + workload_variable( + "config_file", + default="https://raw.githubusercontent.com/icl-utk-edu/hpcc/1.5.0/_hpccinf.txt", + description="Default config file", + workloads=["standard"], + ) + + workload_variable( + "out_file", + default="{experiment_run_dir}/hpccoutf.txt", + description="Output file for results", + workloads=["standard"], + ) + + output_sections = [ + "HPL", + "LatencyBandwidth", + "MPIFFT", + "MPIRandomAccess_LCG", + "MPIRandomAccess", + "PTRANS", + "SingleDGEMM", + "SingleFFT", + "SingleRandomAccess_LCG", + "SingleRandomAccess", + "SingleSTREAM", + "StarDGEMM", + "StarFFT", + "StarRandomAccess_LCG", + "StarRandomAccess", + "StarSTREAM", + "Summary", + ] - context_regex = 'Begin of Summary section' - figure_of_merit_context('Summary', regex=context_regex, output_format='Summary') + context_regex = "Begin of Summary section" + figure_of_merit_context( + "Summary", regex=context_regex, output_format="Summary" + ) summary_metrics = [ - ('HPL_Tflops', 'Tflops'), - ('StarDGEMM_Gflops', 'Gflops'), - ('SingleDGEMM_Gflops', 'Gflops'), - ('PTRANS_GBs', 'GB/s'), - ('MPIRandomAccess_GUPs', 'GUP/s'), - ('MPIRandomAccess_LCG_GUPs', 'GUP/s'), - ('StarRandomAccess_GUPs', 'GUP/s'), - ('SingleRandomAccess_GUPs', 'GUP/s'), - ('StarSTREAM_Copy', 'GB/s'), - ('StarSTREAM_Scale', 'GB/s'), - ('StarSTREAM_Add', 'GB/s'), - ('StarSTREAM_Triad', 'GB/s'), - ('SingleSTREAM_Copy', 'GB/s'), - ('SingleSTREAM_Scale', 'GB/s'), - ('SingleSTREAM_Add', 'GB/s'), - ('SingleSTREAM_Triad', 'GB/s'), - ('StarFFT_Gflops', 'Gflops'), - ('SingleFFT_Gflops', 'Gflops'), - ('MPIFFT_Gflops', 'Gflops'), - ('MaxPingPongLatency_usec', 'usec'), - ('RandomlyOrderedRingLatency_usec', 'usec'), - ('MinPingPongBandwidth_GBytes', 'GB/s'), - ('NaturallyOrderedRingBandwidth_GBytes', 'GB/s'), - ('RandomlyOrderedRingBandwidth_GBytes', 'GB/s') + ("HPL_Tflops", "Tflops"), + ("StarDGEMM_Gflops", "Gflops"), + ("SingleDGEMM_Gflops", "Gflops"), + ("PTRANS_GBs", "GB/s"), + ("MPIRandomAccess_GUPs", "GUP/s"), + ("MPIRandomAccess_LCG_GUPs", "GUP/s"), + ("StarRandomAccess_GUPs", "GUP/s"), + ("SingleRandomAccess_GUPs", "GUP/s"), + ("StarSTREAM_Copy", "GB/s"), + ("StarSTREAM_Scale", "GB/s"), + ("StarSTREAM_Add", "GB/s"), + ("StarSTREAM_Triad", "GB/s"), + ("SingleSTREAM_Copy", "GB/s"), + ("SingleSTREAM_Scale", "GB/s"), + ("SingleSTREAM_Add", "GB/s"), + ("SingleSTREAM_Triad", "GB/s"), + ("StarFFT_Gflops", "Gflops"), + ("SingleFFT_Gflops", "Gflops"), + ("MPIFFT_Gflops", "Gflops"), + ("MaxPingPongLatency_usec", "usec"), + ("RandomlyOrderedRingLatency_usec", "usec"), + ("MinPingPongBandwidth_GBytes", "GB/s"), + ("NaturallyOrderedRingBandwidth_GBytes", "GB/s"), + ("RandomlyOrderedRingBandwidth_GBytes", "GB/s"), ] - log_str = Expander.expansion_str('out_file') + log_str = Expander.expansion_str("out_file") for metric, unit in summary_metrics: - summary_regex = metric + r'=(?P[0-9]+\.[0-9]+)' - figure_of_merit(metric, - log_file=log_str, - fom_regex=summary_regex, - group_name='val', units=unit, - contexts=['Summary'] - ) - ''' + summary_regex = metric + r"=(?P[0-9]+\.[0-9]+)" + figure_of_merit( + metric, + log_file=log_str, + fom_regex=summary_regex, + group_name="val", + units=unit, + contexts=["Summary"], + ) + """ # Below is a list of the full metrics available in the output. The current # implementation captures the summary metrics, which is sufficient, but it # is possible a feature user wants to expand the FOM capture to include the @@ -168,4 +200,4 @@ class Hpcc(SpackApplication): # Min Ping Pong Bandwidth: 10958.338341 MB/s # Naturally Ordered Ring Bandwidth: 13421.772800 MB/s # Randomly Ordered Ring Bandwidth: 12860.856132 MB/s - ''' + """ diff --git a/var/ramble/repos/builtin/applications/hpcg/application.py b/var/ramble/repos/builtin/applications/hpcg/application.py index 2e91f8574..9f2fd6273 100644 --- a/var/ramble/repos/builtin/applications/hpcg/application.py +++ b/var/ramble/repos/builtin/applications/hpcg/application.py @@ -12,85 +12,126 @@ class Hpcg(SpackApplication): - '''Define HPCG application''' - name = 'hpcg' - - maintainers('douglasjacobsen') - - tags('benchmark-app', 'mini-app', 'benchmark') - - define_compiler('gcc9', spack_spec='gcc@9.3.0') - - software_spec('impi2018', - spack_spec='intel-mpi@2018.4.274') - - software_spec('hpcg', - spack_spec='hpcg@3.1 +openmp', - compiler='gcc9') - - required_package('hpcg') - - executable('execute', 'xhpcg', use_mpi=True) - - executable('move-log', 'mv HPCG-Benchmark*.txt {out_file}', - use_mpi=False) - - workload('standard', executables=['execute', 'move-log']) - - workload_variable('matrix_size', default='104 104 104', - description='Dimensions of the matrix to use', - workloads=['standard']) - - workload_variable('iterations', default='60', - description='Number of iterations to perform', - workloads=['standard']) - - workload_variable('out_file', default='{experiment_run_dir}/hpcg_result.out', - description='Output file for results', - workloads=['standard']) - - log_str = Expander.expansion_str('out_file') - - figure_of_merit('Status', log_file=log_str, - fom_regex=r'Final Summary::HPCG result is (?P[a-zA-Z]+) with a GFLOP/s rating of=(?P[0-9]+\.[0-9]+)', - group_name='status', units='') - - figure_of_merit('Gflops', log_file=log_str, - fom_regex=r'Final Summary::HPCG result is (?P[a-zA-Z]+) with a GFLOP/s rating of=(?P[0-9]+\.[0-9]+)', - group_name='gflops', units='GFLOP/s') - - figure_of_merit('Time', log_file=log_str, - fom_regex=r'Final Summary::Results are.* execution time.*is=(?P[0-9]+\.[0-9]*)', - group_name='exec_time', units='s') - - figure_of_merit('ComputeDotProductMsg', log_file=log_str, - fom_regex=r'Final Summary::Reference version of ComputeDotProduct used.*=(?P.*)', - group_name='msg', units='') - - figure_of_merit('ComputeSPMVMsg', log_file=log_str, - fom_regex=r'Final Summary::Reference version of ComputeSPMV used.*=(?P.*)', - group_name='msg', units='') - - figure_of_merit('ComputeMGMsg', log_file=log_str, - fom_regex=r'Final Summary::Reference version of ComputeMG used.*=(?P.*)', - group_name='msg', units='') - - figure_of_merit('ComputeWAXPBYMsg', log_file=log_str, - fom_regex=r'Final Summary::Reference version of ComputeWAXPBY used.*=(?P.*)', - group_name='msg', units='') - - figure_of_merit('HPCG 2.4 Rating', log_file=log_str, - fom_regex=r'Final Summary::HPCG 2\.4 rating.*=(?P[0-9]+\.*[0-9]*)', - group_name='rating', units='') + """Define HPCG application""" + + name = "hpcg" + + maintainers("douglasjacobsen") + + tags("benchmark-app", "mini-app", "benchmark") + + define_compiler("gcc9", spack_spec="gcc@9.3.0") + + software_spec("impi2018", spack_spec="intel-mpi@2018.4.274") + + software_spec("hpcg", spack_spec="hpcg@3.1 +openmp", compiler="gcc9") + + required_package("hpcg") + + executable("execute", "xhpcg", use_mpi=True) + + executable("move-log", "mv HPCG-Benchmark*.txt {out_file}", use_mpi=False) + + workload("standard", executables=["execute", "move-log"]) + + workload_variable( + "matrix_size", + default="104 104 104", + description="Dimensions of the matrix to use", + workloads=["standard"], + ) + + workload_variable( + "iterations", + default="60", + description="Number of iterations to perform", + workloads=["standard"], + ) + + workload_variable( + "out_file", + default="{experiment_run_dir}/hpcg_result.out", + description="Output file for results", + workloads=["standard"], + ) + + log_str = Expander.expansion_str("out_file") + + figure_of_merit( + "Status", + log_file=log_str, + fom_regex=r"Final Summary::HPCG result is (?P[a-zA-Z]+) with a GFLOP/s rating of=(?P[0-9]+\.[0-9]+)", + group_name="status", + units="", + ) + + figure_of_merit( + "Gflops", + log_file=log_str, + fom_regex=r"Final Summary::HPCG result is (?P[a-zA-Z]+) with a GFLOP/s rating of=(?P[0-9]+\.[0-9]+)", + group_name="gflops", + units="GFLOP/s", + ) + + figure_of_merit( + "Time", + log_file=log_str, + fom_regex=r"Final Summary::Results are.* execution time.*is=(?P[0-9]+\.[0-9]*)", + group_name="exec_time", + units="s", + ) + + figure_of_merit( + "ComputeDotProductMsg", + log_file=log_str, + fom_regex=r"Final Summary::Reference version of ComputeDotProduct used.*=(?P.*)", + group_name="msg", + units="", + ) + + figure_of_merit( + "ComputeSPMVMsg", + log_file=log_str, + fom_regex=r"Final Summary::Reference version of ComputeSPMV used.*=(?P.*)", + group_name="msg", + units="", + ) + + figure_of_merit( + "ComputeMGMsg", + log_file=log_str, + fom_regex=r"Final Summary::Reference version of ComputeMG used.*=(?P.*)", + group_name="msg", + units="", + ) + + figure_of_merit( + "ComputeWAXPBYMsg", + log_file=log_str, + fom_regex=r"Final Summary::Reference version of ComputeWAXPBY used.*=(?P.*)", + group_name="msg", + units="", + ) + + figure_of_merit( + "HPCG 2.4 Rating", + log_file=log_str, + fom_regex=r"Final Summary::HPCG 2\.4 rating.*=(?P[0-9]+\.*[0-9]*)", + group_name="rating", + units="", + ) def _make_experiments(self, workspace, app_inst=None): super()._make_experiments(workspace) - input_path = os.path.join(self.expander.expand_var_name('experiment_run_dir'), - 'hpcg.dat') - - with open(input_path, 'w+') as f: - f.write('HPCG benchmark input file\n') - f.write('Sandia National Laboratories; University of Tennessee, Knoxville\n') - f.write(self.expander.expand_var_name('matrix_size') + '\n') - f.write(self.expander.expand_var_name('iterations') + '\n') + input_path = os.path.join( + self.expander.expand_var_name("experiment_run_dir"), "hpcg.dat" + ) + + with open(input_path, "w+") as f: + f.write("HPCG benchmark input file\n") + f.write( + "Sandia National Laboratories; University of Tennessee, Knoxville\n" + ) + f.write(self.expander.expand_var_name("matrix_size") + "\n") + f.write(self.expander.expand_var_name("iterations") + "\n") diff --git a/var/ramble/repos/builtin/applications/hpl/application.py b/var/ramble/repos/builtin/applications/hpl/application.py index 589132df0..5ccf15ac2 100644 --- a/var/ramble/repos/builtin/applications/hpl/application.py +++ b/var/ramble/repos/builtin/applications/hpl/application.py @@ -14,222 +14,326 @@ def pad_value(val, desc): - return ('{:<14}'.format(val) + desc) + return "{:<14}".format(val) + desc class Hpl(SpackApplication): - '''Define HPL application''' - name = 'hpl' - - maintainers('douglasjacobsen') - - tags('benchmark-app', 'benchmark', 'linpack') - - define_compiler('gcc9', spack_spec='gcc@9.3.0') - - software_spec('impi_2018', spack_spec='intel-mpi@2018.4.274') - - software_spec('hpl', spack_spec='hpl@2.3 +openmp', compiler='gcc9') - - required_package('hpl') - - executable('execute', 'xhpl', use_mpi=True) - - workload('standard', executables=['execute']) - workload('calculator', executables=['execute']) - - workload_variable('output_file', - default=pad_value('HPL.out', 'output file name (if any)'), - description='Output file name (if any)', - workloads=['standard']) - workload_variable('device_out', - default=pad_value('6', 'device out (6=stdout,7=stderr,file)'), - description='Output device', - workloads=['standard']) - workload_variable('N-Ns', - default=pad_value('4', 'Number of problems sizes (N)'), - description='Number of problems sizes', - workloads=['standard']) - workload_variable('Ns', - default=pad_value('29 30 34 35', 'Ns'), - description='Problem sizes', - workloads=['standard']) - workload_variable('N-NBs', - default=pad_value('4', 'Number of NBs'), - description='Number of NBs', - workloads=['standard']) - workload_variable('NBs', - default=pad_value('1 2 3 4', 'NBs'), - description='NB values', - workloads=['standard']) - workload_variable('PMAP', - default=pad_value('0', 'PMAP process mapping (0=Row-,1=Column-major)'), - description='PMAP Process mapping. (0=Row-, 1=Column-Major)', - workloads=['standard']) - workload_variable('N-Grids', - default=pad_value('3', 'Number of process grids (P x Q)'), - description='Number of process grids (P x Q)', - workloads=['standard']) - workload_variable('Ps', - default=pad_value('2 1 4', 'Ps'), - description='P values', - workloads=['standard']) - workload_variable('Qs', - default=pad_value('2 4 1', 'Qs'), - description='Q values', - workloads=['standard']) - workload_variable('threshold', - default=pad_value('16.0', 'threshold'), - description='Residual threshold', - workloads=['standard']) - workload_variable('NPFACTs', - default=pad_value('3', 'Number of PFACTs, panel fact'), - description='Number of PFACTs', - workloads=['standard']) - workload_variable('PFACTs', - default=pad_value('0 1 2', 'PFACTs (0=left, 1=Crout, 2=Right)'), - description='PFACT Values', - workloads=['standard']) - workload_variable('N-NBMINs', - default=pad_value('2', 'Number of NBMINs, recursive stopping criteria'), - description='Number of NBMINs', - workloads=['standard']) - workload_variable('NBMINs', - default=pad_value('2 4', 'NBMINs (>= 1)'), - description='NBMIN values', - workloads=['standard']) - workload_variable('N-NDIVs', - default=pad_value('1', 'Number of NDIVs, panels in recursion'), - description='Number of NDIVs', - workloads=['standard']) - workload_variable('NDIVs', - default=pad_value('2', 'NDIVs'), - description='NDIV values', - workloads=['standard']) - workload_variable('N-RFACTs', - default=pad_value('3', 'Number of RFACTs, recursive panel fact.'), - description='Number of RFACTs', - workloads=['standard']) - workload_variable('RFACTs', - default=pad_value('0 1 2', 'RFACTs (0=left, 1=Crout, 2=Right)'), - description='RFACT values', - workloads=['standard']) - workload_variable('N-BCASTs', - default=pad_value('1', 'Number of BCASTs, broadcast'), - description='Number of BCASTs', - workloads=['standard']) - workload_variable('BCASTs', - default=pad_value('0', 'BCASTs (0=1rg,1=1rM,2=2rg,3=2rM,4=Lng,5=LnM)'), - description='BCAST values', - workloads=['standard']) - workload_variable('N-DEPTHs', - default=pad_value('1', 'Number of DEPTHs, lookahead depth'), - description='Number of DEPTHs', - workloads=['standard']) - workload_variable('DEPTHs', - default=pad_value('0', 'DEPTHs (>=0)'), - description='DEPTH values', - workloads=['standard']) - workload_variable('SWAP', - default=pad_value('2', 'SWAP (0=bin-exch,1=long,2=mix)'), - description='Swapping algorithm', - workloads=['standard']) - workload_variable('swapping_threshold', - default=pad_value('64', 'swapping threshold'), - description='Swapping threshold', - workloads=['standard']) - workload_variable('L1', - default=pad_value('0', 'L1 in (0=transposed,1=no-transposed) form'), - description='Storage for upper triangular portion of columns', - workloads=['standard']) - workload_variable('U', - default=pad_value('0', 'U in (0=transposed,1=no-transposed) form'), - description='Storage for the rows of U', - workloads=['standard']) - workload_variable('Equilibration', - default=pad_value('1', 'Equilibration (0=no,1=yes)'), - description='Determines if equilibration should be enabled or disabled.', - workloads=['standard']) - workload_variable('mem_alignment', - default=pad_value('8', 'memory alignment in double (> 0)'), - description='Sets the alignment in doubles for memory addresses', - workloads=['standard']) + """Define HPL application""" + + name = "hpl" + + maintainers("douglasjacobsen") + + tags("benchmark-app", "benchmark", "linpack") + + define_compiler("gcc9", spack_spec="gcc@9.3.0") + + software_spec("impi_2018", spack_spec="intel-mpi@2018.4.274") + + software_spec("hpl", spack_spec="hpl@2.3 +openmp", compiler="gcc9") + + required_package("hpl") + + executable("execute", "xhpl", use_mpi=True) + + workload("standard", executables=["execute"]) + workload("calculator", executables=["execute"]) + + workload_variable( + "output_file", + default=pad_value("HPL.out", "output file name (if any)"), + description="Output file name (if any)", + workloads=["standard"], + ) + workload_variable( + "device_out", + default=pad_value("6", "device out (6=stdout,7=stderr,file)"), + description="Output device", + workloads=["standard"], + ) + workload_variable( + "N-Ns", + default=pad_value("4", "Number of problems sizes (N)"), + description="Number of problems sizes", + workloads=["standard"], + ) + workload_variable( + "Ns", + default=pad_value("29 30 34 35", "Ns"), + description="Problem sizes", + workloads=["standard"], + ) + workload_variable( + "N-NBs", + default=pad_value("4", "Number of NBs"), + description="Number of NBs", + workloads=["standard"], + ) + workload_variable( + "NBs", + default=pad_value("1 2 3 4", "NBs"), + description="NB values", + workloads=["standard"], + ) + workload_variable( + "PMAP", + default=pad_value("0", "PMAP process mapping (0=Row-,1=Column-major)"), + description="PMAP Process mapping. (0=Row-, 1=Column-Major)", + workloads=["standard"], + ) + workload_variable( + "N-Grids", + default=pad_value("3", "Number of process grids (P x Q)"), + description="Number of process grids (P x Q)", + workloads=["standard"], + ) + workload_variable( + "Ps", + default=pad_value("2 1 4", "Ps"), + description="P values", + workloads=["standard"], + ) + workload_variable( + "Qs", + default=pad_value("2 4 1", "Qs"), + description="Q values", + workloads=["standard"], + ) + workload_variable( + "threshold", + default=pad_value("16.0", "threshold"), + description="Residual threshold", + workloads=["standard"], + ) + workload_variable( + "NPFACTs", + default=pad_value("3", "Number of PFACTs, panel fact"), + description="Number of PFACTs", + workloads=["standard"], + ) + workload_variable( + "PFACTs", + default=pad_value("0 1 2", "PFACTs (0=left, 1=Crout, 2=Right)"), + description="PFACT Values", + workloads=["standard"], + ) + workload_variable( + "N-NBMINs", + default=pad_value( + "2", "Number of NBMINs, recursive stopping criteria" + ), + description="Number of NBMINs", + workloads=["standard"], + ) + workload_variable( + "NBMINs", + default=pad_value("2 4", "NBMINs (>= 1)"), + description="NBMIN values", + workloads=["standard"], + ) + workload_variable( + "N-NDIVs", + default=pad_value("1", "Number of NDIVs, panels in recursion"), + description="Number of NDIVs", + workloads=["standard"], + ) + workload_variable( + "NDIVs", + default=pad_value("2", "NDIVs"), + description="NDIV values", + workloads=["standard"], + ) + workload_variable( + "N-RFACTs", + default=pad_value("3", "Number of RFACTs, recursive panel fact."), + description="Number of RFACTs", + workloads=["standard"], + ) + workload_variable( + "RFACTs", + default=pad_value("0 1 2", "RFACTs (0=left, 1=Crout, 2=Right)"), + description="RFACT values", + workloads=["standard"], + ) + workload_variable( + "N-BCASTs", + default=pad_value("1", "Number of BCASTs, broadcast"), + description="Number of BCASTs", + workloads=["standard"], + ) + workload_variable( + "BCASTs", + default=pad_value("0", "BCASTs (0=1rg,1=1rM,2=2rg,3=2rM,4=Lng,5=LnM)"), + description="BCAST values", + workloads=["standard"], + ) + workload_variable( + "N-DEPTHs", + default=pad_value("1", "Number of DEPTHs, lookahead depth"), + description="Number of DEPTHs", + workloads=["standard"], + ) + workload_variable( + "DEPTHs", + default=pad_value("0", "DEPTHs (>=0)"), + description="DEPTH values", + workloads=["standard"], + ) + workload_variable( + "SWAP", + default=pad_value("2", "SWAP (0=bin-exch,1=long,2=mix)"), + description="Swapping algorithm", + workloads=["standard"], + ) + workload_variable( + "swapping_threshold", + default=pad_value("64", "swapping threshold"), + description="Swapping threshold", + workloads=["standard"], + ) + workload_variable( + "L1", + default=pad_value("0", "L1 in (0=transposed,1=no-transposed) form"), + description="Storage for upper triangular portion of columns", + workloads=["standard"], + ) + workload_variable( + "U", + default=pad_value("0", "U in (0=transposed,1=no-transposed) form"), + description="Storage for the rows of U", + workloads=["standard"], + ) + workload_variable( + "Equilibration", + default=pad_value("1", "Equilibration (0=no,1=yes)"), + description="Determines if equilibration should be enabled or disabled.", + workloads=["standard"], + ) + workload_variable( + "mem_alignment", + default=pad_value("8", "memory alignment in double (> 0)"), + description="Sets the alignment in doubles for memory addresses", + workloads=["standard"], + ) # calculator workload-specific variables: - workload_variable('percent_mem', default='85', - description='Percent of memory to use (default 85)', - workloads=['calculator']) - - workload_variable('memory_per_node', default='240', - description='Memory per node in GB', - workloads=['calculator']) - - workload_variable('block_size', default='384', - description='Size of each block', - workloads=['calculator']) - - workload_variable('pfact', default='0', - description='PFACT for optimized calculator', - workloads=['calculator']) - - workload_variable('nbmin', default='2', - description='NBMIN for optimized calculator', - workloads=['calculator']) - - workload_variable('rfact', default='0', - description='RFACT for optimized calculator', - workloads=['calculator']) - - workload_variable('bcast', default='0', - description='BCAST for optimized calculator', - workloads=['calculator']) - - workload_variable('depth', default='0', - description='DEPTH for optimized calculator', - workloads=['calculator']) + workload_variable( + "percent_mem", + default="85", + description="Percent of memory to use (default 85)", + workloads=["calculator"], + ) + + workload_variable( + "memory_per_node", + default="240", + description="Memory per node in GB", + workloads=["calculator"], + ) + + workload_variable( + "block_size", + default="384", + description="Size of each block", + workloads=["calculator"], + ) + + workload_variable( + "pfact", + default="0", + description="PFACT for optimized calculator", + workloads=["calculator"], + ) + + workload_variable( + "nbmin", + default="2", + description="NBMIN for optimized calculator", + workloads=["calculator"], + ) + + workload_variable( + "rfact", + default="0", + description="RFACT for optimized calculator", + workloads=["calculator"], + ) + + workload_variable( + "bcast", + default="0", + description="BCAST for optimized calculator", + workloads=["calculator"], + ) + + workload_variable( + "depth", + default="0", + description="DEPTH for optimized calculator", + workloads=["calculator"], + ) # FOMs: - log_str = os.path.join(Expander.expansion_str('experiment_run_dir'), - Expander.expansion_str('experiment_name') + '.out') - - figure_of_merit('Time', log_file=log_str, - fom_regex=r'.*\s+(?P[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[0-9]+)\s+(?P[0-9]+)\s+(?P

[\w:]+)\s+:\s+(?P[0-9]+\.[0-9]+)s CPU\s+' + \ - r'(?P[0-9]+\.[0-9]+)s WALL \(\s+(?P[0-9]+) calls\).*' - figure_of_merit_context('Profile section', regex=profile_regex, - output_format='{section}') - - figure_of_merit('Section CPU Time', fom_regex=profile_regex, - group_name='cpu', log_file=log_str, units='s', - contexts=['Profile section']) - - figure_of_merit('Section Wall Time', fom_regex=profile_regex, - group_name='wall', log_file=log_str, units='s', - contexts=['Profile section']) - - figure_of_merit('Section Number of Calls', fom_regex=profile_regex, - group_name='calls', log_file=log_str, units='', - contexts=['Profile section']) - - success_criteria('job_done', mode='string', - match=r'.*JOB DONE\..*', - file=log_str) + workload( + wl_name, executables=["copy_inputs", "execute"], inputs=[wl_name] + ) + workload_variable( + "input_path", + default=Expander.expansion_str(wl_name), + description="Path to inputs for " + wl_name + " workload", + workloads=[wl_name], + ) + workload_variable( + "input_file", + default=input_file, + description="Name of input file for " + wl_name + " workload", + workloads=[wl_name], + ) + workload_variable( + "flags", + default="", + description="Flags for Quantum Espresso", + workloads=[wl_name], + ) + + workload( + "WATER_EXX", + executables=["copy_potential1", "copy_potential2", "execute"], + inputs=["WATER_EXX", "WATER_EXX_H", "WATER_EXX_O"], + ) + workload_variable( + "input_file", + default=Expander.expansion_str("WATER_EXX"), + description="Path to WATER_EXX input", + workloads=["WATER_EXX"], + ) + workload_variable( + "potential1", + default=Expander.expansion_str("WATER_EXX_H"), + description="Path to WATER_EXX H potential file", + workloads=["WATER_EXX"], + ) + workload_variable( + "potential2", + default=Expander.expansion_str("WATER_EXX_O"), + description="Path to WATER_EXX O potential file", + workloads=["WATER_EXX"], + ) + + workload_variable( + "flags", + default="", + description="Flags for Quantum Espresso", + workloads=["WATER_EXX"], + ) + + log_str = Expander.expansion_str("log_file") + figure_of_merit( + "Total CPU time", + fom_regex=r"\s*total cpu time spent up to now is\s+(?P