From f0c5719b4d930083938782845e4f17a74dde7156 Mon Sep 17 00:00:00 2001 From: Douglas Jacobsen Date: Fri, 13 Sep 2024 11:34:34 -0600 Subject: [PATCH] Create tutorial triggers for the remaining tutorials --- .../docs/tutorials/10_using_modifiers.rst | 6 +- .../tutorials/ramble-tutorial-10.yaml | 83 +++++++++++++++++++ .../tutorials/ramble-tutorial-11.yaml | 68 +++++++++++++++ .../tutorials/ramble-tutorial-4.yaml | 65 +++++++++++++++ .../tutorials/ramble-tutorial-5.yaml | 65 +++++++++++++++ .../tutorials/ramble-tutorial-6.yaml | 64 ++++++++++++++ .../tutorials/ramble-tutorial-7.yaml | 78 +++++++++++++++++ .../tutorials/ramble-tutorial-8.yaml | 72 ++++++++++++++++ .../tutorials/ramble-tutorial-9.yaml | 65 +++++++++++++++ 9 files changed, 563 insertions(+), 3 deletions(-) create mode 100644 share/ramble/cloud-build/tutorials/ramble-tutorial-10.yaml create mode 100644 share/ramble/cloud-build/tutorials/ramble-tutorial-11.yaml create mode 100644 share/ramble/cloud-build/tutorials/ramble-tutorial-4.yaml create mode 100644 share/ramble/cloud-build/tutorials/ramble-tutorial-5.yaml create mode 100644 share/ramble/cloud-build/tutorials/ramble-tutorial-6.yaml create mode 100644 share/ramble/cloud-build/tutorials/ramble-tutorial-7.yaml create mode 100644 share/ramble/cloud-build/tutorials/ramble-tutorial-8.yaml create mode 100644 share/ramble/cloud-build/tutorials/ramble-tutorial-9.yaml diff --git a/lib/ramble/docs/tutorials/10_using_modifiers.rst b/lib/ramble/docs/tutorials/10_using_modifiers.rst index 6ee3b6d9d..98d6b4417 100644 --- a/lib/ramble/docs/tutorials/10_using_modifiers.rst +++ b/lib/ramble/docs/tutorials/10_using_modifiers.rst @@ -65,7 +65,7 @@ To discover which modifiers are available, execute: .. code-block:: console - $ ramble mods list + $ ramble list --type modifiers Which might output the following: @@ -82,7 +82,7 @@ about the ``lscpu`` modifier, execute: .. code-block:: console - $ ramble mods info lscpu + $ ramble info --type modifiers lscpu This modifier adds the execution of ``lscpu`` to each experiment in a workspace (to capture additional platform level details, such as the CPU model), and @@ -157,7 +157,7 @@ To get information about the ``intel-aps`` modifier, execute: .. code-block:: console - $ ramble mods info intel-aps + $ ramble info --type modifiers intel-aps In the output from this command, you should see a ``mode`` named ``mpi``. One additional difference relateive to ``lscpu`` is that the ``Software Specs:`` diff --git a/share/ramble/cloud-build/tutorials/ramble-tutorial-10.yaml b/share/ramble/cloud-build/tutorials/ramble-tutorial-10.yaml new file mode 100644 index 000000000..3ad61d120 --- /dev/null +++ b/share/ramble/cloud-build/tutorials/ramble-tutorial-10.yaml @@ -0,0 +1,83 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + + +steps: + - name: gcr.io/cloud-builders/git + args: + - fetch + - '--unshallow' + id: ramble-clone + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest + args: + - '-c' + - | + cd /workspace + + export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH} + + . /opt/spack/share/spack/setup-env.sh + . /workspace/share/ramble/setup-env.sh + + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + + spack mirror add ci_cache ${_CI_CACHE} + spack buildcache keys --install --trust + + set -e + + ramble workspace create -d modifiers_wrf -c /workspace/examples/tutorial_10_lscpu_config.yaml + + ramble workspace activate ./modifiers_wrf + + ramble config add "variables:processes_per_node:4" + + ramble list --type modifiers + + ramble info --type modifiers lscpu + + ramble workspace info + + ramble workspace setup --where '{n_nodes} == 1' + + ramble on --where '{n_nodes} == 1' + + ramble workspace analyze --where '{n_nodes} == 1' + + ramble info --type modifiers intel-aps + + cp /workspace/exmaples/tutorial_10_aps_error_config.yaml modifiers_wrf/configs/ramble.yaml + + set +e + + # Expected to error + ramble workspace setup --dry-run + + set -e + + cp /workspace/exmaples/tutorial_10_aps_final_config.yaml modifiers_wrf/configs/ramble.yaml + + ramble config add "variables:processes_per_node:4" + + ramble workspace setup --where '{n_nodes} == 1' + ramble on --where '{n_nodes} == 1' + ramble workspace analyze --where '{n_nodes} == 1' + + ramble workspace deactivate + id: ramble-tutorial-test + entrypoint: /bin/bash +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' + _CI_CACHE: gs://spack/latest +timeout: 1500s +options: + machineType: N1_HIGHCPU_8 diff --git a/share/ramble/cloud-build/tutorials/ramble-tutorial-11.yaml b/share/ramble/cloud-build/tutorials/ramble-tutorial-11.yaml new file mode 100644 index 000000000..6bd5a415d --- /dev/null +++ b/share/ramble/cloud-build/tutorials/ramble-tutorial-11.yaml @@ -0,0 +1,68 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + + +steps: + - name: gcr.io/cloud-builders/git + args: + - fetch + - '--unshallow' + id: ramble-clone + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest + args: + - '-c' + - | + cd /workspace + + export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH} + + . /opt/spack/share/spack/setup-env.sh + . /workspace/share/ramble/setup-env.sh + + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + + spack mirror add ci_cache ${_CI_CACHE} + spack buildcache keys --install --trust + + set -e + + ramble workspace create -d internals_wrf -c /workspace/examples/tutorial_11_new_exec_config.yaml + + ramble workspace activate ./internals_wrf + + ramble workspace info + + ramble workspace setup --dry-run + + grep "date +" internals_wrf/experiments/wrfv4/CONUS_12km/scaling_1/execute_experiment + + cp /workspace/examples/tutorial_11_exec_order_config.yaml internals_wrf/configs/ramble.yaml + + ramble workspace setup --dry-run + + grep "date +" internals_wrf/experiments/wrfv4/CONUS_12km/scaling_1/execute_experiment + + cp /workspace/examples/tutorial_11_exec_injection_config.yaml internals_wrf/configs/ramble.yaml + + ramble workspace setup --dry-run + + grep "date +" internals_wrf/experiments/wrfv4/CONUS_12km/scaling_1/execute_experiment + + ramble workspace deactivate + id: ramble-tutorial-test + entrypoint: /bin/bash +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' + _CI_CACHE: gs://spack/latest +timeout: 1500s +options: + machineType: N1_HIGHCPU_8 diff --git a/share/ramble/cloud-build/tutorials/ramble-tutorial-4.yaml b/share/ramble/cloud-build/tutorials/ramble-tutorial-4.yaml new file mode 100644 index 000000000..14f3202ed --- /dev/null +++ b/share/ramble/cloud-build/tutorials/ramble-tutorial-4.yaml @@ -0,0 +1,65 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + + +steps: + - name: gcr.io/cloud-builders/git + args: + - fetch + - '--unshallow' + id: ramble-clone + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest + args: + - '-c' + - | + cd /workspace + + export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH} + + . /opt/spack/share/spack/setup-env.sh + . /workspace/share/ramble/setup-env.sh + + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + + spack mirror add ci_cache ${_CI_CACHE} + spack buildcache keys --install --trust + + set -e + + ramble workspace create -d basic_gromacs -c /workspace/examples/basic_gromacs_config.yaml + + ramble workspace activate ./basic_gromacs + + ramble workspace info + + ramble workspace info --expansions + + ramble workspace info -vvv + + cp /workspace/examples/vector_matrix_gromacs_config.yaml basic_gromacs/config/ramble.yaml + + ramble workspace setup + + ramble on + + ramble workspace analyze + + ramble workspace deactivate + + id: ramble-tutorial-test + entrypoint: /bin/bash +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' + _CI_CACHE: gs://spack/latest +timeout: 7200s +options: + machineType: N1_HIGHCPU_8 diff --git a/share/ramble/cloud-build/tutorials/ramble-tutorial-5.yaml b/share/ramble/cloud-build/tutorials/ramble-tutorial-5.yaml new file mode 100644 index 000000000..a8fb18d0e --- /dev/null +++ b/share/ramble/cloud-build/tutorials/ramble-tutorial-5.yaml @@ -0,0 +1,65 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + + +steps: + - name: gcr.io/cloud-builders/git + args: + - fetch + - '--unshallow' + id: ramble-clone + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest + args: + - '-c' + - | + cd /workspace + + export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH} + + . /opt/spack/share/spack/setup-env.sh + . /workspace/share/ramble/setup-env.sh + + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + + spack mirror add ci_cache ${_CI_CACHE} + spack buildcache keys --install --trust + + set -e + + ramble workspace create -d vector_gromacs -c /workspace/examples/vector_matrix_gromacs_config.yaml + + ramble workspace activate ./vector_gromacs + + ramble workspace info + + spack info gromacs + + cp /workspace/examples/vector_gromacs_software_config.yaml vector_gromacs/configs/ramble.yaml + + ramble workspace info + + ramble workspace setup --dry-run + + # Test no package manager works. + ramble -c "variables:gromacs_path:/not/a/path" -c "variants:package_manager:null" workspace setup --dry-run + + ramble list --type package_managers + + ramble workspace deactivate + id: ramble-tutorial-test + entrypoint: /bin/bash +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' + _CI_CACHE: gs://spack/latest +timeout: 7200s +options: + machineType: N1_HIGHCPU_8 diff --git a/share/ramble/cloud-build/tutorials/ramble-tutorial-6.yaml b/share/ramble/cloud-build/tutorials/ramble-tutorial-6.yaml new file mode 100644 index 000000000..3895673a2 --- /dev/null +++ b/share/ramble/cloud-build/tutorials/ramble-tutorial-6.yaml @@ -0,0 +1,64 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + + +steps: + - name: gcr.io/cloud-builders/git + args: + - fetch + - '--unshallow' + id: ramble-clone + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest + args: + - '-c' + - | + cd /workspace + + export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH} + + . /opt/spack/share/spack/setup-env.sh + . /workspace/share/ramble/setup-env.sh + + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + + spack mirror add ci_cache ${_CI_CACHE} + spack buildcache keys --install --trust + + set -e + + ramble workspace create -d scaling_wrf -c /workspace/examples/tutorial_6_config.yaml + + ramble workspace activate ./scaling_wrf + + ramble config add "variables:processes_per_node:4" + + ramble info wrfv4 + + ramble workspace info + + ramble workspace info --expansions + + ramble workspace setup + + ramble on --where '{n_nodes} == 1' + + ramble workspace analyze + + ramble workspace deactivate + id: ramble-tutorial-test + entrypoint: /bin/bash +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' + _CI_CACHE: gs://spack/latest +timeout: 7200s +options: + machineType: N1_HIGHCPU_8 diff --git a/share/ramble/cloud-build/tutorials/ramble-tutorial-7.yaml b/share/ramble/cloud-build/tutorials/ramble-tutorial-7.yaml new file mode 100644 index 000000000..99ad0aca1 --- /dev/null +++ b/share/ramble/cloud-build/tutorials/ramble-tutorial-7.yaml @@ -0,0 +1,78 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + + +steps: + - name: gcr.io/cloud-builders/git + args: + - fetch + - '--unshallow' + id: ramble-clone + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest + args: + - '-c' + - | + cd /workspace + + export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH} + + . /opt/spack/share/spack/setup-env.sh + . /workspace/share/ramble/setup-env.sh + + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + + spack mirror add ci_cache ${_CI_CACHE} + spack buildcache keys --install --trust + + set -e + + ramble workspace create -d zips_and_matrices_wrf -c /workspace/examples/wrf_scaling_config.yaml + + ramble workspace activate ./zips_and_matrices_wrf + + ramble workspace info + + set +e + + cp /workspace/examples/tutorial_7_base_config.yaml zips_and_matrices_wrf/configs/ramble.yaml + + # Expected to give an error + ramble workspace info + + + cp /workspace/examples/tutorial_7_matrix_config.yaml zips_and_matrices_wrf/configs/ramble.yaml + + # Expected to give an error + ramble workspace info + + set -e + + + cp /workspace/examples/tutorial_7_final_config.yaml zips_and_matrices_wrf/configs/ramble.yaml + + ramble workspace info + + ramble workspace setup + + ramble workspace setup --where '{n_ranks} >= 20' + + # Skip execution / analysis because cloud-build machines don't have the right number of cores. + + ramble workspace deactivate + id: ramble-tutorial-test + entrypoint: /bin/bash +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' + _CI_CACHE: gs://spack/latest +timeout: 1500s +options: + machineType: N1_HIGHCPU_8 diff --git a/share/ramble/cloud-build/tutorials/ramble-tutorial-8.yaml b/share/ramble/cloud-build/tutorials/ramble-tutorial-8.yaml new file mode 100644 index 000000000..80f5dfc1c --- /dev/null +++ b/share/ramble/cloud-build/tutorials/ramble-tutorial-8.yaml @@ -0,0 +1,72 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + + +steps: + - name: gcr.io/cloud-builders/git + args: + - fetch + - '--unshallow' + id: ramble-clone + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest + args: + - '-c' + - | + cd /workspace + + export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH} + + . /opt/spack/share/spack/setup-env.sh + . /workspace/share/ramble/setup-env.sh + + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + + spack mirror add ci_cache ${_CI_CACHE} + spack buildcache keys --install --trust + + set -e + + ramble workspace create -d var_expanison_and_indirection -c /workspace/examples/tutorial_8_base_config.yaml + + ramble workspace activate ./var_expanison_and_indirection + + ramble workspace info + + set +e + + cp /workspace/examples/tutorial_8_mpi_config.yaml var_expansion_and_indirection/configs/ramble.yaml + #Expected to error + ramble workspace info + + cp /workspace/examples/tutorial_8_mpi_matrix_config.yaml var_expansion_and_indirection/configs/ramble.yaml + # Expected to error + ramble workspace info + + cp /workspace/examples/tutorial_8_expansion_indirection_config.yaml var_expansion_and_indirection/configs/ramble.yaml + # Expected to error + ramble workspace info + set -e + + cp /workspace/examples/tutorial_8_software_environments_config.yaml var_expansion_and_indirection/configs/ramble.yaml + ramble workspace info + + ramble workspace setup --dry-run + + ramble workspace deactivate + id: ramble-tutorial-test + entrypoint: /bin/bash +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' + _CI_CACHE: gs://spack/latest +timeout: 1500s +options: + machineType: N1_HIGHCPU_8 diff --git a/share/ramble/cloud-build/tutorials/ramble-tutorial-9.yaml b/share/ramble/cloud-build/tutorials/ramble-tutorial-9.yaml new file mode 100644 index 000000000..3a373de32 --- /dev/null +++ b/share/ramble/cloud-build/tutorials/ramble-tutorial-9.yaml @@ -0,0 +1,65 @@ +# Copyright 2022-2024 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + + +steps: + - name: gcr.io/cloud-builders/git + args: + - fetch + - '--unshallow' + id: ramble-clone + - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest + args: + - '-c' + - | + cd /workspace + + export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH} + + . /opt/spack/share/spack/setup-env.sh + . /workspace/share/ramble/setup-env.sh + + echo "Spack version is $(spack --version)" + echo "Python version is $(python3 --version)" + + spack mirror add ci_cache ${_CI_CACHE} + spack buildcache keys --install --trust + + set -e + + ramble workspace create -d success_wrf -c /workspace/examples/tutorial_9_regex_criteria_config.yaml + + ramble workspace activate ./success_wrf + + ramble workspace info + + cp /workspace/examples/tutorial_9_fom_criteria_config.yaml success_wrf/configs/ramble.yaml + + # Cloud build VMs only have 4 cores + ramble config add "variables:processes_per_node:4" + + ramble workspace info + + ramble workspace setup + + ramble on --where '{n_nodes} == 1' + + ramble workspace analyze --always-print-foms --where '{n_nodes} == 1' + + ramble workspace deactivate + id: ramble-tutorial-test + entrypoint: /bin/bash +substitutions: + _SPACK_REF: v0.21.2 + _CONDA_VER: 22.11.1 + _BASE_IMG: centos + _BASE_VER: '7' + _CI_CACHE: gs://spack/latest +timeout: 7200s +options: + machineType: N1_HIGHCPU_8