Merge pull request #421 from GoogleCloudPlatform/develop

Merge develop into main for v0.4.0 release

CLAs were all met at commit time.

.flake8

@@ -33,6 +33,7 @@ max-line-length = 99
 
 per-file-ignores =
   var/ramble/repos/*/application.py:F405,F821
+  var/ramble/repos/*/modifier.py:F405,F821
 
 # exclude things we usually do not want linting for.
 # These still get linted when passed explicitly, as when spack flake8 passes

.github/workflows/spelling.yaml

@@ -0,0 +1,19 @@
+name: ramble spelling
+
+on:
+  pull_request: []
+  push:
+    branches-ignore:
+    - main
+    - gh-pages
+
+jobs:
+  formatting:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Check Spelling
+      uses: crate-ci/typos@548ac37a5de9ce84871bf4db3c9b8c462896d480 # v1.16.24
+      with:
+        files: ./lib/ramble/ramble ./lib/ramble/docs ./examples ./share ./bin ./etc ./var ./README.md
+        config: ./.typos.toml

.gitignore

@@ -2,3 +2,4 @@ __pycache__/
 *.x
 *.txt
 var/ramble/cache
+var/ramble/workspaces

.typos.toml

@@ -0,0 +1,25 @@
+[files]
+extend-exclude = [
+    ".git/",
+    # Copy of https://github.com/spack/spack/tree/develop/lib/spack
+    "lib/ramble/external/",
+    "lib/ramble/llnl/",
+    "lib/ramble/spack/",
+]
+ignore-hidden = false
+
+[default]
+extend-ignore-re = [
+    "\\bFOMs",
+]
+
+[default.extend-words]
+"fom" = "fom"
+"namd" = "namd"
+"reord" = "reord"
+
+[default.extend-identifiers]
+"ATPase" = "ATPase"
+"cachable" = "cachable"
+"clen" = "clen"
+"enable_deck_based_FOMs" = "enable_deck_based_FOMs"

LICENSE-APACHE

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2022-2023 Google LLC.
+   Copyright 2022-2024 Google LLC.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -27,13 +27,21 @@ e.g.
 
     $ pip install -r requirements.txt
 
+We recommend Python >= 3.7 for Ramble, but a best effort attempt is made to
+support Python 3.6 as it is used by older operating systems such as Centos7.
+Specifically, you might need to update `pip` and downgrade `protobuf` when
+using Python 3.6.
+
 Outside of these requirements, ramble requires an existing installation of
 spack for some application definition. See
 [Spack’s documentation](https://github.com/spack/spack#-spack) to install Spack.
 
 Documentation
 ----------------
 
+Ramble’s documentation can be viewed at
+[https://googlecloudplatform.github.io/ramble/](https://googlecloudplatform.github.io/ramble/).
+
 For help with Ramble’s commands, run `ramble help` or `ramble help --all`.
 
 For more information on concepts in Ramble, see Ramble’s

bin/ramble

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright 2022-2023 Google LLC
+# Copyright 2022-2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
 # https://www.apache.org/licenses/LICENSE-2.0> or the MIT license

bin/ramble-python

@@ -1,4 +1,4 @@
-# Copyright 2022-2023 Google LLC
+# Copyright 2022-2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
 # https://www.apache.org/licenses/LICENSE-2.0> or the MIT license

etc/ramble/defaults/config.yaml

@@ -14,14 +14,21 @@
 #   ~/.ramble/config.yaml
 # -------------------------------------------------------------------------
 config:
-    test_stage: ~/.ramble/test
-    locks: true
-    verify_ssl: true
-    connect_timeout: 10
-    checksum: true
-    shell: 'sh'
-    spack_flags:
-      install: '--reuse'
-      concretize: '--reuse'
-    input_cache: $ramble/var/ramble/cache
-    workspace_dirs: $ramble/var/ramble/workspaces
+  test_stage: ~/.ramble/test
+  locks: true
+  verify_ssl: true
+  connect_timeout: 10
+  checksum: true
+  shell: sh
+  input_cache: $ramble/var/ramble/cache
+  workspace_dirs: $ramble/var/ramble/workspaces
+  include_phase_dependencies: false
+  spack:
+    install:
+      flags: --reuse
+    concretize:
+      flags: --reuse
+    buildcache:
+      flags: ''
+    env_create:
+      flags: ''

etc/ramble/defaults/formatted_executables.yaml

@@ -0,0 +1,3 @@
+formatted_executables:
+  command:
+    join_separator: '\n'

etc/ramble/defaults/modifier_repos.yaml

@@ -0,0 +1,14 @@
+# -------------------------------------------------------------------------
+# This is the default ramble repository configuration. It includes the
+# builtin ramble application repository.
+#
+# Users can override these settings by editing the following files.
+#
+# Per-ramble-instance settings (overrides defaults):
+#   $RAMBLE_ROOT/etc/ramble/repos.yaml
+#
+# Per-user settings (overrides default and site settings):
+#   ~/.ramble/repos.yaml
+# -------------------------------------------------------------------------
+modifier_repos:
+  - $ramble/var/ramble/repos/builtin

examples/README.md

@@ -33,7 +33,7 @@ or
 ```bash
 ./all_experiments
 ```
-excuted from the root of the workspace.
+executed from the root of the workspace.
 
 ## Variable syntax:
 
@@ -74,7 +74,7 @@ This configuration changes from a pre-existing binary to using spack to build gr
 ### [Basic Expansion Config](basic_expansion_config.yaml)
 The basic expansion configuration file shows some examples of the experiment generation syntax.
 
-It changes from using groamcs to using OpenFoam and WRF.
+It changes from using gromacs to using OpenFoam and WRF.
 
 Additionally, it shows how a configuration file can contain experiments for
 multiple applications and / or workloads at the same time.
@@ -84,8 +84,8 @@ multiple applications and / or workloads at the same time.
 The full expansion configuration file shows a more complete example of
 experiment generation. It extends the basic expansion config file by showing two main aspects:
 
-1) How to define environment variables (`env-vars`)
-2) How to generate experiments using different binaries for the same application name (`spec_name`)
+1) How to define environment variables (`env_vars`)
+2) How to generate experiments using different binaries for the same application name (`env_name`)
 
 ## Available Template Files:
 

examples/basic_expansion_config.yaml

@@ -1,6 +1,6 @@
 ramble:
   variables:
-    mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node} -hosfile hostfile'
+    mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}'
     batch_submit: '{execute_experiment}'
     n_ranks: '{processes_per_node}*{n_nodes}'
   applications:
@@ -17,7 +17,7 @@ ramble:
                 wrf_path: execute_experiment in wrfv4.CONUS_2p5km.new_test
               matrices:
                 - - n_nodes
-                #^-- (partiton, processes_per_node, n_nodes) ->
+                #^-- (partition, processes_per_node, n_nodes) ->
                                 #(part1, 16, 2), (part1, 16, 4)
                                 #(part2, 32, 2), (part2, 32, 4)
     wrfv4:
@@ -28,33 +28,34 @@ ramble:
               variables:
                 n_ranks: '1'
                 n_nodes: '1'
-spack:
-  concretized: true
-  packages:
-    gcc9:
-      spack_spec: gcc@9.3.0 target=x86_64
-    ompi412:
-      spack_spec: openmpi@4.1.2 +legacylaunchers +pmi +thread_multiple +cxx target=x86_64
-      compiler: gcc9
-    impi2018:
-      spack_spec: intel-mpi@2018.4.274
-      compiler: gcc9
-    openfoam:
-      spack_spec: openfoam-org@7
-      compiler: gcc9
-    flex:
-      spack_spec: flex@2.6.4
-      compiler: gcc9
-    wrfv4:
-      spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem ~pnetcdf
-      compiler: gcc9
-  environments:
-    openfoam:
-      packages:
-      - ompi412
-      - flex
-      - openfoam
-    wrfv4:
-      packages:
-      - impi2018
-      - wrfv4
+  spack:
+    concretized: true
+    packages:
+      gcc9:
+        spack_spec: gcc@9.3.0 target=x86_64
+        compiler_spec: gcc@9.3.0
+      ompi412:
+        spack_spec: openmpi@4.1.2 +legacylaunchers +pmi +thread_multiple +cxx target=x86_64
+        compiler: gcc9
+      impi2018:
+        spack_spec: intel-mpi@2018.4.274
+        compiler: gcc9
+      openfoam:
+        spack_spec: openfoam-org@7
+        compiler: gcc9
+      flex:
+        spack_spec: flex@2.6.4
+        compiler: gcc9
+      wrfv4:
+        spack_spec: wrf@4.2 build_type=dm+sm compile_type=em_real nesting=basic ~chem ~pnetcdf
+        compiler: gcc9
+    environments:
+      openfoam:
+        packages:
+        - ompi412
+        - flex
+        - openfoam
+      wrfv4:
+        packages:
+        - impi2018
+        - wrfv4

examples/basic_gromacs_config.yaml

@@ -1,7 +1,10 @@
 ramble:
+  env_vars:
+    set:
+      OMP_NUM_THREADS: '{n_threads}'
   variables:
     processes_per_node: 16
-    mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node} -hostfile hostfile'
+    mpi_command: 'mpirun -n {n_ranks} -ppn {processes_per_node}'
     batch_submit: '{execute_experiment}'
   applications:
     gromacs: # Application name
@@ -28,17 +31,18 @@ ramble:
                 n_threads: '1'
                 size: '0003'
                 type: 'pme'
-              rf_single_rank:
-                variables:
-                  n_ranks: '1'
-                  n_threads: '1'
-                  size: '0003'
-                  type: 'rf'
-spack:
+            rf_single_rank:
+              variables:
+                n_ranks: '1'
+                n_threads: '1'
+                size: '0003'
+                type: 'rf'
+  spack:
     concretized: true
     packages:
       gcc9:
-        spack_spec: gcc@9.3.0 target=x86_64
+        spack_spec: gcc@9.4.0 target=x86_64
+        compiler_spec: gcc@9.4.0
       impi2018:
         spack_spec: intel-mpi@2018.4.274 target=x86_64
         compiler: gcc9

examples/basic_hostname_config.yaml

@@ -20,7 +20,7 @@
 #               n_ranks: '{processes_per_node}'
 
 ramble:
-  env-vars:
+  env_vars:
     set:
       OMP_NUM_THREADS: '{n_threads}'
   variables:
@@ -43,7 +43,7 @@ ramble:
                 n_ranks: '2'
                 n_nodes: '1'
                 processes_per_node: '16'
-spack:
-  concretized: true
-  packages: {}
-  environments: {}
+  spack:
+    concretized: true
+    packages: {}
+    environments: {}