Merged changes from develop

bgunnar5 · Nov 26, 2024 · 63b7b51 · 63b7b51
2 parents 352e7df + e1655df
commit 63b7b51
Show file tree

Hide file tree

Showing 117 changed files with 10,097 additions and 1,002 deletions.
diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml
@@ -10,7 +10,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0 # Checkout the whole history, in case the target is way far behind
 
@@ -40,14 +40,14 @@ jobs:
       MAX_COMPLEXITY: 15
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
 
     - name: Check cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
@@ -95,14 +95,14 @@ jobs:
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Check cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ${{ env.pythonLocation }}
         key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
@@ -112,8 +112,7 @@ jobs:
         python3 -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         pip3 install -r requirements/dev.txt
-        pip freeze
-    
+
     - name: Install singularity
       run: |
         sudo apt-get update && sudo apt-get install -y \
@@ -145,48 +144,31 @@ jobs:
         merlin example feature_demo
         pip3 install -r feature_demo/requirements.txt
 
-    - name: Run pytest over unit test suite
-      run: |
-        python3 -m pytest -v --order-scope=module tests/unit/
-
     - name: Run integration test suite for local tests
       run: |
         python3 tests/integration/run_tests.py --verbose --local
 
-  Distributed-test-suite:
+  Unit-tests:
     runs-on: ubuntu-latest
-    services:
-      # rabbitmq:
-      #   image: rabbitmq:latest
-      #   ports:
-      #     - 5672:5672
-      #   options: --health-cmd "rabbitmqctl node_health_check" --health-interval 10s --health-timeout 5s --health-retries 5
-      # Label used to access the service container
-      redis:
-        # Docker Hub image
-        image: redis
-        # Set health checks to wait until redis has started
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 6379:6379
+    env: 
+      GO_VERSION: 1.18.1
+      SINGULARITY_VERSION: 3.9.9
+      OS: linux
+      ARCH: amd64
 
     strategy:
       matrix:
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Check cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: ${{ env.pythonLocation }}
         key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
@@ -197,33 +179,104 @@ jobs:
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         pip3 install -r requirements/dev.txt
 
-    - name: Install merlin and setup redis as the broker
+    - name: Install singularity
+      run: |
+        sudo apt-get update && sudo apt-get install -y \
+          build-essential \
+          libssl-dev \
+          uuid-dev \
+          libgpgme11-dev \
+          squashfs-tools \
+          libseccomp-dev \
+          pkg-config
+        wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz
+        sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
+        rm go$GO_VERSION.$OS-$ARCH.tar.gz
+        export PATH=$PATH:/usr/local/go/bin
+        wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz
+        tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz
+        cd singularity-ce-$SINGULARITY_VERSION
+        ./mconfig && \
+          make -C ./builddir && \
+          sudo make -C ./builddir install
+
+    - name: Install merlin to run unit tests
       run: |
         pip3 install -e .
-        merlin config --broker redis
+        merlin config
 
     - name: Install CLI task dependencies generated from the 'feature demo' workflow
       run: |
         merlin example feature_demo
         pip3 install -r feature_demo/requirements.txt
 
+    - name: Run pytest over unit test suite
+      run: |
+        python3 -m pytest -v --order-scope=module tests/unit/
+
+  Integration-tests:
+    runs-on: ubuntu-latest
+    env: 
+      GO_VERSION: 1.18.1
+      SINGULARITY_VERSION: 3.9.9
+      OS: linux
+      ARCH: amd64
+
+    strategy:
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Check cache
+      uses: actions/cache@v4
+      with:
+        path: ${{ env.pythonLocation }}
+        key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
+
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip3 install -r requirements/dev.txt
+
+    - name: Install merlin
+      run: |
+        pip3 install -e .
+        merlin config
+
+    - name: Install singularity
+      run: |
+        sudo apt-get update && sudo apt-get install -y \
+          build-essential \
+          libssl-dev \
+          uuid-dev \
+          libgpgme11-dev \
+          squashfs-tools \
+          libseccomp-dev \
+          pkg-config
+        wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz
+        sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
+        rm go$GO_VERSION.$OS-$ARCH.tar.gz
+        export PATH=$PATH:/usr/local/go/bin
+        wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz
+        tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz
+        cd singularity-ce-$SINGULARITY_VERSION
+        ./mconfig && \
+          make -C ./builddir && \
+          sudo make -C ./builddir install
+
+    - name: Install CLI task dependencies generated from the 'feature demo' workflow
+      run: |
+        merlin example feature_demo
+        pip3 install -r feature_demo/requirements.txt
+
+    # TODO remove the --ignore statement once those tests are fixed
     - name: Run integration test suite for distributed tests
-      env:
-        REDIS_HOST: redis
-        REDIS_PORT: 6379
-      run: |
-        python3 tests/integration/run_tests.py --verbose --distributed
-
-    # - name: Setup rabbitmq config
-    #   run: |
-    #     merlin config --test rabbitmq
-
-    # - name: Run integration test suite for rabbitmq
-    #   env:
-    #     AMQP_URL: amqp://localhost:${{ job.services.rabbitmq.ports[5672] }}
-    #     RABBITMQ_USER: Jimmy_Space
-    #     RABBITMQ_PASS: Alexander_Rules
-    #   ports:
-    #     - ${{ job.services.rabbitmq.ports['5672'] }}
-    #   run: |
-    #     python3 tests/integration/run_tests.py --verbose --ids 31 32
+      run: |
+        python3 -m pytest -v --ignore tests/integration/test_celeryadapter.py tests/integration/
diff --git a/.gitignore b/.gitignore
@@ -39,8 +39,9 @@ flux.out
 slurm*.out
 docs/build/
 
-# Tox files
+# Test files
 .tox/*
+.coverage
 
 # Jupyter
 jupyter/.ipynb_checkpoints

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,11 +4,23 @@ All notable changes to Merlin will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [unreleased]
+## [Unreleased]
 ### Added
 - Merlin manager capability to monitor celery workers.
+- Added additional tests for the `merlin run` and `merlin purge` commands
+- Aliased types to represent different types of pytest fixtures
+- New test condition `StepFinishedFilesCount` to help search for `MERLIN_FINISHED` files in output workspaces
+- Added "Unit-tests" GitHub action to run the unit test suite
+- Added `CeleryTaskManager` context manager to the test suite to ensure tasks are safely purged from queues if tests fail
+- Added `command-tests`, `workflow-tests`, and `integration-tests` to the Makefile
 
-## [1.12.2b1]
+### Changed
+- Ported all distributed tests of the integration test suite to pytest
+  - There is now a `commands/` directory and a `workflows/` directory under the integration suite to house these tests
+  - Removed the "Distributed-tests" GitHub action as these tests will now be run under "Integration-tests"
+- Removed `e2e-distributed*` definitions from the Makefile
+
+## [1.12.2]
 ### Added
 - Conflict handler option to the `dict_deep_merge` function in `utils.py`
 - Ability to add module-specific pytest fixtures
@@ -23,6 +35,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - New step return code `$(MERLIN_RAISE_ERROR)` to force an error to be raised by a task (mainly for testing)
   - Added description of this to docs
 - New test to ensure a single failed task won't break a workflow
+- Several new unit tests for the following subdirectories:
+  - `merlin/common/`
+  - `merlin/config/`
+  - `merlin/examples/`
+  - `merlin/server/`
+- Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures
+  - `RedisServerManager`: context to help with starting/stopping a redis server for tests
+  - `CeleryWorkersManager`: context to help with starting/stopping workers for tests
+- Ability to copy and print the `Config` object from `merlin/config/__init__.py`
+- Equality method to the `ContainerFormatConfig` and `ContainerConfig` objects from `merlin/server/server_util.py`
 
 ### Changed
 - `merlin info` is cleaner and gives python package info
@@ -32,6 +54,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added fixtures for `merlin status` tests that copy the workspace to a temporary directory so you can see exactly what's run in a test
 - Batch block and workers now allow for variables to be used in node settings
 - Task id is now the path to the directory
+- Split the `start_server` and `config_server` functions of `merlin/server/server_commands.py` into multiple functions to make testing easier
+- Split the `create_server_config` function of `merlin/server/server_config.py` into two functions to make testing easier
+- Combined `set_snapshot_seconds` and `set_snapshot_changes` methods of `RedisConfig` into one method `set_snapshot`
 
 ### Fixed
 - Bugfix for output of `merlin example openfoam_wf_singularity`
@@ -99,8 +124,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - this required adding a decent amount of test files to help with the tests; these can be found under the tests/unit/study/status_test_files directory
 - Pytest fixtures in the `conftest.py` file of the integration test suite
   - NOTE: an export command `export LC_ALL='C'` had to be added to fix a bug in the WEAVE CI. This can be removed when we resolve this issue for the `merlin server` command
-- Tests for the `celeryadapter.py` module
-- New CeleryTestWorkersManager context to help with starting/stopping workers for tests
+- Coverage to the test suite. This includes adding tests for:
+  - `merlin/common/`
+  - `merlin/config/`
+  - `merlin/examples/`
+  - `celeryadapter.py`
+- Context managers for the `conftest.py` file to ensure safe spin up and shutdown of fixtures
+  - `RedisServerManager`: context to help with starting/stopping a redis server for tests
+  - `CeleryWorkersManager`: context to help with starting/stopping workers for tests
+- Ability to copy and print the `Config` object from `merlin/config/__init__.py`
 
 ### Changed
 - Reformatted the entire `merlin status` command
@@ -136,7 +168,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - The `merlin monitor` command will now keep an allocation up if the queues are empty and workers are still processing tasks
 - Add the restart keyword to the specification docs
 - Cyclical imports and config imports that could easily cause ci issues
-
 ## [1.11.1]
 ### Fixed
 - Typo in `batch.py` that caused lsf launches to fail (`ALL_SGPUS` changed to `ALL_GPUS`)

diff --git a/Makefile b/Makefile
@@ -6,7 +6,7 @@
 #
 # LLNL-CODE-797170
 # All rights reserved.
-# This file is part of Merlin, Version: 1.12.2b1.
+# This file is part of Merlin, Version: 1.12.2.
 #
 # For details, see https://github.com/LLNL/merlin.
 #
@@ -34,12 +34,13 @@ include config.mk
 .PHONY : install-workflow-deps
 .PHONY : install-dev
 .PHONY : unit-tests
+.PHONY : command-tests
+.PHONY : workflow-tests
+.PHONY : integration-tests
 .PHONY : e2e-tests
 .PHONY : e2e-tests-diagnostic
 .PHONY : e2e-tests-local
 .PHONY : e2e-tests-local-diagnostic
-.PHONY : e2e-tests-distributed
-.PHONY : e2e-tests-distributed-diagnostic
 .PHONY : tests
 .PHONY : check-flake8
 .PHONY : check-black
@@ -89,6 +90,18 @@ unit-tests:
 	. $(VENV)/bin/activate; \
 	$(PYTHON) -m pytest -v --order-scope=module $(UNIT); \
 
+command-tests:
+	. $(VENV)/bin/activate; \
+	$(PYTHON) -m pytest -v $(TEST)/integration/commands/; \
+
+
+workflow-tests:
+	. $(VENV)/bin/activate; \
+	$(PYTHON) -m pytest -v $(TEST)/integration/workflows/; \
+
+
+integration-tests: command-tests workflow-tests
+
 
 # run CLI tests - these require an active install of merlin in a venv
 e2e-tests:
@@ -111,18 +124,8 @@ e2e-tests-local-diagnostic:
 	$(PYTHON) $(TEST)/integration/run_tests.py --local --verbose
 
 
-e2e-tests-distributed:
-	. $(VENV)/bin/activate; \
-	$(PYTHON) $(TEST)/integration/run_tests.py --distributed; \
-
-
-e2e-tests-distributed-diagnostic:
-	. $(VENV)/bin/activate; \
-	$(PYTHON) $(TEST)/integration/run_tests.py --distributed --verbose
-
-
 # run unit and CLI tests
-tests: unit-tests e2e-tests
+tests: unit-tests integration-tests e2e-tests
 
 
 check-flake8: