playing with new caches for workflow CI

bgunnar5 · Sep 24, 2024 · e38cc93 · e38cc93
1 parent 176ff4d
commit e38cc93
Showing 1 changed file with 74 additions and 62 deletions.
diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml
@@ -82,7 +82,7 @@ jobs:
         python3 -m pylint merlin --rcfile=setup.cfg --exit-zero
         python3 -m pylint tests --rcfile=setup.cfg --exit-zero
 
-  common-setup:
+  Local-test-suite:
     runs-on: ubuntu-latest
     env: 
       GO_VERSION: 1.18.1
@@ -101,18 +101,26 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Check cache
+    - name: Cache Python dependencies
       uses: actions/cache@v2
       with:
-        path: ${{ env.pythonLocation }}
-        key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
+        path: ~/.cache/pip
+        key: ${{ os.runner }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
 
     - name: Install dependencies
       run: |
         python3 -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         pip3 install -r requirements/dev.txt
 
+    - name: Cache Singularity build artifacts
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION
+            /usr/local/go
+          key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
+
     - name: Install singularity
       run: |
         sudo apt-get update && sudo apt-get install -y \
@@ -134,61 +142,6 @@ jobs:
           make -C ./builddir && \
           sudo make -C ./builddir install
 
-  # TODO make this setup inheritable and then create unit/local/integration tests off of it?
-  Local-test-suite:
-    needs: common-setup
-    runs-on: ubuntu-latest
-    # env: 
-    #   GO_VERSION: 1.18.1
-    #   SINGULARITY_VERSION: 3.9.9
-    #   OS: linux
-    #   ARCH: amd64
-
-    # strategy:
-    #   matrix:
-    #     python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
-
-    # steps:
-    # - uses: actions/checkout@v2
-    # - name: Set up Python ${{ matrix.python-version }}
-    #   uses: actions/setup-python@v2
-    #   with:
-    #     python-version: ${{ matrix.python-version }}
-
-    # - name: Check cache
-    #   uses: actions/cache@v2
-    #   with:
-    #     path: ${{ env.pythonLocation }}
-    #     key: ${{ env.pythonLocation }}-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
-
-    # - name: Install dependencies
-    #   run: |
-    #     python3 -m pip install --upgrade pip
-    #     if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    #     pip3 install -r requirements/dev.txt
-    #     pip freeze
-
-    # - name: Install singularity
-    #   run: |
-    #     sudo apt-get update && sudo apt-get install -y \
-    #       build-essential \
-    #       libssl-dev \
-    #       uuid-dev \
-    #       libgpgme11-dev \
-    #       squashfs-tools \
-    #       libseccomp-dev \
-    #       pkg-config
-    #     wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz
-    #     sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
-    #     rm go$GO_VERSION.$OS-$ARCH.tar.gz
-    #     export PATH=$PATH:/usr/local/go/bin
-    #     wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz
-    #     tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz
-    #     cd singularity-ce-$SINGULARITY_VERSION
-    #     ./mconfig && \
-    #       make -C ./builddir && \
-    #       sudo make -C ./builddir install
-    steps:
     - name: Install merlin to run unit tests
       run: |
         pip3 install -e .
@@ -207,14 +160,73 @@ jobs:
       run: |
         python3 tests/integration/run_tests.py --verbose --local
 
+  # TODO get this and local-test-suite working
+  # - probably need to move the python 3.x call to this below
+  # - not sure if we need --broker redis in the Install step below
   Integration-tests:
-    needs: common-setup
     runs-on: ubuntu-latest
+    env: 
+      GO_VERSION: 1.18.1
+      SINGULARITY_VERSION: 3.9.9
+      OS: linux
+      ARCH: amd64
+
+    strategy:
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+
     steps:
-    - name: Install merlin and setup redis as the broker
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache Python dependencies
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ os.runner }}-pip-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
+
+    - name: Install dependencies
+      run: |
+        python3 -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip3 install -r requirements/dev.txt
+
+    - name: Install merlin
       run: |
         pip3 install -e .
-        merlin config --broker redis
+        merlin config
+
+    - name: Cache Singularity build artifacts
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ github.workspace }}/singularity-ce-$SINGULARITY_VERSION
+            /usr/local/go
+          key: ${{ runner.os }}-singularity-${{ hashFiles('requirements/release.txt') }}-${{ hashFiles('requirements/dev.txt') }}
+
+    - name: Install singularity
+      run: |
+        sudo apt-get update && sudo apt-get install -y \
+          build-essential \
+          libssl-dev \
+          uuid-dev \
+          libgpgme11-dev \
+          squashfs-tools \
+          libseccomp-dev \
+          pkg-config
+        wget https://go.dev/dl/go$GO_VERSION.$OS-$ARCH.tar.gz
+        sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz
+        rm go$GO_VERSION.$OS-$ARCH.tar.gz
+        export PATH=$PATH:/usr/local/go/bin
+        wget https://github.com/sylabs/singularity/releases/download/v$SINGULARITY_VERSION/singularity-ce-$SINGULARITY_VERSION.tar.gz
+        tar -xzf singularity-ce-$SINGULARITY_VERSION.tar.gz
+        cd singularity-ce-$SINGULARITY_VERSION
+        ./mconfig && \
+          make -C ./builddir && \
+          sudo make -C ./builddir install
 
     - name: Install CLI task dependencies generated from the 'feature demo' workflow
       run: |