Add support for marking and skipping slow tests, temporarily mark pyd…

…antic tests as slow (NVIDIA#589)

### Description
These pydantic CLI tests are currently taking an inordinate amount of
time to run. This PR adds support for using `pytest.mark.slow` to flag
tests not to run during pre-merge testing, but will still be run during
nightly tests.

Also adds some cleanup to the azure runner test suite to remove the
docker container from the GPU runner since these are not ephemeral
workers, and updates the cache mounting location for the container job
to re-use downloads.

See https://nvidia.slack.com/archives/C074Z808N05/p1736460988702949
https://jirasw.nvidia.com/browse/BIONEMO-553

### Type of changes
<!-- Mark the relevant option with an [x] -->

- [x]  Bug fix (non-breaking change which fixes an issue)
- [ ]  New feature (non-breaking change which adds functionality)
- [ ]  Refactor
- [ ]  Documentation update
- [ ]  Other (please describe):

### CI Pipeline Configuration
Configure CI behavior by checking relevant boxes below. This will
automatically apply labels.

- [ ]
[SKIP_CI](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/user-guide/contributing/contributing.md#skip_ci)
- Skip all continuous integration tests
- [ ]
[INCLUDE_NOTEBOOKS_TESTS](https://github.com/NVIDIA/bionemo-framework/blob/main/docs/docs/user-guide/contributing/contributing.md#include_notebooks_tests)
- Execute notebook validation tests in pytest

> [!NOTE]
> By default, the notebooks validation tests are skipped unless
explicitly enabled.

### Usage
<!--- How does a user interact with the changed code -->
```python
TODO: Add code snippet
```

### Pre-submit Checklist
<!--- Ensure all items are completed before submitting -->

 - [ ] I have tested these changes locally
 - [ ] I have updated the documentation accordingly
 - [ ] I have added/updated tests as needed
 - [ ] All existing tests pass successfully

---------

Signed-off-by: Peter St. John <pstjohn@nvidia.com>
Signed-off-by: Peter St. John <peterc.stjohn@gmail.com>

.github/workflows/unit-tests.yml

@@ -88,15 +88,17 @@ jobs:
       image: nemoci.azurecr.io/bionemo:${{ github.run_id }}
       options: --gpus all
       volumes:
-        - /home/azureuser/actions-runner-bionemo/cache:/root/.cache
+        - /home/azureuser/actions-runner-bionemo/cache:/github/home/.cache
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           path: ${{ github.run_id }}
 
       - name: Run tests
-        run: ./ci/scripts/run_pytest.sh --no-nbval
+        env:
+          BIONEMO_DATA_SOURCE: ngc
+        run: ./ci/scripts/run_pytest.sh --no-nbval --skip-slow
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
@@ -110,5 +112,14 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           working-directory: ${{ github.run_id }}
+
+  clean-up:
+    needs: run-tests
+    runs-on: self-hosted-nemo-gpus-1
+    if: ${{ always() }}
+    steps:
+      - name: clean up image
+        run: docker rmi nemoci.azurecr.io/bionemo:${{ github.run_id }}
+
 # TODO: exclude tests from base image; run tests from github workspace mounted in the image.
-# TODO: upload coverage report to codecov.io
+# TODO: figure out way of cleaning up working directory (requires sudo or for us to fix file ownership from release container)

ci/scripts/run_pytest.sh

@@ -27,6 +27,7 @@ Usage: $(basename "$0") [OPTIONS]
 Options:
     --skip-docs    Skip running tests in the docs directory
     --no-nbval     Skip jupyter notebook validation tests
+    --skip-slow    Skip tests marked as slow (@pytest.mark.slow)
 
 Note: Documentation tests (docs/) are only run when notebook validation
       is enabled (--no-nbval not set) and docs are not skipped
@@ -48,13 +49,15 @@ export BIONEMO_DATA_SOURCE PYTHONDONTWRITEBYTECODE PYTORCH_CUDA_ALLOC_CONF
 declare -a coverage_files
 SKIP_DOCS=false
 NO_NBVAL=false
+SKIP_SLOW=false
 error=false
 
 # Parse command line arguments
 while (( $# > 0 )); do
     case "$1" in
         --skip-docs) SKIP_DOCS=true ;;
         --no-nbval) NO_NBVAL=true ;;
+        --skip-slow) SKIP_SLOW=true ;;
         -h|--help) usage ;;
         *) echo "Unknown option: $1" >&2; usage 1 ;;
     esac
@@ -83,6 +86,7 @@ PYTEST_OPTIONS=(
     --cov-report=xml:coverage.xml
 )
 [[ "$NO_NBVAL" != true ]] && PYTEST_OPTIONS+=(--nbval-lax)
+[[ "$SKIP_SLOW" == true ]] && PYTEST_OPTIONS+=(-m "not slow")
 
 # Define test directories
 TEST_DIRS=(./sub-packages/bionemo-*/)

sub-packages/bionemo-esm2/tests/bionemo/esm2/scripts/test_pydantic_train.py

@@ -51,6 +51,8 @@ def dummy_parquet_train_val_inputs(tmp_path):
     return train_cluster_path, valid_cluster_path
 
 
+# TODO: These tests currently take an inordinate amount of time. See https://jirasw.nvidia.com/browse/BIONEMO-553
+@pytest.mark.slow
 def test_pretrain_pydantic_cli(dummy_protein_dataset, dummy_parquet_train_val_inputs, tmpdir):
     result_dir = tmpdir.mkdir("results")
     train_cluster_path, valid_cluster_path = dummy_parquet_train_val_inputs

sub-packages/bionemo-geneformer/tests/bionemo/geneformer/scripts/test_pydantic_train.py

@@ -42,6 +42,8 @@ def test_bionemo2_rootdir(data_path):
     assert data_path.is_dir(), f"Test data directory is supposed to be a directory.\n{data_error_str}"
 
 
+# TODO: These tests currently take an inordinate amount of time. See https://jirasw.nvidia.com/browse/BIONEMO-553
+@pytest.mark.slow
 def test_pretrain_cli_from_ckpt(tmpdir, data_path):
     # Same as test_pretrain, but includes a checkpoint to initialize from.
     result_dir = Path(tmpdir.mkdir("results"))
@@ -83,6 +85,8 @@ def test_pretrain_cli_from_ckpt(tmpdir, data_path):
     assert (result_dir / "test-experiment").exists(), "Could not find test experiment directory."
 
 
+# TODO: These tests currently take an inordinate amount of time. See https://jirasw.nvidia.com/browse/BIONEMO-553
+@pytest.mark.slow
 def test_pretrain_cli(tmpdir, data_path):
     """trains from scratch"""
     # data_path: Path = load("single_cell/testdata-20240506") / "cellxgene_2023-12-15_small" / "processed_data"
@@ -123,6 +127,8 @@ def test_pretrain_cli(tmpdir, data_path):
     assert (result_dir / "test-experiment").exists(), "Could not find test experiment directory."
 
 
+# TODO: These tests currently take an inordinate amount of time. See https://jirasw.nvidia.com/browse/BIONEMO-553
+@pytest.mark.slow
 def test_finetune_cli(tmpdir, data_path):
     """Uses CLI to invoke the entrypoint"""
     # data_path: Path = load("single_cell/testdata-20240506") / "cellxgene_2023-12-15_small" / "processed_data"