From 4ab74a3b2903c5f9f1e7fa7488e2adc25a005165 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Thu, 13 Jul 2023 13:56:51 -0400
Subject: [PATCH 01/37] add some more linting checks

---
 .pre-commit-config.yaml | 12 ++++++++++++
 .yamllint.yaml          |  8 ++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 .yamllint.yaml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 36f5d891..752f5bf7 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,6 +15,13 @@ repos:
       - id: end-of-file-fixer
         exclude: .ipynb|.po|.xml
       - id: debug-statements
+      - id: check-yaml
+        args: [ '--allow-multiple-documents' ]
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.32.0
+    hooks:
+      - id: yamllint
+        args: [ '--config-file=.yamllint.yaml' ]
   - repo: https://github.com/psf/black
     rev: 23.3.0
     hooks:
@@ -42,6 +49,11 @@ repos:
       - id: nbstripout
         files: ".ipynb"
         args: [ '--keep-output', '--keep-count' ]
+  - repo: https://github.com/keewis/blackdoc
+    rev: v0.3.8
+    hooks:
+    - id: blackdoc
+      additional_dependencies: ['black==23.3.0']
   - repo: meta
     hooks:
       - id: check-hooks-apply
diff --git a/.yamllint.yaml b/.yamllint.yaml
new file mode 100644
index 00000000..2f3b4a3d
--- /dev/null
+++ b/.yamllint.yaml
@@ -0,0 +1,8 @@
+---
+
+rules:
+  document-start: disable
+  line-length:
+    max: 120
+    level: warning
+  truthy: disable

From 1d618c9d3d78a1cbe43986e5d289e53a82a8eee2 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Thu, 13 Jul 2023 13:58:44 -0400
Subject: [PATCH 02/37] try a docker-based testing approach using dagger-io

---
 .github/workflows/dagger-docker.yml | 39 +++++++++++++++++++++++++++++
 tests/ci/test.py                    | 25 ++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 .github/workflows/dagger-docker.yml
 create mode 100644 tests/ci/test.py

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
new file mode 100644
index 00000000..32cef05b
--- /dev/null
+++ b/.github/workflows/dagger-docker.yml
@@ -0,0 +1,39 @@
+name: Dagger/Docker Testing Suite
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  lint:
+    name: Linting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install pre-commit
+          pre-commit install
+      - name: Run pre-commit
+        run: |
+          pre-commit run --all-files --show-diff-on-failure
+
+  integration:
+    name: Integration Tests
+    needs: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Dagger
+        uses: dagger/dagger-for-github@v4
+        with:
+          workdir: ./tests/ci
+          cmds: python test.py
diff --git a/tests/ci/test.py b/tests/ci/test.py
new file mode 100644
index 00000000..1750f5a2
--- /dev/null
+++ b/tests/ci/test.py
@@ -0,0 +1,25 @@
+"""Execute a command."""
+
+import sys
+
+import anyio
+import dagger
+
+
+async def test():
+    async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
+        python = (
+            client.container()
+            # pull container
+            .from_("python:3.11-slim-buster")
+            # get Python version
+            .with_exec(["python", "-V"])
+        )
+
+        # execute
+        version = await python.stdout()
+
+    print(f"Hello from Dagger and {version}")
+
+
+anyio.run(test)

From 025218c28697a290a1ff04235cdf7db1f395ecb3 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Thu, 13 Jul 2023 13:59:12 -0400
Subject: [PATCH 03/37] update sphinx-rtd-theme package name

---
 environment-dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/environment-dev.yml b/environment-dev.yml
index ab772295..1b0b5a39 100644
--- a/environment-dev.yml
+++ b/environment-dev.yml
@@ -11,4 +11,4 @@ dependencies:
   - pytest
   - sphinx
   - sphinx-intl
-  - sphinx_rtd_theme >=1.0
+  - sphinx-rtd-theme >=1.0

From 3ba5ea2adf14cd792862d3f97d96159d7233a141 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Thu, 13 Jul 2023 14:51:24 -0400
Subject: [PATCH 04/37] Change LICENSE to BSD-3-Clause, move license to
 top-level, rename release_notes, add python package metadata

---
 LICENSE                                       | 28 +++++++++++++++++++
 docs/source/index.rst                         |  5 +---
 docs/source/license.rst                       | 16 +----------
 .../{releasenotes.rst => release_notes.rst}   |  0
 setup.py                                      | 13 +++++++--
 5 files changed, 41 insertions(+), 21 deletions(-)
 create mode 100644 LICENSE
 rename docs/source/{releasenotes.rst => release_notes.rst} (100%)

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 00000000..d35a2dc1
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2017, Ouranos, CRIM
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 77ef6e32..1eb7ee8a 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -41,13 +41,10 @@ Contents
    arch/index
    provenance/index
    support
-   releasenotes
+   release_notes
    license
    todo
 
-
-
-
 Indices and tables
 ==================
 
diff --git a/docs/source/license.rst b/docs/source/license.rst
index 99c7b579..98f78498 100644
--- a/docs/source/license.rst
+++ b/docs/source/license.rst
@@ -2,18 +2,4 @@
 License
 =======
 
-
-Copyright (c) 2017, Ouranos, CRIM
-All rights reserved.
-
-Permission to use, copy, modify, and/or distribute this software for any
-purpose with or without fee is hereby granted, provided that the above
-copyright notice and this permission notice appear in all copies.
-
-THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
-ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+.. include:: ../../LICENSE
diff --git a/docs/source/releasenotes.rst b/docs/source/release_notes.rst
similarity index 100%
rename from docs/source/releasenotes.rst
rename to docs/source/release_notes.rst
diff --git a/setup.py b/setup.py
index 357771f0..5f99d265 100644
--- a/setup.py
+++ b/setup.py
@@ -3,8 +3,13 @@
 CLASSIFIERS = [
     "Intended Audience :: Science/Research",
     "License :: OSI Approved :: BSD License",
-    "Programming Language :: Python",
     "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python",
     "Topic :: Scientific/Engineering :: Atmospheric Science",
 ]
 
@@ -24,7 +29,11 @@
             "sphinx-jsonschema",
             "sphinx-rtd-theme>=1.0",
             "sphinx>=1.4",
-        ]
+        ],
+        "dagger": [
+            "anyio",
+            "dagger",
+        ],
     },
     author="CRIM/Ouranos",
     author_email="pavics@ouranos.ca",

From 05578edca1bc53af96b64b7f8e7f794b40933866 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Thu, 13 Jul 2023 16:24:51 -0400
Subject: [PATCH 05/37] adopt PEP517 and PEP 621, add more metadata, add
 pre-commit hooks for toml files, remove setup.py

---
 .pre-commit-config.yaml | 12 ++++++---
 pavics_sdi/__init__.py  |  4 +++
 pyproject.toml          | 58 +++++++++++++++++++++++++++++++++++++++++
 setup.py                | 44 -------------------------------
 4 files changed, 71 insertions(+), 47 deletions(-)
 create mode 100644 pavics_sdi/__init__.py
 create mode 100644 pyproject.toml
 delete mode 100644 setup.py

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 752f5bf7..e91c041f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ repos:
     rev: v3.8.0
     hooks:
       - id: pyupgrade
-        args: [ '--py37-plus' ]
+        args: [ '--py38-plus' ]
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
@@ -15,8 +15,14 @@ repos:
       - id: end-of-file-fixer
         exclude: .ipynb|.po|.xml
       - id: debug-statements
+        exclude: .ipynb
+      - id: check-toml
       - id: check-yaml
         args: [ '--allow-multiple-documents' ]
+  - repo: https://github.com/pappasam/toml-sort
+    rev: v0.23.1
+    hooks:
+      - id: toml-sort-fix
   - repo: https://github.com/adrienverge/yamllint.git
     rev: v1.32.0
     hooks:
@@ -35,10 +41,10 @@ repos:
     rev: 1.7.0
     hooks:
       - id: nbqa-black
-        args: [ '--target-version=py37' ]
+        args: [ '--target-version=py38' ]
         additional_dependencies: [ black==23.3.0 ]
       - id: nbqa-pyupgrade
-        args: [ '--py37-plus' ]
+        args: [ '--py38-plus' ]
         additional_dependencies: [ pyupgrade==v3.3.1 ]
       - id: nbqa-isort
         args: [ '--profile=black' ]
diff --git a/pavics_sdi/__init__.py b/pavics_sdi/__init__.py
new file mode 100644
index 00000000..655aef37
--- /dev/null
+++ b/pavics_sdi/__init__.py
@@ -0,0 +1,4 @@
+"""PAVICS SDI package."""
+
+
+__version__ = "1.3.0"
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000..8afc8841
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,58 @@
+[build-system]
+requires = ["flit_core >=3.8,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "pavics_sdi"
+requires-python = ">=3.8"
+authors = [
+  {name = "CRIM/Ouranos", email = "pavics@ouranos.ca"}
+]
+maintainers = [
+  {name = "Tuan Long Vu", email = "vu.long@ouranos.ca"},
+  {name = "David Huard", email = "huard.davis@ouranos.ca"},
+  {name = "Trevor James Smith", email = "smith.trevorj@ouranos.ca"},
+  {name = "Francis Charette-Migneault", email = "francis.charette.migneault@gmail.com"}
+]
+readme = {file = "README.rst", content-type = "text/x-rst"}
+description = "Scientific gateway for climate data analytics."
+license = {file = "LICENSE"}
+classifiers = [
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: BSD License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Topic :: Scientific/Engineering :: Atmospheric Science"
+]
+dynamic = ["version"]
+dependencies = []
+
+[project.optional-dependencies]
+docs = [
+  "jinja2",
+  "jupyter",
+  "jupytext",
+  "myst-nb",
+  "requests",
+  "sphinx-intl",
+  "sphinx-jsonschema",
+  "sphinx-rtd-theme>=1.0",
+  "sphinx>=1.4"
+]
+dagger = [
+  "anyio; python_version >= '3.10'",
+  "dagger; python_version >= '3.10'"
+]
+
+[project.urls]
+"Homepage" = "https://pavics-sdi.readthedocs.io"
+"Source" = "https://github.com/Ouranosinc/pavics-sdi/"
+"Changelog" = "https://pavics-sdi.readthedocs.io/en/latest/release_notes.html"
+"Issue tracker" = "https://github.com/Ouranosinc/pavics-sdi/issues"
+"About CRIM" = "https://www.crim.ca/en/"
+"About Ouranos" = "https://www.ouranos.ca/en/"
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 5f99d265..00000000
--- a/setup.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from setuptools import setup
-
-CLASSIFIERS = [
-    "Intended Audience :: Science/Research",
-    "License :: OSI Approved :: BSD License",
-    "Operating System :: OS Independent",
-    "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python",
-    "Topic :: Scientific/Engineering :: Atmospheric Science",
-]
-
-setup(
-    name="pavics_sdi",
-    version="0.1",
-    url="https://pavics-sdi.readthedocs.io",
-    install_requires=[],
-    extras_require={
-        "docs": [
-            "jinja2",
-            "jupyter",
-            "jupytext",
-            "myst-nb",
-            "requests",
-            "sphinx-intl",
-            "sphinx-jsonschema",
-            "sphinx-rtd-theme>=1.0",
-            "sphinx>=1.4",
-        ],
-        "dagger": [
-            "anyio",
-            "dagger",
-        ],
-    },
-    author="CRIM/Ouranos",
-    author_email="pavics@ouranos.ca",
-    description="Scientific gateway for climate data analytics.",
-    long_description=open("README.rst").read(),
-    license="BSD",
-    classifiers=CLASSIFIERS,
-)

From 57dae29a55bbbf47d6b637821ff34f746daa5d33 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 11:44:17 -0400
Subject: [PATCH 06/37] try a different approach using a Dockerfile and build
 arguments

---
 .github/workflows/dagger-docker.yml | 19 +++++++++---
 Dockerfile                          | 36 +++++++++++++++++++++++
 ci/dagger-pipeline.py               | 45 +++++++++++++++++++++++++++++
 environment-dev.yml                 |  1 +
 tests/ci/test.py                    | 25 ----------------
 5 files changed, 97 insertions(+), 29 deletions(-)
 create mode 100644 Dockerfile
 create mode 100644 ci/dagger-pipeline.py
 delete mode 100644 tests/ci/test.py

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 32cef05b..0e58f56e 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -29,11 +29,22 @@ jobs:
     name: Integration Tests
     needs: lint
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+        matrix:
+            python-version: [3.9]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Dagger
-        uses: dagger/dagger-for-github@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
         with:
-          workdir: ./tests/ci
-          cmds: python test.py
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install -r requirements-dev.txt
+      - name: Dagger testing
+        run: |
+          python ./ci/dagger-pipeline.py
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..b8267a71
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,36 @@
+# vim:set ft=dockerfile:
+ARG BASE_IMAGE_TAG
+FROM pavics/workflow-tests:${BASE_IMAGE_TAG:-latest}
+ARG DEBIAN_FRONTEND=noninteractive
+ENV PIP_ROOT_USER_ACTION=ignore
+ARG PYTHON_VERSION
+ENV PYTHON_VERSION=${PYTHON_VERSION:-3.8}
+LABEL org.opencontainers.image.authors="https://github.com/bird-house/finch"
+LABEL Description="Finch WPS" Vendor="Birdhouse" Version="0.11.1"
+
+# root-level commands
+USER root
+
+# Install build tools
+RUN apt-get install --yes build-essential
+
+# Switch to /code directory
+WORKDIR /code
+
+# Build finch environment
+COPY environment-dev.yml /code
+RUN mamba install -n birdy python=$PYTHON_VERSION --yes \
+    && mamba env update -n birdy -f environment-dev.yml \
+    && mamba clean --all --yes
+
+# Add the finch conda environment to the path
+ENV PATH /opt/conda/envs/birdy/bin:$PATH
+
+# Copy finch source code
+COPY . /code
+
+# Install pavics-sdi package
+RUN pip install . --no-deps
+
+# Build the documentation
+CMD ["make", "--directory=/code/docs", "html"]
diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
new file mode 100644
index 00000000..b7535800
--- /dev/null
+++ b/ci/dagger-pipeline.py
@@ -0,0 +1,45 @@
+"""Build the documentation within a docker pipeline to test integration and notebooks."""
+
+import sys
+from pathlib import Path
+
+try:
+    import anyio
+    import dagger
+except ImportError:
+    raise ImportError(
+        "Dagger is not installed. Please install it with `pip install dagger-io` first."
+    )
+
+
+async def main():
+    async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
+        top_level_dir = client.host().directory(Path(__file__).parent.parent.as_posix())
+
+        image_ref = await top_level_dir.docker_build()
+
+        sources = (
+            client.container()
+            # pull container
+            .from_("localpavics-sdi:latest").with_workdir("/code")
+        )
+
+        username = sources.with_exec(["whoami"])
+        whats_here = sources.with_exec(["ls", "-la", "/code"])
+        python_version = sources.with_exec(["python", "-V"])
+        docs = sources.with_exec(["make", "--directory=/code/docs", "html"])
+
+        # execute
+        user = await username.stdout()
+        contents = await whats_here.stdout()
+        version = await python_version.stdout()
+        docs_built = await docs.stdout()
+
+    print(user)
+    print(contents)
+
+    print(f"Hello from Dagger and {version}")
+    print(docs_built)
+
+
+anyio.run(main)
diff --git a/environment-dev.yml b/environment-dev.yml
index 1b0b5a39..0d957c67 100644
--- a/environment-dev.yml
+++ b/environment-dev.yml
@@ -4,6 +4,7 @@ channels:
   - defaults
 dependencies:
   - python >=3.8,<3.12
+  - flit  # solely needed for installing pavics-sdi
   - jinja2 <=3.0.3
   - jupyter
   - myst-nb
diff --git a/tests/ci/test.py b/tests/ci/test.py
deleted file mode 100644
index 1750f5a2..00000000
--- a/tests/ci/test.py
+++ /dev/null
@@ -1,25 +0,0 @@
-"""Execute a command."""
-
-import sys
-
-import anyio
-import dagger
-
-
-async def test():
-    async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
-        python = (
-            client.container()
-            # pull container
-            .from_("python:3.11-slim-buster")
-            # get Python version
-            .with_exec(["python", "-V"])
-        )
-
-        # execute
-        version = await python.stdout()
-
-    print(f"Hello from Dagger and {version}")
-
-
-anyio.run(test)

From 65ceb12d3a0bf5a68206a6d9948fd610d2ded080 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 11:48:45 -0400
Subject: [PATCH 07/37] Tryo to build docker image in GitHub

---
 .github/workflows/dagger-docker.yml | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 0e58f56e..71908d34 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -38,13 +38,18 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Setup Python
-        uses: actions/setup-python@v4
+      - uses: docker/setup-buildx-action@v2
+      - uses: docker/build-push-action@v4
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          python -m pip install -r requirements-dev.txt
+          context: .
+          file: "Dockerfile"
+          build-args: |
+            PYTHON_VERSION=${{ matrix.python-version }}
+          tags: localpavics-sdi:latest
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: false
       - name: Dagger testing
         run: |
           python ./ci/dagger-pipeline.py

From 60cd5272a33f95402b8ac603eb1061221d6f3a32 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 11:55:44 -0400
Subject: [PATCH 08/37] use python3.8

---
 .github/workflows/dagger-docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 71908d34..6cd5b6a9 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -34,7 +34,7 @@ jobs:
         shell: bash -l {0}
     strategy:
         matrix:
-            python-version: [3.9]
+            python-version: [3.8]
     steps:
       - name: Checkout
         uses: actions/checkout@v3

From 949e9c1ee80faa21d8db29925cc0680a4455015e Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 12:00:49 -0400
Subject: [PATCH 09/37] make dagger config local and CI compatible

---
 ci/dagger-pipeline.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index b7535800..bdd29d57 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -1,5 +1,6 @@
 """Build the documentation within a docker pipeline to test integration and notebooks."""
 
+import os
 import sys
 from pathlib import Path
 
@@ -16,13 +17,14 @@ async def main():
     async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
         top_level_dir = client.host().directory(Path(__file__).parent.parent.as_posix())
 
-        image_ref = await top_level_dir.docker_build()
-
-        sources = (
-            client.container()
-            # pull container
-            .from_("localpavics-sdi:latest").with_workdir("/code")
-        )
+        if os.getenv("CI"):
+            sources = (
+                client.container()
+                # pull container
+                .from_("localpavics-sdi:latest").with_workdir("/code")
+            )
+        else:
+            sources = await top_level_dir.docker_build()
 
         username = sources.with_exec(["whoami"])
         whats_here = sources.with_exec(["ls", "-la", "/code"])

From f47cd203588a7c6d6ce2234711432929d025cc0c Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 12:01:40 -0400
Subject: [PATCH 10/37] add run-cancelling action

---
 .github/workflows/dagger-docker.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 6cd5b6a9..f21a9e06 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -11,6 +11,10 @@ jobs:
     name: Linting
     runs-on: ubuntu-latest
     steps:
+      - name: Cancel previous runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+            access_token: ${{ github.token }}
       - name: Checkout
         uses: actions/checkout@v3
       - name: Setup Python

From cdd8d45e5bb0acee5aba053f879abdeb9af59075 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 12:46:08 -0400
Subject: [PATCH 11/37] install dagger in running kernel

---
 .github/workflows/dagger-docker.yml | 5 +++++
 Dockerfile                          | 3 ---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index f21a9e06..8129b7f7 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -42,6 +42,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
       - uses: docker/setup-buildx-action@v2
       - uses: docker/build-push-action@v4
         with:
@@ -56,4 +60,5 @@ jobs:
           push: false
       - name: Dagger testing
         run: |
+          pip install .[dagger]
           python ./ci/dagger-pipeline.py
diff --git a/Dockerfile b/Dockerfile
index b8267a71..67046baf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -29,8 +29,5 @@ ENV PATH /opt/conda/envs/birdy/bin:$PATH
 # Copy finch source code
 COPY . /code
 
-# Install pavics-sdi package
-RUN pip install . --no-deps
-
 # Build the documentation
 CMD ["make", "--directory=/code/docs", "html"]

From 73e6e342259997c09d9d541d6f464d4602a43f90 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 13:17:04 -0400
Subject: [PATCH 12/37] fix python versions and metadata

---
 .github/workflows/dagger-docker.yml | 10 ++++++++--
 Dockerfile                          |  8 +++-----
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 8129b7f7..6f822fe9 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -5,6 +5,12 @@ on:
     branches:
       - master
   pull_request:
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: 'Image tag to use for the Docker image'
+        required: true
+        default: 'latest'
 
 jobs:
   lint:
@@ -38,7 +44,7 @@ jobs:
         shell: bash -l {0}
     strategy:
         matrix:
-            python-version: [3.8]
+            python-version: [3.10]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -52,7 +58,7 @@ jobs:
           context: .
           file: "Dockerfile"
           build-args: |
-            PYTHON_VERSION=${{ matrix.python-version }}
+            BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }} || latest
           tags: localpavics-sdi:latest
           load: true
           cache-from: type=gha
diff --git a/Dockerfile b/Dockerfile
index 67046baf..7ca0729a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,10 +3,8 @@ ARG BASE_IMAGE_TAG
 FROM pavics/workflow-tests:${BASE_IMAGE_TAG:-latest}
 ARG DEBIAN_FRONTEND=noninteractive
 ENV PIP_ROOT_USER_ACTION=ignore
-ARG PYTHON_VERSION
-ENV PYTHON_VERSION=${PYTHON_VERSION:-3.8}
-LABEL org.opencontainers.image.authors="https://github.com/bird-house/finch"
-LABEL Description="Finch WPS" Vendor="Birdhouse" Version="0.11.1"
+LABEL org.opencontainers.image.authors="https://github.com/ouranosinc/pavics-sdi"
+LABEL Description="PAVICS-SDI-TESTING-IMAGE" Vendor="Birdhouse" Version="1.3.0"
 
 # root-level commands
 USER root
@@ -19,7 +17,7 @@ WORKDIR /code
 
 # Build finch environment
 COPY environment-dev.yml /code
-RUN mamba install -n birdy python=$PYTHON_VERSION --yes \
+RUN mamba install -n birdy python=3.8 --yes \
     && mamba env update -n birdy -f environment-dev.yml \
     && mamba clean --all --yes
 

From bbeaad058cb89ade6efa33ecd6b087ce48f5f1e5 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 14:47:38 -0400
Subject: [PATCH 13/37] fix version number

---
 .github/workflows/dagger-docker.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 6f822fe9..6c6788a0 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -44,7 +44,7 @@ jobs:
         shell: bash -l {0}
     strategy:
         matrix:
-            python-version: [3.10]
+            python-version: ["3.10"]
     steps:
       - name: Checkout
         uses: actions/checkout@v3

From 251b1c0572a9000b8bd148ca8d4b12c21c1b8f94 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 15:05:18 -0400
Subject: [PATCH 14/37] fix triggering

---
 .github/workflows/dagger-docker.yml | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 6c6788a0..10de82f4 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -53,17 +53,31 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - uses: docker/setup-buildx-action@v2
+
       - uses: docker/build-push-action@v4
+        if: github.event_name == 'workflow_dispatch'
         with:
           context: .
           file: "Dockerfile"
           build-args: |
-            BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }} || latest
+            BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }}
+          tags: localpavics-sdi:latest
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: false
+
+      - uses: docker/build-push-action@v4
+        if: github.event_name != 'workflow_dispatch'
+        with:
+          context: .
+          file: "Dockerfile"
           tags: localpavics-sdi:latest
           load: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
           push: false
+
       - name: Dagger testing
         run: |
           pip install .[dagger]

From 6566b9ca4093ffff5aafcdca868120c8e024d4b7 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 15:42:00 -0400
Subject: [PATCH 15/37] fix dagger package name

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 8afc8841..dd21d4dc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,8 +45,8 @@ docs = [
   "sphinx>=1.4"
 ]
 dagger = [
-  "anyio; python_version >= '3.10'",
-  "dagger; python_version >= '3.10'"
+  "anyio",
+  "dagger-io"
 ]
 
 [project.urls]

From ca1a85bbf4a6c609916255e43a23ae80aaecf016 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 16:40:24 -0400
Subject: [PATCH 16/37] try a different approach, typo fix

---
 .github/workflows/dagger-docker.yml | 27 +--------------------------
 Dockerfile                          |  4 +++-
 ci/dagger-pipeline.py               | 18 ++++++++++--------
 pyproject.toml                      |  2 +-
 4 files changed, 15 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 10de82f4..02cec617 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -52,33 +52,8 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - uses: docker/setup-buildx-action@v2
-
-      - uses: docker/build-push-action@v4
-        if: github.event_name == 'workflow_dispatch'
-        with:
-          context: .
-          file: "Dockerfile"
-          build-args: |
-            BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }}
-          tags: localpavics-sdi:latest
-          load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          push: false
-
-      - uses: docker/build-push-action@v4
-        if: github.event_name != 'workflow_dispatch'
-        with:
-          context: .
-          file: "Dockerfile"
-          tags: localpavics-sdi:latest
-          load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          push: false
-
       - name: Dagger testing
         run: |
+          export BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }}
           pip install .[dagger]
           python ./ci/dagger-pipeline.py
diff --git a/Dockerfile b/Dockerfile
index 7ca0729a..d3ac19bb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,7 +10,9 @@ LABEL Description="PAVICS-SDI-TESTING-IMAGE" Vendor="Birdhouse" Version="1.3.0"
 USER root
 
 # Install build tools
-RUN apt-get install --yes build-essential
+RUN apt-get install --yes build-essential \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # Switch to /code directory
 WORKDIR /code
diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index bdd29d57..e3353164 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -17,14 +17,16 @@ async def main():
     async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
         top_level_dir = client.host().directory(Path(__file__).parent.parent.as_posix())
 
-        if os.getenv("CI"):
-            sources = (
-                client.container()
-                # pull container
-                .from_("localpavics-sdi:latest").with_workdir("/code")
-            )
-        else:
-            sources = await top_level_dir.docker_build()
+        # if os.getenv("CI"):
+        #     sources = (
+        #         client.container()
+        #         # pull container
+        #         .from_("localpavics-sdi:latest").with_workdir("/code")
+        #     )
+        # else:
+        sources = await top_level_dir.docker_build(
+            build_args=os.getenv("BASE_IMAGE_TAG")
+        )
 
         username = sources.with_exec(["whoami"])
         whats_here = sources.with_exec(["ls", "-la", "/code"])
diff --git a/pyproject.toml b/pyproject.toml
index dd21d4dc..f98dec3d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,7 +10,7 @@ authors = [
 ]
 maintainers = [
   {name = "Tuan Long Vu", email = "vu.long@ouranos.ca"},
-  {name = "David Huard", email = "huard.davis@ouranos.ca"},
+  {name = "David Huard", email = "huard.david@ouranos.ca"},
   {name = "Trevor James Smith", email = "smith.trevorj@ouranos.ca"},
   {name = "Francis Charette-Migneault", email = "francis.charette.migneault@gmail.com"}
 ]

From c1049c0fe3efbbce0998ddd6269a7a19352368b5 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 16:50:47 -0400
Subject: [PATCH 17/37] fix approach

---
 ci/dagger-pipeline.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index e3353164..6795e944 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -7,6 +7,7 @@
 try:
     import anyio
     import dagger
+    from dagger import BuildArg
 except ImportError:
     raise ImportError(
         "Dagger is not installed. Please install it with `pip install dagger-io` first."
@@ -25,7 +26,9 @@ async def main():
         #     )
         # else:
         sources = await top_level_dir.docker_build(
-            build_args=os.getenv("BASE_IMAGE_TAG")
+            build_args=BuildArg("BASE_IMAGE_TAG", os.getenv("BASE_IMAGE_TAG"))
+            if os.getenv("BASE_IMAGE_TAG")
+            else None,
         )
 
         username = sources.with_exec(["whoami"])

From 646587c9c196a9993160795f8fb1a2786ef332d3 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 17:08:23 -0400
Subject: [PATCH 18/37] cleanup

---
 ci/dagger-pipeline.py | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 6795e944..eec81bec 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -18,34 +18,28 @@ async def main():
     async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
         top_level_dir = client.host().directory(Path(__file__).parent.parent.as_posix())
 
-        # if os.getenv("CI"):
-        #     sources = (
-        #         client.container()
-        #         # pull container
-        #         .from_("localpavics-sdi:latest").with_workdir("/code")
-        #     )
-        # else:
+        # build container
         sources = await top_level_dir.docker_build(
             build_args=BuildArg("BASE_IMAGE_TAG", os.getenv("BASE_IMAGE_TAG"))
             if os.getenv("BASE_IMAGE_TAG")
             else None,
         )
 
-        username = sources.with_exec(["whoami"])
-        whats_here = sources.with_exec(["ls", "-la", "/code"])
+        # smoke tests
         python_version = sources.with_exec(["python", "-V"])
+        username = sources.with_exec(["whoami"])
+
+        # build docs
         docs = sources.with_exec(["make", "--directory=/code/docs", "html"])
 
         # execute
         user = await username.stdout()
-        contents = await whats_here.stdout()
         version = await python_version.stdout()
         docs_built = await docs.stdout()
 
-    print(user)
-    print(contents)
-
     print(f"Hello from Dagger and {version}")
+    print(f"Running commands as {user} user.")
+
     print(docs_built)
 
 

From 343cc539dbf9cbdb32ce93d3ba0caa77086cb076 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Mon, 17 Jul 2023 17:34:33 -0400
Subject: [PATCH 19/37] better information

---
 ci/dagger-pipeline.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index eec81bec..29fef461 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -8,6 +8,7 @@
     import anyio
     import dagger
     from dagger import BuildArg
+    from dagger.engine._version import CLI_VERSION as __dagger_version__  # noqa
 except ImportError:
     raise ImportError(
         "Dagger is not installed. Please install it with `pip install dagger-io` first."
@@ -37,9 +38,11 @@ async def main():
         version = await python_version.stdout()
         docs_built = await docs.stdout()
 
-    print(f"Hello from Dagger and {version}")
-    print(f"Running commands as {user} user.")
-
+    print("\n")
+    print(
+        f"Hello from Dagger {__dagger_version__} and {'.'.join([str(v) for v in sys.version_info[0:3]])}\n"
+    )
+    print(f"Running commands as `{user.strip()}` user in {version.strip()}.\n")
     print(docs_built)
 
 

From 0b0e94581452a94d34d62ef16beef49b5211dbdd Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 13:28:48 -0400
Subject: [PATCH 20/37] implement workflow_dispatch with inputs, do not modify
 base image

---
 .github/workflows/dagger-docker.yml | 17 ++++++-
 ci/dagger-pipeline.py               | 69 ++++++++++++++++++++++++-----
 2 files changed, 73 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 02cec617..b1c0a140 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -7,10 +7,19 @@ on:
   pull_request:
   workflow_dispatch:
     inputs:
-      image_tag:
+      BASE_IMAGE_TAG:
         description: 'Image tag to use for the Docker image'
         required: true
         default: 'latest'
+      PAVICS_HOST_URL:
+        description: 'Pavics URL to test against'
+        required: true
+        default: 'https://pavics.ouranos.ca/'
+      SANITIZE_FILE_URL:
+        description: 'URL for the sanitizer configuration'
+        required: true
+        default: 'https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/raw/master/notebooks/output-sanitize.cfg'
+
 
 jobs:
   lint:
@@ -52,6 +61,12 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Set Environment Variables
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+            echo "BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }}" >> $GITHUB_ENV
+            echo "PAVICS_HOST_URL=${{ github.event.inputs.PAVICS_HOST_URL }}" >> $GITHUB_ENV
+            echo "SANITIZE_FILE_URL=${{ github.event.inputs.SANITIZE_FILE_URL }}" >> $GITHUB_ENV
       - name: Dagger testing
         run: |
           export BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }}
diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 29fef461..5170a8b6 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -1,5 +1,6 @@
-"""Build the documentation within a docker pipeline to test integration and notebooks."""
+"""Run notebook tests within a docker pipeline to test integration."""
 
+import logging
 import os
 import sys
 from pathlib import Path
@@ -15,35 +16,79 @@
     )
 
 
+BASE_IMAGE_TAG = os.getenv("BASE_IMAGE_TAG", "latest")
+PAVICS_HOST_URL = os.getenv("PAVICS_HOST_URL", "https://pavics.ouranos.ca")
+SANITIZE_FILE_URL = os.getenv(
+    "SANITIZE_FILE_URL",
+    "https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/raw/master/notebooks/output-sanitize.cfg",
+)
+
+
 async def main():
     async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
-        top_level_dir = client.host().directory(Path(__file__).parent.parent.as_posix())
+        top_level_dir = Path(__file__).parent.parent.as_posix()
 
-        # build container
-        sources = await top_level_dir.docker_build(
-            build_args=BuildArg("BASE_IMAGE_TAG", os.getenv("BASE_IMAGE_TAG"))
-            if os.getenv("BASE_IMAGE_TAG")
-            else None,
+        sources = (
+            # pull container
+            client.container()
+            .from_(f"pavics/workflow-tests:{BASE_IMAGE_TAG}")
+            # set env vars
+            .with_env_variable("PAVICS_HOST_URL", PAVICS_HOST_URL)
+            .with_env_variable("SANITIZE_FILE_URL", SANITIZE_FILE_URL)
+            # copy files to container
+            .with_directory(
+                "/code", client.host().directory(top_level_dir, exclude=[".git", "ci"])
+            )
         )
 
+        # run notebooks
+        notebooks = sources.with_exec(
+            notebook_sanitizer("/code/docs/source/notebooks")
+        ).with_exec(test_notebooks("/code/docs/source/notebooks"))
+
         # smoke tests
         python_version = sources.with_exec(["python", "-V"])
         username = sources.with_exec(["whoami"])
 
-        # build docs
-        docs = sources.with_exec(["make", "--directory=/code/docs", "html"])
-
         # execute
         user = await username.stdout()
         version = await python_version.stdout()
-        docs_built = await docs.stdout()
+        notebook_tests = await notebooks.stdout()
 
     print("\n")
     print(
         f"Hello from Dagger {__dagger_version__} and {'.'.join([str(v) for v in sys.version_info[0:3]])}\n"
     )
     print(f"Running commands as `{user.strip()}` user in {version.strip()}.\n")
-    print(docs_built)
+    print(notebook_tests)
+
+
+def notebook_sanitizer(notebook_path: str) -> list[str]:
+    logging.debug("Copying notebook output sanitizer ...")
+
+    cmd = [
+        "curl",
+        "-L",
+        "$SANITIZE_FILE_URL",
+        "-o",
+        f"{notebook_path}/output-sanitize.cfg",
+        "--silent",
+    ]
+    return cmd
+
+
+def test_notebooks(notebook_path: str) -> list[str]:
+    logging.debug("Running notebook-based tests ...")
+
+    cmd = [
+        "pytest" "--nbval" "--verbose",
+        notebook_path,
+        "--nbval-sanitize-with",
+        f"$SANITIZE_FILE_URL/output-sanitize.cfg",
+        "--ignore",
+        f"{notebook_path}/.ipynb_checkpoints",
+    ]
+    return cmd
 
 
 anyio.run(main)

From 1485340b658b9ef89a98fa0000abb100901ce019 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 14:06:26 -0400
Subject: [PATCH 21/37] debugging

---
 ci/dagger-pipeline.py | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 5170a8b6..af0e9ae2 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -28,7 +28,7 @@ async def main():
     async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
         top_level_dir = Path(__file__).parent.parent.as_posix()
 
-        sources = (
+        sources = await (
             # pull container
             client.container()
             .from_(f"pavics/workflow-tests:{BASE_IMAGE_TAG}")
@@ -37,30 +37,33 @@ async def main():
             .with_env_variable("SANITIZE_FILE_URL", SANITIZE_FILE_URL)
             # copy files to container
             .with_directory(
-                "/code", client.host().directory(top_level_dir, exclude=[".git", "ci"])
+                ".", client.host().directory(top_level_dir, exclude=[".git", "ci"])
             )
         )
 
         # run notebooks
-        notebooks = sources.with_exec(
-            notebook_sanitizer("/code/docs/source/notebooks")
-        ).with_exec(test_notebooks("/code/docs/source/notebooks"))
-
-        # smoke tests
-        python_version = sources.with_exec(["python", "-V"])
-        username = sources.with_exec(["whoami"])
+        # notebooks = sources.with_exec(
+        #     notebook_sanitizer("./docs/source/notebooks")
+        # ).with_exec(test_notebooks("./docs/source/notebooks"))
 
         # execute
-        user = await username.stdout()
-        version = await python_version.stdout()
-        notebook_tests = await notebooks.stdout()
+        whoami = await sources.with_exec(["whoami"]).stdout()
+        version = await sources.with_exec(["python", "-V"]).stdout()
+        whereami = await sources.with_exec(["pwd"]).stdout()
+        whatshere = await sources.with_exec(["ls", "-la"]).stdout()
+        # notebook_tests = await notebooks.stdout()
 
     print("\n")
     print(
-        f"Hello from Dagger {__dagger_version__} and {'.'.join([str(v) for v in sys.version_info[0:3]])}\n"
+        f"Hello from Dagger {__dagger_version__}"
+        " and "
+        f"{'.'.join([str(v) for v in sys.version_info[0:3]])}"
+        " in "
+        f"{whereami.strip()}\n"
+        f"{whatshere.strip()}\n"
     )
-    print(f"Running commands as `{user.strip()}` user in {version.strip()}.\n")
-    print(notebook_tests)
+    print(f"Running commands as `{whoami.strip()}` user in {version.strip()}.\n")
+    # print(notebook_tests)
 
 
 def notebook_sanitizer(notebook_path: str) -> list[str]:

From f39f2a973b2b16966eda778a1b3bc3bd01d04379 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 14:13:37 -0400
Subject: [PATCH 22/37] fix double env-var setting

---
 .github/workflows/dagger-docker.yml | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index b1c0a140..3d9bf002 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -34,15 +34,8 @@ jobs:
         uses: actions/checkout@v3
       - name: Setup Python
         uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-      - name: Install dependencies
-        run: |
-          python -m pip install pre-commit
-          pre-commit install
       - name: Run pre-commit
-        run: |
-          pre-commit run --all-files --show-diff-on-failure
+        uses: pre-commit/action@v3.0.0
 
   integration:
     name: Integration Tests
@@ -69,6 +62,5 @@ jobs:
             echo "SANITIZE_FILE_URL=${{ github.event.inputs.SANITIZE_FILE_URL }}" >> $GITHUB_ENV
       - name: Dagger testing
         run: |
-          export BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }}
           pip install .[dagger]
           python ./ci/dagger-pipeline.py

From 0b88ef17edd36a37363f3eec70f7bca24e5bfe22 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 14:21:18 -0400
Subject: [PATCH 23/37] re-enable notebook tests

---
 .github/workflows/dagger-docker.yml |  4 +++-
 ci/dagger-pipeline.py               | 13 +++++++------
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 3d9bf002..235ee940 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -34,11 +34,13 @@ jobs:
         uses: actions/checkout@v3
       - name: Setup Python
         uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
       - name: Run pre-commit
         uses: pre-commit/action@v3.0.0
 
   integration:
-    name: Integration Tests
+    name: Integration Tests (Python${{ matrix.python-version }})
     needs: lint
     runs-on: ubuntu-latest
     defaults:
diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index af0e9ae2..36ae0880 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -37,21 +37,22 @@ async def main():
             .with_env_variable("SANITIZE_FILE_URL", SANITIZE_FILE_URL)
             # copy files to container
             .with_directory(
-                ".", client.host().directory(top_level_dir, exclude=[".git", "ci"])
+                "/code", client.host().directory(top_level_dir, exclude=[".git", "ci"])
             )
+            .workdir("/code")
         )
 
         # run notebooks
-        # notebooks = sources.with_exec(
-        #     notebook_sanitizer("./docs/source/notebooks")
-        # ).with_exec(test_notebooks("./docs/source/notebooks"))
+        notebooks = sources.with_exec(
+            notebook_sanitizer("/code/docs/source/notebooks")
+        ).with_exec(test_notebooks("/code/docs/source/notebooks"))
 
         # execute
         whoami = await sources.with_exec(["whoami"]).stdout()
         version = await sources.with_exec(["python", "-V"]).stdout()
         whereami = await sources.with_exec(["pwd"]).stdout()
         whatshere = await sources.with_exec(["ls", "-la"]).stdout()
-        # notebook_tests = await notebooks.stdout()
+        notebook_tests = await notebooks.stdout()
 
     print("\n")
     print(
@@ -63,7 +64,7 @@ async def main():
         f"{whatshere.strip()}\n"
     )
     print(f"Running commands as `{whoami.strip()}` user in {version.strip()}.\n")
-    # print(notebook_tests)
+    print(notebook_tests)
 
 
 def notebook_sanitizer(notebook_path: str) -> list[str]:

From 2fd7a0acb9193cb15cc4ef673d6fcfd4f114943d Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 14:29:18 -0400
Subject: [PATCH 24/37] better organization

---
 ci/dagger-pipeline.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 36ae0880..b98df02d 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -25,9 +25,11 @@
 
 
 async def main():
-    async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
-        top_level_dir = Path(__file__).parent.parent.as_posix()
-
+    async with dagger.Connection(
+        dagger.Config(
+            log_output=sys.stderr, workdir=Path(__file__).parent.parent.as_posix()
+        )
+    ) as client:
         sources = await (
             # pull container
             client.container()
@@ -37,9 +39,8 @@ async def main():
             .with_env_variable("SANITIZE_FILE_URL", SANITIZE_FILE_URL)
             # copy files to container
             .with_directory(
-                "/code", client.host().directory(top_level_dir, exclude=[".git", "ci"])
+                "/code", client.host().directory(".", exclude=[".git", "ci"])
             )
-            .workdir("/code")
         )
 
         # run notebooks
@@ -51,7 +52,7 @@ async def main():
         whoami = await sources.with_exec(["whoami"]).stdout()
         version = await sources.with_exec(["python", "-V"]).stdout()
         whereami = await sources.with_exec(["pwd"]).stdout()
-        whatshere = await sources.with_exec(["ls", "-la"]).stdout()
+        whatshere = await sources.entries().stdout()
         notebook_tests = await notebooks.stdout()
 
     print("\n")
@@ -85,10 +86,12 @@ def test_notebooks(notebook_path: str) -> list[str]:
     logging.debug("Running notebook-based tests ...")
 
     cmd = [
-        "pytest" "--nbval" "--verbose",
+        "pytest",
+        "--nbval",
+        "--verbose",
         notebook_path,
         "--nbval-sanitize-with",
-        f"$SANITIZE_FILE_URL/output-sanitize.cfg",
+        "$SANITIZE_FILE_URL/output-sanitize.cfg",
         "--ignore",
         f"{notebook_path}/.ipynb_checkpoints",
     ]

From 5313a10dcaedc1138907dac3cffdcb17d9098512 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 14:35:26 -0400
Subject: [PATCH 25/37] typo fix

---
 ci/dagger-pipeline.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index b98df02d..cd60a1b7 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -52,7 +52,7 @@ async def main():
         whoami = await sources.with_exec(["whoami"]).stdout()
         version = await sources.with_exec(["python", "-V"]).stdout()
         whereami = await sources.with_exec(["pwd"]).stdout()
-        whatshere = await sources.entries().stdout()
+        whatshere = await sources.with_exec(["ls", "-alh"]).stdout()
         notebook_tests = await notebooks.stdout()
 
     print("\n")

From 2ea3caccc5a0f377d775637992094fc1b5889ff5 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 14:56:49 -0400
Subject: [PATCH 26/37] cleaner

---
 ci/dagger-pipeline.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index cd60a1b7..ff92053a 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -32,11 +32,7 @@ async def main():
     ) as client:
         sources = await (
             # pull container
-            client.container()
-            .from_(f"pavics/workflow-tests:{BASE_IMAGE_TAG}")
-            # set env vars
-            .with_env_variable("PAVICS_HOST_URL", PAVICS_HOST_URL)
-            .with_env_variable("SANITIZE_FILE_URL", SANITIZE_FILE_URL)
+            client.container().from_(f"pavics/workflow-tests:{BASE_IMAGE_TAG}")
             # copy files to container
             .with_directory(
                 "/code", client.host().directory(".", exclude=[".git", "ci"])
@@ -44,9 +40,11 @@ async def main():
         )
 
         # run notebooks
-        notebooks = sources.with_exec(
-            notebook_sanitizer("/code/docs/source/notebooks")
-        ).with_exec(test_notebooks("/code/docs/source/notebooks"))
+        notebooks = (
+            sources.with_exec(notebook_sanitizer("/code/docs/source/notebooks"))
+            .with_env_variable("PAVICS_HOST_URL", PAVICS_HOST_URL)
+            .with_exec(test_notebooks("/code/docs/source/notebooks"))
+        )
 
         # execute
         whoami = await sources.with_exec(["whoami"]).stdout()
@@ -72,9 +70,11 @@ def notebook_sanitizer(notebook_path: str) -> list[str]:
     logging.debug("Copying notebook output sanitizer ...")
 
     cmd = [
+        "sh",
+        "-c",
         "curl",
         "-L",
-        "$SANITIZE_FILE_URL",
+        SANITIZE_FILE_URL,
         "-o",
         f"{notebook_path}/output-sanitize.cfg",
         "--silent",
@@ -91,7 +91,7 @@ def test_notebooks(notebook_path: str) -> list[str]:
         "--verbose",
         notebook_path,
         "--nbval-sanitize-with",
-        "$SANITIZE_FILE_URL/output-sanitize.cfg",
+        f"{SANITIZE_FILE_URL}/output-sanitize.cfg",
         "--ignore",
         f"{notebook_path}/.ipynb_checkpoints",
     ]

From 44448e5424222c760b55d81d039fe3824593b403 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 15:04:46 -0400
Subject: [PATCH 27/37] more typos

---
 ci/dagger-pipeline.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index ff92053a..c82235ea 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -70,8 +70,6 @@ def notebook_sanitizer(notebook_path: str) -> list[str]:
     logging.debug("Copying notebook output sanitizer ...")
 
     cmd = [
-        "sh",
-        "-c",
         "curl",
         "-L",
         SANITIZE_FILE_URL,
@@ -91,7 +89,7 @@ def test_notebooks(notebook_path: str) -> list[str]:
         "--verbose",
         notebook_path,
         "--nbval-sanitize-with",
-        f"{SANITIZE_FILE_URL}/output-sanitize.cfg",
+        f"{notebook_path}/output-sanitize.cfg",
         "--ignore",
         f"{notebook_path}/.ipynb_checkpoints",
     ]

From fd9dc13396eeb9ebf804737804297ac4a3244b05 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 15:05:10 -0400
Subject: [PATCH 28/37] remove Dockerfile

---
 Dockerfile | 33 ---------------------------------
 1 file changed, 33 deletions(-)
 delete mode 100644 Dockerfile

diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index d3ac19bb..00000000
--- a/Dockerfile
+++ /dev/null
@@ -1,33 +0,0 @@
-# vim:set ft=dockerfile:
-ARG BASE_IMAGE_TAG
-FROM pavics/workflow-tests:${BASE_IMAGE_TAG:-latest}
-ARG DEBIAN_FRONTEND=noninteractive
-ENV PIP_ROOT_USER_ACTION=ignore
-LABEL org.opencontainers.image.authors="https://github.com/ouranosinc/pavics-sdi"
-LABEL Description="PAVICS-SDI-TESTING-IMAGE" Vendor="Birdhouse" Version="1.3.0"
-
-# root-level commands
-USER root
-
-# Install build tools
-RUN apt-get install --yes build-essential \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-# Switch to /code directory
-WORKDIR /code
-
-# Build finch environment
-COPY environment-dev.yml /code
-RUN mamba install -n birdy python=3.8 --yes \
-    && mamba env update -n birdy -f environment-dev.yml \
-    && mamba clean --all --yes
-
-# Add the finch conda environment to the path
-ENV PATH /opt/conda/envs/birdy/bin:$PATH
-
-# Copy finch source code
-COPY . /code
-
-# Build the documentation
-CMD ["make", "--directory=/code/docs", "html"]

From 2a12d9d67fbb99b5d2bb6d6dd78e86b1e77ce8c9 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 15:26:46 -0400
Subject: [PATCH 29/37] give jenkins rights to the folder

---
 ci/dagger-pipeline.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index c82235ea..5f06c0f3 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -35,13 +35,16 @@ async def main():
             client.container().from_(f"pavics/workflow-tests:{BASE_IMAGE_TAG}")
             # copy files to container
             .with_directory(
-                "/code", client.host().directory(".", exclude=[".git", "ci"])
+                "/code",
+                client.host().directory(".", exclude=[".git", "ci"], owner="jenkins"),
             )
         )
 
         # run notebooks
         notebooks = (
-            sources.with_exec(notebook_sanitizer("/code/docs/source/notebooks"))
+            sources.with_exec(
+                notebook_sanitizer(SANITIZE_FILE_URL, "/code/docs/source/notebooks")
+            )
             .with_env_variable("PAVICS_HOST_URL", PAVICS_HOST_URL)
             .with_exec(test_notebooks("/code/docs/source/notebooks"))
         )
@@ -66,13 +69,13 @@ async def main():
     print(notebook_tests)
 
 
-def notebook_sanitizer(notebook_path: str) -> list[str]:
+def notebook_sanitizer(file_url: str, notebook_path: str) -> list[str]:
     logging.debug("Copying notebook output sanitizer ...")
 
     cmd = [
         "curl",
         "-L",
-        SANITIZE_FILE_URL,
+        file_url,
         "-o",
         f"{notebook_path}/output-sanitize.cfg",
         "--silent",

From 2176210132dd1e54745d9ba519fb4184283242b5 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 15:35:03 -0400
Subject: [PATCH 30/37] place kwargs in proper context

---
 ci/dagger-pipeline.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 5f06c0f3..048e6623 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -36,7 +36,9 @@ async def main():
             # copy files to container
             .with_directory(
                 "/code",
-                client.host().directory(".", exclude=[".git", "ci"], owner="jenkins"),
+                client.host().directory("."),
+                exclude=[".git", "ci"],
+                owner="jenkins",
             )
         )
 

From b9c964bcbaa576a04841e1560822d742f6138ff1 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 18 Jul 2023 15:52:02 -0400
Subject: [PATCH 31/37] fix arguments

---
 ci/dagger-pipeline.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 048e6623..5880408d 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -93,10 +93,8 @@ def test_notebooks(notebook_path: str) -> list[str]:
         "--nbval",
         "--verbose",
         notebook_path,
-        "--nbval-sanitize-with",
-        f"{notebook_path}/output-sanitize.cfg",
-        "--ignore",
-        f"{notebook_path}/.ipynb_checkpoints",
+        f"--sanitize-with={notebook_path}/output-sanitize.cfg",
+        f"--ignore={notebook_path}/.ipynb_checkpoints",
     ]
     return cmd
 

From de8b00473dc11293283e5f8f48a463d8fd3f8205 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Wed, 19 Jul 2023 10:18:19 -0400
Subject: [PATCH 32/37] fix PAVICS_HOST env var

---
 .github/workflows/dagger-docker.yml | 4 ++--
 ci/dagger-pipeline.py               | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 235ee940..4e1838ab 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -11,7 +11,7 @@ on:
         description: 'Image tag to use for the Docker image'
         required: true
         default: 'latest'
-      PAVICS_HOST_URL:
+      PAVICS_HOST:
         description: 'Pavics URL to test against'
         required: true
         default: 'https://pavics.ouranos.ca/'
@@ -60,7 +60,7 @@ jobs:
         if: ${{ github.event_name == 'workflow_dispatch' }}
         run: |
             echo "BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }}" >> $GITHUB_ENV
-            echo "PAVICS_HOST_URL=${{ github.event.inputs.PAVICS_HOST_URL }}" >> $GITHUB_ENV
+            echo "PAVICS_HOST=${{ github.event.inputs.PAVICS_HOST }}" >> $GITHUB_ENV
             echo "SANITIZE_FILE_URL=${{ github.event.inputs.SANITIZE_FILE_URL }}" >> $GITHUB_ENV
       - name: Dagger testing
         run: |
diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 5880408d..9d674a73 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -17,7 +17,7 @@
 
 
 BASE_IMAGE_TAG = os.getenv("BASE_IMAGE_TAG", "latest")
-PAVICS_HOST_URL = os.getenv("PAVICS_HOST_URL", "https://pavics.ouranos.ca")
+PAVICS_HOST = os.getenv("PAVICS_HOST", "https://pavics.ouranos.ca")
 SANITIZE_FILE_URL = os.getenv(
     "SANITIZE_FILE_URL",
     "https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/raw/master/notebooks/output-sanitize.cfg",
@@ -47,7 +47,7 @@ async def main():
             sources.with_exec(
                 notebook_sanitizer(SANITIZE_FILE_URL, "/code/docs/source/notebooks")
             )
-            .with_env_variable("PAVICS_HOST_URL", PAVICS_HOST_URL)
+            .with_env_variable("PAVICS_HOST", PAVICS_HOST)
             .with_exec(test_notebooks("/code/docs/source/notebooks"))
         )
 

From a08eb64f0cc84b0766572cb48e9568c72c68096c Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 25 Jul 2023 14:50:10 -0400
Subject: [PATCH 33/37] add a conftest.py, use pytest_collectstart to ignore
 widget and javascript comparisons, run tests nightly at midnight-ish EST

---
 .github/workflows/dagger-docker.yml |  7 ++++---
 ci/dagger-pipeline.py               | 17 +++++++++++++++--
 tests/conftest.py                   | 12 ++++++++++++
 3 files changed, 31 insertions(+), 5 deletions(-)
 create mode 100644 tests/conftest.py

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 4e1838ab..7b689c14 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -1,10 +1,11 @@
 name: Dagger/Docker Testing Suite
 
 on:
-  push:
-    branches:
-      - master
+  # FIXME: Remove Pull Request trigger before merging
   pull_request:
+  schedule:
+    # Every night at 4:00 AM UTC
+    - cron: '0 4 * * *'
   workflow_dispatch:
     inputs:
       BASE_IMAGE_TAG:
diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 9d674a73..e7ef4261 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -48,7 +48,12 @@ async def main():
                 notebook_sanitizer(SANITIZE_FILE_URL, "/code/docs/source/notebooks")
             )
             .with_env_variable("PAVICS_HOST", PAVICS_HOST)
-            .with_exec(test_notebooks("/code/docs/source/notebooks"))
+            .with_exec(
+                test_notebooks(
+                    notebook_path="/code/docs/source/notebooks",
+                    conftest_dir="/code/tests",
+                )
+            )
         )
 
         # execute
@@ -85,17 +90,25 @@ def notebook_sanitizer(file_url: str, notebook_path: str) -> list[str]:
     return cmd
 
 
-def test_notebooks(notebook_path: str) -> list[str]:
+def test_notebooks(
+    notebook_path: str,
+    conftest_dir: str | None,
+) -> list[str]:
     logging.debug("Running notebook-based tests ...")
 
     cmd = [
         "pytest",
+        f"--confcutdir={conftest_dir}" if conftest_dir else None,
         "--nbval",
         "--verbose",
         notebook_path,
         f"--sanitize-with={notebook_path}/output-sanitize.cfg",
         f"--ignore={notebook_path}/.ipynb_checkpoints",
     ]
+
+    while None in cmd:
+        cmd.remove(None)
+
     return cmd
 
 
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 00000000..ad5e11da
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,12 @@
+"""Pytest configuration file."""
+
+
+def pytest_collectstart(collector):  # noqa
+    # Make sure ancestor folder name do not end with `.ipynb`, else we have
+    # AttributeError: 'Session' object has no attribute 'skip_compare'.
+    if collector.fspath and collector.fspath.ext == ".ipynb":
+        collector.skip_compare += (
+            "text/html",
+            "application/javascript",
+            "application/vnd.holoviews_load.v0+json",
+        )

From f1f580c8be6ab5169b88bd98b14d4de0411dbacc Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Wed, 26 Jul 2023 10:25:34 -0400
Subject: [PATCH 34/37] try a different approach

---
 ci/dagger-pipeline.py | 7 ++++---
 pyproject.toml        | 6 ++++++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index e7ef4261..5162c3cf 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -51,7 +51,7 @@ async def main():
             .with_exec(
                 test_notebooks(
                     notebook_path="/code/docs/source/notebooks",
-                    conftest_dir="/code/tests",
+                    conftest_dir="/code",
                 )
             )
         )
@@ -98,10 +98,11 @@ def test_notebooks(
 
     cmd = [
         "pytest",
-        f"--confcutdir={conftest_dir}" if conftest_dir else None,
+        # f"--confcutdir={conftest_dir}" if conftest_dir else None,
+        # f"--rootdir={conftest_dir}" if conftest_dir else None,
         "--nbval",
         "--verbose",
-        notebook_path,
+        # notebook_path,
         f"--sanitize-with={notebook_path}/output-sanitize.cfg",
         f"--ignore={notebook_path}/.ipynb_checkpoints",
     ]
diff --git a/pyproject.toml b/pyproject.toml
index f98dec3d..cd44393d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,3 +56,9 @@ dagger = [
 "Issue tracker" = "https://github.com/Ouranosinc/pavics-sdi/issues"
 "About CRIM" = "https://www.crim.ca/en/"
 "About Ouranos" = "https://www.ouranos.ca/en/"
+
+[tool.pytest.ini_options]
+testpaths = [
+  "tests",
+  "docs/source/notbooks"
+]

From 09774b93468e8a1ef9d948be0daab989a9ab2d28 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Wed, 26 Jul 2023 11:00:18 -0400
Subject: [PATCH 35/37] re-add testing dirs to cli calls

---
 ci/dagger-pipeline.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 5162c3cf..63091933 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -98,11 +98,11 @@ def test_notebooks(
 
     cmd = [
         "pytest",
-        # f"--confcutdir={conftest_dir}" if conftest_dir else None,
-        # f"--rootdir={conftest_dir}" if conftest_dir else None,
+        f"--confcutdir={conftest_dir}" if conftest_dir else None,
+        f"--rootdir={conftest_dir}" if conftest_dir else None,
         "--nbval",
         "--verbose",
-        # notebook_path,
+        notebook_path,
         f"--sanitize-with={notebook_path}/output-sanitize.cfg",
         f"--ignore={notebook_path}/.ipynb_checkpoints",
     ]

From 929530b7ec3e001c94c980fa73bffb5c17d5074b Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 15 Aug 2023 16:34:38 -0400
Subject: [PATCH 36/37] BASE_IMAGE_TAG must always be supplied to
 dagger-pipeline.py

---
 .github/workflows/dagger-docker.yml | 7 ++++++-
 ci/dagger-pipeline.py               | 5 ++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/dagger-docker.yml b/.github/workflows/dagger-docker.yml
index 7b689c14..81675fc6 100644
--- a/.github/workflows/dagger-docker.yml
+++ b/.github/workflows/dagger-docker.yml
@@ -11,7 +11,6 @@ on:
       BASE_IMAGE_TAG:
         description: 'Image tag to use for the Docker image'
         required: true
-        default: 'latest'
       PAVICS_HOST:
         description: 'Pavics URL to test against'
         required: true
@@ -21,6 +20,8 @@ on:
         required: true
         default: 'https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/raw/master/notebooks/output-sanitize.cfg'
 
+env:
+  BASE_IMAGE_TAG: 230601
 
 jobs:
   lint:
@@ -58,6 +59,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Set Environment Variables
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+        run: |
+            echo "BASE_IMAGE_TAG=${{ env.BASE_IMAGE_TAG }}" >> $GITHUB_ENV
+      - name: Set Environment Variables (Workflow Dispatch)
         if: ${{ github.event_name == 'workflow_dispatch' }}
         run: |
             echo "BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }}" >> $GITHUB_ENV
diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 63091933..949e6cff 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -16,7 +16,7 @@
     )
 
 
-BASE_IMAGE_TAG = os.getenv("BASE_IMAGE_TAG", "latest")
+BASE_IMAGE_TAG = os.getenv("BASE_IMAGE_TAG")
 PAVICS_HOST = os.getenv("PAVICS_HOST", "https://pavics.ouranos.ca")
 SANITIZE_FILE_URL = os.getenv(
     "SANITIZE_FILE_URL",
@@ -30,6 +30,9 @@ async def main():
             log_output=sys.stderr, workdir=Path(__file__).parent.parent.as_posix()
         )
     ) as client:
+        if not BASE_IMAGE_TAG:
+            raise ValueError("BASE_IMAGE_TAG environment variable is not set.")
+
         sources = await (
             # pull container
             client.container().from_(f"pavics/workflow-tests:{BASE_IMAGE_TAG}")

From e0cc4d4a875b3099d9f2ce8645fd7da6d9034ab8 Mon Sep 17 00:00:00 2001
From: Zeitsperre <10819524+Zeitsperre@users.noreply.github.com>
Date: Tue, 15 Aug 2023 17:22:54 -0400
Subject: [PATCH 37/37] deal with changing API

---
 ci/dagger-pipeline.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ci/dagger-pipeline.py b/ci/dagger-pipeline.py
index 949e6cff..0927a35f 100644
--- a/ci/dagger-pipeline.py
+++ b/ci/dagger-pipeline.py
@@ -9,12 +9,16 @@
     import anyio
     import dagger
     from dagger import BuildArg
-    from dagger.engine._version import CLI_VERSION as __dagger_version__  # noqa
 except ImportError:
     raise ImportError(
         "Dagger is not installed. Please install it with `pip install dagger-io` first."
     )
 
+try:
+    from dagger._engine._version import CLI_VERSION as __dagger_version__  # noqa
+except ModuleNotFoundError:
+    __dagger_version__ = "unknown"
+
 
 BASE_IMAGE_TAG = os.getenv("BASE_IMAGE_TAG")
 PAVICS_HOST = os.getenv("PAVICS_HOST", "https://pavics.ouranos.ca")