Add a docker/dagger-based testing workflow, adopt PEP 517/621

.github/workflows/dagger-docker.yml

@@ -0,0 +1,74 @@
+name: Dagger/Docker Testing Suite
+
+on:
+  # FIXME: Remove Pull Request trigger before merging
+  pull_request:
+  schedule:
+    # Every night at 4:00 AM UTC
+    - cron: '0 4 * * *'
+  workflow_dispatch:
+    inputs:
+      BASE_IMAGE_TAG:
+        description: 'Image tag to use for the Docker image'
+        required: true
+      PAVICS_HOST:
+        description: 'Pavics URL to test against'
+        required: true
+        default: 'https://pavics.ouranos.ca/'
+      SANITIZE_FILE_URL:
+        description: 'URL for the sanitizer configuration'
+        required: true
+        default: 'https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/raw/master/notebooks/output-sanitize.cfg'
+
+env:
+  BASE_IMAGE_TAG: 230601
+
+jobs:
+  lint:
+    name: Linting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel previous runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+            access_token: ${{ github.token }}
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Run pre-commit
+        uses: pre-commit/action@v3.0.0
+
+  integration:
+    name: Integration Tests (Python${{ matrix.python-version }})
+    needs: lint
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+        matrix:
+            python-version: ["3.10"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Set Environment Variables
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+        run: |
+            echo "BASE_IMAGE_TAG=${{ env.BASE_IMAGE_TAG }}" >> $GITHUB_ENV
+      - name: Set Environment Variables (Workflow Dispatch)
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+            echo "BASE_IMAGE_TAG=${{ github.event.inputs.image_tag }}" >> $GITHUB_ENV
+            echo "PAVICS_HOST=${{ github.event.inputs.PAVICS_HOST }}" >> $GITHUB_ENV
+            echo "SANITIZE_FILE_URL=${{ github.event.inputs.SANITIZE_FILE_URL }}" >> $GITHUB_ENV
+      - name: Dagger testing
+        run: |
+          pip install .[dagger]
+          python ./ci/dagger-pipeline.py

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
     rev: v3.14.0
     hooks:
       - id: pyupgrade
-        args: [ '--py37-plus' ]
+        args: [ '--py38-plus' ]
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
@@ -15,6 +15,19 @@ repos:
       - id: end-of-file-fixer
         exclude: .ipynb|.po|.xml
       - id: debug-statements
+        exclude: .ipynb
+      - id: check-toml
+      - id: check-yaml
+        args: [ '--allow-multiple-documents' ]
+  - repo: https://github.com/pappasam/toml-sort
+    rev: v0.23.1
+    hooks:
+      - id: toml-sort-fix
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.32.0
+    hooks:
+      - id: yamllint
+        args: [ '--config-file=.yamllint.yaml' ]
   - repo: https://github.com/psf/black
     rev: 23.9.1
     hooks:
@@ -28,10 +41,9 @@ repos:
     rev: 1.7.0
     hooks:
       - id: nbqa-black
-        args: [ '--target-version=py37' ]
         additional_dependencies: [ 'black==23.9.1' ]
       - id: nbqa-pyupgrade
-        args: [ '--py37-plus' ]
+        args: [ '--py38-plus' ]
         additional_dependencies: [ 'pyupgrade==v3.14.0' ]
       - id: nbqa-isort
         args: [ '--profile=black' ]
@@ -42,6 +54,11 @@ repos:
       - id: nbstripout
         files: ".ipynb"
         args: [ '--keep-output', '--keep-count' ]
+  - repo: https://github.com/keewis/blackdoc
+    rev: v0.3.8
+    hooks:
+    - id: blackdoc
+      additional_dependencies: ['black==23.3.0']
   - repo: meta
     hooks:
       - id: check-hooks-apply

.yamllint.yaml

@@ -0,0 +1,8 @@
+---
+
+rules:
+  document-start: disable
+  line-length:
+    max: 120
+    level: warning
+  truthy: disable

LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2017, Ouranos, CRIM
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ci/dagger-pipeline.py

@@ -0,0 +1,123 @@
+"""Run notebook tests within a docker pipeline to test integration."""
+
+import logging
+import os
+import sys
+from pathlib import Path
+
+try:
+    import anyio
+    import dagger
+    from dagger import BuildArg
+except ImportError:
+    raise ImportError(
+        "Dagger is not installed. Please install it with `pip install dagger-io` first."
+    )
+
+try:
+    from dagger._engine._version import CLI_VERSION as __dagger_version__  # noqa
+except ModuleNotFoundError:
+    __dagger_version__ = "unknown"
+
+
+BASE_IMAGE_TAG = os.getenv("BASE_IMAGE_TAG")
+PAVICS_HOST = os.getenv("PAVICS_HOST", "https://pavics.ouranos.ca")
+SANITIZE_FILE_URL = os.getenv(
+    "SANITIZE_FILE_URL",
+    "https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/raw/master/notebooks/output-sanitize.cfg",
+)
+
+
+async def main():
+    async with dagger.Connection(
+        dagger.Config(
+            log_output=sys.stderr, workdir=Path(__file__).parent.parent.as_posix()
+        )
+    ) as client:
+        if not BASE_IMAGE_TAG:
+            raise ValueError("BASE_IMAGE_TAG environment variable is not set.")
+
+        sources = await (
+            # pull container
+            client.container().from_(f"pavics/workflow-tests:{BASE_IMAGE_TAG}")
+            # copy files to container
+            .with_directory(
+                "/code",
+                client.host().directory("."),
+                exclude=[".git", "ci"],
+                owner="jenkins",
+            )
+        )
+
+        # run notebooks
+        notebooks = (
+            sources.with_exec(
+                notebook_sanitizer(SANITIZE_FILE_URL, "/code/docs/source/notebooks")
+            )
+            .with_env_variable("PAVICS_HOST", PAVICS_HOST)
+            .with_exec(
+                test_notebooks(
+                    notebook_path="/code/docs/source/notebooks",
+                    conftest_dir="/code",
+                )
+            )
+        )
+
+        # execute
+        whoami = await sources.with_exec(["whoami"]).stdout()
+        version = await sources.with_exec(["python", "-V"]).stdout()
+        whereami = await sources.with_exec(["pwd"]).stdout()
+        whatshere = await sources.with_exec(["ls", "-alh"]).stdout()
+        notebook_tests = await notebooks.stdout()
+
+    print("\n")
+    print(
+        f"Hello from Dagger {__dagger_version__}"
+        " and "
+        f"{'.'.join([str(v) for v in sys.version_info[0:3]])}"
+        " in "
+        f"{whereami.strip()}\n"
+        f"{whatshere.strip()}\n"
+    )
+    print(f"Running commands as `{whoami.strip()}` user in {version.strip()}.\n")
+    print(notebook_tests)
+
+
+def notebook_sanitizer(file_url: str, notebook_path: str) -> list[str]:
+    logging.debug("Copying notebook output sanitizer ...")
+
+    cmd = [
+        "curl",
+        "-L",
+        file_url,
+        "-o",
+        f"{notebook_path}/output-sanitize.cfg",
+        "--silent",
+    ]
+    return cmd
+
+
+def test_notebooks(
+    notebook_path: str,
+    conftest_dir: str | None,
+) -> list[str]:
+    logging.debug("Running notebook-based tests ...")
+
+    cmd = [
+        "pytest",
+        f"--confcutdir={conftest_dir}" if conftest_dir else None,
+        f"--rootdir={conftest_dir}" if conftest_dir else None,
+        "--nbval",
+        "--verbose",
+        notebook_path,
+        f"--sanitize-with={notebook_path}/output-sanitize.cfg",
+        f"--ignore={notebook_path}/.ipynb_checkpoints",
+    ]
+
+    while None in cmd:
+        cmd.remove(None)
+
+    return cmd
+
+
+anyio.run(main)

docs/source/index.rst

@@ -41,13 +41,10 @@ Contents
    arch/index
    provenance/index
    support
-   releasenotes
+   release_notes
    license
    todo
 
-
-
-
 Indices and tables
 ==================
 

docs/source/license.rst

@@ -2,18 +2,4 @@
 License
 =======
 
-
-Copyright (c) 2017, Ouranos, CRIM
-All rights reserved.
-
-Permission to use, copy, modify, and/or distribute this software for any
-purpose with or without fee is hereby granted, provided that the above
-copyright notice and this permission notice appear in all copies.
-
-THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
-ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+.. include:: ../../LICENSE

environment-dev.yml

@@ -4,11 +4,12 @@ channels:
   - defaults
 dependencies:
   - python >=3.8,<3.12
+  - flit  # solely needed for installing pavics-sdi
   - jinja2 <=3.0.3
   - jupyter
   - myst-nb
   - nbval
   - pytest
   - sphinx
   - sphinx-intl
-  - sphinx_rtd_theme >=1.0
+  - sphinx-rtd-theme >=1.0

pavics_sdi/__init__.py

@@ -0,0 +1,4 @@
+"""PAVICS SDI package."""
+
+
+__version__ = "1.3.0"