Configurable servers

CONTRIBUTING.rst

@@ -3,5 +3,5 @@ Contributing
 
 Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given.
 
-Please read the Birdhouse `Developer Guide <https://birdhouse.readthedocs.io/en/latest/dev_guide.html>`_
-and the `Raven Documentation <https://pavics-raven.readthedocs.io/en/latest>`_ to get started.
+Please read the Birdhouse `Developer Guide <https://birdhouse.readthedocs.io/en/latest/guidelines.html>`_
+and the `Raven Documentation <https://pavics-sdi.readthedocs.io/projects/raven/en/latest/>`_ to get started.

Dockerfile

@@ -9,7 +9,7 @@ WORKDIR /code
 
 # Create conda environment
 COPY environment.yml .
-RUN mamba env create -n raven -f environment.yml && mamba install -n raven gunicorn  && mamba clean --all --yes
+RUN mamba env create -n raven -f environment.yml && mamba install -n raven gunicorn && mamba clean --all --yes
 
 # Add the raven conda environment to the path
 ENV PATH=/opt/conda/envs/raven/bin:$PATH
@@ -20,7 +20,7 @@ COPY . /code
 # Install raven
 RUN pip install . --no-deps
 
-# Start WPS service on port 9099 on 0.0.0.0
+# Start WPS service on port 9099
 EXPOSE 9099
 
 CMD ["gunicorn", "--bind=0.0.0.0:9099", "raven.wsgi:application"]

Makefile

@@ -7,19 +7,23 @@ APP_NAME := raven-wps
 
 OS := $(shell uname)
 
-WPS_PORT := 9099
-WPS_URL := http://0.0.0.0:$(WPS_PORT)
+WPS_PORT ?= 9099
+WPS_URL ?= http://0.0.0.0:$(WPS_PORT)
 
 # If WPS_URL is overridden, this should also be overridden to match.
-WPS_OUTPUT_URL := http://localhost:$(WPS_PORT)/outputs
+WPS_OUTPUT_URL ?= http://localhost:$(WPS_PORT)/outputs
 
-# This will only work on Linux (not macOS/homebrew GDAL)
-GDAL_VERSION := $(shell gdal-config --version)
+# This will only work for Linux and macOS/homebrew
+ifeq ($(OS),"Linux")
+	GDAL_VERSION := $(shell gdal-config --version)
+else ifeq ($(OS),"Darwin")
+	GDAL_VERSION := $(shell gdalinfo --version | awk '{print $2}' | sed s'/.$//')
+endif
 
 # Used in target refresh-notebooks to make it looks like the notebooks have
 # been refreshed from the production server below instead of from the local dev
 # instance so the notebooks can also be used as tutorial notebooks.
-OUTPUT_URL = https://pavics.ouranos.ca/wpsoutputs/raven
+OUTPUT_URL ?= https://pavics.ouranos.ca/wpsoutputs/raven
 
 SANITIZE_FILE := https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/raw/master/notebooks/output-sanitize.cfg
 

docker-compose.yml

@@ -28,4 +28,4 @@ services:
       - ./:/opt/wps
     environment:
       - RAVENPY_TESTDATA_PATH=/testdata
-    command: bash -c "source activate wps && pytest -x"
+    command: bash -c "python -m pip install pytest && pytest -x"

docs/source/dev_guide.rst

@@ -9,19 +9,17 @@ Developer Guide
 
 .. WARNING:: To create new processes, look at examples in Emu_.
 
-
 Re-create a fresh environment
 -----------------------------
 
-.. code-block:: console
-
-  make stop  # in case you previously did 'make start'
-  conda deactivate  # exit the current 'raven' conda env so we can destroy it
-  conda env remove -n raven  # destroy the current conda env to recreate one from scratch
-  conda env create -f environment.yml
-  conda activate raven
-  make develop  # install raven-wps and additional dev tools
+.. code-block:: shell
 
+    make stop  # in case you previously did 'make start'
+    conda deactivate  # exit the current 'raven' conda env so we can destroy it
+    conda env remove -n raven  # destroy the current conda env to recreate one from scratch
+    conda env create -f environment.yml
+    conda activate raven
+    make develop  # install raven-wps and additional dev tools
 
 Building the docs
 -----------------
@@ -49,10 +47,7 @@ First activate the ``raven`` Conda environment and install ``pytest``.
 
 .. code-block:: console
 
-   $ source activate raven
-   $ pip install -r requirements_dev.txt  # if not already installed
-   OR
-   $ make develop
+    make develop
 
 Run quick tests (skip slow and online):
 
@@ -87,8 +82,7 @@ Do the same as above using the ``Makefile``.
 Running notebooks tests
 -----------------------
 
-Assuming that the ``raven`` conda env has already been created and is up-to-date and
-raven-wps has been installed with ``make develop``:
+Assuming that the ``raven`` conda env has already been created and is up-to-date and raven-wps has been installed with ``make develop``:
 
     .. code-block:: console
 
@@ -98,28 +92,28 @@ raven-wps has been installed with ``make develop``:
         # to test all notebooks
         make test-notebooks
 
- Or:
+    Or, alternatively:
 
     .. code-block:: console
 
         # to test a single notebook (note the .run at the end of the notebook path)
         make docs/source/notebooks/Subset_climate_data_over_watershed.ipynb.run
 
 
-The notebooks may also require other WPS services (Finch and Flyingpigeon).
+The notebooks may also require other WPS services (`finch`).
 By default these are from the production server but we can point the notebooks to local servers if needed for development purposes:
 
     .. code-block:: console
 
         # to test all notebooks
-        make FLYINGPIGEON_WPS_URL=http://localhost:8093 FINCH_WPS_URL=http://localhost:5000 test-notebooks
+        make FINCH_WPS_URL=http://localhost:5000 test-notebooks
 
-Or:
+    Or, alternatively:
 
     .. code-block:: console
 
         # to test a single notebook (note the .run at the end of the notebook path)
-        make FLYINGPIGEON_WPS_URL=http://localhost:8093 FINCH_WPS_URL=http://localhost:5000 docs/source/notebooks/Subset_climate_data_over_watershed.ipynb.run
+        make FINCH_WPS_URL=http://localhost:5000 docs/source/notebooks/Subset_climate_data_over_watershed.ipynb.run
 
 If instead we want to run the notebooks against the production raven-wps server or any other raven-wps servers:
 
@@ -128,20 +122,19 @@ If instead we want to run the notebooks against the production raven-wps server
         # to test all notebooks
         make WPS_URL=https://pavics.ouranos.ca/twitcher/ows/proxy/raven/wps test-notebooks
 
-Or:
+    Or, alternatively:
 
     .. code-block:: console
 
         # to test a single notebook (note the .run at the end of the notebook path)
         make WPS_URL=https://pavics.ouranos.ca/twitcher/ows/proxy/raven/wps docs/source/notebooks/Subset_climate_data_over_watershed.ipynb.run
 
-We can also override all three of the server variables (WPS_URL, FINCH_WPS_URL, FLYINGPIGEON_WPS_URL) to pick and choose any servers/services from anywhere we want.
+We can also override all three of the server variables (``WPS_URL``, ``FINCH_WPS_URL``) to pick and choose any servers/services from anywhere we want.
 
 Starting local Jupyter server to edit/develop notebooks
 -------------------------------------------------------
 
-Assuming that the ``raven`` conda env has already been created and is up-to-date and
-raven-wps has been installed with ``make develop``:
+Assuming that the ``raven`` conda env has already been created and is up-to-date and raven-wps has been installed with ``make develop``:
 
 .. code-block:: console
 
@@ -151,23 +144,19 @@ raven-wps has been installed with ``make develop``:
     # to start local jupyter notebook server listing all current notebooks
     make notebook  # Control-C to terminate once done
 
-    # Can also override all three WPS_URL, FINCH_WPS_URL and FLYINGPIGEON_WPS_URL here as well,
+    # Can also override WPS_URL and FINCH_WPS_URL here as well,
     # just like 'make test-notebooks' to be able to pick and choose any servers anywhere we want.
 
     # By overriding these variables at the 'make notebook' step, we will not need to
     # override them one by one in each notebook as each notebook will also look
     # for those variables as environment variables.
 
-
 Bulk refresh all notebooks output
 ---------------------------------
 
-This automated refresh only works for notebooks that passed ``make
-test-notebooks`` above.  For those that failed, manually starting a local
-Jupyter server and refresh them manually.
+This automated refresh only works for notebooks that passed ``make test-notebooks`` above.  For those that failed, manually starting a local Jupyter server and refresh them manually.
 
-Assuming that the ``raven`` conda env has already been created and is up-to-date and
-raven-wps has been installed with ``make develop``:
+Assuming that the ``raven`` conda env has already been created and is up-to-date and raven-wps has been installed with ``make develop``:
 
     .. code-block:: console
 
@@ -177,25 +166,24 @@ raven-wps has been installed with ``make develop``:
         # to refresh all notebooks
         make refresh-notebooks
 
-Or:
+    Or, alternatively:
 
     .. code-block:: console
 
         # to refresh a single notebook (note the .refresh at the end of the notebook path)
         make docs/source/notebooks/Assess_probabilistic_flood_risk.ipynb.refresh
 
-        # Can also override all three of the server variables (WPS_URL, FINCH_WPS_URL and FLYINGPIGEON_WPS_URL) here as well,
+        # Can also override the server variables (WPS_URL, FINCH_WPS_URL) here as well,
         # just like 'make test-notebooks' to be able to pick and choose any servers/services from anywhere we want.
 
-
 Prepare a release
 -----------------
 
 Update the Conda specification file to build identical environments_ on a specific OS.
 
 .. note:: You should run this on your target OS, in our case Linux.
 
-.. code-block:: console
+.. code-block:: shell
 
   conda env create -f environment.yml
   source activate raven
@@ -205,7 +193,6 @@ Update the Conda specification file to build identical environments_ on a specif
 
 .. _environments: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#building-identical-conda-environments
 
-
 Bump a new version
 ------------------
 

environment-dev.yml

@@ -5,19 +5,20 @@ channels:
 dependencies:
   - python >=3.9,<3.13
   - flit >=3.8,<4.0
-  - birdy
+  - birdy >=0.8.6
   - black >=24.10.0
   - bump-my-version >=0.26.0
   - flake8 >=7.0
-  - flake8-rst-docstrings
+  - flake8-rst-docstrings >=0.3.0
   - ipykernel
   - ipywidgets
   - isort >=5.13.2
-  - nbconvert
-  - nbval
-  - pip >=23.3.0
+  - nbconvert >=7.13.0
+  - nbval >=0.10.0
+  - pip >=24.2.0
   - pre-commit >=3.6.0
-  - pymetalink
+  - pymetalink >=6.5.2
   - pytest >=8.0.0
   - pytest-xdist >=3.2
-  - python-build
+  - python-build >=1.0
+  - yamllint >=1.33.0

environment.yml

@@ -5,31 +5,30 @@ channels:
 dependencies:
   - python >=3.9,<3.13
   - pywps >=4.6
-  - affine
-  - anyascii
+  - affine >=2.4.0
+  - anyascii >=0.3.2
   - cartopy >=0.23.0
   - click >=8.1.7
   - fiona >=1.9
   - geopandas >=0.12.0
-  - geojson
+  - geojson >=3.1.0
   - gdal >=3.0
   - jinja2 >=3.1.4
-  - matplotlib
-  - nbval
-  - netcdf4
+  - matplotlib-base >=3.6.0
+  - nbval >=0.10.0
+  - netcdf4 >=1.6.5
   - numpy >=1.23.0
-  - owslib
+  - owslib >=0.29.0
   - pandas >=2.2
   - psutil >=6.0.0
   - psycopg2
-  - pymetalink
+  - pymetalink >=6.5.2
   - pyogrio >=0.7.2
   - pyproj >=3.4
-  - rasterio
-  - rasterstats
-  - requests
-  - rioxarray
+  - rasterio >=1.4.0
+  - rasterstats >=0.20.0
+  - requests >=2.31.0
+  - rioxarray >=0.15.0
   - shapely >=2.0
-  - urlpath
   - xarray >=2023.11.0
   - xclim >=0.48.2