diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 2c0bf7280..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: 2 - -setup: &setup - docker: - - image: continuumio/miniconda3:4.10.3 - environment: - # QGIS complains when setting up an QgsApplicatoin if `QT_QPA_PLATFORM` is not - # set to `offscreen`: - # qt.qpa.xcb: could not connect to display - # qt.qpa.plugin: Could not load the Qt platform plugin "xcb" in "" even - # though it was found. - # This application failed to start because no Qt - # platform plugin could be initialized. Reinstalling the application - # may fix this problem. - # - # Available platform plugins are: eglfs, minimal, minimalegl, - # offscreen, vnc, webgl, xcb. - # - # Fatal Python error: Aborted - QT_QPA_PLATFORM: offscreen - working_directory: ~/qgreenland - -jobs: - test: - <<: *setup - steps: - - checkout - - run: - name: 'Apt install libgl1-mesa-glx' - command: | - # Install libgl1-mesa-glx. Import errors occur otherwise. - # See: https://app.circleci.com/jobs/github/nsidc/qgreenland/72/parallel-runs/0/steps/0-102 - apt-get update && apt-get install -y libgl1-mesa-glx - - run: - name: 'Run all tests (lint, config validation, etc.)' - command: | - conda env create --quiet -f environment-lock.yml - conda init bash - /bin/bash --login -c "conda activate qgreenland && inv test.ci" - - trigger_build: - <<: *setup - steps: - - run: - name: 'Trigger Jenkins to build production package' - command: | - REF="${CIRCLE_TAG}" - REQ_URL="${JENKINS_PROD_BUILD_JOB_URL}/buildWithParameters?ref=${REF}&delay=5sec" - - wget "$REQ_URL" - -workflows: - version: 2 - - # For commits on any branch, only run tests. - # For tags vX.Y.Z*, run tests then trigger a Jenkins build. - test_and_sometimes_trigger_build: - jobs: - - test: - filters: - tags: - only: /^v\d+\.\d+\.\d+.*$/ - - trigger_build: - requires: - - test - filters: - tags: - only: /^v\d+\.\d+\.\d+.*$/ - branches: - ignore: /.*/ diff --git a/.flake8 b/.flake8 deleted file mode 100644 index a6b39db73..000000000 --- a/.flake8 +++ /dev/null @@ -1,26 +0,0 @@ -[flake8] -max-line-length = 90 -max-complexity = 8 -inline-quotes = " - -# flake8-import-order -application_import_names = qgreenland -import_order_style = pycharm - -# D1: Ignore errors requiring docstrings on everything. -# W503: Line breaks should occur after the binary operator to keep all variable names aligned. -# E731: Lambda assignments are OK, use your best judgement. -ignore = D1,W503,E731 - -# E501: Line too long. Long strings, e.g. URLs, are common in config. -# F821: Undefined name. Expected in some templates and scripts. -# FS003: f-string missing prefix. Useful for config templates. -per-file-ignores = - qgreenland/ancillary/templates/layer_cfg.py: F821 - qgreenland/config/datasets/**/*.py: FS003, E501 - qgreenland/config/datasets/*.py: FS003, E501 - qgreenland/config/helpers/**/*.py: FS003 - qgreenland/config/layers/**/*.py: FS003, E501 - qgreenland/test/util/test_config.py: FS003 - qgreenland/test/util/test_runtime_vars.py: FS003 - scripts/qgis_examples/*.py: F821 diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml new file mode 100644 index 000000000..44e5c7406 --- /dev/null +++ b/.github/workflows/test-and-build.yml @@ -0,0 +1,74 @@ +name: "Test and (if tag) build" + +on: + push: + branches: + - "*" + tags: + - "v[0-9]+.[0-9]+.[0-9]+*" + pull_request: + + +# Default to bash in login mode; key to activating conda environment +# https://github.com/mamba-org/provision-with-micromamba#IMPORTANT +defaults: + run: + shell: "bash -l {0}" + + +jobs: + test: + runs-on: "ubuntu-latest" + steps: + - name: "Check out repository" + uses: "actions/checkout@v3" + + - name: "Apt install libgl1-mesa-glx" + run: | + # Install libgl1-mesa-glx. Import errors occur otherwise. + # See: https://app.circleci.com/jobs/github/nsidc/qgreenland/72/parallel-runs/0/steps/0-102 + sudo apt-get update + sudo apt-get install -y libgl1-mesa-glx + + # Rename lock file; mamba expects files ending in `-lock.yml` to have a different format :( + # https://github.com/mamba-org/mamba/issues/1209#issuecomment-1447546266 + - name: "HACK: Rename lockfile" + run: "mv environment-lock.yml environment-ci.yml" + + - name: "Install Conda environment" + uses: "mamba-org/setup-micromamba@v1.4.1" + with: + environment-file: "environment-ci.yml" + cache-environment: true + + - name: "Run tests" + run: "inv test.ci" + env: + # QGIS complains when setting up an QgsApplicatoin if `QT_QPA_PLATFORM` is not + # set to `offscreen`: + # qt.qpa.xcb: could not connect to display + # qt.qpa.plugin: Could not load the Qt platform plugin "xcb" in "" even + # though it was found. + # This application failed to start because no Qt + # platform plugin could be initialized. Reinstalling the application + # may fix this problem. + # + # Available platform plugins are: eglfs, minimal, minimalegl, + # offscreen, vnc, webgl, xcb. + # + # Fatal Python error: Aborted + QT_QPA_PLATFORM: offscreen + + + build: + runs-on: "ubuntu-latest" + needs: ["test"] + if: "github.ref_type == 'tag'" + steps: + - name: "Trigger Jenkins to build QGreenland Core" + run: | + JOB_NAME="qgreenland_C3_Production_Build_QGreenland_Package" + JOB_URL="${{ secrets.JENKINS_URL }}/job/${JOB_NAME}" + JOB_BUILD_URL="${JOB_URL}/buildWithParameters?ref=${{ github.ref_name }}&delay=5sec" + + wget "$JOB_BUILD_URL" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..39f36fa2a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,44 @@ +default_language_version: + python: "python3.10" + + +repos: + + - repo: "https://github.com/pre-commit/pre-commit-hooks" + rev: "v4.4.0" + hooks: + - id: "check-added-large-files" + - id: "check-vcs-permalinks" + - id: "end-of-file-fixer" + + - repo: "https://github.com/charliermarsh/ruff-pre-commit" + rev: "v0.0.269" + hooks: + - id: "ruff" + # NOTE: "--exit-non-zero-on-fix" is important for CI to function + # correctly! + args: ["--fix", "--exit-non-zero-on-fix"] + + - repo: "https://github.com/psf/black" + rev: "23.3.0" + hooks: + - id: "black" + + - repo: "https://github.com/jendrikseipp/vulture" + rev: "v2.7" + hooks: + - id: "vulture" + + # Commented because it requires Docker, which is not available in all CI + # runners. Works on GitHub actions but not CircleCI. + # - repo: "https://github.com/koalaman/shellcheck-precommit" + # rev: "v0.9.0" + # hooks: + # - id: "shellcheck" + # # args: ["--severity=warning"] # Optionally only show errors and warnings + # + + - repo: "https://github.com/shellcheck-py/shellcheck-py" + rev: "v0.9.0.5" + hooks: + - id: "shellcheck" diff --git a/doc/_notes/20200115_notes.org b/doc/_notes/20200115_notes.org index d2e274848..b2bf287df 100644 --- a/doc/_notes/20200115_notes.org +++ b/doc/_notes/20200115_notes.org @@ -58,4 +58,3 @@ - Place names - DEMs - Contour lines - diff --git a/doc/contributor-how-to/how-to-dev-plugin.md b/doc/contributor-how-to/how-to-dev-plugin.md index 4b973a2fd..a291e79a3 100644 --- a/doc/contributor-how-to/how-to-dev-plugin.md +++ b/doc/contributor-how-to/how-to-dev-plugin.md @@ -44,4 +44,4 @@ Navigate to **Plugins -> Plugin Reloader -> Reload Plugin: qgreenland-plugin**. You should receive a message notifying you that the plugin has been reloaded. After reloading the plugin, you are all set to begin using QGreenland Custom. Reference the user guide, -**How to install and use QGreenland Custom**, to get started. \ No newline at end of file +**How to install and use QGreenland Custom**, to get started. diff --git a/doc/contributor-how-to/use-our-tooling.md b/doc/contributor-how-to/use-our-tooling.md new file mode 100644 index 000000000..4e67c7612 --- /dev/null +++ b/doc/contributor-how-to/use-our-tooling.md @@ -0,0 +1,25 @@ +# How to use QGreenland development tooling + +## Linting and formatting + +This project uses [pre-commit](https://pre-commit.com/) for linting and code formatting. +This dependency is already part of the QGreenland Conda environment! To set it up, +simply: + +``` +pre-commit install +``` + +This will configure Git hooks which will trigger when you make a commit. + + +## Testing and other stuff + +We use [invoke](https://www.pyinvoke.org/) for other miscellaneous tasks, like: + +* Environment locking (`inv env.lock`) +* Interactive docs building (`inv docs.watch`) +* Typechecking (`inv test.typecheck`) +* ... and much, much more! + +Use `inv --list` to view a list of available tasks. diff --git a/doc/reference/glossary/index.rst b/doc/reference/glossary/index.rst index 4f9b277ae..152c06c75 100644 --- a/doc/reference/glossary/index.rst +++ b/doc/reference/glossary/index.rst @@ -9,4 +9,4 @@ Glossary :glob: ./user-glossary.md - ./contributor-glossary.md \ No newline at end of file + ./contributor-glossary.md diff --git a/doc/reference/glossary/user-glossary.md b/doc/reference/glossary/user-glossary.md index e4e0a540b..d6af83fde 100644 --- a/doc/reference/glossary/user-glossary.md +++ b/doc/reference/glossary/user-glossary.md @@ -58,4 +58,4 @@ targeted use. Modules include: value. - **Vector**: A layer type consisting of either points, lines, or polygons, where each point, - line, or polygon has a unique set of attribute values. \ No newline at end of file + line, or polygon has a unique set of attribute values. diff --git a/doc/tutorials/analyze-ice-sheet-volume.md b/doc/tutorials/analyze-ice-sheet-volume.md index d7edc6a3c..080da03bd 100644 --- a/doc/tutorials/analyze-ice-sheet-volume.md +++ b/doc/tutorials/analyze-ice-sheet-volume.md @@ -76,4 +76,3 @@ sheet thickness data included in the QGreenland core package to calculate the volume of the Greenland ice sheet. Having accomplished this, the user is now ready to explore other geospatial processing tools included in the **Processing Toolbox** to further analyze other QGreenland data. - diff --git a/environment-lock.yml b/environment-lock.yml index 1f29192d2..dab675462 100644 --- a/environment-lock.yml +++ b/environment-lock.yml @@ -15,18 +15,18 @@ dependencies: - backcall=0.2.0=pyh9f0ad1d_0 - backports=1.0=pyhd8ed1ab_3 - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0 - - black=22.12.0=py310hff52083_0 - - blosc=1.21.3=hafa529b_0 + - blosc=1.21.4=h0f2a231_0 - boost-cpp=1.78.0=h6582d0a_3 - brotlipy=0.7.0=py310h5764c6d_1005 - bump2version=1.0.1=pyh9f0ad1d_0 - bzip2=1.0.8=h7f98852_4 - - c-ares=1.18.1=h7f98852_0 - - ca-certificates=2022.12.7=ha878542_0 + - c-ares=1.19.1=hd590300_0 + - ca-certificates=2023.5.7=hbcca054_0 - cairo=1.16.0=h35add3b_1015 - ceres-solver=2.1.0=hf302a74_1 - - certifi=2022.12.7=pyhd8ed1ab_0 + - certifi=2023.5.7=pyhd8ed1ab_0 - cffi=1.15.1=py310h255011f_3 + - cfgv=3.3.1=pyhd8ed1ab_0 - cfitsio=4.2.0=hd9d235c_0 - cftime=1.6.2=py310hde88566_1 - chardet=4.0.0=py310hff52083_3 @@ -34,12 +34,13 @@ dependencies: - click-plugins=1.1.1=py_0 - cligj=0.7.2=pyhd8ed1ab_1 - colorama=0.4.6=pyhd8ed1ab_0 - - coverage=7.2.5=py310h2372a71_0 - - cryptography=40.0.2=py310h34c0648_0 - - curl=8.0.1=h588be90_0 + - coverage=7.2.7=py310h2372a71_0 + - cryptography=41.0.1=py310h75e40e8_0 + - curl=8.1.2=h409715c_0 - dataclasses=0.8=pyhc8e2a94_3 - dbus=1.13.6=h5008d03_3 - decorator=5.1.1=pyhd8ed1ab_0 + - distlib=0.3.6=pyhd8ed1ab_0 - docutils=0.17.1=py310hff52083_3 - draco=1.5.6=hf52228f_0 - eigen=3.4.0=h4bd325d_0 @@ -47,15 +48,8 @@ dependencies: - executing=1.2.0=pyhd8ed1ab_0 - exiv2=0.27.6=hb9a316c_1 - expat=2.5.0=hcb278e6_1 - - fftw=3.3.10=nompi_hc118613_107 + - filelock=3.12.0=pyhd8ed1ab_0 - fiona=1.8.22=py310ha325b7b_5 - - flake8=3.8.4=py_0 - - flake8-bugbear=21.9.2=pyhd8ed1ab_0 - - flake8-comprehensions=3.2.3=py_0 - - flake8-docstrings=1.5.0=py_0 - - flake8-import-order=0.18.2=pyhd8ed1ab_0 - - flake8-polyfill=1.0.2=py_0 - - flake8-quotes=2.1.1=py_0 - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 - font-ttf-inconsolata=3.000=h77eed37_0 - font-ttf-source-code-pro=2.038=h77eed37_0 @@ -67,41 +61,38 @@ dependencies: - freexl=1.0.6=h166bdaf_1 - funcy=1.18=pyhd8ed1ab_0 - future=0.18.3=pyhd8ed1ab_0 - - gdal=3.6.4=py310hf0ca374_1 + - gdal=3.6.4=py310hf0ca374_2 - geos=3.11.2=hcb278e6_0 - geotiff=1.7.1=h480ec47_8 - gettext=0.21.1=h27087fc_0 - gflags=2.2.2=he1b5a44_1004 - giflib=5.2.1=h0b41bf4_3 - - glib=2.76.2=hfc55251_0 - - glib-tools=2.76.2=hfc55251_0 + - glib=2.76.3=hfc55251_0 + - glib-tools=2.76.3=hfc55251_0 - glog=0.6.0=h6f12383_0 - gmp=6.2.1=h58526e2_0 - graphite2=1.3.13=h58526e2_1001 - gsl=2.7=he838d99_0 - gst-plugins-base=1.22.0=h4243ec0_2 - gstreamer=1.22.0=h25f0c4b_2 - - gstreamer-orc=0.4.33=h166bdaf_0 - - harfbuzz=6.0.0=h3ff4399_1 + - harfbuzz=7.3.0=hdb3a94d_0 - hdf4=4.2.15=h501b40f_6 - hdf5=1.14.0=nompi_hb72d44e_103 - httplib2=0.22.0=pyhd8ed1ab_0 - humanize=2.6.0=py_0 - icu=72.1=hcb278e6_0 + - identify=2.5.24=pyhd8ed1ab_0 - idna=2.10=pyh9f0ad1d_0 - imagesize=1.4.1=pyhd8ed1ab_0 - importlib-metadata=6.6.0=pyha770c72_0 - - importlib_metadata=6.6.0=hd8ed1ab_0 - iniconfig=2.0.0=pyhd8ed1ab_0 - invoke=1.4.1=py_0 - ipdb=0.13.13=pyhd8ed1ab_0 - - ipython=8.13.2=pyh41d4057_0 - - isort=5.12.0=pyhd8ed1ab_1 - - jack=1.9.22=h11f4161_0 + - ipython=8.14.0=pyh41d4057_0 - jedi=0.18.2=pyhd8ed1ab_0 - jinja2=3.1.2=pyhd8ed1ab_1 - json-c=0.16=hc379101_0 - - kealib=1.5.0=he7a6254_1 + - kealib=1.5.1=h3845be2_3 - keyutils=1.6.1=h166bdaf_0 - krb5=1.20.1=h81ceb04_0 - lame=3.100=h166bdaf_1003 @@ -111,57 +102,53 @@ dependencies: - ld_impl_linux-64=2.40=h41732ed_0 - lerc=4.0.0=h27087fc_0 - libaec=1.0.6=hcb278e6_1 - - libblas=3.9.0=16_linux64_openblas + - libblas=3.9.0=17_linux64_openblas - libcap=2.67=he9d0100_0 - - libcblas=3.9.0=16_linux64_openblas - - libclang=16.0.3=default_h83cc7fd_0 - - libclang13=16.0.3=default_hd781213_0 + - libcblas=3.9.0=17_linux64_openblas + - libclang=16.0.3=default_h1cdf331_2 + - libclang13=16.0.3=default_h4d60ac6_2 - libcups=2.3.3=h36d4200_3 - - libcurl=8.0.1=h588be90_0 - - libdb=6.2.32=h9c3ff4c_0 + - libcurl=8.1.2=h409715c_0 - libdeflate=1.18=h0b41bf4_0 - libedit=3.1.20191231=he28a2e2_2 - libev=4.33=h516909a_1 - - libevent=2.1.10=h28343ad_4 + - libevent=2.1.12=hf998b51_1 - libexpat=2.5.0=hcb278e6_1 - libffi=3.4.2=h7f98852_5 - libflac=1.4.2=h27087fc_0 - - libgcc-ng=12.2.0=h65d4601_19 + - libgcc-ng=13.1.0=he5830b7_0 - libgcrypt=1.10.1=h166bdaf_0 - - libgdal=3.6.4=h7239d12_1 - - libgfortran-ng=12.2.0=h69a702a_19 - - libgfortran5=12.2.0=h337968e_19 - - libglib=2.76.2=hebfc3b9_0 - - libgomp=12.2.0=h65d4601_19 + - libgdal=3.6.4=hada8d5e_2 + - libgfortran-ng=13.1.0=h69a702a_0 + - libgfortran5=13.1.0=h15d22d2_0 + - libglib=2.76.3=hebfc3b9_0 + - libgomp=13.1.0=he5830b7_0 - libgpg-error=1.46=h620e276_0 - - libhwloc=2.9.1=hd6dc26d_0 + - libhwloc=2.9.1=hf312287_1 - libiconv=1.17=h166bdaf_0 - libjpeg-turbo=2.1.5.1=h0b41bf4_0 - libkml=1.3.0=h37653c0_1015 - - liblapack=3.9.0=16_linux64_openblas - - libllvm15=15.0.7=hadd5161_1 + - liblapack=3.9.0=17_linux64_openblas - libllvm16=16.0.3=hbf9e925_1 - libnetcdf=4.9.2=nompi_hdf9a29f_104 - libnghttp2=1.52.0=h61bc06f_0 - libnsl=2.0.0=h7f98852_0 - libogg=1.3.4=h7f98852_1 - - libopenblas=0.3.21=pthreads_h78a6416_3 + - libopenblas=0.3.23=pthreads_h80387f5_0 - libopus=1.3.1=h7f98852_1 - libpng=1.6.39=h753d276_0 - - libpq=15.2=hb675445_0 + - libpq=15.3=hbcd7760_0 - libprotobuf=3.21.12=h3eb15da_0 - librttopo=1.1.0=h0d5128d_13 - libsecret=0.18.8=h329b89f_2 - libsndfile=1.2.0=hb75c966_0 - libspatialindex=1.9.3=h9c3ff4c_4 - libspatialite=5.0.1=h7d1ca68_25 - - libsqlite=3.40.0=h753d276_1 - - libssh2=1.10.0=hf14f497_3 - - libstdcxx-ng=12.2.0=h46fd767_19 + - libsqlite=3.42.0=h2797004_0 + - libssh2=1.11.0=h0841786_0 + - libstdcxx-ng=13.1.0=hfd8a6a1_0 - libsystemd0=253=h8c4010b_1 - libtiff=4.5.0=ha587672_6 - - libtool=2.4.7=h27087fc_0 - - libudev1=253=h0b41bf4_1 - libuuid=2.38.1=h0b41bf4_0 - libvorbis=1.3.7=h9c3ff4c_0 - libwebp=1.3.0=hb47c5f0_0 @@ -178,50 +165,49 @@ dependencies: - lz4-c=1.9.4=hcb278e6_0 - markdown=3.3.7=pyhd8ed1ab_0 - markdown-it-py=1.1.0=pyhd8ed1ab_0 - - markupsafe=2.1.2=py310h1fa729e_0 + - markupsafe=2.1.3=py310h2372a71_0 - matplotlib-inline=0.1.6=pyhd8ed1ab_0 - - mccabe=0.6.1=py_1 - mdit-py-plugins=0.2.8=pyhd8ed1ab_0 - metis=5.1.0=h58526e2_1006 - mock=5.0.2=pyhd8ed1ab_0 - mpfr=4.2.0=hb012696_0 - mpg123=1.31.3=hcb278e6_0 - - munch=2.5.0=py_0 + - munch=3.0.0=pyhd8ed1ab_0 - mypy=1.2.0=py310h1fa729e_0 - mypy_extensions=1.0.0=pyha770c72_0 - - mysql-common=8.0.32=ha901b37_1 - - mysql-libs=8.0.32=hd7da12d_1 + - mysql-common=8.0.32=hf1915f5_2 + - mysql-libs=8.0.32=hca2cd23_2 - myst-parser=0.15.2=pyhd8ed1ab_0 - ncurses=6.3=h27087fc_1 - netcdf4=1.6.3=nompi_py310h2d0b64f_102 - - nitro=2.7.dev6=hcb278e6_5 + - nitro=2.7.dev8=h59595ed_0 + - nodeenv=1.8.0=pyhd8ed1ab_0 - nose2=0.9.2=py_0 - nspr=4.35=h27087fc_0 - nss=3.89=he45b914_0 - numpy=1.24.3=py310ha4c1d20_0 - openjpeg=2.5.0=hfec8fc6_2 - openpyxl=3.1.2=py310h2372a71_0 - - openssl=3.1.0=hd590300_3 - - owslib=0.29.1=pyhd8ed1ab_0 + - openssl=3.1.1=hd590300_1 + - owslib=0.29.2=pyhd8ed1ab_0 - packaging=23.1=pyhd8ed1ab_0 - pandas=1.5.3=py310h9b08913_1 - parso=0.8.3=pyhd8ed1ab_0 - - pathspec=0.11.1=pyhd8ed1ab_0 - pcre2=10.40=hc3806b6_0 - - pdal=2.5.3=h09aa857_0 - - pep8-naming=0.9.1=py_0 + - pdal=2.5.4=h3283ea9_0 - perl=5.32.1=2_h7f98852_perl5 - pexpect=4.8.0=pyh1a96a4e_2 - pickleshare=0.7.5=py_1003 - pip=23.1.2=pyhd8ed1ab_0 - pixman=0.40.0=h36c2ea0_0 - - platformdirs=3.5.0=pyhd8ed1ab_0 + - platformdirs=3.5.1=pyhd8ed1ab_0 - plotly=5.14.1=pyhd8ed1ab_0 - pluggy=1.0.0=pyhd8ed1ab_5 - ply=3.11=py_1 - - poppler=23.04.0=hf052cbe_1 + - poppler=23.05.0=hd18248d_1 - poppler-data=0.4.12=hd8ed1ab_0 - - postgresql=15.2=h3248436_0 + - postgresql=15.3=h814edd5_0 + - pre-commit=3.3.2=pyha770c72_0 - proj=9.2.0=h8ffa02c_0 - prometheus_client=0.5.0=py_0 - prompt-toolkit=3.0.38=pyha770c72_0 @@ -230,18 +216,13 @@ dependencies: - psycopg2=2.9.3=py310h416cc33_2 - pthread-stubs=0.4=h36c2ea0_1001 - ptyprocess=0.7.0=pyhd3deb0d_0 - - pulseaudio=16.1=hcb278e6_3 - - pulseaudio-client=16.1=h5195f5e_3 - - pulseaudio-daemon=16.1=ha8d29e2_3 + - pulseaudio-client=16.1=hb77b528_4 - pure_eval=0.2.2=pyhd8ed1ab_0 - py=1.11.0=pyh6c4a22f_0 - - pycodestyle=2.6.0=pyh9f0ad1d_0 - pycparser=2.21=pyhd8ed1ab_0 - - pydantic=1.10.7=py310h1fa729e_0 - - pydocstyle=6.3.0=pyhd8ed1ab_0 - - pyflakes=2.2.0=pyh9f0ad1d_0 + - pydantic=1.10.8=py310h2372a71_0 - pygments=2.15.1=pyhd8ed1ab_0 - - pyopenssl=23.1.1=pyhd8ed1ab_0 + - pyopenssl=23.2.0=pyhd8ed1ab_1 - pyparsing=3.0.9=pyhd8ed1ab_0 - pyproj=3.5.0=py310hb814896_1 - pyqt=5.15.7=py310hab646b1_3 @@ -250,25 +231,24 @@ dependencies: - pysocks=1.7.1=pyha2e5f31_6 - pytest=6.2.5=py310hff52083_3 - pytest-cov=2.12.1=pyhd8ed1ab_0 - - python=3.10.10=he550d4f_0_cpython + - python=3.10.11=he550d4f_0_cpython - python-daemon=3.0.1=pyhd8ed1ab_1 - python-dateutil=2.8.2=pyhd8ed1ab_0 - python_abi=3.10=3_cp310 - pytz=2023.3=pyhd8ed1ab_0 - pyyaml=5.4.1=py310h5764c6d_4 - qca=2.3.6=h4a6f7a0_0 - - qgis=3.28.6=py310h2db2bc6_2 + - qgis=3.28.7=py310h053775b_0 - qjson=0.9.0=hb2b434f_1009 - qscintilla2=2.14.0=py310h704022c_0 - - qt-main=5.15.8=h5c52f38_10 - - qtkeychain=0.13.2=h7ba989a_2 + - qt-main=5.15.8=haa3a1c2_11 + - qtkeychain=0.14.1=hbc31b07_0 - qtwebkit=5.212=h7b3bce5_10 - - qwt=6.2.0=hc45b483_5 + - qwt=6.2.0=h16b76b2_6 - readline=8.2=h8228510_1 - requests=2.25.1=pyhd3deb0d_0 - setuptools=67.7.2=pyhd8ed1ab_0 - shapely=2.0.1=py310h056c13c_1 - - shellcheck=0.7.2=ha770c72_1 - sip=6.7.9=py310hc6cd4ac_0 - six=1.16.0=pyh6c4a22f_0 - snappy=1.1.10=h9fff704_0 @@ -281,7 +261,7 @@ dependencies: - sphinxcontrib-jsmath=1.0.1=py_0 - sphinxcontrib-qthelp=1.0.3=py_0 - sphinxcontrib-serializinghtml=1.1.5=pyhd8ed1ab_2 - - sqlite=3.40.0=h4ff8645_1 + - sqlite=3.42.0=h2c6b66d_0 - stack_data=0.6.2=pyhd8ed1ab_0 - suitesparse=5.10.1=h9e50725_1 - tbb=2021.9.0=hf52228f_0 @@ -290,17 +270,18 @@ dependencies: - tk=8.6.12=h27826a3_0 - toml=0.10.2=pyhd8ed1ab_0 - tomli=2.0.1=pyhd8ed1ab_0 - - tornado=6.3=py310h1fa729e_0 + - tornado=6.3.2=py310h2372a71_0 - traitlets=5.9.0=pyhd8ed1ab_0 - types-markdown=3.3.31=pyhd8ed1ab_0 - types-pyyaml=5.4.12=pyhd8ed1ab_0 - types-requests=2.25.12=pyhd8ed1ab_0 - - typing-extensions=4.5.0=hd8ed1ab_0 - - typing_extensions=4.5.0=pyha770c72_0 + - typing-extensions=4.6.3=hd8ed1ab_0 + - typing_extensions=4.6.3=pyha770c72_0 - tzcode=2023c=h0b41bf4_0 - tzdata=2023c=h71feb2d_0 + - ukkonen=1.0.1=py310hbf28c38_3 - urllib3=1.26.15=pyhd8ed1ab_0 - - vulture=1.6=pyh9f0ad1d_0 + - virtualenv=20.23.0=pyhd8ed1ab_0 - wcwidth=0.2.6=pyhd8ed1ab_0 - wheel=0.40.0=pyhd8ed1ab_0 - xcb-util=0.4.0=h516909a_0 @@ -311,10 +292,10 @@ dependencies: - xerces-c=3.2.4=h8d71039_2 - xkeyboard-config=2.38=h0b41bf4_0 - xorg-kbproto=1.0.7=h7f98852_1002 - - xorg-libice=1.0.10=h7f98852_0 - - xorg-libsm=1.2.3=hd9c2040_1000 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 - xorg-libx11=1.8.4=h0b41bf4_0 - - xorg-libxau=1.0.9=h7f98852_0 + - xorg-libxau=1.0.11=hd590300_0 - xorg-libxdmcp=1.1.3=h7f98852_0 - xorg-libxext=1.3.4=h0b41bf4_2 - xorg-libxrender=0.9.10=h7f98852_1003 @@ -329,7 +310,5 @@ dependencies: - zstd=1.5.2=h3eb15da_6 - pip: - autodoc-pydantic==1.5.1 - - flake8-debugger==3.2.1 - - flake8-use-fstring==1.4 - sphinx-rtd-theme==1.0.0 - sphinx-selective-exclude==1.0.3 diff --git a/environment.yml b/environment.yml index b15bd21b7..5717a9264 100644 --- a/environment.yml +++ b/environment.yml @@ -39,9 +39,9 @@ dependencies: - latexmk ~=4.55 - myst-parser ~=0.15.2 - sphinx-click ~=3.0.1 - # TODO: What does this do? Do we need it? - # - sphinx-autodoc-typehints ~=1.12.0 - # SEE PIP DEPENDENCIES FOR MORE + # TODO: What does this do? Do we need it? + # - sphinx-autodoc-typehints ~=1.12.0 + # SEE PIP DEPENDENCIES FOR MORE # task-runners - invoke ~=1.4.0 @@ -50,17 +50,8 @@ dependencies: - pytest ~=6.2 - pytest-cov ~=2.12 - # static analysis - - flake8 ~=3.8.3 - - flake8-bugbear ~=21.9.2 - - flake8-comprehensions ~=3.2.2 - - flake8-docstrings ~=1.5.0 - - flake8-import-order ~=0.18.1 - - flake8-quotes ~=2.1.1 - - pep8-naming ~=0.9.1 - - vulture ~=1.0 - - shellcheck ~=0.7.1 - # SEE PIP DEPENDENCIES FOR MORE + # linting + - pre-commit # typechecking - mypy ~=1.2.0 @@ -69,15 +60,11 @@ dependencies: - bump2version - ipython - ipdb - - pip - - black ~=22.0 - - isort ~=5.0 + - pip # Pip dependencies could be imported or non-imported :( - pip: - - flake8-debugger ~=3.2 - - flake8-use-fstring ~=1.0 - sphinx-rtd-theme ~=1.0.0 - autodoc-pydantic ~=1.5.1 - sphinx-selective-exclude ~=1.0.3 diff --git a/pyproject.toml b/pyproject.toml index 3ddd0e117..e3bf8da05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,78 @@ [tool.black] -target-version = ['py310'] +target-version = ["py310"] -[tool.isort] -profile = "black" -known_third_party = "luigi" -known_first_party = "qgreenland" + +[tool.vulture] +paths = ["qgreenland", "scripts"] +min_confidence = 80 + + +[tool.ruff] +target-version = "py310" +select = [ + "F", + "E", + "W", + "C4", + "C90", + "I", + "N", + "D", + "UP", + "YTT", + "B", + "A", + "C4", + "T10", + "RUF", +] + +ignore = [ + # D1: Ignore errors requiring docstrings on everything. + # D203: "1 blank line required before class docstring" + # D213: "Multi-line docstring summary should start at the second line" + # E731: Lambda assignments are OK, use your best judgement. + # RUF010: !a, !r, !s are too magical for me. + "D1", "D203", "D213", "E731", "RUF010", + + # Rules ignored on switch to `ruff` + # TODO: re-enable and fix these! + # A0: All rules related to names shadowing python built-ins + # B904: Within an `except` clause, raise exceptions with `raise ... from err` + # or `raise ... from None` to distinguish them from errors in exception + # handling. + # B905: `zip()` without an explicit `strict=` parameter + # N806: Variable name should be lowercase + # RUF001: String contains ambiguous unicode character `’` + # UP007: Use `X | Y` for type annotations + "A0", "B904", "B905", "N806", "RUF001", "UP007", + +] +exclude = [ + "vulture_allowlist.py", + # TODO: Remove tasks/docs excludes: + "tasks/*", + "doc/*", +] + +[tool.ruff.per-file-ignores] +# E402: Module level import not at top of file +# E501: Line too long. Long strings, e.g. URLs, are common in config. +# F821: Undefined name. Expected in some templates and scripts. +"scripts/qgis_examples/*.py" = ["F821"] +"tasks/test.py" = ["E402"] +"qgreenland/ancillary/templates/layer_cfg.py" = ["F821"] +"qgreenland/config/datasets/**/*.py" = ["E501"] +"qgreenland/config/datasets/*.py" = ["E501"] +"qgreenland/config/layers/**/*.py" = ["E501"] +"qgreenland/config/helpers/**/*.py" = ["E501"] + +[tool.ruff.isort] +known-first-party = ["qgreenland"] +known-third-party = ["luigi"] + +[tool.ruff.mccabe] +max-complexity = 8 + +[tool.ruff.flake8-quotes] +inline-quotes = "double" diff --git a/qgreenland/assets/arctic_circle.geojson b/qgreenland/assets/arctic_circle.geojson index a3b928a81..63866f7e6 100644 --- a/qgreenland/assets/arctic_circle.geojson +++ b/qgreenland/assets/arctic_circle.geojson @@ -13,4 +13,4 @@ } } ] -} \ No newline at end of file +} diff --git a/qgreenland/assets/wmm2020_geomagnetic_north_pole.geojson b/qgreenland/assets/wmm2020_geomagnetic_north_pole.geojson index 9b29d1c93..0aa78afb4 100644 --- a/qgreenland/assets/wmm2020_geomagnetic_north_pole.geojson +++ b/qgreenland/assets/wmm2020_geomagnetic_north_pole.geojson @@ -11,4 +11,4 @@ } } ] -} \ No newline at end of file +} diff --git a/qgreenland/cli/cleanup.py b/qgreenland/cli/cleanup.py index a3aef0303..5223aefa0 100644 --- a/qgreenland/cli/cleanup.py +++ b/qgreenland/cli/cleanup.py @@ -66,10 +66,7 @@ def _print_and_run(cmd, *, dry_run): "delete_fetch_by_pattern", "--delete-fetch-by-pattern", "-f", - help=( - "Delete fetched dataset assets matching PATTERN" - " (`{dataset_id}.{asset_id}`)" # noqa: FS003 - ), + help="Delete fetched dataset assets matching PATTERN (`{dataset_id}.{asset_id}`)", multiple=True, metavar="PATTERN", ) diff --git a/qgreenland/cli/config_template.py b/qgreenland/cli/config_template.py index e1d3cd5cd..b4fdbe9aa 100644 --- a/qgreenland/cli/config_template.py +++ b/qgreenland/cli/config_template.py @@ -4,7 +4,7 @@ def _print_template(template_fn: str) -> None: - contents = open(TEMPLATES_DIR / template_fn, "r").read() + contents = open(TEMPLATES_DIR / template_fn).read() # Don't print the extra newline, so the contents can be redirected without # change. diff --git a/qgreenland/config/datasets/wmm.py b/qgreenland/config/datasets/wmm.py index d9a204f29..2fbece0f2 100644 --- a/qgreenland/config/datasets/wmm.py +++ b/qgreenland/config/datasets/wmm.py @@ -19,20 +19,20 @@ HttpAsset( id="geomagnetic_coordinates", urls=[ - "ftp://ftp.ngdc.noaa.gov/geomag/wmm/wmm2020/shapefiles/WMM2020_geomagnetic_coordinate_shapefiles.zip", # noqa:E501 + "ftp://ftp.ngdc.noaa.gov/geomag/wmm/wmm2020/shapefiles/WMM2020_geomagnetic_coordinate_shapefiles.zip", ], ), HttpAsset( id="blackout_zones", urls=[ - "ftp://ftp.ngdc.noaa.gov/geomag/wmm/wmm2020/shapefiles/WMM2020-2025_BoZ_Shapefile.zip", # noqa:E501 + "ftp://ftp.ngdc.noaa.gov/geomag/wmm/wmm2020/shapefiles/WMM2020-2025_BoZ_Shapefile.zip", ], ), *[ HttpAsset( id=str(year), urls=[ - f"ftp://ftp.ngdc.noaa.gov/geomag/wmm/wmm2020/shapefiles/{year}/WMM_{year}_all_shape_geographic.zip", # noqa:E501 + f"ftp://ftp.ngdc.noaa.gov/geomag/wmm/wmm2020/shapefiles/{year}/WMM_{year}_all_shape_geographic.zip", ], ) for year in range(2020, 2025 + 1) diff --git a/qgreenland/config/helpers/ancillary/sea_ice_age_params.json b/qgreenland/config/helpers/ancillary/sea_ice_age_params.json index 9343cc08f..e5334c122 100644 --- a/qgreenland/config/helpers/ancillary/sea_ice_age_params.json +++ b/qgreenland/config/helpers/ancillary/sea_ice_age_params.json @@ -109,4 +109,4 @@ "date_range": "September 16-22" } } -} \ No newline at end of file +} diff --git a/qgreenland/config/helpers/layers/esa_cci_surface_elev.py b/qgreenland/config/helpers/layers/esa_cci_surface_elev.py index f792130a4..307335340 100644 --- a/qgreenland/config/helpers/layers/esa_cci_surface_elev.py +++ b/qgreenland/config/helpers/layers/esa_cci_surface_elev.py @@ -25,7 +25,6 @@ def surface_elevation_layer( end_year: int, variable: SurfaceElevVar, ) -> Layer: - if variable == "SEC": description = "Rate of surface elevation change in meters per year." style = "surface_elevation_change" diff --git a/qgreenland/config/helpers/layers/lonlat.py b/qgreenland/config/helpers/layers/lonlat.py index 79e7428a1..8c6f52e13 100644 --- a/qgreenland/config/helpers/layers/lonlat.py +++ b/qgreenland/config/helpers/layers/lonlat.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Union, cast +from typing import Literal, Union, cast from qgreenland.config.datasets.lonlat import lonlat as dataset from qgreenland.config.helpers.steps.ogr2ogr import STANDARD_OGR2OGR_ARGS @@ -21,7 +21,7 @@ def _make_lonlat_layer( deg_str = asset.id.rsplit("_", maxsplit=1)[0].split("_", maxsplit=1)[1] deg = deg_str.replace("_", ".") - ogr2ogr_clip_args: List[Union[str, EvalFilePath]] + ogr2ogr_clip_args: list[Union[str, EvalFilePath]] if asset.id.startswith("lat"): title_prefix = "Latitude" segment_max_distance = 1 diff --git a/qgreenland/config/helpers/layers/racmo.py b/qgreenland/config/helpers/layers/racmo.py index 4fcacdf8e..babd35ba6 100644 --- a/qgreenland/config/helpers/layers/racmo.py +++ b/qgreenland/config/helpers/layers/racmo.py @@ -134,7 +134,8 @@ def _make_racmo_wind_speed() -> Layer: RACMO_LAYER_ORDER = [ "racmo_wind_vectors", "racmo_wind_speed", -] + list(_masked_racmo_raster_params.keys()) + *list(_masked_racmo_raster_params.keys()), +] def _make_masked_racmo_layer( diff --git a/qgreenland/config/helpers/layers/sea_ice_age.py b/qgreenland/config/helpers/layers/sea_ice_age.py index 3d2f94441..c279371af 100644 --- a/qgreenland/config/helpers/layers/sea_ice_age.py +++ b/qgreenland/config/helpers/layers/sea_ice_age.py @@ -16,7 +16,7 @@ def _get_layer_params(): - with open(PARAMS_FP, "r") as f: + with open(PARAMS_FP) as f: return json.loads(f.read()) diff --git a/qgreenland/config/helpers/layers/streams_outlets_basins.py b/qgreenland/config/helpers/layers/streams_outlets_basins.py index 8438ba24e..e5f971344 100644 --- a/qgreenland/config/helpers/layers/streams_outlets_basins.py +++ b/qgreenland/config/helpers/layers/streams_outlets_basins.py @@ -1,4 +1,4 @@ -from qgreenland.config.datasets.streams_outlets_basins import ( # noqa: E501 +from qgreenland.config.datasets.streams_outlets_basins import ( streams_outlets_basins as dataset, ) from qgreenland.config.helpers.steps.ogr2ogr import ogr2ogr diff --git a/qgreenland/config/helpers/steps/compress_and_add_overviews.py b/qgreenland/config/helpers/steps/compress_and_add_overviews.py index 8b7974288..02c2c490a 100644 --- a/qgreenland/config/helpers/steps/compress_and_add_overviews.py +++ b/qgreenland/config/helpers/steps/compress_and_add_overviews.py @@ -92,6 +92,6 @@ def compress_and_add_overviews( ), CommandStep( id="build_overviews", - args=copy_into_place + ["&&"] + add_overviews, + args=[*copy_into_place, "&&", *add_overviews], ), ] diff --git a/qgreenland/config/helpers/steps/decompress.py b/qgreenland/config/helpers/steps/decompress.py index ddcc64228..90719a10d 100644 --- a/qgreenland/config/helpers/steps/decompress.py +++ b/qgreenland/config/helpers/steps/decompress.py @@ -30,10 +30,8 @@ def decompress_step( elif decompress_type == "gzip": if decompress_contents_mask: raise NotImplementedError( - ( - "The `decompress_contents_mask` kwarg is not supported for" - " the `gzip` decompression type." - ) + "The `decompress_contents_mask` kwarg is not supported for" + " the `gzip` decompression type." ) args = [ diff --git a/qgreenland/config/helpers/steps/gdal_edit.py b/qgreenland/config/helpers/steps/gdal_edit.py index d9fc0475a..23ac0168e 100644 --- a/qgreenland/config/helpers/steps/gdal_edit.py +++ b/qgreenland/config/helpers/steps/gdal_edit.py @@ -8,7 +8,6 @@ def gdal_edit( output_file: str, gdal_edit_args: StepArgs = (), ) -> list[CommandStep]: - return [ CommandStep( id="gdal_edit", diff --git a/qgreenland/config/helpers/steps/ogr2ogr.py b/qgreenland/config/helpers/steps/ogr2ogr.py index f01318167..23de901de 100644 --- a/qgreenland/config/helpers/steps/ogr2ogr.py +++ b/qgreenland/config/helpers/steps/ogr2ogr.py @@ -31,8 +31,8 @@ def ogr2ogr( return [ CommandStep( id="ogr2ogr", - args=init_args - + [ + args=[ + *init_args, "ogr2ogr", *STANDARD_OGR2OGR_ARGS, "-clipdst", diff --git a/qgreenland/config/helpers/steps/warp.py b/qgreenland/config/helpers/steps/warp.py index ccc96fb36..293dca56d 100644 --- a/qgreenland/config/helpers/steps/warp.py +++ b/qgreenland/config/helpers/steps/warp.py @@ -12,7 +12,6 @@ def warp( resampling_method: ResamplingMethod = "bilinear", warp_args: StepArgs = (), ) -> list[CommandStep]: - return [ CommandStep( args=[ diff --git a/qgreenland/config/layers/Glaciology/Gravimetric mass balance/layers.py b/qgreenland/config/layers/Glaciology/Gravimetric mass balance/layers.py index 9d8a3d624..b7ffcce71 100644 --- a/qgreenland/config/layers/Glaciology/Gravimetric mass balance/layers.py +++ b/qgreenland/config/layers/Glaciology/Gravimetric mass balance/layers.py @@ -1,5 +1,5 @@ import datetime as dt -from typing import Generator +from collections.abc import Generator from qgreenland.config.datasets.esa_cci import ( esa_cci_gravimetric_mass_balance_dtu as dataset, diff --git a/qgreenland/config/project.py b/qgreenland/config/project.py index 53a47ebf5..82734f01e 100644 --- a/qgreenland/config/project.py +++ b/qgreenland/config/project.py @@ -5,10 +5,10 @@ crs=PROJECT_CRS, boundaries={ "background": { - "filepath": "{assets_dir}/latitude_shape_40_degrees.geojson", # noqa: FS003 + "filepath": "{assets_dir}/latitude_shape_40_degrees.geojson", }, "data": { - "filepath": "{assets_dir}/greenland_rectangle.geojson", # noqa: FS003 + "filepath": "{assets_dir}/greenland_rectangle.geojson", }, }, ) diff --git a/qgreenland/models/config/asset.py b/qgreenland/models/config/asset.py index d06002572..d0f1d90df 100644 --- a/qgreenland/models/config/asset.py +++ b/qgreenland/models/config/asset.py @@ -89,7 +89,7 @@ class RepositoryAsset(DatasetAsset): # TODO: Move the assets into the config directory??? filepath: EvalFilePath - """The location of the asset, e.g. `{assets_dir}/foo.txt`.""" # noqa: FS003 + """The location of the asset, e.g. `{assets_dir}/foo.txt`.""" @validator("filepath") @classmethod diff --git a/qgreenland/models/config/layer.py b/qgreenland/models/config/layer.py index 969ebd4d0..042e523c6 100644 --- a/qgreenland/models/config/layer.py +++ b/qgreenland/models/config/layer.py @@ -65,7 +65,7 @@ def style_file_exists(cls, value): style_filepath = _style_filepath(value) if not style_filepath.is_file(): raise exc.QgrInvalidConfigError( - (f"Style file does not exist: {style_filepath}") + f"Style file does not exist: {style_filepath}" ) return value diff --git a/qgreenland/models/config/project.py b/qgreenland/models/config/project.py index 4080b9f70..a4e861d89 100644 --- a/qgreenland/models/config/project.py +++ b/qgreenland/models/config/project.py @@ -66,7 +66,7 @@ def calculate_bbox(cls, values) -> dict[str, Any]: # NOTE: Import inside the method to avoid a cycle. The config subpackage # imports from the models subpackage, so the models can't import from # config. - from qgreenland.config.constants import PROJECT_CRS # noqa + from qgreenland.config.constants import PROJECT_CRS if (boundary_crs := meta["crs"]["init"].upper()) != PROJECT_CRS.upper(): raise exc.QgrInvalidConfigError( diff --git a/qgreenland/models/config/step.py b/qgreenland/models/config/step.py index b4fdf1b67..bb891c9d9 100644 --- a/qgreenland/models/config/step.py +++ b/qgreenland/models/config/step.py @@ -33,7 +33,7 @@ class CommandStep(QgrBaseModel, LayerStep): type: Literal["command"] = "command" args: list[EvalStr] - """The command arguments, e.g. ['cat', '{input_dir}/foo.txt'].""" # noqa:FS003 + """The command arguments, e.g. ['cat', '{input_dir}/foo.txt'].""" # We use a root validator here because with a regular validator, we would # not have access to the `args` field, because field order matters to diff --git a/qgreenland/runners/__init__.py b/qgreenland/runners/__init__.py index 99ce8d629..43c63f23a 100644 --- a/qgreenland/runners/__init__.py +++ b/qgreenland/runners/__init__.py @@ -1,13 +1,13 @@ """A runner is a function which executes a certain type of step.""" -from typing import Any, Type +from typing import Any from qgreenland.models.config.step import AnyStep, CommandStep from qgreenland.runners.command import command_runner # Each runner corresponds to a type of "step" available in the layer # configuration file. -RUNNERS: dict[Type[AnyStep], Any] = { +RUNNERS: dict[type[AnyStep], Any] = { CommandStep: command_runner, # 'python': 'TODO', } diff --git a/qgreenland/test/data/sample_module_zerodiv.py b/qgreenland/test/data/sample_module_zerodiv.py index 72dca4d5e..f173d6fc1 100644 --- a/qgreenland/test/data/sample_module_zerodiv.py +++ b/qgreenland/test/data/sample_module_zerodiv.py @@ -1 +1 @@ -1 / 0 +1 / 0 # noqa diff --git a/qgreenland/test/util/test_qgis.py b/qgreenland/test/util/test_qgis.py index b05a42bd2..6b573e71c 100644 --- a/qgreenland/test/util/test_qgis.py +++ b/qgreenland/test/util/test_qgis.py @@ -10,87 +10,86 @@ from qgreenland.test.constants import MOCK_COMPILE_PACKAGE_DIR -def test_make_map_layer_online(setup_teardown_qgis_app, online_layer_node): - result = qgl.make_map_layer(online_layer_node) - - assert "https://demo.mapserver.org" in result.source() - assert result.dataProvider().name() == "wms" - assert result.name() == online_layer_node.layer_cfg.title - - -@patch( - "qgreenland.util.layer.COMPILE_PACKAGE_DIR", - new=MOCK_COMPILE_PACKAGE_DIR, -) -def test_make_map_layer_raster(setup_teardown_qgis_app, raster_layer_node): - result = qgl.make_map_layer(raster_layer_node) - - # The result is a a raster layer - assert isinstance(result, qgc.QgsRasterLayer) - - # Has the expected path to the data on disk. - expected_raster_path = ( - MOCK_COMPILE_PACKAGE_DIR - / "Group" - / "Subgroup" - / "Example raster" - / "example.tif" +@pytest.mark.usefixtures("setup_teardown_qgis_app") +class TestQgisApp: + def test_make_map_layer_online(self, online_layer_node): + result = qgl.make_map_layer(online_layer_node) + + assert "https://demo.mapserver.org" in result.source() + assert result.dataProvider().name() == "wms" + assert result.name() == online_layer_node.layer_cfg.title + + @patch( + "qgreenland.util.layer.COMPILE_PACKAGE_DIR", + new=MOCK_COMPILE_PACKAGE_DIR, ) - assert result.source() == str(expected_raster_path) - - # With the expected shape. - result_shape = (result.dataProvider().xSize(), result.dataProvider().ySize()) - expected_shape = (2, 2) - assert result_shape == expected_shape - - # The title is correctly set. - assert result.name() == raster_layer_node.layer_cfg.title - - -@patch( - "qgreenland.util.layer.COMPILE_PACKAGE_DIR", - new=MOCK_COMPILE_PACKAGE_DIR, -) -def test_add_layer_metadata(setup_teardown_qgis_app, raster_layer_node): - mock_raster_layer = qgl.make_map_layer(raster_layer_node) - - qgl.add_layer_metadata(mock_raster_layer, raster_layer_node.layer_cfg) - - # The abstract gets set with the value returned by `qgis.build_abstract`. - assert mock_raster_layer.metadata().abstract() == qgm.build_layer_metadata( - raster_layer_node.layer_cfg + def test_make_map_layer_raster(self, raster_layer_node): + result = qgl.make_map_layer(raster_layer_node) + + # The result is a a raster layer + assert isinstance(result, qgc.QgsRasterLayer) + + # Has the expected path to the data on disk. + expected_raster_path = ( + MOCK_COMPILE_PACKAGE_DIR + / "Group" + / "Subgroup" + / "Example raster" + / "example.tif" + ) + assert result.source() == str(expected_raster_path) + + # With the expected shape. + result_shape = (result.dataProvider().xSize(), result.dataProvider().ySize()) + expected_shape = (2, 2) + assert result_shape == expected_shape + + # The title is correctly set. + assert result.name() == raster_layer_node.layer_cfg.title + + @patch( + "qgreenland.util.layer.COMPILE_PACKAGE_DIR", + new=MOCK_COMPILE_PACKAGE_DIR, ) - - actual_title = mock_raster_layer.metadata().title() - expected_title = raster_layer_node.layer_cfg.title - assert actual_title == expected_title - - # Sets the spatial extent based on the the layer extent. - expected_extent = mock_raster_layer.extent() - # extent metadata contains both spatial and temporal elements. These are - # lists. We set one overall extent for the dataset, so take the first element. - meta_extent = mock_raster_layer.metadata().extent().spatialExtents()[0] - # The `expected_extent` is a QgsRectangle. - assert expected_extent == meta_extent.bounds.toRectangle() - - -@patch( - "qgreenland.util.layer.COMPILE_PACKAGE_DIR", - new=MOCK_COMPILE_PACKAGE_DIR, -) -def test__add_layers_and_groups(setup_teardown_qgis_app, raster_layer_node): - # Test that _add_layers_and_groups works without error - project = qgc.QgsProject.instance() - prj._add_layers_and_groups(project, raster_layer_node.root) - added_layers = list(project.mapLayers().values()) - assert len(added_layers) == 1 - assert added_layers[0].name() == "Example raster" - - # Clear the project for the next test... - project.clear() - - # Test that an exception is raised when parent groups of a layer are not - # created first - with pytest.raises(exc.QgrQgsLayerTreeGroupError): - prj._add_layers_and_groups(project, raster_layer_node) - project.clear() + def test_add_layer_metadata(self, raster_layer_node): + mock_raster_layer = qgl.make_map_layer(raster_layer_node) + + qgl.add_layer_metadata(mock_raster_layer, raster_layer_node.layer_cfg) + + # The abstract gets set with the value returned by `qgis.build_abstract`. + assert mock_raster_layer.metadata().abstract() == qgm.build_layer_metadata( + raster_layer_node.layer_cfg + ) + + actual_title = mock_raster_layer.metadata().title() + expected_title = raster_layer_node.layer_cfg.title + assert actual_title == expected_title + + # Sets the spatial extent based on the the layer extent. + expected_extent = mock_raster_layer.extent() + # extent metadata contains both spatial and temporal elements. These are + # lists. We set one overall extent for the dataset, so take the first element. + meta_extent = mock_raster_layer.metadata().extent().spatialExtents()[0] + # The `expected_extent` is a QgsRectangle. + assert expected_extent == meta_extent.bounds.toRectangle() + + @patch( + "qgreenland.util.layer.COMPILE_PACKAGE_DIR", + new=MOCK_COMPILE_PACKAGE_DIR, + ) + def test__add_layers_and_groups(self, raster_layer_node): + # Test that _add_layers_and_groups works without error + project = qgc.QgsProject.instance() + prj._add_layers_and_groups(project, raster_layer_node.root) + added_layers = list(project.mapLayers().values()) + assert len(added_layers) == 1 + assert added_layers[0].name() == "Example raster" + + # Clear the project for the next test... + project.clear() + + # Test that an exception is raised when parent groups of a layer are not + # created first + with pytest.raises(exc.QgrQgsLayerTreeGroupError): + prj._add_layers_and_groups(project, raster_layer_node) + project.clear() diff --git a/qgreenland/util/cli/validate.py b/qgreenland/util/cli/validate.py index 977589bb0..88929e12d 100644 --- a/qgreenland/util/cli/validate.py +++ b/qgreenland/util/cli/validate.py @@ -6,7 +6,7 @@ def validate_ambiguous_command(kwargs): """Validate for conflicting options and suggest a fix.""" msg = ( "Ambiguous command! You have requested both to delete all" - " {resource}s _and_ to delete {resource}s" # noqa: FS003 + " {resource}s _and_ to delete {resource}s" " by PATTERN. Please choose only one." ) diff --git a/qgreenland/util/command.py b/qgreenland/util/command.py index c7b883fa5..f4aca5051 100644 --- a/qgreenland/util/command.py +++ b/qgreenland/util/command.py @@ -1,6 +1,6 @@ import logging import subprocess -from typing import Sequence +from collections.abc import Sequence import qgreenland.exceptions as exc from qgreenland.util.runtime_vars import EvalStr diff --git a/qgreenland/util/json.py b/qgreenland/util/json.py index 8787f03dd..d65c7f184 100644 --- a/qgreenland/util/json.py +++ b/qgreenland/util/json.py @@ -15,4 +15,4 @@ def default(self, o): return str(o) if hasattr(o, "__json__") and callable(o.__json__): return o.__json__() - return super(MagicJSONEncoder, self).default(o) + return super().default(o) diff --git a/qgreenland/util/luigi/__init__.py b/qgreenland/util/luigi/__init__.py index cee1446e5..1bad75823 100644 --- a/qgreenland/util/luigi/__init__.py +++ b/qgreenland/util/luigi/__init__.py @@ -1,5 +1,5 @@ +from collections.abc import Generator from functools import cache -from typing import Generator, Type import luigi @@ -25,7 +25,7 @@ from qgreenland.util.luigi.tasks.main import ChainableTask, FinalizeTask # TODO: Make "fetch" tasks into Python "steps"? -ASSET_TYPE_TASKS: dict[Type[AnyAsset], Type[FetchTask]] = { +ASSET_TYPE_TASKS: dict[type[AnyAsset], type[FetchTask]] = { CmrAsset: FetchCmrGranule, CommandAsset: FetchDataWithCommand, HttpAsset: FetchDataFiles, diff --git a/qgreenland/util/luigi/target.py b/qgreenland/util/luigi/target.py index 22823ea18..b78e6d26c 100644 --- a/qgreenland/util/luigi/target.py +++ b/qgreenland/util/luigi/target.py @@ -1,7 +1,7 @@ import shutil +from collections.abc import Generator from contextlib import contextmanager from pathlib import Path -from typing import Generator import luigi diff --git a/qgreenland/util/luigi/tasks/fetch.py b/qgreenland/util/luigi/tasks/fetch.py index b5f69be6d..b28cde5f4 100644 --- a/qgreenland/util/luigi/tasks/fetch.py +++ b/qgreenland/util/luigi/tasks/fetch.py @@ -103,7 +103,6 @@ def output(self): def run(self): if isinstance(self.asset_cfg, RepositoryAsset): - with temporary_path_dir(self.output()) as temp_path: evaluated_filepath = self.asset_cfg.filepath.eval() @@ -122,7 +121,7 @@ def run(self): class FetchDataWithCommand(FetchTask): - """Fetch data using a command, writing to '{output_dir}'.""" # noqa: FS003 + """Fetch data using a command, writing to '{output_dir}'.""" def output(self): return luigi.LocalTarget( diff --git a/qgreenland/util/luigi/tasks/pipeline.py b/qgreenland/util/luigi/tasks/pipeline.py index 73fc7eed0..6f9fb2b7d 100644 --- a/qgreenland/util/luigi/tasks/pipeline.py +++ b/qgreenland/util/luigi/tasks/pipeline.py @@ -75,8 +75,7 @@ def requires(self): fetch_only=self.fetch_only, ) - for task in tasks: - yield task + yield from tasks class LayerManifest(luigi.Task): diff --git a/qgreenland/util/misc.py b/qgreenland/util/misc.py index 2f1725080..ca3410b26 100644 --- a/qgreenland/util/misc.py +++ b/qgreenland/util/misc.py @@ -1,5 +1,5 @@ from collections import Counter -from typing import Iterator +from collections.abc import Iterator def find_duplicates(items: Iterator[str]) -> list[str]: diff --git a/qgreenland/util/model_validators.py b/qgreenland/util/model_validators.py index 3cf8aafda..2c242c792 100644 --- a/qgreenland/util/model_validators.py +++ b/qgreenland/util/model_validators.py @@ -1,4 +1,5 @@ -from typing import Any, Callable +from collections.abc import Callable +from typing import Any from pydantic import validator diff --git a/qgreenland/util/module.py b/qgreenland/util/module.py index cd3151547..daa132683 100644 --- a/qgreenland/util/module.py +++ b/qgreenland/util/module.py @@ -2,7 +2,7 @@ import inspect from pathlib import Path from types import ModuleType -from typing import Type, TypeVar +from typing import TypeVar def module_from_path(module_path: Path) -> ModuleType: @@ -33,7 +33,7 @@ def module_from_path(module_path: Path) -> ModuleType: def load_objects_from_paths_by_class( module_paths: list[Path], *, - target_class: Type[T], + target_class: type[T], ) -> list[T]: """Return all objects of class `model_class` in `module_paths`.""" found_models = [] @@ -51,7 +51,7 @@ def load_objects_from_paths_by_class( def _find_in_module_by_class( module: ModuleType, - target_class: Type[T], + target_class: type[T], ) -> list[T]: """Find all objects of class `model_class` among `module` members.""" module_members = inspect.getmembers(module) diff --git a/qgreenland/util/qgis/layer.py b/qgreenland/util/qgis/layer.py index 9cccfd740..be8b5fbae 100644 --- a/qgreenland/util/qgis/layer.py +++ b/qgreenland/util/qgis/layer.py @@ -1,7 +1,8 @@ import functools import tempfile +from collections.abc import Callable from pathlib import Path -from typing import Callable, Union +from typing import Union from xml.sax.saxutils import escape import qgis.core as qgc diff --git a/qgreenland/util/request.py b/qgreenland/util/request.py index 75f85bdb9..9f88cbe75 100644 --- a/qgreenland/util/request.py +++ b/qgreenland/util/request.py @@ -43,7 +43,6 @@ def fetch_and_write_file( # noqa:C901 stream=True, headers={"User-Agent": "QGreenland"}, ) as resp: - # Try to extract the filename from the `content-disposition` header if ( disposition := resp.headers.get("content-disposition") diff --git a/qgreenland/util/runtime_vars.py b/qgreenland/util/runtime_vars.py index 1151d96e1..92041bd04 100644 --- a/qgreenland/util/runtime_vars.py +++ b/qgreenland/util/runtime_vars.py @@ -8,7 +8,7 @@ # TODO: Make this a dataclass? :shrug: # TODO: Move to _typing module/package? -class EvalPath(object): +class EvalPath: """Path with `eval` method for runtime string interpolation.""" val: str @@ -52,8 +52,7 @@ class EvalFilePath(EvalPath): @classmethod def __get_validators__(cls): """Pydantic magic method for implicit validation and conversion.""" - for v in super().__get_validators__(): - yield v + yield from super().__get_validators__() yield cls.validate_exists @classmethod @@ -114,10 +113,9 @@ def __json__(self) -> str: def eval( self, *, - input_dir: Optional[str] = "{input_dir}", # noqa:FS003 - output_dir: Optional[str] = "{output_dir}", # noqa:FS003 + input_dir: Optional[str] = "{input_dir}", + output_dir: Optional[str] = "{output_dir}", ) -> str: - return self.format( input_dir=input_dir, output_dir=output_dir, diff --git a/qgreenland/util/template.py b/qgreenland/util/template.py index ae9037ce9..9d9fc2c34 100644 --- a/qgreenland/util/template.py +++ b/qgreenland/util/template.py @@ -17,7 +17,7 @@ def load_template(fn: str) -> Template: """ template_path = TEMPLATES_DIR / fn - with open(template_path, "r") as f: + with open(template_path) as f: template_str = "".join(f.readlines()) return Template(template_str) diff --git a/qgreenland/util/tree.py b/qgreenland/util/tree.py index 2317f80b6..283a3f5c3 100644 --- a/qgreenland/util/tree.py +++ b/qgreenland/util/tree.py @@ -207,10 +207,10 @@ def _manual_ordering_strategy( for s in settings.order: try: if s.startswith(":"): - matcher = lambda x: isinstance(x, Layer) and x.id == s[1:] + matcher = lambda x, s=s: isinstance(x, Layer) and x.id == s[1:] thing_desc = f'layer id "{s[1:]}"' else: - matcher = lambda x: isinstance(x, Path) and x.name == s + matcher = lambda x, s=s: isinstance(x, Path) and x.name == s thing_desc = f'group/directory "{s}"' matches = funcy.lfilter(matcher, layers_and_groups) @@ -271,7 +271,11 @@ def _ordered_layers_and_groups( Groups are represented as directories and are returned unchanged. """ - (layer_and_group_paths, settings, settings_path,) = _handle_layer_config_directory( + ( + layer_and_group_paths, + settings, + settings_path, + ) = _handle_layer_config_directory( the_dir, is_root=is_root, ) @@ -469,7 +473,7 @@ def _delete_node( Yes, this is the right way :) https://github.com/c0fec0de/anytree/issues/152 """ - node_path = list(node.group_name_path) + [node.name] + node_path = [*list(node.group_name_path), node.name] node_name = "/".join(node_path) logger.warn(f"{msg}: /{node_name}") node.parent = None diff --git a/scripts/analyze_logs.py b/scripts/analyze_logs.py index 1937691c0..207fede32 100755 --- a/scripts/analyze_logs.py +++ b/scripts/analyze_logs.py @@ -52,6 +52,7 @@ class Parser: """Read log lines and build up stats for reporting. Example: + ------- { 'v0.1.2': { 'downloads': 0, 'bytes': 0} 'v0.23.0dev': { 'downloads': 131, 'bytes': 31549151} @@ -142,7 +143,7 @@ def report(self): logs_fp = logs_file_path(sys.argv) parser = Parser() - with open(logs_fp, "r") as logs_file: + with open(logs_fp) as logs_file: print(f"Parsing log file: {logs_fp}...") lines = logs_file.readlines() diff --git a/scripts/data/generate_lat_lon_geojson.py b/scripts/data/generate_lat_lon_geojson.py index 3aa21a91a..54c4ddd0d 100644 --- a/scripts/data/generate_lat_lon_geojson.py +++ b/scripts/data/generate_lat_lon_geojson.py @@ -6,7 +6,7 @@ from shapely.geometry import LineString OUT_DIR = "./out" -degrees_template = "{deg}° {min}' {sec}\"" # noqa: FS003 +degrees_template = "{deg}° {min}' {sec}\"" def _decimal_degrees_to_dms(decimal, *, lat_or_lon, include_secs=True): diff --git a/scripts/data/make_sea_ice_age_params.py b/scripts/data/make_sea_ice_age_params.py index 86e1707ae..0b70014f4 100644 --- a/scripts/data/make_sea_ice_age_params.py +++ b/scripts/data/make_sea_ice_age_params.py @@ -29,10 +29,8 @@ def _get_min_max_rankings_file() -> bytes: """Fetch the Sea Ice Index Min/Max rankings spreadsheet and return its content.""" response = requests.get( - ( - "https://masie_web.apps.nsidc.org/pub/DATASETS/NOAA/G02135/" - "seaice_analysis/Sea_Ice_Index_Min_Max_Rankings_G02135_v3.0.xlsx" - ) + "https://masie_web.apps.nsidc.org/pub/DATASETS/NOAA/G02135/" + "seaice_analysis/Sea_Ice_Index_Min_Max_Rankings_G02135_v3.0.xlsx" ) return response.content diff --git a/scripts/export_config_csv.py b/scripts/export_config_csv.py index e4cf099a2..f4673462c 100644 --- a/scripts/export_config_csv.py +++ b/scripts/export_config_csv.py @@ -1,7 +1,7 @@ """Exports layer configuration as a CSV file.""" # Hack to import from qgreenland -import os # noqa: E401 +import os import sys THIS_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/tasks/__init__.py b/tasks/__init__.py index 1bd741423..2bb357dc8 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -1,7 +1,6 @@ from invoke import Collection from . import config, docs, env -from . import format as _format from . import test ns = Collection() @@ -9,4 +8,3 @@ ns.add_collection(docs) ns.add_collection(env) ns.add_collection(test) -ns.add_collection(_format) diff --git a/tasks/format.py b/tasks/format.py deleted file mode 100644 index 6aefd7477..000000000 --- a/tasks/format.py +++ /dev/null @@ -1,19 +0,0 @@ -from invoke import task - -from .util import PROJECT_DIR, print_and_run - - -@task(default=True) -def format(ctx): - """Apply formatting standards to the codebase.""" - # isort 5.10.1 does not support "magic trailing comma" feature of Black, so it will - # combine multiple imports to one line if they fit. - # - # https://github.com/PyCQA/isort/issues/1683 - print_and_run(f"isort {PROJECT_DIR}") - - # Black 22.1 has problems with string handling. We can work around those with - # `fmt: on` and `fmt: off` comments, but that's not fun. - # - # https://github.com/psf/black/issues/2188 - print_and_run(f"black {PROJECT_DIR}") diff --git a/tasks/test.py b/tasks/test.py index b5a5a02c8..2d47a6350 100644 --- a/tasks/test.py +++ b/tasks/test.py @@ -20,31 +20,10 @@ @task -def shellcheck(ctx): - # It's unclear why, but the return code seems to be getting swallowed. - print_and_run( - f"cd {PROJECT_DIR} &&" - f' for file in $(find {SCRIPTS_DIR} -type f -name "*.sh"); do' - " shellcheck $file;" - " done;", - pty=True, - ) - - -@task( - pre=[shellcheck], - aliases=["flake8"], -) def lint(ctx): - """Run flake8 and vulture linting.""" + """Run linting: ruff, black, vulture, shellcheck via pre-commit.""" print_and_run( - f"cd {PROJECT_DIR} &&" f" flake8 {PACKAGE_DIR} {SCRIPTS_DIR}", - pty=True, - ) - print_and_run( - f"cd {PROJECT_DIR} &&" - f" vulture --min-confidence 100 {PACKAGE_DIR} {SCRIPTS_DIR}" - " vulture_allowlist.py", + f"cd {PROJECT_DIR} && pre-commit run --all-files", pty=True, ) print("🎉🙈 Linting passed.") @@ -136,19 +115,9 @@ def typecheck(ctx, check_config=False): print("🎉🦆 Type checking passed.") -@task -def formatcheck(ctx): - """Check that the code conforms to formatting standards.""" - print_and_run(f"isort --check-only {PROJECT_DIR}") - print_and_run(f"black --check {PROJECT_DIR}") - - print("🎉🙈 Format check passed.") - - @task( pre=[ lint, - formatcheck, call(typecheck, check_config=True), validate, ]