From 750597032c3d8b83ad1668574b3779178ff2b696 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Wed, 27 Nov 2024 20:33:36 +0000 Subject: [PATCH 1/9] Support Python 3.9, pandas 1.4, and numpy 1.23 --- .github/workflows/pytest-coverage.yaml | 37 +++++++++++++++++++++----- python/ccao/vars_funs.py | 13 ++++----- python/pyproject.toml | 20 +++++++------- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pytest-coverage.yaml b/.github/workflows/pytest-coverage.yaml index e758284..554ec3d 100644 --- a/.github/workflows/pytest-coverage.yaml +++ b/.github/workflows/pytest-coverage.yaml @@ -15,7 +15,24 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + version: + # Test the lower bound of pandas/numpy support for Python 3.9, + # and test the upper bound for all other Python versions + - python: "3.9" + pandas: "1.4.*" + numpy: "1.23.*" + - python: "3.10" + pandas: "2.2.*" + numpy: "2.1.*" + - python: "3.11" + pandas: "2.2.*" + numpy: "2.1.*" + - python: "3.12" + pandas: "2.2.*" + numpy: "2.1.*" + - python: "3.13" + pandas: "2.2.*" + numpy: "2.1.*" steps: - name: Checkout code uses: actions/checkout@v4 @@ -27,26 +44,34 @@ jobs: cache-dependency-glob: python/pyproject.toml cache-suffix: test - - name: Install Python ${{ matrix.python-version }} + - name: Install Python ${{ matrix.version.python }} uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.version.python }} - name: Install Python dependencies working-directory: python shell: bash run: uv pip install .[dev,docs] + - name: Install pandas ${{ matrix.version.pandas }} and numpy ${{ matrix.version.numpy }} + working-directory: python + shell: bash + run: | + uv pip install \ + pandas==${{ matrix.version.pandas }} \ + numpy==${{ matrix.version.numpy}} + - name: Run pytest working-directory: python shell: bash run: | pytest -v --doctest-modules \ - --junitxml=junit/test-results-${{ matrix.python-version }}.xmlpytest + --junitxml=junit/test-results-${{ matrix.version.python }}.xmlpytest - name: Upload artifacts if: failure() uses: actions/upload-artifact@v4 with: - name: python/pytest-results-${{ matrix.python-version }} - path: python/junit/test-results-${{ matrix.python-version }}.xml + name: python/pytest-results-${{ matrix.version.python }} + path: python/junit/test-results-${{ matrix.version.python }}.xml diff --git a/python/ccao/vars_funs.py b/python/ccao/vars_funs.py index 892ae29..5afed5a 100644 --- a/python/ccao/vars_funs.py +++ b/python/ccao/vars_funs.py @@ -1,5 +1,6 @@ # Functions for translating variables between different data sources import importlib.resources +import typing import pandas as pd @@ -14,12 +15,12 @@ def vars_rename( - data: list[str] | pd.DataFrame, + data: typing.Union[typing.List[str], pd.DataFrame], names_from: str, names_to: str, output_type: str = "inplace", - dictionary: pd.DataFrame | None = None, -) -> list[str] | pd.DataFrame: + dictionary: typing.Optional[pd.DataFrame] = None, +) -> typing.Union[typing.List[str], pd.DataFrame]: """ Rename variables from one naming convention to another. @@ -130,10 +131,10 @@ def vars_rename( def vars_recode( data: pd.DataFrame, - cols: list[str] | None = None, + cols: typing.Optional[typing.List[str]] = None, code_type: str = "long", as_factor: bool = True, - dictionary: pd.DataFrame | None = None, + dictionary: typing.Optional[pd.DataFrame] = None, ) -> pd.DataFrame: """ Replace numerically coded variables with human-readable values. @@ -249,7 +250,7 @@ def vars_recode( # vars dict def transform_column( col: pd.Series, var_name: str, values_to: str, as_factor: bool - ) -> pd.Series | pd.Categorical: + ) -> typing.Union[pd.Series, pd.Categorical]: if var_name in dict_long["var_name"].values: var_rows = dict_long[dict_long["var_name"] == var_name] # Get a dictionary mapping the possible codes to their values. diff --git a/python/pyproject.toml b/python/pyproject.toml index 733619b..6f50f47 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -3,26 +3,28 @@ name = "ccao" version = "1.3.0" description = "Convenience Functions and Datasets for the Cook County Assessor's Office" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.9" authors = [ {name = "Jean Cochrane", email="jean.cochrane@cookcountyil.gov"}, {name = "Dan Snow", email="daniel.snow@cookcountyil.gov"}, ] dependencies = [ - "pandas>=2.2.3", + "pandas>=1.4.0,<=2.3.0", + "numpy>=1.23.0,<=2.2.0" ] [project.optional-dependencies] dev = [ - "mypy>=1.13.0", - "pytest>=8.3.3", - "ruff>=0.7.4", + "mypy", + "pytest", + "ruff", ] docs = [ - "Sphinx>=8.1.3", - "myst-parser>=4.0.0", - "pydata-sphinx-theme>=0.16.0", - "sphinx-pyproject>=0.3.0" + "Sphinx", + "myst-parser", + "pydata-sphinx-theme", + "sphinx-pyproject", + "sphinx-autobuild" ] [tool.setuptools] From df3af62a7fadae5956661d7a21272cefa7407db1 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Wed, 27 Nov 2024 20:39:21 +0000 Subject: [PATCH 2/9] Try installing pandas/numpy before the other dependencies in pytest-coverage workflow --- .github/workflows/pytest-coverage.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest-coverage.yaml b/.github/workflows/pytest-coverage.yaml index 554ec3d..d3a6fce 100644 --- a/.github/workflows/pytest-coverage.yaml +++ b/.github/workflows/pytest-coverage.yaml @@ -49,11 +49,6 @@ jobs: with: python-version: ${{ matrix.version.python }} - - name: Install Python dependencies - working-directory: python - shell: bash - run: uv pip install .[dev,docs] - - name: Install pandas ${{ matrix.version.pandas }} and numpy ${{ matrix.version.numpy }} working-directory: python shell: bash @@ -62,6 +57,11 @@ jobs: pandas==${{ matrix.version.pandas }} \ numpy==${{ matrix.version.numpy}} + - name: Install Python dependencies + working-directory: python + shell: bash + run: uv pip install .[dev,docs] + - name: Run pytest working-directory: python shell: bash From e34b6d7856636d0cbbcee4f5e7a95db7c7cab417 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Mon, 2 Dec 2024 18:03:34 +0000 Subject: [PATCH 3/9] Try building and testing Python package with tox --- .github/workflows/pytest-coverage.yaml | 77 -------------------- .github/workflows/python-build-and-test.yaml | 44 +++++++++++ python/pyproject.toml | 52 ++++++++++--- 3 files changed, 86 insertions(+), 87 deletions(-) delete mode 100644 .github/workflows/pytest-coverage.yaml create mode 100644 .github/workflows/python-build-and-test.yaml diff --git a/.github/workflows/pytest-coverage.yaml b/.github/workflows/pytest-coverage.yaml deleted file mode 100644 index d3a6fce..0000000 --- a/.github/workflows/pytest-coverage.yaml +++ /dev/null @@ -1,77 +0,0 @@ -on: - pull_request: - push: - branches: [main, master] - -name: pytest-coverage - -env: - PYTHONUNBUFFERED: "1" - UV_SYSTEM_PYTHON: 1 - -jobs: - pytest-coverage: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - version: - # Test the lower bound of pandas/numpy support for Python 3.9, - # and test the upper bound for all other Python versions - - python: "3.9" - pandas: "1.4.*" - numpy: "1.23.*" - - python: "3.10" - pandas: "2.2.*" - numpy: "2.1.*" - - python: "3.11" - pandas: "2.2.*" - numpy: "2.1.*" - - python: "3.12" - pandas: "2.2.*" - numpy: "2.1.*" - - python: "3.13" - pandas: "2.2.*" - numpy: "2.1.*" - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v3 - with: - enable-cache: true - cache-dependency-glob: python/pyproject.toml - cache-suffix: test - - - name: Install Python ${{ matrix.version.python }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.version.python }} - - - name: Install pandas ${{ matrix.version.pandas }} and numpy ${{ matrix.version.numpy }} - working-directory: python - shell: bash - run: | - uv pip install \ - pandas==${{ matrix.version.pandas }} \ - numpy==${{ matrix.version.numpy}} - - - name: Install Python dependencies - working-directory: python - shell: bash - run: uv pip install .[dev,docs] - - - name: Run pytest - working-directory: python - shell: bash - run: | - pytest -v --doctest-modules \ - --junitxml=junit/test-results-${{ matrix.version.python }}.xmlpytest - - - name: Upload artifacts - if: failure() - uses: actions/upload-artifact@v4 - with: - name: python/pytest-results-${{ matrix.version.python }} - path: python/junit/test-results-${{ matrix.version.python }}.xml diff --git a/.github/workflows/python-build-and-test.yaml b/.github/workflows/python-build-and-test.yaml new file mode 100644 index 0000000..e7f483c --- /dev/null +++ b/.github/workflows/python-build-and-test.yaml @@ -0,0 +1,44 @@ +on: + pull_request: + push: + branches: [main, master] + +name: python-build-and-test + +env: + PYTHONUNBUFFERED: "1" + +jobs: + build-and-test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: python/pyproject.toml + cache-suffix: test + + - name: Install Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install tox + shell: bash + run: | + uv tool install tox --with tox-uv,tox-gh-actions + tox --version + + - name: Build and test with tox + shell: bash + working-directory: python + run: tox r diff --git a/python/pyproject.toml b/python/pyproject.toml index 6f50f47..d24e369 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -9,22 +9,22 @@ authors = [ {name = "Dan Snow", email="daniel.snow@cookcountyil.gov"}, ] dependencies = [ - "pandas>=1.4.0,<=2.3.0", - "numpy>=1.23.0,<=2.2.0" + "pandas>=1.4.3", + "numpy>=1.23.1" ] [project.optional-dependencies] dev = [ - "mypy", - "pytest", - "ruff", + "mypy>=1.0.0", + "pytest>=7.0.0", + "ruff>=0.8.0", ] docs = [ - "Sphinx", - "myst-parser", - "pydata-sphinx-theme", - "sphinx-pyproject", - "sphinx-autobuild" + "Sphinx>=7.0", + "myst-parser>=1.0.0", + "pydata-sphinx-theme>=0.16.0", + "sphinx-pyproject>=0.3.0", + "sphinx-autobuild>=2024.10.3" ] [tool.setuptools] @@ -57,3 +57,35 @@ highlight_language = "none" html_theme = "pydata_sphinx_theme" html_logo = "../images/logo.png" html_show_copyright = false + +[tool.pytest.ini_options] +minversion = "7.0" +addopts = "-v --cache-clear -rf --doctest-modules" +console_output_style = "count" + +[tool.tox] +legacy_tox_ini = """ +[tox] +min_version = 4.0 +envlist = + py{39, 310, 311}-lowest + py{39, 310, 311, 312, 313} + +[gh-actions] +python = + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: py312 + 3.13: py313 + +[testenv] +extras = dev,docs +commands = pytest + +[testenv:py{39, 310, 311}-lowest] +uv_resolution = lowest-direct + +[testenv:py{310, 311, 312, 313}] +uv_resolution = highest +""" From 56371586b18cb0c3e79ba1d3be4c254f2ac1f817 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Mon, 2 Dec 2024 19:47:27 +0000 Subject: [PATCH 4/9] Add UV_CACHE_DIR to tox env to see if it speeds up builds --- python/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index d24e369..1a5c766 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -82,6 +82,8 @@ python = [testenv] extras = dev,docs commands = pytest +passenv = + UV_CACHE_DIR [testenv:py{39, 310, 311}-lowest] uv_resolution = lowest-direct From 0e29e6feab426c35bb5d9ccf0a67eb1ada136422 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Mon, 2 Dec 2024 20:03:07 +0000 Subject: [PATCH 5/9] Revert "Add UV_CACHE_DIR to tox env to see if it speeds up builds" This reverts commit 56371586b18cb0c3e79ba1d3be4c254f2ac1f817. --- python/pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 1a5c766..d24e369 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -82,8 +82,6 @@ python = [testenv] extras = dev,docs commands = pytest -passenv = - UV_CACHE_DIR [testenv:py{39, 310, 311}-lowest] uv_resolution = lowest-direct From b410f6e028975666e03a82d9271628b67ba70e94 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Mon, 2 Dec 2024 20:03:56 +0000 Subject: [PATCH 6/9] Restrict tox envs since 3.11 seems to need to build a dep from source --- python/pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index d24e369..2f70d09 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -68,7 +68,7 @@ legacy_tox_ini = """ [tox] min_version = 4.0 envlist = - py{39, 310, 311}-lowest + py{39, 310}-lowest py{39, 310, 311, 312, 313} [gh-actions] @@ -83,9 +83,9 @@ python = extras = dev,docs commands = pytest -[testenv:py{39, 310, 311}-lowest] +[testenv:py{39, 310}-lowest] uv_resolution = lowest-direct -[testenv:py{310, 311, 312, 313}] +[testenv:py{39, 310, 311, 312, 313}] uv_resolution = highest """ From b9f300cd025d74cbee42f1617e038811294a20dc Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Mon, 2 Dec 2024 22:55:03 +0000 Subject: [PATCH 7/9] Fix wheel caching on CI when using uv in Python package --- .github/workflows/python-build-and-test.yaml | 4 ++-- python/pyproject.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-build-and-test.yaml b/.github/workflows/python-build-and-test.yaml index e7f483c..fd338ec 100644 --- a/.github/workflows/python-build-and-test.yaml +++ b/.github/workflows/python-build-and-test.yaml @@ -21,11 +21,11 @@ jobs: uses: actions/checkout@v4 - name: Install uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@v4 with: enable-cache: true cache-dependency-glob: python/pyproject.toml - cache-suffix: test + cache-suffix: ${{ matrix.python-version }}-test - name: Install Python ${{ matrix.python-version }} uses: actions/setup-python@v5 diff --git a/python/pyproject.toml b/python/pyproject.toml index 2f70d09..ab21060 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -68,7 +68,7 @@ legacy_tox_ini = """ [tox] min_version = 4.0 envlist = - py{39, 310}-lowest + py{39, 310, 311}-lowest py{39, 310, 311, 312, 313} [gh-actions] @@ -83,7 +83,7 @@ python = extras = dev,docs commands = pytest -[testenv:py{39, 310}-lowest] +[testenv:py{39, 310, 311}-lowest] uv_resolution = lowest-direct [testenv:py{39, 310, 311, 312, 313}] From 815b6a828763bbadf3b939ca483502361e45fd45 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Mon, 2 Dec 2024 22:56:48 +0000 Subject: [PATCH 8/9] Speed up Python install with uv in docs.yaml --- .github/workflows/docs.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index d1b8bfd..bcf02a3 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -33,16 +33,12 @@ jobs: needs: website - name: Install uv - uses: astral-sh/setup-uv@v3 - with: - enable-cache: true - cache-dependency-glob: python/pyproject.toml - cache-suffix: docs + uses: astral-sh/setup-uv@v4 - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version-file: python/pyproject.toml + shell: bash + working-directory: python + run: uv venv - name: Install Python dependencies working-directory: python @@ -54,7 +50,7 @@ jobs: shell: Rscript {0} - name: Build Python docs - run: sphinx-build python/docs/source docs/python + run: uv run sphinx-build python/docs/source docs/python - name: Configure pages uses: actions/configure-pages@v5 From 0890d98b349475d6b9389ad88156463897a276b9 Mon Sep 17 00:00:00 2001 From: Jean Cochrane Date: Mon, 2 Dec 2024 23:11:50 +0000 Subject: [PATCH 9/9] Pass env vars to tox defensively --- python/pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index ab21060..2f3a81f 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -82,6 +82,9 @@ python = [testenv] extras = dev,docs commands = pytest +passenv = + UV_CACHE_DIR + PYTHONUNBUFFERED [testenv:py{39, 310, 311}-lowest] uv_resolution = lowest-direct