From 5fe5cd8118e338816a87860b9e6f572ca5e9cddb Mon Sep 17 00:00:00 2001 From: Ben Jeffery Date: Thu, 11 Jul 2024 12:37:00 +0100 Subject: [PATCH] Upgrade actions, build with numpy2 --- .circleci/config.yml | 1 + .github/workflows/docker/shared.env | 4 +- .github/workflows/docs.yml | 10 +-- .github/workflows/tests.yml | 40 ++++++---- .mergify.yml | 12 +-- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 2 + pyproject.toml | 75 ++++++++++++++++++- requirements/CI-docs/requirements.txt | 18 ++--- .../CI-tests-complete/requirements.txt | 19 ++--- requirements/CI-tests-conda/requirements.txt | 12 +-- setup.py | 12 +-- tests/tsutil.py | 2 +- 13 files changed, 142 insertions(+), 67 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08c58217..d670bd16 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -115,6 +115,7 @@ jobs: command: | python -m venv venv source venv/bin/activate + pip install numpy>2 pip install dist/*.tar.gz python -c 'import tsinfer; print(tsinfer.__version__)' diff --git a/.github/workflows/docker/shared.env b/.github/workflows/docker/shared.env index 6f613fb8..c10e471e 100644 --- a/.github/workflows/docker/shared.env +++ b/.github/workflows/docker/shared.env @@ -1,6 +1,6 @@ PYTHON_VERSIONS=( + cp312-cp312 + cp311-cp311 cp310-cp310 cp39-cp39 - cp38-cp38 - cp37-cp37m ) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 69249897..50f53c18 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -19,23 +19,23 @@ jobs: runs-on: ubuntu-latest steps: - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" - - uses: actions/cache@v3 + - uses: actions/cache@v4 id: venv-cache with: path: venv - key: docs-venv-v5-${{ hashFiles('requirements/CI-docs/requirements.txt') }} + key: docs-venv-v6-${{ hashFiles('requirements/CI-docs/requirements.txt') }} - name: Create venv and install deps (one by one to avoid conflict errors) if: steps.venv-cache.outputs.cache-hit != 'true' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c710dcfc..efd6c499 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,11 +11,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - name: install clang-format @@ -31,28 +31,28 @@ jobs: strategy: fail-fast: false matrix: - python: [ 3.9, "3.10" ] + python: [ 3.9, "3.12" ] os: [ macos-latest, ubuntu-latest, windows-latest ] defaults: run: shell: bash steps: - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true - name: Cache conda and dependancies id: cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CONDA }}/envs - key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.python}}-conda-v1-${{ hashFiles('requirements/CI-tests-conda/requirements.txt')}} + key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.python}}-conda-v5-${{ hashFiles('requirements/CI-tests-conda/requirements.txt')}} - name: Install Miniconda with Mamba uses: conda-incubator/setup-miniconda@v3 @@ -61,10 +61,10 @@ jobs: activate-environment: anaconda-client-env python-version: ${{ matrix.python }} channels: conda-forge - channel-priority: strict +# channel-priority: strict auto-update-conda: true - mamba-version: "*" - use-mamba: true +# mamba-version: "*" +# use-mamba: true - name: Fix windows .profile if: steps.cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest' @@ -76,29 +76,41 @@ jobs: - name: Install compiler from conda if: steps.cache.outputs.cache-hit != 'true' shell: bash -l {0} #We need a login shell to get conda - run: mamba install --yes c-compiler + run: conda install --yes c-compiler - name: Install conda deps if: steps.cache.outputs.cache-hit != 'true' shell: bash -l {0} #We need a login shell to get conda - run: mamba install --yes --file=requirements/CI-tests-conda/requirements.txt + run: conda install --yes --file=requirements/CI-tests-conda/requirements.txt - name: Install cyvcf2 #Fails if done via conda due to no windows support. if: steps.cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest' run: | source ~/.profile conda activate anaconda-client-env - pip install cyvcf2==0.30.18 + #Install these by pip so we don't pull in cbgen with conda as it isn't available on 3.12 + pip install sgkit==0.9.0 cyvcf2==0.31.1 yarl==1.9.4 aiohttp==3.9.5 requests==2.32.3 + - name: Install sgkit only on windows + if: steps.cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest' + run: | + source ~/.profile + conda activate anaconda-client-env + #Install these by pip so we don't pull in cbgen with conda as it isn't available on 3.12 + pip install sgkit==0.9.0 - name: Build module run: | source ~/.profile conda activate anaconda-client-env + # Use numpy2 to build the module + pip install "numpy>=2" python setup.py build_ext --inplace - name: Run tests run: | source ~/.profile conda activate anaconda-client-env + # Test with numpy<2 for numba + pip install "numpy<2" python -m pytest -xv diff --git a/.mergify.yml b/.mergify.yml index 142f566b..51382016 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -5,11 +5,11 @@ queue_rules: - "#changes-requested-reviews-by=0" - status-success=Lint - status-success=Python (3.9, macos-latest) - - status-success=Python (3.10, macos-latest) + - status-success=Python (3.12, macos-latest) - status-success=Python (3.9, ubuntu-latest) - - status-success=Python (3.10, ubuntu-latest) + - status-success=Python (3.12, ubuntu-latest) - status-success=Python (3.9, windows-latest) - - status-success=Python (3.10, windows-latest) + - status-success=Python (3.12, windows-latest) - "status-success=ci/circleci: build" pull_request_rules: @@ -22,11 +22,11 @@ pull_request_rules: - label=AUTOMERGE-REQUESTED - status-success=Lint - status-success=Python (3.9, macos-latest) - - status-success=Python (3.10, macos-latest) + - status-success=Python (3.12, macos-latest) - status-success=Python (3.9, ubuntu-latest) - - status-success=Python (3.10, ubuntu-latest) + - status-success=Python (3.12, ubuntu-latest) - status-success=Python (3.9, windows-latest) - - status-success=Python (3.10, windows-latest) + - status-success=Python (3.12, windows-latest) - "status-success=ci/circleci: build" actions: queue: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6728e341..716cd77d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: rev: v3.2.2 hooks: - id: pyupgrade - args: [ --py3-plus, --py37-plus ] + args: [ --py3-plus, --py39-plus ] - repo: https://github.com/psf/black rev: 22.10.0 hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index 12671f76..43ca4043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ In development - Remove the `uuid` field from SampleData. SampleData equality is now purely based on data. ({pr}`748`, {user}`benjeffery`) +- tsinfer now requires Python 3.9 or later + **Performance improvements** - Reduce memory usage when running `match_samples` against large cohorts diff --git a/pyproject.toml b/pyproject.toml index 4a7ef3cf..39c06e77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,10 +3,79 @@ requires = [ "setuptools>=42", "setuptools_scm", "wheel", - "oldest-supported-numpy" + "numpy>=2" ] - build-backend = "setuptools.build_meta" [tool.setuptools_scm] -write_to = "tsinfer/_version.py" \ No newline at end of file +write_to = "tsinfer/_version.py" + +[project] +name = "tsinfer" +dynamic = ["version"] +authors = [ + {name = "Tskit Developers", email = "admin@tskit.dev"}, +] +description = "Infer tree sequences from genetic variation data." +readme = "README.md" +requires-python = ">=3.9" +license = {text = "GNU GPLv3+"} +classifiers = [ + "Programming Language :: C", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3 :: Only", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Development Status :: 3 - Alpha", + "Environment :: Other Environment", + "Intended Audience :: Science/Research", + "Operating System :: POSIX", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Bio-Informatics", +] +keywords = [ + "population genetics", + "tree sequence", + "ancestral recombination graph", + "evolutionary tree", + "inference", + "tsinfer", +] +dependencies = [ + "numpy>=1.23.5", + "six", + "tqdm", + "humanize", + "daiquiri", + "tskit>=0.5.3", + "numcodecs>=0.6", + "zarr>=2.2,!=2.11.0,!=2.11.1,!=2.11.2,<3", + "lmdb", + "sortedcontainers", + "attrs>=19.2.0", + "dask[array]", + "numba", +] + +[project.urls] +Homepage = "https://tskit.dev/tsinfer" +Documentation = "https://tskit.dev/tsinfer/docs/stable" +Changelog = "https://tskit.dev/tsinfer/docs/stable/CHANGELOG.html" +"Bug Tracker" = "https://github.com/tskit-dev/tsinfer/issues" +"Source Code" = "https://github.com/tskit-dev/tsinfer/" + +[project.scripts] +tsinfer = "tsinfer.__main__:main" + +[tool.setuptools] +packages = ["tsinfer"] +include-package-data = true + +[tool.pytest.ini_options] +testpaths = ["tests"] \ No newline at end of file diff --git a/requirements/CI-docs/requirements.txt b/requirements/CI-docs/requirements.txt index 839b181f..1a8ed794 100644 --- a/requirements/CI-docs/requirements.txt +++ b/requirements/CI-docs/requirements.txt @@ -1,11 +1,11 @@ -jupyter-book==0.15.1 -sphinx-issues==3.0.1 +jupyter-book==1.0.2 +sphinx-issues==4.1.0 sphinx-argparse==0.4.0 -humanize==4.7.0 -lmdb==1.4.1 -tqdm==4.65.0 -daiquiri==3.2.1 -msprime==1.2.0 -sgkit[vcf]==0.7.0 -ipywidgets==8.1.0 +humanize==4.10.0 +lmdb==1.5.1 +tqdm==4.66.4 +daiquiri==3.2.5.1 +msprime==1.3.2 +sgkit[vcf]==0.9.0 +ipywidgets==8.1.3 sphinx-book-theme #Unpinned to allow easy updating. diff --git a/requirements/CI-tests-complete/requirements.txt b/requirements/CI-tests-complete/requirements.txt index 4dd3ff66..e4ed8489 100644 --- a/requirements/CI-tests-complete/requirements.txt +++ b/requirements/CI-tests-complete/requirements.txt @@ -1,14 +1,15 @@ build==1.2.1 colorama==0.4.6 -daiquiri==3.0.0 -humanize==4.9.0 -lmdb==1.4.1 -matplotlib==3.8.4 -meson==1.4.0 -msprime==1.3.1 -pytest==8.2.0 +daiquiri==3.2.5.1 +humanize==4.10.0 +lmdb==1.5.1 +matplotlib==3.9.1 +meson==1.5.0 +msprime==1.3.2 +pytest==8.2.2 pytest-cov==5.0.0 seaborn==0.13.2 -sgkit[vcf]==0.8.0 +sgkit[vcf]==0.9.0 +tskit==0.5.8 tqdm==4.66.4 -twine==5.0.0 +twine==5.1.1 diff --git a/requirements/CI-tests-conda/requirements.txt b/requirements/CI-tests-conda/requirements.txt index 76ec8615..c1532a1e 100644 --- a/requirements/CI-tests-conda/requirements.txt +++ b/requirements/CI-tests-conda/requirements.txt @@ -1,10 +1,10 @@ -pytest==8.2.0 -msprime==1.3.1 -humanize==4.9.0 -python-lmdb==1.3.0 +pytest==8.2.2 +msprime==1.3.2 +humanize==4.10.0 +python-lmdb==1.4.1 tqdm==4.66.4 daiquiri==3.0.0 -sgkit[vcf]==0.8.0 -matplotlib==3.8.4 +matplotlib==3.9.1 seaborn==0.13.2 colorama==0.4.6 +tskit==0.5.8 \ No newline at end of file diff --git a/setup.py b/setup.py index f0579d28..02a63bba 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,6 @@ from setuptools import Extension from setuptools import setup - IS_WINDOWS = platform.system() == "Windows" libdir = "lib" @@ -22,8 +21,6 @@ "err.c", "avl.c", ] -# We're not actually using very much of tskit at the moment, so -# just build the stuff we need. tsk_source_files = ["core.c"] kas_source_files = ["kastore.c"] @@ -34,24 +31,17 @@ + [os.path.join(kasdir, f) for f in kas_source_files] ) -libraries = [] -if IS_WINDOWS: - # Needed for generating UUIDs in tskit - libraries.append("Advapi32") +libraries = ["Advapi32"] if IS_WINDOWS else [] _tsinfer_module = Extension( "_tsinfer", sources=sources, extra_compile_args=["-std=c99"], libraries=libraries, - # Enable asserts by default. undef_macros=["NDEBUG"], include_dirs=includes + [numpy.get_include()], ) setup( - # The package name along with all the other metadata is specified in setup.cfg - # However, GitHub's dependency graph can't see the package unless we put this here. - name="tsinfer", ext_modules=[_tsinfer_module], ) diff --git a/tests/tsutil.py b/tests/tsutil.py index 3050653f..423acd46 100644 --- a/tests/tsutil.py +++ b/tests/tsutil.py @@ -252,7 +252,7 @@ def make_ts_and_zarr(path, add_optional=False, shuffle_alleles=True): # dataset, but for testing it is desirable to have sgkit code write as much of the # data as possible. with open(path / "data.vcf", "w") as f: - ts.write_vcf(f) + ts.write_vcf(f, allow_position_zero=True) sgkit.io.vcf.vcf_to_zarr( path / "data.vcf", path / "data.zarr",