From 74eba83043073776061e0c7cb6c75ab61ff94f22 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 16:56:45 +0100 Subject: [PATCH 01/18] chore: drop 3.6 from nox --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index da07652bc9..2b6f11dbd9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,6 +1,6 @@ import nox -ALL_PYTHONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] +ALL_PYTHONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] nox.options.sessions = ["lint", "tests"] From cdd06be49cc01619fc49ea0d6951a58a127c1d1e Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:05:17 +0100 Subject: [PATCH 02/18] chore: drop 3.6 from CI --- .github/workflows/build-test.yml | 7 ------- .github/workflows/tests.yml | 4 ++-- .github/workflows/wheels.yml | 10 ++-------- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c95129be53..70abe7a8d8 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -36,7 +36,6 @@ jobs: - '3.9' - '3.8' - '3.7' - - '3.6' python-architecture: - x64 @@ -91,7 +90,6 @@ jobs: - '3.9' - '3.8' - '3.7' - - '3.6' runs-on: macOS-11 @@ -148,11 +146,6 @@ jobs: - '3.9' - '3.8' - '3.7' - - '3.6' - - include: - - python-version: '3.6' - numpy-version: 1.13.1 runs-on: ubuntu-20.04 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2d71294408..44873a53d1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,8 +61,8 @@ jobs: - uses: pypa/cibuildwheel@v2.9.0 env: CIBW_ARCHS_MACOS: universal2 - CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 - CIBW_BUILD: cp39-win_amd64 cp36-manylinux_x86_64 cp38-macosx_universal2 + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2010 + CIBW_BUILD: cp39-win_amd64 cp37-manylinux_x86_64 cp38-macosx_universal2 - uses: pypa/cibuildwheel@v2.9.0 if: matrix.os == 'ubuntu-latest' diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ee0048b1d1..1fe38f41b9 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -34,12 +34,6 @@ jobs: build: ["cp*", "pp*"] include: - - os: ubuntu-latest - type: ManyLinux1 - arch: auto64 - build: "cp{36,37,38}-*" - CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 - - os: macos-latest type: "Universal" arch: universal2 @@ -51,7 +45,7 @@ jobs: - os: windows-latest arch: auto32 - build: "cp{36,37,38,39}-*" + build: "cp{37,38,39}-*" steps: @@ -76,7 +70,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: [36, 37, 38, 39, 310, 311] + python: [37, 38, 39, 310, 311] arch: [aarch64] steps: From b310afb12b1b6e3ed4796a96df702fd4003f83f5 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:06:31 +0100 Subject: [PATCH 03/18] chore: switch to manylinux2014 manylinux2014 support was added in pip 19.3. Python 3.7 ships with pip 20 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 44873a53d1..212146a0b5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,7 @@ jobs: - uses: pypa/cibuildwheel@v2.9.0 env: CIBW_ARCHS_MACOS: universal2 - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2010 + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_BUILD: cp39-win_amd64 cp37-manylinux_x86_64 cp38-macosx_universal2 - uses: pypa/cibuildwheel@v2.9.0 From 6163d7e320fe73e6f8a549a3ea9268bf1a5bd6ad Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:12:50 +0100 Subject: [PATCH 04/18] chore: drop 3.6 from pyprojec.toml Bump manylinux version --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2446f7893a..7fb790d8ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ PIP_ONLY_BINARY = "cmake,numpy" [[tool.cibuildwheel.overrides]] select = "cp3?-*" -manylinux-x86_64-image = "manylinux2010" +manylinux-x86_64-image = "manylinux2014" [tool.pytest.ini_options] @@ -79,7 +79,7 @@ filterwarnings = [ log_cli_level = "info" [tool.pylint] -master.py-version = "3.6" +master.py-version = "3.7" master.jobs = "0" master.ignore-paths = [ "src/awkward/_typeparser/generated_parser.py" From 58ac3dde14fb41d905c6a19ab535d0bf06929a91 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:13:12 +0100 Subject: [PATCH 05/18] chore: bump Python version in requirements & package info --- requirements-dev.txt | 8 ++++---- setup.cfg | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3a9e3cc8d0..93071bbf29 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,10 +1,10 @@ autograd flake8 -fsspec;python_version > "3.6" and sys_platform != "win32" -jax>=0.2.7;sys_platform != "win32" and python_version > "3.6" and python_version < "3.11" -jaxlib>=0.1.57,!=0.1.68;sys_platform != "win32" and python_version > "3.6" and python_version < "3.11" +fsspec;sys_platform != "win32" +jax>=0.2.7;sys_platform != "win32" and python_version < "3.11" +jaxlib>=0.1.57,!=0.1.68;sys_platform != "win32" and python_version < "3.11" numba>=0.50.0;python_version < "3.11" numexpr;python_version < "3.11" pandas>=0.24.0 -pyarrow>=7.0.0;python_version > "3.6" and sys_platform != "win32" and python_version < "3.11" +pyarrow>=7.0.0;sys_platform != "win32" and python_version < "3.11" PyYAML diff --git a/setup.cfg b/setup.cfg index 4eb0d3e1e7..44aab294e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,6 @@ classifiers = Programming Language :: Python Programming Language :: Python :: 3 Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -46,7 +45,7 @@ project_urls = [options] packages = find: -python_requires = >=3.6 +python_requires = >=3.7 include_package_data = True package_dir = =src From 1942c9a601d2d0e371da3a78ee536c586988bceb Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:14:33 +0100 Subject: [PATCH 06/18] chore: bump Python version in docs --- docs-sphinx/index.rst | 4 ++-- docs-src/quickstart.md | 2 +- docs-src/requirements.txt | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs-sphinx/index.rst b/docs-sphinx/index.rst index f0400f31ae..1955008864 100644 --- a/docs-sphinx/index.rst +++ b/docs-sphinx/index.rst @@ -20,8 +20,8 @@ :alt: Conda-Forge :target: https://github.com/conda-forge/awkward-feedstock -.. image:: https://img.shields.io/badge/python-3.6%E2%80%923.10-blue - :alt: Python 3.6‒3.10 +.. image:: https://img.shields.io/badge/python-3.7%E2%80%923.10-blue + :alt: Python 3.7‒3.10 :target: https://www.python.org .. image:: https://img.shields.io/badge/license-BSD%203--Clause-blue.svg diff --git a/docs-src/quickstart.md b/docs-src/quickstart.md index 6cc88b8685..e8ce77e78a 100644 --- a/docs-src/quickstart.md +++ b/docs-src/quickstart.md @@ -15,7 +15,7 @@ kernelspec: [![PyPI version](https://badge.fury.io/py/awkward.svg)](https://pypi.org/project/awkward) [![Conda-Forge](https://img.shields.io/conda/vn/conda-forge/awkward)](https://github.com/conda-forge/awkward-feedstock) -[![Python 3.6‒3.10](https://img.shields.io/badge/python-3.6%E2%80%923.10-blue)](https://www.python.org) +[![Python 3.7‒3.10](https://img.shields.io/badge/python-3.7%E2%80%923.10-blue)](https://www.python.org) [![BSD-3 Clause License](https://img.shields.io/badge/license-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Continuous integration tests](https://img.shields.io/azure-devops/build/jpivarski/Scikit-HEP/3/main?label=tests)](https://dev.azure.com/jpivarski/Scikit-HEP/_build) diff --git a/docs-src/requirements.txt b/docs-src/requirements.txt index d426619f2b..64e2a45f01 100644 --- a/docs-src/requirements.txt +++ b/docs-src/requirements.txt @@ -10,5 +10,5 @@ matplotlib uproot uproot3 jupyter-book -jax>=0.2.7;python_version>="3.6" and sys_platform != "win32" -jaxlib>=0.1.57;python_version>="3.6" and sys_platform != "win32" +jax>=0.2.7;sys_platform != "win32" +jaxlib>=0.1.57;sys_platform != "win32" From cb39b8fa3a7bd79fd8097da4b518c7baebdf2019 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:14:44 +0100 Subject: [PATCH 07/18] chore: bump Python version in README and CONTRIBUTING --- CONTRIBUTING.md | 6 +++--- README-pypi.md | 2 +- README.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e914bf38d..7332196ece 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -267,11 +267,11 @@ Templating is only used for integer specialization. ### Python standard -We target Python 3.6 and above. Import statements can assume Python 3 names, string-checking can assume Python 3 meanings of `str` and `bytes`, Unicode literals don't need to be prefixed by `u`, and dict order can be assumed to be stable. Python's f-strings can now be used, but not with equals signs (e.g. `f"{something = }"` rather than `f"something = {something}"`) because that's a Python 3.8 feature (its main use is in debugging, anyway). +We target Python 3.7 and above. Import statements can assume Python 3 names, string-checking can assume Python 3 meanings of `str` and `bytes`, Unicode literals don't need to be prefixed by `u`, and dict order can be assumed to be stable. Python's f-strings can now be used, but not with equals signs (e.g. `f"{something = }"` rather than `f"something = {something}"`) because that's a Python 3.8 feature (its main use is in debugging, anyway). -If you see any outdated (pre-Python 3.6) code, you can safely clean them up. Some strings are not easier to read as f-strings or require some work to make them more readable; it should be a case-by-case basis. +If you see any outdated (pre-Python 3.7) code, you can safely clean them up. Some strings are not easier to read as f-strings or require some work to make them more readable; it should be a case-by-case basis. -Python 3.6 will be the standard until it becomes "painful" to keep it (like, when CI and build tools no longer support it, or if we adopt type annotations). +Python 3.7 will be the standard until it becomes "painful" to keep it (like, when CI and build tools no longer support it, or if we adopt type annotations). ### Third party dependencies diff --git a/README-pypi.md b/README-pypi.md index f39dfb1509..abd4c074d2 100644 --- a/README-pypi.md +++ b/README-pypi.md @@ -2,7 +2,7 @@ [![PyPI version](https://badge.fury.io/py/awkward.svg)](https://pypi.org/project/awkward) [![Conda-Forge](https://img.shields.io/conda/vn/conda-forge/awkward)](https://github.com/conda-forge/awkward-feedstock) -[![Python 3.6‒3.10](https://img.shields.io/badge/python-3.6%E2%80%923.10-blue)](https://www.python.org) +[![Python 3.7‒3.10](https://img.shields.io/badge/python-3.7%E2%80%923.10-blue)](https://www.python.org) [![BSD-3 Clause License](https://img.shields.io/badge/license-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Continuous integration tests](https://img.shields.io/azure-devops/build/jpivarski/Scikit-HEP/3/main?label=tests)](https://dev.azure.com/jpivarski/Scikit-HEP/_build) diff --git a/README.md b/README.md index 591302ffc5..97b7d67041 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI version](https://badge.fury.io/py/awkward.svg)](https://pypi.org/project/awkward) [![Conda-Forge](https://img.shields.io/conda/vn/conda-forge/awkward)](https://github.com/conda-forge/awkward-feedstock) -[![Python 3.6‒3.10](https://img.shields.io/badge/python-3.6%E2%80%923.10-blue)](https://www.python.org) +[![Python 3.7‒3.10](https://img.shields.io/badge/python-3.7%E2%80%923.10-blue)](https://www.python.org) [![BSD-3 Clause License](https://img.shields.io/badge/license-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Build Test](https://github.com/scikit-hep/awkward/actions/workflows/build-test.yml/badge.svg?branch=main)](https://github.com/scikit-hep/awkward/actions/workflows/build-test.yml) From 23b102af45a3e9762481f3e1a72205873693b049 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:15:49 +0100 Subject: [PATCH 08/18] chore: test numpy==1.13.1 on 3.7 --- .github/workflows/build-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 70abe7a8d8..1e5b5ae4c9 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -146,6 +146,9 @@ jobs: - '3.9' - '3.8' - '3.7' + include: + - python-version: '3.7' + numpy-version: 1.13.1 runs-on: ubuntu-20.04 From 78112709adf60b95f00a70237653313483b70519 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:17:38 +0100 Subject: [PATCH 09/18] chore: remove reference to un-specified CIBW variable --- .github/workflows/wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 1fe38f41b9..959f3e7b5c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -56,7 +56,6 @@ jobs: - uses: pypa/cibuildwheel@v2.9.0 env: CIBW_BUILD: ${{ matrix.build }} - CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.CIBW_MANYLINUX_X86_64_IMAGE }} CIBW_ARCHS: ${{ matrix.arch }} - name: Upload wheels From 7b93a36a1d45808874c1deb9049f292056df6ea5 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:19:49 +0100 Subject: [PATCH 10/18] chore: be explicit about manylinux version --- .github/workflows/wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 959f3e7b5c..2ef30bee1e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -57,6 +57,7 @@ jobs: env: CIBW_BUILD: ${{ matrix.build }} CIBW_ARCHS: ${{ matrix.arch }} + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - name: Upload wheels uses: actions/upload-artifact@v3 From 912ca0deb76f418b991631d7525f2f7ab1d9515d Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:27:07 +0100 Subject: [PATCH 11/18] chore: drop Python 3.6 guard in jax connect --- src/awkward/_v2/_connect/jax/__init__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/awkward/_v2/_connect/jax/__init__.py b/src/awkward/_v2/_connect/jax/__init__.py index b4732f6c1c..3e963f6e0e 100644 --- a/src/awkward/_v2/_connect/jax/__init__.py +++ b/src/awkward/_v2/_connect/jax/__init__.py @@ -1,7 +1,6 @@ # BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE import awkward as ak -import sys try: import jax @@ -55,13 +54,6 @@ def import_jax(name="Awkward Arrays with JAX"): if jax is None: raise ak._v2._util.error(ModuleNotFoundError(error_message.format(name))) - if not (sys.version_info.major == 3 and sys.version_info.minor >= 7): - raise ak._v2._util.error( - AssertionError( - "Using Awkward Arrays with JAX requires Python version >= 3.7" - ) - ) - global pytrees_registered if not pytrees_registered: From 9dd34d54f61c99bdd8ccc588366f6d86aea25394 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:32:42 +0100 Subject: [PATCH 12/18] fix: use oldest-supported-numpy for numpy test --- .github/workflows/build-test.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 1e5b5ae4c9..46701b3b16 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -146,9 +146,10 @@ jobs: - '3.9' - '3.8' - '3.7' + numpy-package: "numpy" include: - python-version: '3.7' - numpy-version: 1.13.1 + numpy-package: "oldest-supported-numpy" runs-on: ubuntu-20.04 @@ -166,7 +167,7 @@ jobs: with: key: >- ${{ github.job - }}-${{matrix.python-version}}-${{matrix.numpy-version}} + }}-${{matrix.python-version}}-${{matrix.numpy-package}} - name: Use ccache run: | @@ -179,12 +180,7 @@ jobs: python-version: '${{ matrix.python-version }}' - name: Install NumPy - run: | - if [[ -z "${NUMPY_VERSION}" ]]; then - python -m pip install numpy; - else - python -m pip install "numpy==${NUMPY_VERSION}"; - fi + run: python -m pip install "${{ matrix.numpy-package }}" - name: Build run: 'python -m pip install -v .[test,dev]' From e0424a8092074650fc1325655fd67aae8169fa0c Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Thu, 1 Sep 2022 17:34:09 +0100 Subject: [PATCH 13/18] fix: workflow syntax --- .github/workflows/build-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 46701b3b16..5e6a04b470 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -146,7 +146,8 @@ jobs: - '3.9' - '3.8' - '3.7' - numpy-package: "numpy" + numpy-package: + - "numpy" include: - python-version: '3.7' numpy-package: "oldest-supported-numpy" From 5edbf7f5b2dd2f358c748d66fb1bfa8898a74655 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Sat, 3 Sep 2022 10:46:52 +0100 Subject: [PATCH 14/18] docs: increase Python UB in README --- README-pypi.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README-pypi.md b/README-pypi.md index abd4c074d2..599f43d70c 100644 --- a/README-pypi.md +++ b/README-pypi.md @@ -2,7 +2,7 @@ [![PyPI version](https://badge.fury.io/py/awkward.svg)](https://pypi.org/project/awkward) [![Conda-Forge](https://img.shields.io/conda/vn/conda-forge/awkward)](https://github.com/conda-forge/awkward-feedstock) -[![Python 3.7‒3.10](https://img.shields.io/badge/python-3.7%E2%80%923.10-blue)](https://www.python.org) +[![Python 3.7‒3.11](https://img.shields.io/badge/python-3.7%E2%80%923.11-blue)](https://www.python.org) [![BSD-3 Clause License](https://img.shields.io/badge/license-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Continuous integration tests](https://img.shields.io/azure-devops/build/jpivarski/Scikit-HEP/3/main?label=tests)](https://dev.azure.com/jpivarski/Scikit-HEP/_build) diff --git a/README.md b/README.md index 97b7d67041..ee54df8caf 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![PyPI version](https://badge.fury.io/py/awkward.svg)](https://pypi.org/project/awkward) [![Conda-Forge](https://img.shields.io/conda/vn/conda-forge/awkward)](https://github.com/conda-forge/awkward-feedstock) -[![Python 3.7‒3.10](https://img.shields.io/badge/python-3.7%E2%80%923.10-blue)](https://www.python.org) +[![Python 3.7‒3.11](https://img.shields.io/badge/python-3.7%E2%80%923.11-blue)](https://www.python.org) [![BSD-3 Clause License](https://img.shields.io/badge/license-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Build Test](https://github.com/scikit-hep/awkward/actions/workflows/build-test.yml/badge.svg?branch=main)](https://github.com/scikit-hep/awkward/actions/workflows/build-test.yml) From 0e70a56efeb18f33f9bf0682b166ee3c78bcfa85 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Sat, 3 Sep 2022 10:49:29 +0100 Subject: [PATCH 15/18] docs: use CPython EoL --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7332196ece..f1bb78a46d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -231,7 +231,7 @@ As we change the code, we should keep in mind the following priorities, in this Above all, the purpose of any programming language is to be read by humans; if we were only concerned with operating the machine, we would be flipping individual bits. It should be organized in stanzas that highlight similarities and differences by grouping them on the screen. -We adhere to an 80-character line width, which is a [standard in the industry](https://github.com/scikit-hep/awkward-1.0/pull/183), despite the fact that we don't write punch-cards anymore. The standardized width allows several window columns to be examined side-by-side. Exceptions to the 80-character limit follow [PEP 8](https://www.python.org/dev/peps/pep-0008/): we don't split URLs or similar tokens that must be read as a unit. +We adhere to an 80-character line width, which is a [stand<65;53;42Mard in the industry](https://github.com/scikit-hep/awkward-1.0/pull/183), despite the fact that we don't write punch-cards anymore. The standardized width allows several window columns to be examined side-by-side. Exceptions to the 80-character limit follow [PEP 8](https://www.python.org/dev/peps/pep-0008/): we don't split URLs or similar tokens that must be read as a unit. Unit tests do not need to adhere to the 80-character limit. @@ -271,7 +271,7 @@ We target Python 3.7 and above. Import statements can assume Python 3 names, str If you see any outdated (pre-Python 3.7) code, you can safely clean them up. Some strings are not easier to read as f-strings or require some work to make them more readable; it should be a case-by-case basis. -Python 3.7 will be the standard until it becomes "painful" to keep it (like, when CI and build tools no longer support it, or if we adopt type annotations). +Awkward Array follows [CPython's EoL schedule](https://endoflife.date/python), and will drop support for out-of-date versions of Python accordingly. ### Third party dependencies From 2b4aca2c342b964d3f9c4bc10d6cc4819a1e655d Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Sat, 3 Sep 2022 10:51:50 +0100 Subject: [PATCH 16/18] ci: remove redundant CIBW configuration from CI --- .github/workflows/tests.yml | 1 - .github/workflows/wheels.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 212146a0b5..a6128ccf16 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,6 @@ jobs: - uses: pypa/cibuildwheel@v2.9.0 env: CIBW_ARCHS_MACOS: universal2 - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_BUILD: cp39-win_amd64 cp37-manylinux_x86_64 cp38-macosx_universal2 - uses: pypa/cibuildwheel@v2.9.0 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2ef30bee1e..959f3e7b5c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -57,7 +57,6 @@ jobs: env: CIBW_BUILD: ${{ matrix.build }} CIBW_ARCHS: ${{ matrix.arch }} - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 - name: Upload wheels uses: actions/upload-artifact@v3 From df5a48ed435f7588ad5dad9467969a7f658db790 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Sat, 3 Sep 2022 13:38:52 +0100 Subject: [PATCH 17/18] chore: update pre-commit config --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80e3b2efb9..05db0f64ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: rev: "v2.37.3" hooks: - id: pyupgrade - args: ["--py36-plus"] + args: ["--py37-plus"] - repo: https://github.com/psf/black rev: 22.6.0 From fc9ca84d2deafb889b54dfed4d4d1111322e4a8e Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Sat, 3 Sep 2022 13:19:20 -0500 Subject: [PATCH 18/18] Scanned through the text and brought a lot of things up to date. --- CONTRIBUTING.md | 83 ++++++++++++++----------------------------------- 1 file changed, 24 insertions(+), 59 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f1bb78a46d..6472b6f043 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,6 +32,20 @@ That said, most of the commits on a pull request/git branch should be from a sin As such, you should name your branch starting with your GitHub userid and a slash, such as `jpivarski/write-contributing-md`. If you start a pull request with a branch that doesn't follow convention, though, you don't need to fix it. +The titles of pull requests (and therefore the merge commit messages) should follow [these conventions](https://www.conventionalcommits.org/en/v1.0.0/#summary). Mostly, this means prefixing the title with one of these words and a colon: + + * feat: new feature + * fix: bug-fix + * perf: code change that improves performance + * refactor: code change that neither fixes a bug nor adds a feature + * style: changes that do not affect the meaning of the code + * test: adding missing tests or correcting existing tests + * build: changes that affect the build system or external dependencies + * docs: documentation only changes + * ci: changes to our CI configuration files and scripts + * chore: other changes that don't modify src or test files + * revert: reverts a previous commit + Most pull requests are merged with the "squash and merge" feature, so details about commit history within a pull request are lost. Feel free, therefore, to commit with any frequency you're comfortable with. I like to make frequent commits to avoid losing work to a dead laptop, and to have more save-points to recover from. It is unnecessary to manually edit (rebase) your commit history. If, however, you do want to save a pull request as multiple commits on `main`, ask me and we'll discuss. (The default branch is named `main`, not `master`.) @@ -55,15 +69,7 @@ python -m pytest -vv -rs tests-spec python -m pytest -vv -rs tests-cpu-kernels ``` -Furthermore, if you have an Nvidia GPU, you can build and locally install the experimental CUDA plug-in with - -```bash -pip uninstall -y awkward-cuda-kernels -python dev/generate-cuda.py -./cuda-build.sh --install -``` - -The `--install` does a local `pip install` on your system, which is the only way to use it. You can run its tests with +Furthermore, if you have an Nvidia GPU and CuPy installed, you can run the CUDA tests with ```bash python dev/generate-tests.py @@ -114,45 +120,12 @@ Before re-building documentation, you might want to delete the files that were g rm -rf docs-shinx/_auto docs_sphinx/_build docs-sphinx/_static ``` -### Continuous integration +### Continuous testing Pull requests must pass all [continuous integration](https://dev.azure.com/jpivarski/Scikit-HEP/_build?definitionId=3&_a=summary) tests before they are merged. I will sometimes cancel non-essential builds to give priority to pull requests that are almost ready to be merged. If you needed the result of the build as a diagnostic, you can ask me to restart your job or make a trivial change to trigger a new build. Currently, we only run merge builds (the state of your branch if merged with `main`), not regular branch builds (the state of your branch as-is), because only merge builds can be made to run for pull requests from external forks and it makes better use of our limited execution time on Azure. If you want to enable regular branch builds, you can turn it on for your branch by editing `trigger/branches/exclude` in [.ci/azure-buildtest-awkwrad.yml](https://github.com/scikit-hep/awkward-1.0/blob/9b6fca3f6e6456860ae40979171f762e0045ce7c/.ci/azure-buildtest-awkward.yml#L1-L5). The merge build trigger is not controlled by the YAML file. It is better, however, to keep up-to-date with `git merge main`. -### Semi-automated testing of CUDA kernels - -For development on the cuda-kernels, an AWS VM with a GPU has been set up to run tests in the `tests-cuda` directory. You can email jpivarski at GMail for more information on how to get permissions to launch the AWS instance. - -To see if the AWS instance is running, use the launcher: - -```bash -% ssh -i ~/.ssh/awkward1-cuda-test.pem ubuntu@WW.XX.YY.ZZ ./list-instances.py -i-0295dd31185ce78f7 WW.XX.YY.ZZ running ['launcher'] -i-0a1ec83c261607797 AA.BB.CC.DD running ['awkward1-cuda-test'] -i-0a9dd553fe19610fb (none) stopped ['awkward1-cuda-test-2gpus'] -``` - -If it is `stopped`, launch it with: - -```bash -ssh -i ~/.ssh/awkward1-cuda-test.pem ubuntu@WW.XX.YY.ZZ aws ec2 start-instances --instance-ids i-0a1ec83c261607797 -``` - -Once it's `running`, use its IP address (`AA.BB.CC.DD`) to run the tests for your branch (`my-branch-name`): - -```bash -ssh -i ~/.ssh/awkward1-cuda-test.pem ubuntu@AA.BB.CC.DD ./cuda-run-tests.sh my-branch-name -``` - -When you are done, be sure to put it back into the `stopped` state: - -```bash -ssh -i ~/.ssh/awkward1-cuda-test.pem ubuntu@WW.XX.YY.ZZ aws ec2 stop-instances --instance-ids i-0a1ec83c261607797 -``` - -and verify with `list-instances`. While the GPU instance is running, you can ssh into it interactively for debugging. - ### The main branch The Awkward Array `main` branch must be kept in an unbroken state. Although the recommended way to install Awkward Array is through pip or conda, the `main` branch on GitHub must always be functional. Pull requests for bug fixes and new features are based on `main`, so it has to work for users to test our proposed changes. @@ -167,6 +140,10 @@ Committing directly to `main` is not allowed except for and only by me. +### The main-v1 branch + +The `main-v1` branch was split from `main` just before Awkward 1.x code was removed, so it exists to make 1.10.x bug-fix releases. These commits must be be drawn from `main-v1`, not `main`, and pull requests must target `main-v1` (not the GitHub default). A single commit cannot be applied to both `main` and `main-v1` because they have diverged too much. If a bug-fix needs to be applied to both (unlikely), it will have to be reimplemented on both. + ### Releases Similarly, only I publish releases (or a team of maintainers, in the future). Publishing releases starts the deployment procedure, updating the package that users will get when they pip-install. @@ -190,16 +167,6 @@ Contributing to each part of the codebase has a different flavor: A Contribution might only touch one layer of the code or it might involve more than one. -### Deployment products - -Awkward Array consists of two (or three) shared libraries, - - * `libawkward-cpu-kernels.so` (or `.dylib`, etc.) - * `libawkward-cuda-kernels.so` (if compiled with CUDA support) - * `libawkward.so` - -and a Python library shipped as binary wheels or (in rare cases) a source tarball to be compiled. The pip and conda deployments include all shared libraries with the Python package and put C++ headers in the system-dependent location for `includes`, in a directory named `awkward`. - ### Performance considerations The conventional model is that Python is for a good user interface and C++ is for performance. In the case of Awkward Array, even the C++ layer is not intended for high performance; this is pushed down to the CPU and GPU kernels. In typical applications, the number of C++ objects is small (hundreds to thousands of instances) while the size of array buffers sent to CPU and GPU kernels is large (billions of elements). @@ -231,7 +198,7 @@ As we change the code, we should keep in mind the following priorities, in this Above all, the purpose of any programming language is to be read by humans; if we were only concerned with operating the machine, we would be flipping individual bits. It should be organized in stanzas that highlight similarities and differences by grouping them on the screen. -We adhere to an 80-character line width, which is a [stand<65;53;42Mard in the industry](https://github.com/scikit-hep/awkward-1.0/pull/183), despite the fact that we don't write punch-cards anymore. The standardized width allows several window columns to be examined side-by-side. Exceptions to the 80-character limit follow [PEP 8](https://www.python.org/dev/peps/pep-0008/): we don't split URLs or similar tokens that must be read as a unit. +We adhere to an 80-character line width, which is a [standard in the industry](https://github.com/scikit-hep/awkward-1.0/pull/183), despite the fact that we don't write punch-cards anymore. The standardized width allows several window columns to be examined side-by-side. Exceptions to the 80-character limit follow [PEP 8](https://www.python.org/dev/peps/pep-0008/): we don't split URLs or similar tokens that must be read as a unit. Unit tests do not need to adhere to the 80-character limit. @@ -259,11 +226,9 @@ As stated above, most of the C++ features we use are "old," such as virtual inhe A quick scan of the code would reveal that we make extensive use of `std::shared_ptr`. Raw pointers are used in extremely limited circumstances, and never passed out of the scope that holds a shared pointer to the same resource (guaranteeing its lifetime). The main use of raw pointers is for down-casting with `dynamic_cast`. -Class instances are passed between functions as generic types for flexibility. For example, `toListOffsetArray64` returns a list array converted to the `ListOffsetArray64` class, but it is passed as a smart pointer to the common superclass, `Content`. This is because passing as a `ListOffsetArray64` exposes typeinfo symbols that violate visibility rules. - Class hierarchies are intentionally simple. If any inheritance is involved, the one superclass is abstract and all subclasses are direct children of that abstract superclass. -Templating is only used for integer specialization. +Templating is mostly just used for integer specialization. ### Python standard @@ -281,11 +246,11 @@ The Python codebase only strictly depends on NumPy 1.13.1, the first version wit Other third party libraries are used if they exist (can be imported), and we only accept certain versions of these libraries. Both the test-import and any version-testing must be within runtime code, not startup code, so that they're only invoked when users explicitly call for the feature that requires them. -Versions can be explicitly tested with `distutils.version.LooseVersion`, though it's better to test for features (existence of classes, methods, and attributes) than to test for explicit version numbers. +Versions can be explicitly tested with `packaging.version.parse`, though it's better to test for features (existence of classes, methods, and attributes) than to test for explicit version numbers. ### Array object details -Arrays are (or soon will be: [#176](https://github.com/scikit-hep/awkward-1.0/issues/176) and [#177](https://github.com/scikit-hep/awkward-1.0/issues/177)) immutable objects. Only the high-level `ak.Array` changes its state in-place in response to user choices (such as `__setitem__`, which replaces its `layout` using the pure function `ak.with_field`). This is not a performance liability but usually a benefit because it means that we can freely share data among array objects without worrying about long-distance modifications. +Arrays are immutable objects. Only the high-level `ak.Array` changes its state in-place in response to user choices (such as `__setitem__`, which replaces its `layout` using the pure function `ak.with_field`). This is not a performance liability but usually a benefit because it means that we can freely share data among array objects without worrying about long-distance modifications. Users can break this model by wrapping NumPy arrays as Awkward Arrays and changing the original NumPy arrays in-place, but they are encouraged not to.