Skip to content

Commit

Permalink
Cleanups (#276)
Browse files Browse the repository at this point in the history
- prettier
- stricter mypy
- remove split-reduce
  • Loading branch information
dcherian authored Oct 14, 2023
1 parent 9dd126c commit a2b46b0
Show file tree
Hide file tree
Showing 30 changed files with 15,194 additions and 299 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
jobs:
benchmark:
# if: ${{ contains( github.event.pull_request.labels.*.name, 'run-benchmark') && github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} # Run if the PR has been labelled correctly.
if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} # Always run.
if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} # Always run.
name: Linux
runs-on: ubuntu-20.04
env:
Expand Down
20 changes: 10 additions & 10 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -59,28 +59,28 @@ jobs:

optional-deps:
name: ${{ matrix.env }}
runs-on: ${{ matrix.os }}
runs-on: "ubuntu-latest"
defaults:
run:
shell: bash -l {0}
strategy:
fail-fast: false
matrix:
os: ["ubuntu-latest"]
env:
[
"no-xarray",
"no-dask",
"minimal-requirements",
]
python-version: ["3.11"]
env: ["no-xarray", "no-dask"]
include:
- env: "no-numba"
python-version: "3.12"
- env: "minimal-requirements"
python-version: "3.9"
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all branches and tags.
- name: Set up conda environment
uses: mamba-org/setup-micromamba@v1
with:
environment-file: ci/environment.yml
environment-file: ci/${{ matrix.env }}.yml
environment-name: flox-tests
init-shell: bash
cache-environment: true
Expand Down Expand Up @@ -110,7 +110,7 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
repository: 'pydata/xarray'
repository: "pydata/xarray"
fetch-depth: 0 # Fetch all history for all branches and tags.
- name: Set up conda environment
uses: mamba-org/setup-micromamba@v1
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pypi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.x'
python-version: "3.x"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
docs/source/generated/
html/
.asv/
asv_bench/pkgs/

# Byte-compiled / optimized / DLL files
__pycache__/
Expand Down
101 changes: 53 additions & 48 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,56 +1,61 @@
ci:
autoupdate_schedule: quarterly
autoupdate_schedule: quarterly

repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: 'v0.0.292'
hooks:
- id: ruff
args: ["--fix", "--show-fixes"]

- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-yaml
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-docstring-first

- repo: https://github.com/psf/black-pre-commit-mirror
rev: 23.9.1
hooks:
- id: black

- repo: https://github.com/executablebooks/mdformat
rev: 0.7.17
hooks:
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: "v0.0.292"
hooks:
- id: ruff
args: ["--fix", "--show-fixes"]

- repo: https://github.com/pre-commit/mirrors-prettier
rev: "v3.0.3"
hooks:
- id: prettier

- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:
- id: check-yaml
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-docstring-first

- repo: https://github.com/psf/black-pre-commit-mirror
rev: 23.9.1
hooks:
- id: black

- repo: https://github.com/executablebooks/mdformat
rev: 0.7.17
hooks:
- id: mdformat
additional_dependencies:
- mdformat-black
- mdformat-myst

- repo: https://github.com/nbQA-dev/nbQA
rev: 1.7.0
hooks:
- id: nbqa-black
- id: nbqa-ruff
args: [--fix]

- repo: https://github.com/kynan/nbstripout
rev: 0.6.1
hooks:
- id: nbstripout
args: [--extra-keys=metadata.kernelspec metadata.language_info.version]

- repo: https://github.com/codespell-project/codespell
rev: v2.2.6
hooks:
- id: codespell
additional_dependencies:
- tomli

- repo: https://github.com/abravalheri/validate-pyproject
rev: v0.14
hooks:
- id: validate-pyproject
- repo: https://github.com/nbQA-dev/nbQA
rev: 1.7.0
hooks:
- id: nbqa-black
- id: nbqa-ruff
args: [--fix]

- repo: https://github.com/kynan/nbstripout
rev: 0.6.1
hooks:
- id: nbstripout
args: [--extra-keys=metadata.kernelspec metadata.language_info.version]

- repo: https://github.com/codespell-project/codespell
rev: v2.2.6
hooks:
- id: codespell
additional_dependencies:
- tomli

- repo: https://github.com/abravalheri/validate-pyproject
rev: v0.15
hooks:
- id: validate-pyproject
192 changes: 96 additions & 96 deletions asv_bench/asv.conf.json
Original file line number Diff line number Diff line change
@@ -1,98 +1,98 @@
{
// The version of the config file format. Do not change, unless
// you know what you are doing.
"version": 1,

// The name of the project being benchmarked
"project": "flox",

// The project's homepage
"project_url": "http://flox.readthedocs.io/",

// The URL or local path of the source code repository for the
// project being benchmarked
"repo": "..",

// The Python project's subdirectory in your repo. If missing or
// the empty string, the project is assumed to be located at the root
// of the repository.
// "repo_subdir": "",

// Customizable commands for building, installing, and
// uninstalling the project. See asv.conf.json documentation.
//
// "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"],
// "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"],
// "build_command": [
// "python setup.py build",
// "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}"
// ],

// List of branches to benchmark. If not provided, defaults to "master"
// (for git) or "default" (for mercurial).
"branches": ["main"], // for git
"dvcs": "git",

// timeout in seconds for installing any dependencies in environment
// defaults to 10 min
"install_timeout": 600,

// the base URL to show a commit for the project.
"show_commit_url": "http://github.com/xarray-contrib/flox/commit/",

// The Pythons you'd like to test against. If not provided, defaults
// to the current version of Python used to run `asv`.
// "pythons": ["3.9"],

"environment_type": "mamba",
"conda_channels": ["conda-forge"],
"conda_environment_file": "../ci/benchmark.yml",

// The directory (relative to the current directory) that benchmarks are
// stored in. If not provided, defaults to "benchmarks"
"benchmark_dir": "benchmarks",

// The directory (relative to the current directory) to cache the Python
// environments in. If not provided, defaults to "env"
"env_dir": ".asv/env",

// The directory (relative to the current directory) that raw benchmark
// results are stored in. If not provided, defaults to "results".
"results_dir": ".asv/results",

// The directory (relative to the current directory) that the html tree
// should be written to. If not provided, defaults to "html".
"html_dir": ".asv/html",

// The number of characters to retain in the commit hashes.
// "hash_length": 8,

// `asv` will cache results of the recent builds in each
// environment, making them faster to install next time. This is
// the number of builds to keep, per environment.
// "build_cache_size": 2,

// The commits after which the regression search in `asv publish`
// should start looking for regressions. Dictionary whose keys are
// regexps matching to benchmark names, and values corresponding to
// the commit (exclusive) after which to start looking for
// regressions. The default is to start from the first commit
// with results. If the commit is `null`, regression detection is
// skipped for the matching benchmark.
//
// "regressions_first_commits": {
// "some_benchmark": "352cdf", // Consider regressions only after this commit
// "another_benchmark": null, // Skip regression detection altogether
// },

// The thresholds for relative change in results, after which `asv
// publish` starts reporting regressions. Dictionary of the same
// form as in ``regressions_first_commits``, with values
// indicating the thresholds. If multiple entries match, the
// maximum is taken. If no entry matches, the default is 5%.
//
// "regressions_thresholds": {
// "some_benchmark": 0.01, // Threshold of 1%
// "another_benchmark": 0.5, // Threshold of 50%
// },
// The version of the config file format. Do not change, unless
// you know what you are doing.
"version": 1,

// The name of the project being benchmarked
"project": "flox",

// The project's homepage
"project_url": "http://flox.readthedocs.io/",

// The URL or local path of the source code repository for the
// project being benchmarked
"repo": "..",

// The Python project's subdirectory in your repo. If missing or
// the empty string, the project is assumed to be located at the root
// of the repository.
// "repo_subdir": "",

// Customizable commands for building, installing, and
// uninstalling the project. See asv.conf.json documentation.
//
// "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"],
// "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"],
// "build_command": [
// "python setup.py build",
// "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}"
// ],

// List of branches to benchmark. If not provided, defaults to "master"
// (for git) or "default" (for mercurial).
"branches": ["main"], // for git
"dvcs": "git",

// timeout in seconds for installing any dependencies in environment
// defaults to 10 min
"install_timeout": 600,

// the base URL to show a commit for the project.
"show_commit_url": "http://github.com/xarray-contrib/flox/commit/",

// The Pythons you'd like to test against. If not provided, defaults
// to the current version of Python used to run `asv`.
// "pythons": ["3.9"],

"environment_type": "mamba",
"conda_channels": ["conda-forge"],
"conda_environment_file": "../ci/benchmark.yml",

// The directory (relative to the current directory) that benchmarks are
// stored in. If not provided, defaults to "benchmarks"
"benchmark_dir": "benchmarks",

// The directory (relative to the current directory) to cache the Python
// environments in. If not provided, defaults to "env"
"env_dir": ".asv/env",

// The directory (relative to the current directory) that raw benchmark
// results are stored in. If not provided, defaults to "results".
"results_dir": ".asv/results",

// The directory (relative to the current directory) that the html tree
// should be written to. If not provided, defaults to "html".
"html_dir": ".asv/html"

// The number of characters to retain in the commit hashes.
// "hash_length": 8,

// `asv` will cache results of the recent builds in each
// environment, making them faster to install next time. This is
// the number of builds to keep, per environment.
// "build_cache_size": 2,

// The commits after which the regression search in `asv publish`
// should start looking for regressions. Dictionary whose keys are
// regexps matching to benchmark names, and values corresponding to
// the commit (exclusive) after which to start looking for
// regressions. The default is to start from the first commit
// with results. If the commit is `null`, regression detection is
// skipped for the matching benchmark.
//
// "regressions_first_commits": {
// "some_benchmark": "352cdf", // Consider regressions only after this commit
// "another_benchmark": null, // Skip regression detection altogether
// },

// The thresholds for relative change in results, after which `asv
// publish` starts reporting regressions. Dictionary of the same
// form as in ``regressions_first_commits``, with values
// indicating the thresholds. If multiple entries match, the
// maximum is taken. If no entry matches, the default is 5%.
//
// "regressions_thresholds": {
// "some_benchmark": 0.01, // Threshold of 1%
// "another_benchmark": 0.5, // Threshold of 50%
// },
}
2 changes: 1 addition & 1 deletion asv_bench/benchmarks/README_CI.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ The `asv` suite can be run for any PR on GitHub Actions (check workflow `.github
We use `asv continuous` to run the job, which runs a relative performance measurement. This means that there's no state to be saved and that regressions are only caught in terms of performance ratio (absolute numbers are available but they are not useful since we do not use stable hardware over time). `asv continuous` will:

- Compile `scikit-image` for _both_ commits. We use `ccache` to speed up the process, and `mamba` is used to create the build environments.
- Run the benchmark suite for both commits, _twice_ (since `processes=2` by default).
- Run the benchmark suite for both commits, _twice_ (since `processes=2` by default).
- Generate a report table with performance ratios:
- `ratio=1.0` -> performance didn't change.
- `ratio<1.0` -> PR made it slower.
Expand Down
2 changes: 1 addition & 1 deletion ci/benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ dependencies:
- asv
- cachey
- dask-core
- numpy>=1.20
- numpy>=1.21
- mamba
- pip
- python=3.10
Expand Down
Loading

0 comments on commit a2b46b0

Please sign in to comment.