Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build platform dependent wheel files #606

Merged
merged 4 commits into from
Aug 21, 2023
Merged

Build platform dependent wheel files #606

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
56dc66c
Build wheels for individual platforms/python versions
gclendenning Jul 20, 2023
a41a640
Uncomment pushing to PyPI
gclendenning Jul 20, 2023
01c23b5
Add python 3.11
gclendenning Jul 24, 2023
df1b1dc
Merge branch 'scikit-learn-contrib:master' into master
gclendenning Aug 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
391 changes: 243 additions & 148 deletions azure-pipelines.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,149 +1,244 @@
# Python package
# Create and test a Python package on multiple Python versions.
# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
# https://docs.microsoft.com/azure/devops/pipelines/languages/python

# Trigger a build when there is a push to the main branch or a tag starts with release-
trigger:
- master

jobs:
- job: Linux
pool:
vmImage: ubuntu-latest
strategy:
matrix:
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
Python39:
python.version: '3.9'

steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'

- script: |
python -m pip install --upgrade pip
pip install -r requirements.txt
displayName: 'Install dependencies'

- script: |
pip install cython
python setup.py develop

- script: |
pip install pytest pytest-azurepipelines
pytest
displayName: 'pytest'

- task: PublishTestResults@2
inputs:
testResultsFiles: 'pytest.xml'
testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)'
condition: succeededOrFailed()

- job: Windows
pool:
vmImage: 'windows-latest'
strategy:
matrix:
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
Python39:
python.version: '3.9'

steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'

- script: |
python -m pip install --upgrade pip
pip install -r requirements.txt
displayName: 'Install dependencies'

- script: |
pip install cython
python setup.py develop

- script: |
pip install pytest pytest-azurepipelines
pytest
displayName: 'pytest'

- job: MacOS
pool:
vmImage: 'macos-latest'
strategy:
matrix:
Python37:
python.version: '3.7'
Python38:
python.version: '3.8'
Python39:
python.version: '3.9'

steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'

- script: |
python -m pip install --upgrade pip
pip install -r requirements.txt
displayName: 'Install dependencies'

- script: |
pip install cython
python setup.py develop

- script: |
pip install pytest pytest-azurepipelines
pytest
displayName: 'pytest'

- job: Coverage
pool:
vmImage: ubuntu-latest
strategy:
matrix:
Python39:
python.version: '3.9'

steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'

- script: |
python -m pip install --upgrade pip
pip install -r requirements.txt
displayName: 'Install dependencies'

- script: |
pip install cython
pip install pytest
pip install pytest-cov
pip install coveralls
pip install codecov
python setup.py develop

- script: |
pip install pytest pytest-azurepipelines
pytest hdbscan/tests --show-capture=no -v --disable-warnings --junitxml=pytest.xml --cov=hdbscan/ --cov-report=xml --cov-report=html
codecov
displayName: 'pytest'

- task: PublishTestResults@2
inputs:
testResultsFiles: 'pytest.xml'
testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)'
condition: succeededOrFailed()
branches:
include:
- master
tags:
include:
- release-*

# Trigger a build when there is a pull request to the main branch
# Ignore PRs that are just updating the docs
pr:
branches:
include:
- master
exclude:
- doc/*
- README.rst

variables:
triggeredByPullRequest: $[eq(variables['Build.Reason'], 'PullRequest')]

stages:
- stage: RunAllTests
displayName: Run test suite
jobs:
- job: run_platform_tests
strategy:
matrix:
mac_py37:
imageName: 'macOS-latest'
python.version: '3.7'
linux_py37:
imageName: 'ubuntu-latest'
python.version: '3.7'
windows_py37:
imageName: 'windows-latest'
python.version: '3.7'
mac_py38:
imageName: 'macOS-latest'
python.version: '3.8'
linux_py38:
imageName: 'ubuntu-latest'
python.version: '3.8'
windows_py38:
imageName: 'windows-latest'
python.version: '3.8'
mac_py39:
imageName: 'macOS-latest'
python.version: '3.9'
linux_py39:
imageName: 'ubuntu-latest'
python.version: '3.9'
windows_py39:
imageName: 'windows-latest'
python.version: '3.9'
mac_py310:
imageName: 'macOS-latest'
python.version: '3.10'
linux_py310:
imageName: 'ubuntu-latest'
python.version: '3.10'
windows_py310:
imageName: 'windows-latest'
python.version: '3.10'
mac_py311:
imageName: 'macOS-latest'
python.version: '3.11'
linux_py311:
imageName: 'ubuntu-latest'
python.version: '3.11'
windows_py311:
imageName: 'windows-latest'
python.version: '3.11'
pool:
vmImage: $(imageName)

steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'

- script: |
python -m pip install --upgrade pip
pip install -r requirements.txt
displayName: 'Install dependencies'

- script: |
pip install -e .
pip install pytest pytest-azurepipelines
pip install pytest-cov
pip install coveralls
displayName: 'Install package'

- script: |
pytest hdbscan/tests --show-capture=no -v --disable-warnings --junitxml=junit/test-results.xml --cov=hdbscan/ --cov-report=xml --cov-report=html
displayName: 'Run tests'

- bash: |
coveralls
displayName: 'Publish to coveralls'
condition: and(succeeded(), eq(variables.triggeredByPullRequest, false)) # Don't run this for PRs because they can't access pipeline secrets
env:
COVERALLS_REPO_TOKEN: $(COVERALLS_TOKEN)

- task: PublishTestResults@2
inputs:
testResultsFiles: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)'
condition: succeededOrFailed()

- stage: BuildPublishArtifact
dependsOn: RunAllTests
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/release-'), eq(variables.triggeredByPullRequest, false))
jobs:
# Need to use manylinux as ubuntu-latest is too new
- job: Manylinux2014Build
pool:
vmImage: 'ubuntu-latest'
container: quay.io/pypa/manylinux2014_x86_64:latest
strategy:
matrix:
linux_py37:
python.version: 'cp37-cp37m'
linux_py38:
python.version: 'cp38-cp38'
linux_py39:
python.version: 'cp39-cp39'
linux_py310:
python.version: 'cp310-cp310'
linux_py311:
python.version: 'cp311-cp311'
steps:
- script: |
"${PYBIN}/python" -m pip install --upgrade pip
"${PYBIN}/python" -m pip install wheel
"${PYBIN}/python" -m pip install -r requirements.txt
displayName: 'Install dependencies and build tools'
env:
PYBIN: /opt/python/$(python.version)/bin
- script: |
"${PYBIN}/python" setup.py sdist bdist_wheel
displayName: 'Build wheels'
env:
PYBIN: /opt/python/$(python.version)/bin
- bash: |
auditwheel repair dist/*linux_x86_64.whl --plat manylinux2014_x86_64 -w wheelhouse-manylinux/
displayName: 'Audit wheels'

- task: DownloadSecureFile@1
name: PYPIRC_CONFIG
displayName: 'Download pypirc'
inputs:
secureFile: 'pypirc'

- bash: |
"${PYBIN}/python" -m pip install twine
"${PYBIN}/python" -m twine upload -r pypi --config-file $(PYPIRC_CONFIG.secureFilePath) --skip-existing --disable-progress-bar wheelhouse-manylinux/*
"${PYBIN}/python" -m twine upload -r pypi --config-file $(PYPIRC_CONFIG.secureFilePath) --skip-existing --disable-progress-bar dist/*.tar.gz
displayName: 'Publish wheel to PyPi'
env:
PYBIN: /opt/python/$(python.version)/bin

- job: BuildWindowsAndMacOSArtifacts
displayName: Build source dists and wheels for windows and macOS
strategy:
matrix:
mac_py37:
imageName: 'macOS-latest'
python.version: '3.7'
windows_py37:
imageName: 'windows-latest'
python.version: '3.7'
mac_py38:
imageName: 'macOS-latest'
python.version: '3.8'
windows_py38:
imageName: 'windows-latest'
python.version: '3.8'
mac_py39:
imageName: 'macOS-latest'
python.version: '3.9'
windows_py39:
imageName: 'windows-latest'
python.version: '3.9'
mac_py310:
imageName: 'macOS-latest'
python.version: '3.10'
windows_py310:
imageName: 'windows-latest'
python.version: '3.10'
mac_py311:
imageName: 'macOS-latest'
python.version: '3.11'
windows_py311:
imageName: 'windows-latest'
python.version: '3.11'
pool:
vmImage: $(imageName)

steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'

- script: |
python -m pip install --upgrade pip
pip install wheel
pip install -r requirements.txt
displayName: 'Install dependencies'

- script: |
pip install -e .
displayName: 'Install package locally'

- bash: |
python setup.py sdist bdist_wheel
displayName: 'Build package'

- bash: |
export PACKAGE_VERSION="$(python setup.py --version)"
echo "Package Version: ${PACKAGE_VERSION}"
echo "##vso[task.setvariable variable=packageVersionFormatted;]release-${PACKAGE_VERSION}"
displayName: 'Get package version'

- script: |
echo "Version in git tag $(Build.SourceBranchName) does not match version derived from setup.py $(packageVersionFormatted)"
exit 1
displayName: Raise error if version doesnt match tag
condition: and(succeeded(), ne(variables['Build.SourceBranchName'], variables['packageVersionFormatted']))

- task: DownloadSecureFile@1
name: PYPIRC_CONFIG
displayName: 'Download pypirc'
inputs:
secureFile: 'pypirc'

- script: |
pip install twine
twine upload -r pypi --config-file $(PYPIRC_CONFIG.secureFilePath) --skip-existing dist/*
displayName: 'Upload to PyPI'
condition: and(succeeded(), eq(variables['Build.SourceBranchName'], variables['packageVersionFormatted']))