Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make python CUDA package pipeline 1ES compliant #23802

Draft
wants to merge 18 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ resources:
include:
- main
branch: main

repositories:
- repository: 1esPipelines
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
parameters:
- name: isReleaseBuild
type: boolean
Expand All @@ -20,7 +24,17 @@ variables:
${{ else }}:
value: onnxruntime-cuda-12

stages:
- template: stages/py-cuda-publishing-stage.yml
extends:
# The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
# For non-production pipelines, use "Unofficial" as defined below.
# For productions pipelines, use "Official".
template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
parameters:
artifact_feed: $(ArtifactFeed)
sdl:
sourceAnalysisPool:
name: onnxruntime-Win-CPU-2022
os: windows
stages:
- template: stages/py-cuda-publishing-stage.yml
parameters:
artifact_feed: $(ArtifactFeed)
50 changes: 33 additions & 17 deletions tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ parameters:
displayName: 'Linux packages cmake build type. Linux Only.'
default: 'Release'
values:
- Debug
- Release
- RelWithDebInfo
- MinSizeRel
- Debug
- Release
- RelWithDebInfo
- MinSizeRel

# Only applies to QNN packages.
- name: qnn_sdk_version
Expand All @@ -63,17 +63,33 @@ parameters:

trigger: none

stages:
- template: stages/py-cpu-packaging-stage.yml
resources:
repositories:
- repository: 1esPipelines
type: git
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release
extends:
# The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
# For non-production pipelines, use "Unofficial" as defined below.
# For productions pipelines, use "Official".
template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
parameters:
enable_linux_cpu: ${{ parameters.enable_linux_cpu }}
enable_windows_cpu: ${{ parameters.enable_windows_cpu }}
enable_mac_cpu: ${{ parameters.enable_mac_cpu }}
enable_linux_arm: ${{ parameters.enable_linux_arm }}
enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }}
enable_windows_arm64ec_qnn: ${{ parameters.enable_windows_arm64ec_qnn }}
enable_windows_x64_qnn: ${{ parameters.enable_windows_x64_qnn }}
enable_linux_x64_qnn: ${{ parameters.enable_linux_x64_qnn }}
build_py_parameters: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
qnn_sdk_version: ${{ parameters.qnn_sdk_version }}
sdl:
sourceAnalysisPool:
name: onnxruntime-Win-CPU-2022
os: windows
stages:
- template: stages/py-cpu-packaging-stage.yml
parameters:
enable_linux_cpu: ${{ parameters.enable_linux_cpu }}
enable_windows_cpu: ${{ parameters.enable_windows_cpu }}
enable_mac_cpu: ${{ parameters.enable_mac_cpu }}
enable_linux_arm: ${{ parameters.enable_linux_arm }}
enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }}
enable_windows_arm64ec_qnn: ${{ parameters.enable_windows_arm64ec_qnn }}
enable_windows_x64_qnn: ${{ parameters.enable_windows_x64_qnn }}
enable_linux_x64_qnn: ${{ parameters.enable_linux_x64_qnn }}
build_py_parameters: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
qnn_sdk_version: ${{ parameters.qnn_sdk_version }}
Original file line number Diff line number Diff line change
Expand Up @@ -151,10 +151,11 @@ stages:
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'

- task: PublishBuildArtifacts@1
- task: 1ES.PublishPipelineArtifact@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
artifactName: onnxruntime-win-$(PythonVersion)
targetPath: '$(Build.ArtifactStagingDirectory)'

- script: |
7z x *.whl
Expand Down Expand Up @@ -199,7 +200,9 @@ stages:
workspace:
clean: all
pool:
vmImage: 'macOS-13'
name: "Azure Pipelines"
image: "macOS-13"
os: macOS
variables:
MACOSX_DEPLOYMENT_TARGET: '13.3'
strategy:
Expand Down Expand Up @@ -251,74 +254,81 @@ stages:
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'

- task: PublishBuildArtifacts@1
- task: 1ES.PublishPipelineArtifact@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime
artifactName: onnxruntime-macos-$(PythonVersion)
targetPath: '$(Build.ArtifactStagingDirectory)'

- template: ../templates/component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'


- ${{ if eq(parameters.enable_linux_arm, true) }}:
- stage: Python_Packaging_Linux_ARM
dependsOn: []
jobs:
- template: ../templates/py-linux.yml
parameters:
arch: 'aarch64'
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
extra_build_arg: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}

- ${{ if eq(parameters.enable_linux_cpu, true) }}:
- stage: Python_Packaging_Linux_CPU
dependsOn: []
jobs:
- ${{ if eq(parameters.enable_linux_arm, true) }}:
- stage: Python_Packaging_Linux_ARM
dependsOn: []
jobs:
- template: ../templates/py-linux.yml
parameters:
arch: 'x86_64'
machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large'
arch: 'aarch64'
machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
extra_build_arg: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
is1ES: true

- ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}:
- stage: Python_Packaging_Windows_ARM64_QNN
dependsOn: []
jobs:
- template: ../templates/py-win-arm64-qnn.yml
- ${{ if eq(parameters.enable_linux_cpu, true) }}:
- stage: Python_Packaging_Linux_CPU
dependsOn: []
jobs:
- template: ../templates/py-linux.yml
parameters:
arch: 'x86_64'
machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large'
extra_build_arg: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
is1ES: true

- ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}:
- stage: Python_Packaging_Windows_ARM64_QNN
dependsOn: []
jobs:
- template: ../templates/py-win-arm64-qnn.yml
parameters:
MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}
is1ES: true

- ${{ if eq(parameters.enable_windows_arm64ec_qnn, true) }}:
- stage: Python_Packaging_Windows_arm64ec_QNN
dependsOn: []
jobs:
- template: ../templates/py-win-arm64ec-qnn.yml
parameters:
MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64'
MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}
is1ES: true

- ${{ if eq(parameters.enable_windows_arm64ec_qnn, true) }}:
- stage: Python_Packaging_Windows_arm64ec_QNN
dependsOn: []
jobs:
- template: ../templates/py-win-arm64ec-qnn.yml
parameters:
MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}

- ${{ if eq(parameters.enable_windows_x64_qnn, true) }}:
- stage: Python_Packaging_Windows_x64_QNN
dependsOn: []
jobs:
- template: ../templates/py-win-x64-qnn.yml
parameters:
MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}

- ${{ if eq(parameters.enable_linux_x64_qnn, true) }}:
- stage: Python_Packaging_Linux_x64_QNN
dependsOn: []
jobs:
- template: ../templates/py-linux-qnn.yml
- ${{ if eq(parameters.enable_windows_x64_qnn, true) }}:
- stage: Python_Packaging_Windows_x64_QNN
dependsOn: []
jobs:
- template: ../templates/py-win-x64-qnn.yml
parameters:
machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
extra_build_arg: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU'
QNN_SDK: ${{ parameters.qnn_sdk_version }}
BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }}
is1ES: true

- ${{ if eq(parameters.enable_linux_x64_qnn, true) }}:
- stage: Python_Packaging_Linux_x64_QNN
dependsOn: []
jobs:
- template: ../templates/py-linux-qnn.yml
parameters:
machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
extra_build_arg: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
is1ES: true
Loading
Loading