diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml index e4499705efdf5..230c391c00ebd 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-publishing-pipeline.yml @@ -7,7 +7,11 @@ resources: include: - main branch: main - + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release parameters: - name: isReleaseBuild type: boolean @@ -20,7 +24,17 @@ variables: ${{ else }}: value: onnxruntime-cuda-12 -stages: -- template: stages/py-cuda-publishing-stage.yml +extends: + # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. + # For non-production pipelines, use "Unofficial" as defined below. + # For productions pipelines, use "Official". + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: - artifact_feed: $(ArtifactFeed) \ No newline at end of file + sdl: + sourceAnalysisPool: + name: onnxruntime-Win-CPU-2022 + os: windows + stages: + - template: stages/py-cuda-publishing-stage.yml + parameters: + artifact_feed: $(ArtifactFeed) \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml index 01d30d0e1ba86..28ddd29ec63e6 100644 --- a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml @@ -50,10 +50,10 @@ parameters: displayName: 'Linux packages cmake build type. Linux Only.' default: 'Release' values: - - Debug - - Release - - RelWithDebInfo - - MinSizeRel + - Debug + - Release + - RelWithDebInfo + - MinSizeRel # Only applies to QNN packages. - name: qnn_sdk_version @@ -63,17 +63,33 @@ parameters: trigger: none -stages: -- template: stages/py-cpu-packaging-stage.yml +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. + # For non-production pipelines, use "Unofficial" as defined below. + # For productions pipelines, use "Official". + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: - enable_linux_cpu: ${{ parameters.enable_linux_cpu }} - enable_windows_cpu: ${{ parameters.enable_windows_cpu }} - enable_mac_cpu: ${{ parameters.enable_mac_cpu }} - enable_linux_arm: ${{ parameters.enable_linux_arm }} - enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }} - enable_windows_arm64ec_qnn: ${{ parameters.enable_windows_arm64ec_qnn }} - enable_windows_x64_qnn: ${{ parameters.enable_windows_x64_qnn }} - enable_linux_x64_qnn: ${{ parameters.enable_linux_x64_qnn }} - build_py_parameters: ${{ parameters.build_py_parameters }} - cmake_build_type: ${{ parameters.cmake_build_type }} - qnn_sdk_version: ${{ parameters.qnn_sdk_version }} + sdl: + sourceAnalysisPool: + name: onnxruntime-Win-CPU-2022 + os: windows + stages: + - template: stages/py-cpu-packaging-stage.yml + parameters: + enable_linux_cpu: ${{ parameters.enable_linux_cpu }} + enable_windows_cpu: ${{ parameters.enable_windows_cpu }} + enable_mac_cpu: ${{ parameters.enable_mac_cpu }} + enable_linux_arm: ${{ parameters.enable_linux_arm }} + enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }} + enable_windows_arm64ec_qnn: ${{ parameters.enable_windows_arm64ec_qnn }} + enable_windows_x64_qnn: ${{ parameters.enable_windows_x64_qnn }} + enable_linux_x64_qnn: ${{ parameters.enable_linux_x64_qnn }} + build_py_parameters: ${{ parameters.build_py_parameters }} + cmake_build_type: ${{ parameters.cmake_build_type }} + qnn_sdk_version: ${{ parameters.qnn_sdk_version }} diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml index 4ff539df9f914..c196bab734ed1 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml @@ -151,10 +151,11 @@ stages: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - ArtifactName: onnxruntime + artifactName: onnxruntime-win-$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' - script: | 7z x *.whl @@ -199,7 +200,9 @@ stages: workspace: clean: all pool: - vmImage: 'macOS-13' + name: "Azure Pipelines" + image: "macOS-13" + os: macOS variables: MACOSX_DEPLOYMENT_TARGET: '13.3' strategy: @@ -251,74 +254,81 @@ stages: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - ArtifactName: onnxruntime + artifactName: onnxruntime-macos-$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' - template: ../templates/component-governance-component-detection-steps.yml parameters: condition: 'succeeded' - - ${{ if eq(parameters.enable_linux_arm, true) }}: - - stage: Python_Packaging_Linux_ARM - dependsOn: [] - jobs: - - template: ../templates/py-linux.yml - parameters: - arch: 'aarch64' - machine_pool: 'onnxruntime-linux-ARM64-CPU-2019' - extra_build_arg: ${{ parameters.build_py_parameters }} - cmake_build_type: ${{ parameters.cmake_build_type }} - - - ${{ if eq(parameters.enable_linux_cpu, true) }}: - - stage: Python_Packaging_Linux_CPU - dependsOn: [] - jobs: +- ${{ if eq(parameters.enable_linux_arm, true) }}: + - stage: Python_Packaging_Linux_ARM + dependsOn: [] + jobs: - template: ../templates/py-linux.yml parameters: - arch: 'x86_64' - machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large' + arch: 'aarch64' + machine_pool: 'onnxruntime-linux-ARM64-CPU-2019' extra_build_arg: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} + is1ES: true - - ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}: - - stage: Python_Packaging_Windows_ARM64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-arm64-qnn.yml +- ${{ if eq(parameters.enable_linux_cpu, true) }}: + - stage: Python_Packaging_Linux_CPU + dependsOn: [] + jobs: + - template: ../templates/py-linux.yml + parameters: + arch: 'x86_64' + machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large' + extra_build_arg: ${{ parameters.build_py_parameters }} + cmake_build_type: ${{ parameters.cmake_build_type }} + is1ES: true + +- ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}: + - stage: Python_Packaging_Windows_ARM64_QNN + dependsOn: [] + jobs: + - template: ../templates/py-win-arm64-qnn.yml + parameters: + MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' + QNN_SDK: ${{ parameters.qnn_sdk_version }} + BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} + is1ES: true + +- ${{ if eq(parameters.enable_windows_arm64ec_qnn, true) }}: + - stage: Python_Packaging_Windows_arm64ec_QNN + dependsOn: [] + jobs: + - template: ../templates/py-win-arm64ec-qnn.yml parameters: - MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' + MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' QNN_SDK: ${{ parameters.qnn_sdk_version }} BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} + is1ES: true - - ${{ if eq(parameters.enable_windows_arm64ec_qnn, true) }}: - - stage: Python_Packaging_Windows_arm64ec_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-arm64ec-qnn.yml - parameters: - MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - - - ${{ if eq(parameters.enable_windows_x64_qnn, true) }}: - - stage: Python_Packaging_Windows_x64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-x64-qnn.yml - parameters: - MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - - - ${{ if eq(parameters.enable_linux_x64_qnn, true) }}: - - stage: Python_Packaging_Linux_x64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-linux-qnn.yml +- ${{ if eq(parameters.enable_windows_x64_qnn, true) }}: + - stage: Python_Packaging_Windows_x64_QNN + dependsOn: [] + jobs: + - template: ../templates/py-win-x64-qnn.yml parameters: - machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU' - extra_build_arg: ${{ parameters.build_py_parameters }} - cmake_build_type: ${{ parameters.cmake_build_type }} + MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' + QNN_SDK: ${{ parameters.qnn_sdk_version }} + BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} + is1ES: true + +- ${{ if eq(parameters.enable_linux_x64_qnn, true) }}: + - stage: Python_Packaging_Linux_x64_QNN + dependsOn: [] + jobs: + - template: ../templates/py-linux-qnn.yml + parameters: + machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU' + extra_build_arg: ${{ parameters.build_py_parameters }} + cmake_build_type: ${{ parameters.cmake_build_type }} + is1ES: true diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml index 347a3145e8c70..8126cda449daa 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux-qnn.yml @@ -6,10 +6,10 @@ parameters: type: string default: 'Release' values: - - Debug - - Release - - RelWithDebInfo - - MinSizeRel + - Debug + - Release + - RelWithDebInfo + - MinSizeRel - name: device type: string @@ -27,68 +27,82 @@ parameters: displayName: QNN SDK version type: string default: 2.31.0.250130 + +- name: is1ES + displayName: 'Whether the pipeline is running in 1ES' + type: boolean + default: false jobs: - job: Linux_py_qnn_Wheels_x64 timeoutInMinutes: 240 workspace: clean: all - pool: ${{ parameters.machine_pool }} + pool: + name: ${{ parameters.machine_pool }} + os: linux variables: - # The build machine pool doesn't have dotnet, so it can't run CG. - - name: skipComponentGovernanceDetection - value: true - - name: ORT_CACHE_DIR - value: $(Agent.TempDirectory)/ort_ccache - - name: TODAY - value: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] - - name: extra_build_args - ${{ if ne(parameters.extra_build_arg, '') }}: - value: -x ${{ parameters.extra_build_arg }} - ${{ if eq(parameters.extra_build_arg, '') }}: - value: '' + # The build machine pool doesn't have dotnet, so it can't run CG. + - name: skipComponentGovernanceDetection + value: true + - name: ORT_CACHE_DIR + value: $(Agent.TempDirectory)/ort_ccache + - name: TODAY + value: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] + - name: extra_build_args + ${{ if ne(parameters.extra_build_arg, '') }}: + value: -x ${{ parameters.extra_build_arg }} + ${{ if eq(parameters.extra_build_arg, '') }}: + value: '' steps: - - checkout: self - clean: true - submodules: none + - checkout: self + clean: true + submodules: none - - template: jobs/download_linux_qnn_sdk.yml - parameters: - QnnSDKVersion: ${{ parameters.QnnSdk }} + - template: jobs/download_linux_qnn_sdk.yml + parameters: + QnnSDKVersion: ${{ parameters.QnnSdk }} - - template: set-nightly-build-option-variable-step.yml + - template: set-nightly-build-option-variable-step.yml - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile - Context: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuildpythonx86_64_qnn + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" + Repository: onnxruntimecpubuildpythonx86_64_qnn - - template: linux-build-step-with-cache.yml - parameters: - WithCache: ${{parameters.with_cache}} - Today: $(TODAY) - AdditionalKey: Linux_py_qnn_Wheels_x64 - CacheDir: $(ORT_CACHE_DIR) - ChangeEveryCommit: true - BuildStep: - - task: Bash@3 - displayName: 'Build Python Wheel' - inputs: - targetType: filePath - filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh - arguments: -i onnxruntimecpubuildpythonx86_64_qnn -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) - env: - ADDITIONAL_DOCKER_PARAMETER: "--volume $(QnnSDKRootDir):/qnn_sdk" + - template: linux-build-step-with-cache.yml + parameters: + WithCache: ${{parameters.with_cache}} + Today: $(TODAY) + AdditionalKey: Linux_py_qnn_Wheels_x64 + CacheDir: $(ORT_CACHE_DIR) + ChangeEveryCommit: true + BuildStep: + - task: Bash@3 + displayName: 'Build Python Wheel' + inputs: + targetType: filePath + filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh + arguments: -i onnxruntimecpubuildpythonx86_64_qnn -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) + env: + ADDITIONAL_DOCKER_PARAMETER: "--volume $(QnnSDKRootDir):/qnn_sdk" + - ${{ if eq(parameters.is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: Linux ONNXRuntime QNN python wheel' + inputs: + targetPath: '$(Build.BinariesDirectory)/dist' + artifactName: onnxruntime-linux-qnn-x64 - - task: PublishBuildArtifacts@1 + - ${{ if eq(parameters.is1ES, false) }}: + - task: PublishPipelineArtifact@1 displayName: 'Publish Artifact: Linux ONNXRuntime QNN python wheel' inputs: - PathtoPublish: '$(Build.BinariesDirectory)/dist' - ArtifactName: onnxruntime-linux-qnn-x64 + targetPath: '$(Build.BinariesDirectory)/dist' + artifactName: onnxruntime-linux-qnn-x64 - - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml index e591b719ecfa9..8d0c4334f4874 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml @@ -9,10 +9,10 @@ parameters: type: string default: 'Release' values: - - Debug - - Release - - RelWithDebInfo - - MinSizeRel + - Debug + - Release + - RelWithDebInfo + - MinSizeRel - name: device type: string @@ -34,76 +34,98 @@ parameters: type: string default: '' +- name: is1ES + displayName: 'Whether the pipeline is running in 1ES' + type: boolean + default: false + jobs: - job: Linux_py_Wheels_${{ parameters.arch }}_${{parameters.ep}} timeoutInMinutes: 240 workspace: clean: all - pool: ${{ parameters.machine_pool }} + pool: + name: ${{ parameters.machine_pool }} + os: 'linux' + ${{ if eq(parameters.arch, 'aarch64') }}: + hostArchitecture: Arm64 variables: - # The build machine pool doesn't have dotnet, so it can't run CG. - - name: skipComponentGovernanceDetection - value: true - - name: ORT_CACHE_DIR - value: $(Agent.TempDirectory)/ort_ccache - - name: TODAY - value: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] - - name: extra_build_args - ${{ if ne(parameters.extra_build_arg, '') }}: - value: '-x ${{ parameters.extra_build_arg }}' - ${{ if eq(parameters.extra_build_arg, '') }}: - value: '' - - name: python_exe_path - ${{ if ne(parameters.python_exe_path, '') }}: - value: '-p ${{ parameters.python_exe_path }}' - ${{ if eq(parameters.python_exe_path, '') }}: - value: '' + # The build machine pool doesn't have dotnet, so it can't run CG. + - name: skipComponentGovernanceDetection + value: true + - name: ORT_CACHE_DIR + value: $(Agent.TempDirectory)/ort_ccache + - name: TODAY + value: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] + - name: extra_build_args + ${{ if ne(parameters.extra_build_arg, '') }}: + value: '-x ${{ parameters.extra_build_arg }}' + ${{ if eq(parameters.extra_build_arg, '') }}: + value: '' + - name: python_exe_path + ${{ if ne(parameters.python_exe_path, '') }}: + value: '-p ${{ parameters.python_exe_path }}' + ${{ if eq(parameters.python_exe_path, '') }}: + value: '' steps: - - checkout: self - clean: true - submodules: none - - - template: set-nightly-build-option-variable-step.yml - - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu/Dockerfile - Context: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuildpython${{ parameters.arch }} - - - template: linux-build-step-with-cache.yml - parameters: - WithCache: ${{parameters.with_cache}} - Today: $(TODAY) - AdditionalKey: Linux_py_Wheels_${{ parameters.arch }} - CacheDir: $(ORT_CACHE_DIR) - ChangeEveryCommit: true - BuildStep: - - task: Bash@3 - displayName: 'Build Python Wheel' - inputs: - targetType: filePath - filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh - arguments: -i onnxruntimecpubuildpython${{ parameters.arch }} -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) $(python_exe_path) - ${{ if eq(parameters.with_cache, 'true') }}: - env: - ADDITIONAL_DOCKER_PARAMETER: "--volume $(ORT_CACHE_DIR):/cache -e CCACHE_DIR=/cache -e ORT_BUILD_WITH_CACHE=1" - - - task: PublishBuildArtifacts@1 + - checkout: self + clean: true + submodules: none + + - template: set-nightly-build-option-variable-step.yml + + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" + Repository: onnxruntimecpubuildpython${{ parameters.arch }} + + - template: linux-build-step-with-cache.yml + parameters: + WithCache: ${{parameters.with_cache}} + Today: $(TODAY) + AdditionalKey: Linux_py_Wheels_${{ parameters.arch }} + CacheDir: $(ORT_CACHE_DIR) + ChangeEveryCommit: true + BuildStep: + - task: Bash@3 + displayName: 'Build Python Wheel' + inputs: + targetType: filePath + filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh + arguments: -i onnxruntimecpubuildpython${{ parameters.arch }} -d "${{ parameters.device }}" -c ${{ parameters.cmake_build_type }} $(extra_build_args) $(python_exe_path) + ${{ if eq(parameters.with_cache, 'true') }}: + env: + ADDITIONAL_DOCKER_PARAMETER: "--volume $(ORT_CACHE_DIR):/cache -e CCACHE_DIR=/cache -e ORT_BUILD_WITH_CACHE=1" + + - ${{ if eq(parameters.is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: - PathtoPublish: '$(Build.BinariesDirectory)/dist' - ArtifactName: onnxruntime-${{ parameters.ep }} - - - task: PublishPipelineArtifact@0 + targetPath: '$(Build.BinariesDirectory)/dist' + artifactName: onnxruntime-${{ parameters.arch }}-${{ parameters.ep }} + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Test Binaries' + inputs: + artifactName: 'drop-linux-cpu-${{ parameters.arch }}-${{ parameters.ep }}' + targetPath: '$(Build.BinariesDirectory)/${{ parameters.cmake_build_type }}' + - ${{ if eq(parameters.is1ES, false) }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + targetPath: '$(Build.BinariesDirectory)/dist' + artifactName: onnxruntime-${{ parameters.arch }}-${{ parameters.ep }} + - task: PublishPipelineArtifact@1 displayName: 'Publish Test Binaries' inputs: artifactName: 'drop-linux-cpu-${{ parameters.arch }}-${{ parameters.ep }}' targetPath: '$(Build.BinariesDirectory)/${{ parameters.cmake_build_type }}' - - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' + + + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml index c475feaef0018..cbbaca95f4e01 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cpu.yml @@ -50,7 +50,7 @@ jobs: artifact: 'drop-linux-cpu-${{ parameters.arch }}-${{parameters.ep}}' - download: current # pipeline resource identifier. - artifact: 'onnxruntime${{ parameters.python_wheel_suffix }}-${{ parameters.ep }}' + artifact: 'onnxruntime-${{ parameters.arch }}-${{ parameters.ep }}' - bash: | set -e -x @@ -64,7 +64,7 @@ jobs: artifact: 'drop-linux-cpu-${{ parameters.arch }}-${{parameters.ep}}' - download: build # pipeline resource identifier. - artifact: 'onnxruntime${{ parameters.python_wheel_suffix }}-${{ parameters.ep }}' + artifact: 'onnxruntime-${{ parameters.arch }}-${{ parameters.ep }}' - bash: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml index 4c9d0dccaf48d..10ea7f6203bb1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64-qnn.yml @@ -19,6 +19,11 @@ parameters: type: string default: '' +- name: is1ES + displayName: 'Whether the pipeline is running in 1ES' + type: boolean + default: false + jobs: - job: Win_py_arm64_qnn_Wheels timeoutInMinutes: 210 @@ -26,6 +31,8 @@ jobs: clean: all pool: name: ${{ parameters.MACHINE_POOL }} + os: windows + hostArchitecture: Arm64 strategy: matrix: Python311_arm64: @@ -41,132 +48,140 @@ jobs: GRADLE_OPTS: '-Dorg.gradle.daemon=false' VSGenerator: 'Visual Studio 17 2022' steps: - - checkout: self - clean: true - submodules: recursive - - - template: telemetry-steps.yml - - - script: | - MKDIR $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64 - XCOPY /s /y /h /e /c /q "$(LocalPythonDir)\*.*" $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64\ - COPY NUL $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64.complete - DIR $(Agent.ToolsDirectory)\Python - DIR $(Agent.ToolsDirectory)\Python\$(PythonVersion) - DIR $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64 - displayName: Copy python $(PythonVersion) version to agent tools directory - - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PythonVersion) - addToPath: true - architecture: 'arm64' - - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: 'Lotus' - - - task: onebranch.pipeline.tsaoptions@1 - displayName: 'OneBranch TSAOptions' - inputs: - tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' - appendSourceBranchName: false - - - task: PythonScript@0 - inputs: - scriptSource: inline - script: | - import subprocess - subprocess.call(['pip', 'install', '-q', 'setuptools', 'wheel']) - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - - template: set-nightly-build-option-variable-step.yml - - - template: jobs/download_win_qnn_sdk.yml - parameters: - QnnSDKVersion: ${{ parameters.QNN_SDK }} - - - task: PythonScript@0 - displayName: 'Generate cmake config' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: > - --config RelWithDebInfo - --build_dir $(Build.BinariesDirectory) - --skip_submodule_sync - --cmake_generator "$(VSGenerator)" - --build_shared_lib - --use_qnn - --qnn_home $(QnnSDKRootDir) - --enable_pybind - --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --update - $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} - workingDirectory: '$(Build.BinariesDirectory)' - - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: 'arm64' - configuration: RelWithDebInfo - msbuildArchitecture: 'arm64' - maximumCpuCount: true - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - createLogFile: true - - # Esrp signing - - template: win-esrp-dll.yml - parameters: - FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' - DisplayName: 'ESRP - Sign Native dlls' - DoEsrp: true - Pattern: '*.pyd' - - - task: PythonScript@0 - displayName: 'Build wheel' - inputs: - scriptPath: '$(Build.SourcesDirectory)\setup.py' - arguments: 'bdist_wheel $(NightlyBuildOption) --wheel_name_suffix=qnn' - workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' - - - task: CopyFiles@2 - displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' - Contents: '*.whl' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime_qnn_arm64 - - - script: | - 7z x *.whl - workingDirectory: '$(Build.ArtifactStagingDirectory)' - displayName: 'unzip the package' - - - task: CredScan@3 - displayName: 'Run CredScan' - inputs: - debugMode: false - continueOnError: true - - - task: BinSkim@4 - displayName: 'Run BinSkim' - inputs: - AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll' - - - task: TSAUpload@2 - displayName: 'TSA upload' - condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) - inputs: - GdnPublishTsaOnboard: false - GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' - - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' + - checkout: self + clean: true + submodules: recursive + + - template: telemetry-steps.yml + + - script: | + MKDIR $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64 + XCOPY /s /y /h /e /c /q "$(LocalPythonDir)\*.*" $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64\ + COPY NUL $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64.complete + DIR $(Agent.ToolsDirectory)\Python + DIR $(Agent.ToolsDirectory)\Python\$(PythonVersion) + DIR $(Agent.ToolsDirectory)\Python\$(PythonVersion)\arm64 + displayName: Copy python $(PythonVersion) version to agent tools directory + + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PythonVersion) + addToPath: true + architecture: 'arm64' + + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'Lotus' + + - task: onebranch.pipeline.tsaoptions@1 + displayName: 'OneBranch TSAOptions' + inputs: + tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' + appendSourceBranchName: false + + - task: PythonScript@0 + inputs: + scriptSource: inline + script: | + import subprocess + subprocess.call(['pip', 'install', '-q', 'setuptools', 'wheel']) + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Install python modules' + + - template: set-nightly-build-option-variable-step.yml + + - template: jobs/download_win_qnn_sdk.yml + parameters: + QnnSDKVersion: ${{ parameters.QNN_SDK }} + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: > + --config RelWithDebInfo + --build_dir $(Build.BinariesDirectory) + --skip_submodule_sync + --cmake_generator "$(VSGenerator)" + --build_shared_lib + --use_qnn + --qnn_home $(QnnSDKRootDir) + --enable_pybind + --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --update + $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} + workingDirectory: '$(Build.BinariesDirectory)' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: 'arm64' + configuration: RelWithDebInfo + msbuildArchitecture: 'arm64' + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + + # Esrp signing + - template: win-esrp-dll.yml + parameters: + FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' + DisplayName: 'ESRP - Sign Native dlls' + DoEsrp: true + Pattern: '*.pyd' + + - task: PythonScript@0 + displayName: 'Build wheel' + inputs: + scriptPath: '$(Build.SourcesDirectory)\setup.py' + arguments: 'bdist_wheel $(NightlyBuildOption) --wheel_name_suffix=qnn' + workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + + - task: CopyFiles@2 + displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' + Contents: '*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - ${{ if eq(parameters.is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_arm64_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ if eq(parameters.is1ES, false) }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + input: + artifactName: onnxruntime_qnn_arm64_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' + + - script: | + 7z x *.whl + workingDirectory: '$(Build.ArtifactStagingDirectory)' + displayName: 'unzip the package' + + - task: CredScan@3 + displayName: 'Run CredScan' + inputs: + debugMode: false + continueOnError: true + + - task: BinSkim@4 + displayName: 'Run BinSkim' + inputs: + AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll' + + - task: TSAUpload@2 + displayName: 'TSA upload' + condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + inputs: + GdnPublishTsaOnboard: false + GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' + + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml index ed29f1e67515e..24321d2a3e1ec 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-arm64ec-qnn.yml @@ -19,6 +19,11 @@ parameters: type: string default: '' +- name: is1ES + displayName: 'Whether the pipeline is running in 1ES' + type: boolean + default: false + jobs: - job: Win_py_x64_qnn_Wheels timeoutInMinutes: 210 @@ -26,6 +31,7 @@ jobs: clean: all pool: name: ${{ parameters.MACHINE_POOL }} + os: windows strategy: matrix: Python310_x64: @@ -40,117 +46,124 @@ jobs: GRADLE_OPTS: '-Dorg.gradle.daemon=false' VSGenerator: 'Visual Studio 17 2022' steps: - - checkout: self - clean: true - submodules: recursive - - - template: telemetry-steps.yml - - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PythonVersion) - addToPath: true - architecture: 'x64' - - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: 'Lotus' - - - task: onebranch.pipeline.tsaoptions@1 - displayName: 'OneBranch TSAOptions' - inputs: - tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' - appendSourceBranchName: fals - - - script: python -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\linux\python\requirements.txt - - - - template: set-nightly-build-option-variable-step.yml - - - template: jobs/download_win_qnn_sdk.yml - parameters: - QnnSDKVersion: ${{ parameters.QNN_SDK }} - - - task: PythonScript@0 - displayName: 'Generate cmake config' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: > - --config RelWithDebInfo - --build_dir $(Build.BinariesDirectory) - --skip_submodule_sync - --cmake_generator "$(VSGenerator)" - --build_shared_lib - --use_qnn - --qnn_home $(QnnSDKRootDir) - --enable_pybind - --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --update --arm64ec - $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} - workingDirectory: '$(Build.BinariesDirectory)' - - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' - platform: 'arm64ec' - configuration: RelWithDebInfo - msbuildArchitecture: 'x64' - maximumCpuCount: true - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - createLogFile: true - - # Esrp signing - - template: win-esrp-dll.yml - parameters: - FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' - DisplayName: 'ESRP - Sign Native dlls' - DoEsrp: true - Pattern: '*.pyd' - - - task: PythonScript@0 - displayName: 'Build wheel' - inputs: - scriptPath: '$(Build.SourcesDirectory)\setup.py' - arguments: 'bdist_wheel $(NightlyBuildOption) --wheel_name_suffix=qnn' - workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' - - - task: CopyFiles@2 - displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' - Contents: '*.whl' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime_qnn_arm64ec - - - script: | - 7z x *.whl - workingDirectory: '$(Build.ArtifactStagingDirectory)' - displayName: 'unzip the package' - - - task: CredScan@3 - displayName: 'Run CredScan' - inputs: - debugMode: false - continueOnError: true - - - task: BinSkim@4 - displayName: 'Run BinSkim' - inputs: - AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll' - - - task: TSAUpload@2 - displayName: 'TSA upload' - condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) - inputs: - GdnPublishTsaOnboard: false - GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' - - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' + - checkout: self + clean: true + submodules: recursive + + - template: telemetry-steps.yml + + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PythonVersion) + addToPath: true + architecture: 'x64' + + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: 'Lotus' + + - task: onebranch.pipeline.tsaoptions@1 + displayName: 'OneBranch TSAOptions' + inputs: + tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' + appendSourceBranchName: fals + + - script: python -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\linux\python\requirements.txt + + + - template: set-nightly-build-option-variable-step.yml + + - template: jobs/download_win_qnn_sdk.yml + parameters: + QnnSDKVersion: ${{ parameters.QNN_SDK }} + + - task: PythonScript@0 + displayName: 'Generate cmake config' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: > + --config RelWithDebInfo + --build_dir $(Build.BinariesDirectory) + --skip_submodule_sync + --cmake_generator "$(VSGenerator)" + --build_shared_lib + --use_qnn + --qnn_home $(QnnSDKRootDir) + --enable_pybind + --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --update --arm64ec + $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} + workingDirectory: '$(Build.BinariesDirectory)' + + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: 'arm64ec' + configuration: RelWithDebInfo + msbuildArchitecture: 'x64' + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + + # Esrp signing + - template: win-esrp-dll.yml + parameters: + FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' + DisplayName: 'ESRP - Sign Native dlls' + DoEsrp: true + Pattern: '*.pyd' + + - task: PythonScript@0 + displayName: 'Build wheel' + inputs: + scriptPath: '$(Build.SourcesDirectory)\setup.py' + arguments: 'bdist_wheel $(NightlyBuildOption) --wheel_name_suffix=qnn' + workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + + - task: CopyFiles@2 + displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' + Contents: '*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - ${{ if eq(parameters.is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_arm64ec_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ if eq(parameters.is1ES, false) }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_arm64ec_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' + - script: | + 7z x *.whl + workingDirectory: '$(Build.ArtifactStagingDirectory)' + displayName: 'unzip the package' + + - task: CredScan@3 + displayName: 'Run CredScan' + inputs: + debugMode: false + continueOnError: true + + - task: BinSkim@4 + displayName: 'Run BinSkim' + inputs: + AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll' + + - task: TSAUpload@2 + displayName: 'TSA upload' + condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + inputs: + GdnPublishTsaOnboard: false + GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' + + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml index 13069846da342..175b343e55d57 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-x64-qnn.yml @@ -19,6 +19,11 @@ parameters: type: string default: '' +- name: is1ES + displayName: 'Whether the pipeline is running in 1ES' + type: boolean + default: false + jobs: - job: Win_py_x64_qnn_Wheels timeoutInMinutes: 210 @@ -116,10 +121,18 @@ jobs: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime_qnn_x64 + - ${{ if eq(parameters.is1ES, true) }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_x64_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' + - ${{ if eq(parameters.is1ES, false) }}: + - task: PublishPipelineArtifact@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + artifactName: onnxruntime_qnn_x64_$(PythonVersion) + targetPath: '$(Build.ArtifactStagingDirectory)' - script: | 7z x *.whl