From 2a6cee12ca44718186091e3313f64b9bb0d47467 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 8 Oct 2024 21:46:46 -0700 Subject: [PATCH 01/40] update --- .../azure-pipelines/py-cuda-package-test-pipeline.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml index 79b69bf34cbe..085a9c6d65c7 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml @@ -12,12 +12,13 @@ stages: - stage: Linux_Test_GPU_x86_64_stage dependsOn: jobs: - - template: stages/jobs/py-linux-cuda-package-test-job.yml + - template: templates/py-packaging-linux-test-cuda.yml parameters: - CudaVersion: '12.2' + arch: 'x86_64' machine_pool: 'Onnxruntime-Linux-GPU' + python_wheel_suffix: '_gpu' timeout: 480 - build_id: ${{ parameters.build_id }} - project: ${{ parameters.project }} - pipeline: ${{ parameters.pipeline }} + docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20240719.1 + trt_version: '10.4.0.26-1.cuda12.6' + cuda_version: '12.2' From 48e3e56910a96fb5c52aacce9f3a10eb60344def Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 9 Oct 2024 17:11:24 +0000 Subject: [PATCH 02/40] update --- .../github/azure-pipelines/py-cuda-package-test-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml index 085a9c6d65c7..062b88a326dc 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml @@ -18,7 +18,7 @@ stages: machine_pool: 'Onnxruntime-Linux-GPU' python_wheel_suffix: '_gpu' timeout: 480 - docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20240719.1 + docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241008.1 trt_version: '10.4.0.26-1.cuda12.6' cuda_version: '12.2' From f91109f3d9a893b4694bf596aa9be33ddf84aad1 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 9 Oct 2024 17:41:30 +0000 Subject: [PATCH 03/40] update --- .../azure-pipelines/templates/py-packaging-linux-test-cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cuda.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cuda.yml index 0c3cd60a712f..4ca462bf962f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cuda.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cuda.yml @@ -100,7 +100,7 @@ jobs: inputs: targetType: filePath filePath: tools/ci_build/github/linux/run_python_dockertest.sh - arguments: -d GPU -c ${{parameters.cmake_build_type}} -i onnxruntimecuda118xtrt86build${{ parameters.arch }} + arguments: -d GPU -c ${{parameters.cmake_build_type}} -i onnxruntimecuda${{ replace(parameters.cuda_version, '.', '') }}xtrt86build${{ parameters.arch }} - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories' From e16e15c99bbf367a9512386cdc3b7c3378edbd52 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 9 Oct 2024 20:24:20 -0700 Subject: [PATCH 04/40] a --- .../templates/mac-cpu-packing-jobs.yml | 2 +- .../templates/py-package-smoking-test.yml | 2 ++ .../templates/py-packaging-stage.yml | 24 +++++++++++++++++++ .../azure-pipelines/templates/py-win-gpu.yml | 11 ++++----- .../linux/build_linux_python_package.sh | 2 +- .../python/cpu/scripts/install_deps.sh | 4 ++-- .../x86_64/python/cpu/scripts/install_deps.sh | 4 ++-- .../python/cuda/scripts/install_deps.sh | 4 ++-- .../docker/scripts/install_python_deps.sh | 2 +- .../docker/scripts/manylinux/install_deps.sh | 4 ++-- .../scripts/manylinux/install_deps_aten.sh | 4 ++-- .../scripts/manylinux/install_deps_eager.sh | 4 ++-- .../scripts/manylinux/install_deps_lort.sh | 2 +- 13 files changed, 47 insertions(+), 22 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml index 3b661d9eb2dc..b057a0aac66c 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml @@ -86,7 +86,7 @@ jobs: popd export PATH=$(Build.BinariesDirectory)/installed/bin:$PATH export ONNX_ML=1 - export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" + export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' displayName: 'Install dependencies' env: diff --git a/tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml b/tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml index ff54dc647447..10a0354979ae 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-package-smoking-test.yml @@ -28,6 +28,8 @@ jobs: PythonVersion: '3.11' Python312: PythonVersion: '3.12' + Python313: + PythonVersion: '3.13' steps: - checkout: none diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index edaae227ee78..afbbcd2856ac 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -96,6 +96,10 @@ stages: PythonVersion: '3.12' MsbuildPlatform: x64 buildArch: x64 + Python313_x64: + PythonVersion: '3.13' + MsbuildPlatform: x64 + buildArch: x64 # Training build cannot support Win32 for now because one or more of its python # dependencies does not support Win32. So, don't build a training package for Win32 ${{ if not(contains(parameters.build_py_parameters, '--enable_training')) }}: @@ -308,6 +312,16 @@ stages: publish_symbols: ${{ parameters.publish_symbols }} cmake_build_type: ${{ parameters.cmake_build_type }} + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-A10' + PYTHON_VERSION: '3.13' + EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" + ENV_SETUP_SCRIPT: setup_env_gpu.bat + EP_NAME: gpu + publish_symbols: ${{ parameters.publish_symbols }} + cmake_build_type: ${{ parameters.cmake_build_type }} + - template: py-win-gpu.yml parameters: MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' @@ -338,6 +352,16 @@ stages: publish_symbols: ${{ parameters.publish_symbols }} cmake_build_type: ${{ parameters.cmake_build_type }} + - template: py-win-gpu.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' + PYTHON_VERSION: '3.13' + EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos + ENV_SETUP_SCRIPT: setup_env.bat + EP_NAME: directml + publish_symbols: ${{ parameters.publish_symbols }} + cmake_build_type: ${{ parameters.cmake_build_type }} + - ${{ if eq(parameters.enable_mac_cpu, true) }}: - stage: Python_Packaging_MacOS dependsOn: [] diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml index e89227d51de3..656912f0a09e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml @@ -251,13 +251,12 @@ stages: SpecificArtifact: ${{ parameters.SpecificArtifact }} BuildId: ${{ parameters.BuildId }} - - powershell: | - pushd onnxruntime/test/python - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - popd - workingDirectory: '$(Build.SourcesDirectory)' + - task: PowerShell@2 displayName: 'Install ONNX' + inputs: + filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1' + workingDirectory: '$(Build.BinariesDirectory)' + arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }} - powershell: | python -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index 568d9a74d75d..43a28b02cb4b 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -9,7 +9,7 @@ EXTRA_ARG="" # Put 3.10 at the last because Ubuntu 22.04 use python 3.10 and we will upload the intermediate build files of this # config to Azure DevOps Artifacts and download them to a Ubuntu 22.04 machine to run the tests. -PYTHON_EXES=("/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp310-cp310/bin/python3.10") +PYTHON_EXES=("/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp310-cp310/bin/python3.10") while getopts "d:p:x:c:" parameter_Option do case "${parameter_Option}" in diff --git a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh index f576b867da73..e612ea26afc0 100755 --- a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh @@ -1,14 +1,14 @@ #!/bin/bash set -e -x pushd . -PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12") +PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13") CURRENT_DIR=$(pwd) if ! [ -x "$(command -v protoc)" ]; then $CURRENT_DIR/install_protobuf.sh fi popd export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh index f576b867da73..e612ea26afc0 100755 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh @@ -1,14 +1,14 @@ #!/bin/bash set -e -x pushd . -PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12") +PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13") CURRENT_DIR=$(pwd) if ! [ -x "$(command -v protoc)" ]; then $CURRENT_DIR/install_protobuf.sh fi popd export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh index f576b867da73..e612ea26afc0 100755 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh @@ -1,14 +1,14 @@ #!/bin/bash set -e -x pushd . -PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12") +PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13") CURRENT_DIR=$(pwd) if ! [ -x "$(command -v protoc)" ]; then $CURRENT_DIR/install_protobuf.sh fi popd export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do diff --git a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh index 1ac1d226deec..7f13c07de2ea 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh @@ -53,7 +53,7 @@ else fi export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" ${PYTHON_EXE} -m pip install -r ${0/%install_python_deps\.sh/requirements\.txt} if [ $DEVICE_TYPE = "gpu" ]; then if [[ $INSTALL_DEPS_TRAINING = true ]]; then diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh index 5b181a484a60..bb47ec5eb337 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh @@ -19,7 +19,7 @@ PARENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &> /dev/null && pwd)" source "$PARENT_DIR/install_dotnet.sh" if [ ! -d "/opt/conda/bin" ]; then - PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12") + PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13") else PYTHON_EXES=("/opt/conda/bin/python") fi @@ -38,7 +38,7 @@ if ! [ -x "$(command -v protoc)" ]; then fi export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh index 0d1b7049df7e..0e7852b12d69 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh @@ -6,7 +6,7 @@ dnf -y install \ graphviz if [ ! -d "/opt/conda/bin" ]; then - PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11") + PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp313-cp313/bin/python3.13") else PYTHON_EXES=("/opt/conda/bin/python") fi @@ -33,7 +33,7 @@ if ! [ -x "$(command -v protoc)" ]; then fi export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh index d8d2fbc06a00..9cf142febfe9 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh @@ -6,7 +6,7 @@ yum -y install \ graphviz if [ ! -d "/opt/conda/bin" ]; then - PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12") + PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13") else PYTHON_EXES=("/opt/conda/bin/python") fi @@ -35,7 +35,7 @@ if ! [ -x "$(command -v protoc)" ]; then fi export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh index 06a117098d3f..8dbccd4582f2 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh @@ -17,7 +17,7 @@ if ! [ -x "$(command -v protoc)" ]; then fi export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" PYTHON_EXE=/usr/bin/python3.9 echo "Installing Pytorch requirements" From 5156eccd50356387b8d3be58f90cf506635f9128 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 10 Oct 2024 03:55:21 +0000 Subject: [PATCH 05/40] revert --- .../github/linux/docker/scripts/install_python_deps.sh | 2 +- .../github/linux/docker/scripts/manylinux/install_deps.sh | 4 ++-- .../linux/docker/scripts/manylinux/install_deps_aten.sh | 4 ++-- .../linux/docker/scripts/manylinux/install_deps_eager.sh | 4 ++-- .../linux/docker/scripts/manylinux/install_deps_lort.sh | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh index 7f13c07de2ea..1ac1d226deec 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh @@ -53,7 +53,7 @@ else fi export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" ${PYTHON_EXE} -m pip install -r ${0/%install_python_deps\.sh/requirements\.txt} if [ $DEVICE_TYPE = "gpu" ]; then if [[ $INSTALL_DEPS_TRAINING = true ]]; then diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh index bb47ec5eb337..5b181a484a60 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh @@ -19,7 +19,7 @@ PARENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &> /dev/null && pwd)" source "$PARENT_DIR/install_dotnet.sh" if [ ! -d "/opt/conda/bin" ]; then - PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13") + PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12") else PYTHON_EXES=("/opt/conda/bin/python") fi @@ -38,7 +38,7 @@ if ! [ -x "$(command -v protoc)" ]; then fi export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh index 0e7852b12d69..0d1b7049df7e 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh @@ -6,7 +6,7 @@ dnf -y install \ graphviz if [ ! -d "/opt/conda/bin" ]; then - PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp313-cp313/bin/python3.13") + PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11") else PYTHON_EXES=("/opt/conda/bin/python") fi @@ -33,7 +33,7 @@ if ! [ -x "$(command -v protoc)" ]; then fi export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh index 9cf142febfe9..d8d2fbc06a00 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh @@ -6,7 +6,7 @@ yum -y install \ graphviz if [ ! -d "/opt/conda/bin" ]; then - PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13") + PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12") else PYTHON_EXES=("/opt/conda/bin/python") fi @@ -35,7 +35,7 @@ if ! [ -x "$(command -v protoc)" ]; then fi export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh index 8dbccd4582f2..06a117098d3f 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh @@ -17,7 +17,7 @@ if ! [ -x "$(command -v protoc)" ]; then fi export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" PYTHON_EXE=/usr/bin/python3.9 echo "Installing Pytorch requirements" From 1c0797ab261a6e820bad202e6408727b8c3748e1 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 10 Oct 2024 04:19:36 +0000 Subject: [PATCH 06/40] update --- .../github/azure-pipelines/templates/py-packaging-stage.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index afbbcd2856ac..d39a6c57d989 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -382,6 +382,9 @@ stages: PythonVersion: '3.11' Python312: PythonVersion: '3.12' + Python313: + PythonVersion: '3.13' + steps: - checkout: self clean: true From bf585a1959ea6080a5dc552f45efa1e2c7768c9c Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 10 Oct 2024 14:44:48 +0000 Subject: [PATCH 07/40] update --- tools/ci_build/github/android/build_aar_and_copy_artifacts.sh | 2 +- .../github/azure-pipelines/orttraining-linux-ci-pipeline.yml | 2 +- .../templates/set-python-manylinux-variables-step.yml | 4 ++++ tools/ci_build/github/linux/build_cuda_ci.sh | 4 ++-- tools/ci_build/github/linux/build_tensorrt_ci.sh | 4 ++-- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh b/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh index 88fb578c591b..77c0c56b75fe 100755 --- a/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh +++ b/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh @@ -6,7 +6,7 @@ set -e set -x -export PATH=/opt/python/cp38-cp38/bin:$PATH +export PATH=/opt/python/cp312-cp312/bin:$PATH ls /build ls /build/deps diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index 04462a60776d..ffe3aefaea21 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -91,7 +91,7 @@ jobs: /bin/bash -c " set -ex; \ ccache -s; \ - /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + /opt/python/cp312-cp312/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator 'Unix Makefiles' \ --config Release \ --skip_submodule_sync \ diff --git a/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml b/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml index 68836117db81..585a78c17e04 100644 --- a/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml +++ b/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml @@ -35,6 +35,10 @@ steps: variables = { "PythonManylinuxDir": "/opt/python/cp312-cp312" } + elif version == "3.13": + variables = { + "PythonManylinuxDir": "/opt/python/cp313-cp313" + } else: raise ValueError("Unsupported Python version: '{}'".format(version)) diff --git a/tools/ci_build/github/linux/build_cuda_ci.sh b/tools/ci_build/github/linux/build_cuda_ci.sh index c8691b3a01e7..6b155da02030 100755 --- a/tools/ci_build/github/linux/build_cuda_ci.sh +++ b/tools/ci_build/github/linux/build_cuda_ci.sh @@ -35,8 +35,8 @@ if [ -x "$(command -v ccache)" ]; then ccache -s; BUILD_ARGS+=("--use_cache") fi -if [ -f /opt/python/cp38-cp38/bin/python3 ]; then - /opt/python/cp38-cp38/bin/python3 tools/ci_build/build.py "${BUILD_ARGS[@]}" +if [ -f /opt/python/cp312-cp312/bin/python3 ]; then + /opt/python/cp312-cp312/bin/python3 tools/ci_build/build.py "${BUILD_ARGS[@]}" else python3 tools/ci_build/build.py "${BUILD_ARGS[@]}" fi diff --git a/tools/ci_build/github/linux/build_tensorrt_ci.sh b/tools/ci_build/github/linux/build_tensorrt_ci.sh index 3002f2c239f1..5b206bc0a92d 100755 --- a/tools/ci_build/github/linux/build_tensorrt_ci.sh +++ b/tools/ci_build/github/linux/build_tensorrt_ci.sh @@ -35,8 +35,8 @@ if [ -x "$(command -v ccache)" ]; then ccache -s; BUILD_ARGS+=("--use_cache") fi -if [ -f /opt/python/cp38-cp38/bin/python3 ]; then - /opt/python/cp38-cp38/bin/python3 tools/ci_build/build.py "${BUILD_ARGS[@]}" +if [ -f /opt/python/cp312-cp312/bin/python3 ]; then + /opt/python/cp312-cp312/bin/python3 tools/ci_build/build.py "${BUILD_ARGS[@]}" else python3 tools/ci_build/build.py "${BUILD_ARGS[@]}" fi From 3bff877fc79d5e7134aa779d327f645d2414e4f8 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 10 Oct 2024 15:47:54 +0000 Subject: [PATCH 08/40] update --- .../azure-pipelines/linux-gpu-ci-pipeline.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 84e953366f9f..8c42afacce49 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -175,13 +175,14 @@ stages: -e NVIDIA_TF32_OVERRIDE=0 \ $(Repository) \ /bin/bash -c ' + set -e nvidia-smi; \ /sbin/ldconfig -N -v $(sed "s/:/ /" <<< $LD_LIBRARY_PATH) 2>/dev/null | grep -E "libcudart.so|libcudnn.so|libnvinfer.so"; \ cat /usr/local/cuda/include/cuda.h | grep -m1 CUDA_VERSION; \ cat /usr/include/cudnn_version.h | grep CUDNN_MAJOR -m1 -A 2; \ - ln -s /opt/python/cp38-cp38/bin/python3 /tmp/python3; \ - /tmp/python3 -m pip install /build/Release/dist/*.whl; \ - /tmp/python3 -u -c "from onnxruntime.capi._pybind_state import (OrtDevice as C_OrtDevice) ; \ + export PATH=/opt/python/cp312-cp312/bin:$PATH \ + python3 -m pip install /build/Release/dist/*.whl; \ + python3 -u -c "from onnxruntime.capi._pybind_state import (OrtDevice as C_OrtDevice) ; \ ort_device = C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0); \ print(ort_device); print(ort_device.device_type(), C_OrtDevice.cuda()); \ assert(ort_device.device_type()==1); assert(C_OrtDevice.cuda()==1);" \ @@ -204,13 +205,13 @@ stages: /bin/bash -c ' set -ex; \ cp /onnxruntime_src/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt /tmp/requirements.txt; \ - ln -s /opt/python/cp38-cp38/bin/python3 /tmp/python3; \ - /tmp/python3 -m pip install -r /tmp/requirements.txt; \ - /tmp/python3 -m pip install /build/Release/dist/*.whl; \ + export PATH=/opt/python/cp312-cp312/bin:$PATH \ + python3 -m pip install -r /tmp/requirements.txt; \ + python3 -m pip install /build/Release/dist/*.whl; \ cd /build/Release && xargs -a /build/Release/perms.txt chmod a+x; \ cd /onnxruntime_src/java && /onnxruntime_src/java/gradlew cmakeCheck -DcmakeBuildDir=/build/Release -DUSE_CUDA=1; \ cd /tmp; \ - /tmp/python3 /onnxruntime_src/tools/ci_build/build.py \ + python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests \ --enable_transformers_tool_test --use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda --cudnn_home=/usr/local/cuda \ --enable_pybind --build_java --ctest_path "" ; \ From cd4ed8ce645be25134a59e35437d5d18df3a7898 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 10 Oct 2024 15:54:21 +0000 Subject: [PATCH 09/40] update --- .../docker/inference/aarch64/python/cpu/scripts/install_deps.sh | 2 +- .../docker/inference/x86_64/python/cpu/scripts/install_deps.sh | 2 +- .../docker/inference/x86_64/python/cuda/scripts/install_deps.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh index e612ea26afc0..81de2abf3ff8 100755 --- a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/install_deps.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -x pushd . -PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13") +PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp313-cp313t/bin/python3.13") CURRENT_DIR=$(pwd) if ! [ -x "$(command -v protoc)" ]; then $CURRENT_DIR/install_protobuf.sh diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh index e612ea26afc0..81de2abf3ff8 100755 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -x pushd . -PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13") +PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp313-cp313t/bin/python3.13") CURRENT_DIR=$(pwd) if ! [ -x "$(command -v protoc)" ]; then $CURRENT_DIR/install_protobuf.sh diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh index e612ea26afc0..81de2abf3ff8 100755 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -x pushd . -PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13") +PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp313-cp313t/bin/python3.13") CURRENT_DIR=$(pwd) if ! [ -x "$(command -v protoc)" ]; then $CURRENT_DIR/install_protobuf.sh From 6a379d38acc23922322f5f733d7886a9e43e6198 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 10 Oct 2024 16:16:57 +0000 Subject: [PATCH 10/40] update --- .../inference/aarch64/python/cpu/scripts/requirements.txt | 2 +- .../docker/inference/x86_64/python/cpu/scripts/requirements.txt | 2 +- .../inference/x86_64/python/cuda/scripts/requirements.txt | 2 +- .../ci_build/github/linux/docker/scripts/lort/requirements.txt | 2 +- .../github/linux/docker/scripts/manylinux/requirements.txt | 2 +- tools/ci_build/github/linux/docker/scripts/requirements.txt | 2 +- .../ortmodule/stage1/requirements_rocm/requirements.txt | 2 +- .../training/ortmodule/stage1/torch_eager_cpu/requirements.txt | 2 +- .../docker/scripts/training/ortmodule/stage2/requirements.txt | 2 +- tools/ci_build/github/windows/eager/requirements.txt | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/requirements.txt b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/requirements.txt index a977ccae1922..07a9f3f481aa 100644 --- a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/scripts/requirements.txt @@ -1,5 +1,5 @@ numpy==1.21.6 ; python_version < '3.9' -numpy==2.0.0 ; python_version >= '3.9' +numpy==2.1.2 ; python_version >= '3.9' mypy pytest setuptools>=68.2.2 diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/requirements.txt b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/requirements.txt index f065ba03243f..ee07c7ef4f52 100644 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/requirements.txt @@ -1,5 +1,5 @@ numpy==1.21.6 ; python_version < '3.9' -numpy==2.0.0 ; python_version >= '3.9' +numpy==2.1.2 ; python_version >= '3.9' mypy pytest setuptools>=68.2.2 diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/requirements.txt b/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/requirements.txt index a977ccae1922..07a9f3f481aa 100644 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/requirements.txt @@ -1,5 +1,5 @@ numpy==1.21.6 ; python_version < '3.9' -numpy==2.0.0 ; python_version >= '3.9' +numpy==2.1.2 ; python_version >= '3.9' mypy pytest setuptools>=68.2.2 diff --git a/tools/ci_build/github/linux/docker/scripts/lort/requirements.txt b/tools/ci_build/github/linux/docker/scripts/lort/requirements.txt index d76a4337e748..ac47d1668ec2 100644 --- a/tools/ci_build/github/linux/docker/scripts/lort/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/lort/requirements.txt @@ -9,7 +9,7 @@ astunparse expecttest!=0.2.0 hypothesis numpy==1.21.6 ; python_version < '3.9' -numpy==2.0.0 ; python_version >= '3.9' +numpy==2.1.2 ; python_version >= '3.9' psutil pyyaml requests diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt b/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt index 12db3bd132bb..d6045c612580 100644 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt @@ -1,5 +1,5 @@ numpy==1.21.6 ; python_version < '3.9' -numpy==2.0.0 ; python_version >= '3.9' +numpy==2.1.2 ; python_version >= '3.9' mypy pytest setuptools>=68.2.2 diff --git a/tools/ci_build/github/linux/docker/scripts/requirements.txt b/tools/ci_build/github/linux/docker/scripts/requirements.txt index 36af6aa71b07..a64ca80665a6 100644 --- a/tools/ci_build/github/linux/docker/scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/requirements.txt @@ -1,6 +1,6 @@ cerberus numpy==1.24.4 ; python_version < '3.9' -numpy==2.0.0; python_version >= '3.9' +numpy==2.1.2; python_version >= '3.9' mypy pytest setuptools==69.0.3 diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_rocm/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_rocm/requirements.txt index 89bda11737d1..051f42dac335 100644 --- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_rocm/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_rocm/requirements.txt @@ -1,2 +1,2 @@ numpy==1.21.6 ; python_version < '3.9' -numpy==2.0.0 ; python_version >= '3.9' +numpy==2.1.2 ; python_version >= '3.9' diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/torch_eager_cpu/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/torch_eager_cpu/requirements.txt index 6858d99dc26a..6346c54decf9 100644 --- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/torch_eager_cpu/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/torch_eager_cpu/requirements.txt @@ -6,6 +6,6 @@ cerberus h5py scikit-learn numpy==1.21.6 ; python_version < '3.9' -numpy==2.0.0 ; python_version >= '3.9' +numpy==2.1.2 ; python_version >= '3.9' pandas parameterized diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage2/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage2/requirements.txt index 3b13a51f18e2..dd86b32f88c7 100644 --- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage2/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage2/requirements.txt @@ -1,7 +1,7 @@ pandas scikit-learn numpy==1.21.6 ; python_version < '3.9' -numpy==2.0.0 ; python_version >= '3.9' +numpy==2.1.2 ; python_version >= '3.9' transformers==v4.36.0 accelerate==0.25.0 rsa==4.9 diff --git a/tools/ci_build/github/windows/eager/requirements.txt b/tools/ci_build/github/windows/eager/requirements.txt index b285defd89f5..d118280c8e6d 100644 --- a/tools/ci_build/github/windows/eager/requirements.txt +++ b/tools/ci_build/github/windows/eager/requirements.txt @@ -1,7 +1,7 @@ setuptools wheel numpy==1.21.6 ; python_version < '3.9' -numpy==2.0.0 ; python_version >= '3.9' +numpy==2.1.2 ; python_version >= '3.9' typing_extensions torch==2.2.0 parameterized From db67ef1836ccd61a40a6bcb2efbaa697af46f772 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 10 Oct 2024 17:06:03 +0000 Subject: [PATCH 11/40] revert --- .../github/linux/docker/scripts/manylinux/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt b/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt index d6045c612580..12db3bd132bb 100644 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt @@ -1,5 +1,5 @@ numpy==1.21.6 ; python_version < '3.9' -numpy==2.1.2 ; python_version >= '3.9' +numpy==2.0.0 ; python_version >= '3.9' mypy pytest setuptools>=68.2.2 From 653aaf7a71c0a8be1a55a872671672850dfd6917 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 10 Oct 2024 12:09:26 -0700 Subject: [PATCH 12/40] update --- tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 8c42afacce49..92a05378d6c9 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -180,7 +180,7 @@ stages: /sbin/ldconfig -N -v $(sed "s/:/ /" <<< $LD_LIBRARY_PATH) 2>/dev/null | grep -E "libcudart.so|libcudnn.so|libnvinfer.so"; \ cat /usr/local/cuda/include/cuda.h | grep -m1 CUDA_VERSION; \ cat /usr/include/cudnn_version.h | grep CUDNN_MAJOR -m1 -A 2; \ - export PATH=/opt/python/cp312-cp312/bin:$PATH \ + export PATH=/opt/python/cp312-cp312/bin:$PATH; \ python3 -m pip install /build/Release/dist/*.whl; \ python3 -u -c "from onnxruntime.capi._pybind_state import (OrtDevice as C_OrtDevice) ; \ ort_device = C_OrtDevice(C_OrtDevice.cuda(), C_OrtDevice.default_memory(), 0); \ From 5705319a8f92ae46e317d5b20b52220446fb4e5e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 10 Oct 2024 13:12:57 -0700 Subject: [PATCH 13/40] update --- .../ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml | 2 +- .../github/azure-pipelines/stages/py-cuda-packaging-stage.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 92a05378d6c9..295549b647dc 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -205,7 +205,7 @@ stages: /bin/bash -c ' set -ex; \ cp /onnxruntime_src/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt /tmp/requirements.txt; \ - export PATH=/opt/python/cp312-cp312/bin:$PATH \ + export PATH=/opt/python/cp312-cp312/bin:$PATH; \ python3 -m pip install -r /tmp/requirements.txt; \ python3 -m pip install /build/Release/dist/*.whl; \ cd /build/Release && xargs -a /build/Release/perms.txt chmod a+x; \ diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml index f9f90b43f0cf..896843cadbbd 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml @@ -48,11 +48,10 @@ parameters: type: object displayName: 'Python versions to build' default: - - '3.8' - - '3.9' - '3.10' - '3.11' - '3.12' + - '3.13' stages: - ${{ if eq(parameters.enable_windows_gpu, true) }}: From da15f9e433afd34f30475835fb16f45b5700b529 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 01:18:03 +0000 Subject: [PATCH 14/40] update --- .../linux/build_linux_python_package.sh | 28 ++++- .../inference/x86_64/python/cpu/Dockerfile | 2 +- .../x86_64/python/cpu/scripts/install_deps.sh | 36 ------ .../python/cpu/scripts/install_protobuf.sh | 108 ------------------ .../python/cpu/scripts/requirements.txt | 11 -- .../inference/x86_64/python/cuda/Dockerfile | 2 +- .../python/cuda/scripts/install_deps.sh | 36 ------ .../python/cuda/scripts/install_protobuf.sh | 108 ------------------ .../cuda/scripts => python}/requirements.txt | 3 +- 9 files changed, 27 insertions(+), 307 deletions(-) delete mode 100755 tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh delete mode 100755 tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_protobuf.sh delete mode 100644 tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/requirements.txt delete mode 100755 tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh delete mode 100755 tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_protobuf.sh rename tools/ci_build/github/linux/{docker/inference/x86_64/python/cuda/scripts => python}/requirements.txt (54%) diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index 43a28b02cb4b..6f8405a07bd9 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -26,10 +26,6 @@ done BUILD_ARGS=("--build_dir" "/build" "--config" "$BUILD_CONFIG" "--update" "--build" "--skip_submodule_sync" "--parallel" "--use_binskim_compliant_compile_flags" "--build_wheel") -if [[ "$EXTRA_ARG" == *"training"* ]]; then - echo "Skip building unit tests because the container is a manylinux docker" - BUILD_ARGS+=("--cmake_extra_defines" "onnxruntime_BUILD_UNIT_TESTS=OFF") -fi if [ "$BUILD_CONFIG" != "Debug" ]; then BUILD_ARGS+=("--enable_lto") @@ -37,6 +33,26 @@ fi ARCH=$(uname -m) +# No release binary for ccache aarch64, so we need to build it from source. +if ! [ -x "$(command -v ccache)" ]; then + ccache_url="https://github.com/ccache/ccache/archive/refs/tags/v4.8.tar.gz" + pushd . + curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o ccache_src.tar.gz $ccache_url + mkdir ccache_main + cd ccache_main + tar -zxf ../ccache_src.tar.gz --strip=1 + + mkdir build + cd build + cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release .. + make + make install + which ccache + popd + rm -f ccache_src.tar.gz + rm -rf ccache_src +fi + echo "EXTRA_ARG:" echo "$EXTRA_ARG" @@ -60,9 +76,13 @@ if [ "$BUILD_DEVICE" == "NPU" ]; then BUILD_ARGS+=("--use_qnn" "--qnn_home=/qnn_sdk") fi +export ONNX_ML=1 +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" + for PYTHON_EXE in "${PYTHON_EXES[@]}" do rm -rf /build/"$BUILD_CONFIG" + ${PYTHON_EXE} -m pip install -r /onnxruntime_src/tools/ci_build/github/linux/python/requirements.txt ${PYTHON_EXE} /onnxruntime_src/tools/ci_build/build.py "${BUILD_ARGS[@]}" cp /build/"$BUILD_CONFIG"/dist/*.whl /build/dist diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile index 4242c29aa2c7..1fd91f693f51 100644 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile @@ -1,7 +1,7 @@ FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20241008.1 ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && rm -rf /tmp/scripts ARG BUILD_UID=1001 ARG BUILD_USER=onnxruntimedev diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh deleted file mode 100755 index 81de2abf3ff8..000000000000 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_deps.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -set -e -x -pushd . -PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp313-cp313t/bin/python3.13") -CURRENT_DIR=$(pwd) -if ! [ -x "$(command -v protoc)" ]; then - $CURRENT_DIR/install_protobuf.sh -fi -popd -export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" - -for PYTHON_EXE in "${PYTHON_EXES[@]}" -do - ${PYTHON_EXE} -m pip install -r requirements.txt -done - -# No release binary for ccache aarch64, so we need to build it from source. -if ! [ -x "$(command -v ccache)" ]; then - ccache_url="https://github.com/ccache/ccache/archive/refs/tags/v4.8.tar.gz" - pushd . - curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o ccache_src.tar.gz $ccache_url - mkdir ccache_main - cd ccache_main - tar -zxf ../ccache_src.tar.gz --strip=1 - - mkdir build - cd build - cmake -DCMAKE_INSTALL_PREFIX=/usr/local _DCMAKE_BUILD_TYPE=Release .. - make - make install - which ccache - popd - rm -f ccache_src.tar.gz - rm -rf ccache_src -fi diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_protobuf.sh b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_protobuf.sh deleted file mode 100755 index 31b5ca6f9e69..000000000000 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_protobuf.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -set -e -x - -INSTALL_PREFIX='/usr' -DEP_FILE_PATH='/tmp/scripts/deps.txt' -while getopts "p:d:" parameter_Option -do case "${parameter_Option}" -in -p) INSTALL_PREFIX=${OPTARG};; -d) DEP_FILE_PATH=${OPTARG};; -esac -done - - - -EXTRA_CMAKE_ARGS="-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_CXX_STANDARD=17" - -case "$(uname -s)" in - Darwin*) - echo 'Building ONNX Runtime on Mac OS X' - EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DCMAKE_OSX_ARCHITECTURES=x86_64;arm64" - GCC_PATH=$(which clang) - GPLUSPLUS_PATH=$(which clang++) - ;; - Linux*) - SYS_LONG_BIT=$(getconf LONG_BIT) - DISTRIBUTOR=$(lsb_release -i -s) - - if [[ ("$DISTRIBUTOR" = "CentOS" || "$DISTRIBUTOR" = "RedHatEnterprise") && $SYS_LONG_BIT = "64" ]]; then - LIBDIR="lib64" - else - LIBDIR="lib" - fi - EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=$LIBDIR" - # Depending on how the compiler has been configured when it was built, sometimes "gcc -dumpversion" shows the full version. - GCC_VERSION=$(gcc -dumpversion | cut -d . -f 1) - #-fstack-clash-protection prevents attacks based on an overlapping heap and stack. - if [ "$GCC_VERSION" -ge 8 ]; then - CFLAGS="$CFLAGS -fstack-clash-protection" - CXXFLAGS="$CXXFLAGS -fstack-clash-protection" - fi - ARCH=$(uname -m) - GCC_PATH=$(which gcc) - GPLUSPLUS_PATH=$(which g++) - if [ "$ARCH" == "x86_64" ] && [ "$GCC_VERSION" -ge 9 ]; then - CFLAGS="$CFLAGS -fcf-protection" - CXXFLAGS="$CXXFLAGS -fcf-protection" - fi - export CFLAGS - export CXXFLAGS - ;; - *) - exit 1 -esac -mkdir -p "$INSTALL_PREFIX" - -if [ -x "$(command -v ninja)" ]; then - EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -G Ninja" -fi -echo "Installing abseil ..." -pushd . -absl_url=$(grep '^abseil_cpp' "$DEP_FILE_PATH" | cut -d ';' -f 2 ) -if [[ "$absl_url" = https* ]]; then - absl_url=$(echo $absl_url | sed 's/\.zip$/\.tar.gz/') - curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o absl_src.tar.gz $absl_url - mkdir abseil - cd abseil - tar -zxf ../absl_src.tar.gz --strip=1 -else - cp $absl_url absl_src.zip - unzip absl_src.zip - cd */ -fi - -CC=$GCC_PATH CXX=$GPLUSPLUS_PATH cmake "." "-DABSL_PROPAGATE_CXX_STD=ON" "-DCMAKE_BUILD_TYPE=Release" "-DBUILD_TESTING=OFF" "-DABSL_USE_EXTERNAL_GOOGLETEST=ON" "-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" "-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX" $EXTRA_CMAKE_ARGS -if [ -x "$(command -v ninja)" ]; then - ninja - ninja install -else - make -j$(getconf _NPROCESSORS_ONLN) - make install -fi -popd - -pushd . -echo "Installing protobuf ..." -protobuf_url=$(grep '^protobuf' $DEP_FILE_PATH | cut -d ';' -f 2 ) -if [[ "$protobuf_url" = https* ]]; then - protobuf_url=$(echo "$protobuf_url" | sed 's/\.zip$/\.tar.gz/') - curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o protobuf_src.tar.gz "$protobuf_url" - mkdir protobuf - cd protobuf - tar -zxf ../protobuf_src.tar.gz --strip=1 -else - cp $protobuf_url protobuf_src.zip - unzip protobuf_src.zip - cd protobuf-* -fi - -CC=$GCC_PATH CXX=$GPLUSPLUS_PATH cmake . "-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB_DEFAULT=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF "-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" $EXTRA_CMAKE_ARGS -Dprotobuf_ABSL_PROVIDER=package -if [ -x "$(command -v ninja)" ]; then - ninja - ninja install -else - make -j$(getconf _NPROCESSORS_ONLN) - make install -fi -popd diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/requirements.txt b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/requirements.txt deleted file mode 100644 index ee07c7ef4f52..000000000000 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -numpy==1.21.6 ; python_version < '3.9' -numpy==2.1.2 ; python_version >= '3.9' -mypy -pytest -setuptools>=68.2.2 -wheel -onnx==1.16.1 -protobuf==4.21.12 -sympy==1.12 -flatbuffers -packaging>=22.0 diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/Dockerfile b/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/Dockerfile index 85b1469a038f..54bdbebbd131 100644 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/Dockerfile @@ -36,7 +36,7 @@ ENV PATH /usr/local/cuda/bin:$PATH ENV CUDA_MODULE_LOADING "LAZY" ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && rm -rf /tmp/scripts ARG BUILD_UID=1001 ARG BUILD_USER=onnxruntimedev diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh deleted file mode 100755 index 81de2abf3ff8..000000000000 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_deps.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -set -e -x -pushd . -PYTHON_EXES=("/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp313-cp313t/bin/python3.13") -CURRENT_DIR=$(pwd) -if ! [ -x "$(command -v protoc)" ]; then - $CURRENT_DIR/install_protobuf.sh -fi -popd -export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" - -for PYTHON_EXE in "${PYTHON_EXES[@]}" -do - ${PYTHON_EXE} -m pip install -r requirements.txt -done - -# No release binary for ccache aarch64, so we need to build it from source. -if ! [ -x "$(command -v ccache)" ]; then - ccache_url="https://github.com/ccache/ccache/archive/refs/tags/v4.8.tar.gz" - pushd . - curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o ccache_src.tar.gz $ccache_url - mkdir ccache_main - cd ccache_main - tar -zxf ../ccache_src.tar.gz --strip=1 - - mkdir build - cd build - cmake -DCMAKE_INSTALL_PREFIX=/usr/local _DCMAKE_BUILD_TYPE=Release .. - make - make install - which ccache - popd - rm -f ccache_src.tar.gz - rm -rf ccache_src -fi diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_protobuf.sh b/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_protobuf.sh deleted file mode 100755 index 31b5ca6f9e69..000000000000 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/install_protobuf.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -set -e -x - -INSTALL_PREFIX='/usr' -DEP_FILE_PATH='/tmp/scripts/deps.txt' -while getopts "p:d:" parameter_Option -do case "${parameter_Option}" -in -p) INSTALL_PREFIX=${OPTARG};; -d) DEP_FILE_PATH=${OPTARG};; -esac -done - - - -EXTRA_CMAKE_ARGS="-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_CXX_STANDARD=17" - -case "$(uname -s)" in - Darwin*) - echo 'Building ONNX Runtime on Mac OS X' - EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DCMAKE_OSX_ARCHITECTURES=x86_64;arm64" - GCC_PATH=$(which clang) - GPLUSPLUS_PATH=$(which clang++) - ;; - Linux*) - SYS_LONG_BIT=$(getconf LONG_BIT) - DISTRIBUTOR=$(lsb_release -i -s) - - if [[ ("$DISTRIBUTOR" = "CentOS" || "$DISTRIBUTOR" = "RedHatEnterprise") && $SYS_LONG_BIT = "64" ]]; then - LIBDIR="lib64" - else - LIBDIR="lib" - fi - EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=$LIBDIR" - # Depending on how the compiler has been configured when it was built, sometimes "gcc -dumpversion" shows the full version. - GCC_VERSION=$(gcc -dumpversion | cut -d . -f 1) - #-fstack-clash-protection prevents attacks based on an overlapping heap and stack. - if [ "$GCC_VERSION" -ge 8 ]; then - CFLAGS="$CFLAGS -fstack-clash-protection" - CXXFLAGS="$CXXFLAGS -fstack-clash-protection" - fi - ARCH=$(uname -m) - GCC_PATH=$(which gcc) - GPLUSPLUS_PATH=$(which g++) - if [ "$ARCH" == "x86_64" ] && [ "$GCC_VERSION" -ge 9 ]; then - CFLAGS="$CFLAGS -fcf-protection" - CXXFLAGS="$CXXFLAGS -fcf-protection" - fi - export CFLAGS - export CXXFLAGS - ;; - *) - exit 1 -esac -mkdir -p "$INSTALL_PREFIX" - -if [ -x "$(command -v ninja)" ]; then - EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -G Ninja" -fi -echo "Installing abseil ..." -pushd . -absl_url=$(grep '^abseil_cpp' "$DEP_FILE_PATH" | cut -d ';' -f 2 ) -if [[ "$absl_url" = https* ]]; then - absl_url=$(echo $absl_url | sed 's/\.zip$/\.tar.gz/') - curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o absl_src.tar.gz $absl_url - mkdir abseil - cd abseil - tar -zxf ../absl_src.tar.gz --strip=1 -else - cp $absl_url absl_src.zip - unzip absl_src.zip - cd */ -fi - -CC=$GCC_PATH CXX=$GPLUSPLUS_PATH cmake "." "-DABSL_PROPAGATE_CXX_STD=ON" "-DCMAKE_BUILD_TYPE=Release" "-DBUILD_TESTING=OFF" "-DABSL_USE_EXTERNAL_GOOGLETEST=ON" "-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" "-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX" $EXTRA_CMAKE_ARGS -if [ -x "$(command -v ninja)" ]; then - ninja - ninja install -else - make -j$(getconf _NPROCESSORS_ONLN) - make install -fi -popd - -pushd . -echo "Installing protobuf ..." -protobuf_url=$(grep '^protobuf' $DEP_FILE_PATH | cut -d ';' -f 2 ) -if [[ "$protobuf_url" = https* ]]; then - protobuf_url=$(echo "$protobuf_url" | sed 's/\.zip$/\.tar.gz/') - curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o protobuf_src.tar.gz "$protobuf_url" - mkdir protobuf - cd protobuf - tar -zxf ../protobuf_src.tar.gz --strip=1 -else - cp $protobuf_url protobuf_src.zip - unzip protobuf_src.zip - cd protobuf-* -fi - -CC=$GCC_PATH CXX=$GPLUSPLUS_PATH cmake . "-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB_DEFAULT=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF "-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" $EXTRA_CMAKE_ARGS -Dprotobuf_ABSL_PROVIDER=package -if [ -x "$(command -v ninja)" ]; then - ninja - ninja install -else - make -j$(getconf _NPROCESSORS_ONLN) - make install -fi -popd diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/requirements.txt b/tools/ci_build/github/linux/python/requirements.txt similarity index 54% rename from tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/requirements.txt rename to tools/ci_build/github/linux/python/requirements.txt index 07a9f3f481aa..f23ac05ea539 100644 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cuda/scripts/requirements.txt +++ b/tools/ci_build/github/linux/python/requirements.txt @@ -1,5 +1,4 @@ -numpy==1.21.6 ; python_version < '3.9' -numpy==2.1.2 ; python_version >= '3.9' +numpy==2.1.2 mypy pytest setuptools>=68.2.2 From a4096a34b65ff5922b0c69a5cef9e5f5e50dd54a Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 01:37:35 +0000 Subject: [PATCH 15/40] update --- tools/ci_build/github/windows/helpers.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/windows/helpers.ps1 b/tools/ci_build/github/windows/helpers.ps1 index 95a36aa24e90..929418029e44 100644 --- a/tools/ci_build/github/windows/helpers.ps1 +++ b/tools/ci_build/github/windows/helpers.ps1 @@ -638,7 +638,7 @@ function Install-ONNX { $temp_dir = Get-TempDirectory $new_requirements_text_file = Join-Path $temp_dir "new_requirements.txt" Write-Host "Installing python packages..." - Get-Content "$src_root\tools\ci_build\github\linux\docker\inference\x86_64\python\cpu\scripts\requirements.txt" | Select-String -pattern 'onnx' -notmatch | Out-File $new_requirements_text_file + Get-Content "$src_root\tools\ci_build\github\linux\python\requirements.txt" | Select-String -pattern 'onnx' -notmatch | Out-File $new_requirements_text_file [string[]]$pip_args = "-m", "pip", "install", "-qq", "--disable-pip-version-check", "-r", $new_requirements_text_file &"python.exe" $pip_args From 41167cfe8075c01b11bd4101915ac0c366f867c9 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 01:40:03 +0000 Subject: [PATCH 16/40] update --- tools/ci_build/github/linux/build_linux_python_package.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index 6f8405a07bd9..9831dc9f7322 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -44,7 +44,7 @@ if ! [ -x "$(command -v ccache)" ]; then mkdir build cd build - cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_BUILD_TYPE=Release .. + cmake -DCMAKE_INSTALL_PREFIX=/build -DCMAKE_BUILD_TYPE=Release .. make make install which ccache @@ -52,6 +52,7 @@ if ! [ -x "$(command -v ccache)" ]; then rm -f ccache_src.tar.gz rm -rf ccache_src fi +export PATH=/build/bin:$PATH echo "EXTRA_ARG:" echo "$EXTRA_ARG" From 237573f83771dfd40305dfd0e5edcbe0a9863984 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 01:49:20 +0000 Subject: [PATCH 17/40] update --- tools/ci_build/github/linux/build_linux_python_package.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index 9831dc9f7322..c322c3837188 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -45,14 +45,15 @@ if ! [ -x "$(command -v ccache)" ]; then mkdir build cd build cmake -DCMAKE_INSTALL_PREFIX=/build -DCMAKE_BUILD_TYPE=Release .. - make + make -j$(nproc) make install + export PATH=/build/bin:$PATH which ccache popd rm -f ccache_src.tar.gz rm -rf ccache_src fi -export PATH=/build/bin:$PATH + echo "EXTRA_ARG:" echo "$EXTRA_ARG" From e181dda9d045884753c810fe6a4095fb38e2728d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 03:42:10 +0000 Subject: [PATCH 18/40] update --- ...arm64-v8a-QNN-crosscompile-ci-pipeline.yml | 5 +--- .../templates/mac-cpu-packing-jobs.yml | 29 ++++++------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml index 7bc1cd669bbf..abdcb1b7610c 100644 --- a/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml @@ -63,9 +63,6 @@ jobs: - script: | set -e -x - rm -rf /tmp/scripts - cp -r tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts /tmp - /tmp/scripts/install_protobuf.sh -p $(Build.BinariesDirectory)/installed -d cmake/deps.txt python3 tools/ci_build/build.py \ --config Release \ --android \ @@ -78,7 +75,7 @@ jobs: --use_qnn \ --qnn_home $(QnnSDKRootDir) \ --cmake_generator=Ninja \ - --skip_tests --path_to_protoc_exe $(Build.BinariesDirectory)/installed/bin/protoc + --skip_tests displayName: Build QNN EP - script: | diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml index b057a0aac66c..9965ab8bef8e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml @@ -71,26 +71,15 @@ jobs: arguments: --new_dir $(Build.BinariesDirectory)/deps workingDirectory: $(Build.BinariesDirectory) - - template: mac-build-step-with-cache.yml - parameters: - WithCache: ${{ parameters.WithCache }} - Today: $(TODAY) - AdditionalKey: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_protobuf.sh' - CacheDir: $(PROTO_CACHE_DIR) - ChangeEveryCommit: false - BuildStep: - - script: | - set -e -x - pushd . - $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/scripts/install_protobuf.sh -d $(Build.SourcesDirectory)/cmake/deps.txt -p $(Build.BinariesDirectory)/installed - popd - export PATH=$(Build.BinariesDirectory)/installed/bin:$PATH - export ONNX_ML=1 - export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" - python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - displayName: 'Install dependencies' - env: - CCACHE_DIR: $(PROTO_CACHE_DIR) + - script: | + set -e -x + export PATH=$(Build.BinariesDirectory)/installed/bin:$PATH + export ONNX_ML=1 + export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" + python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' + displayName: 'Install dependencies' + env: + CCACHE_DIR: $(PROTO_CACHE_DIR) - ${{ if eq(parameters.MacosArch, 'universal2') }}: - template: mac-cpu-packaging-steps.yml From 33377af20047f76712187e21a704d5afc49e6bdd Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 03:45:55 +0000 Subject: [PATCH 19/40] update --- .../azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml | 2 +- tools/ci_build/github/azure-pipelines/templates/win-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml index ef48244bbb29..a8a5f13b1f73 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml @@ -34,7 +34,7 @@ parameters: steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.8' + versionSpec: '3.12' addToPath: true architecture: ${{parameters.BuildArch}} diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 52547fd9a796..2cb7f94470d7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -129,7 +129,7 @@ stages: - task: UsePythonVersion@0 inputs: - versionSpec: '3.8' + versionSpec: '3.12' addToPath: true architecture: ${{ parameters.buildArch }} From dc6906a45ca7bef4a69c3bf33b3eb72d4699f692 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 03:57:46 +0000 Subject: [PATCH 20/40] update --- .../orttraining-linux-ci-pipeline.yml | 16 +--------------- .../templates/mac-cpu-packing-jobs.yml | 4 +--- .../github/linux/build_linux_python_package.sh | 6 +++++- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index ffe3aefaea21..3dac8a7598d1 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -47,11 +47,6 @@ jobs: clean: true submodules: none - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - addToPath: true - - template: templates/get-docker-image-steps.yml parameters: Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile @@ -91,16 +86,7 @@ jobs: /bin/bash -c " set -ex; \ ccache -s; \ - /opt/python/cp312-cp312/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build --cmake_generator 'Unix Makefiles' \ - --config Release \ - --skip_submodule_sync \ - --build_shared_lib \ - --parallel --use_binskim_compliant_compile_flags \ - --build_wheel \ - --enable_onnx_tests \ - --enable_training \ - --use_cache; \ + tools/ci_build/github/linux/build_linux_python_package.sh -p /opt/python/cp312-cp312/bin/python3 -e -x "--enable_training" \ ccache -sv; \ ccache -z" workingDirectory: $(Build.SourcesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml index 9965ab8bef8e..eb48b44db5a1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml @@ -77,9 +77,7 @@ jobs: export ONNX_ML=1 export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF" python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - displayName: 'Install dependencies' - env: - CCACHE_DIR: $(PROTO_CACHE_DIR) + - ${{ if eq(parameters.MacosArch, 'universal2') }}: - template: mac-cpu-packaging-steps.yml diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index c322c3837188..371bf04df295 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -6,7 +6,7 @@ set -e -x mkdir -p /build/dist EXTRA_ARG="" - +ENABLE_CACHE=false # Put 3.10 at the last because Ubuntu 22.04 use python 3.10 and we will upload the intermediate build files of this # config to Azure DevOps Artifacts and download them to a Ubuntu 22.04 machine to run the tests. PYTHON_EXES=("/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp310-cp310/bin/python3.10") @@ -18,6 +18,7 @@ d) BUILD_DEVICE=${OPTARG};; p) PYTHON_EXES=${OPTARG};; x) EXTRA_ARG=${OPTARG};; c) BUILD_CONFIG=${OPTARG};; +e) ENABLE_CACHE=true *) echo "Usage: $0 -d [-p ] [-x ] [-c ]" exit 1;; esac @@ -30,6 +31,9 @@ BUILD_ARGS=("--build_dir" "/build" "--config" "$BUILD_CONFIG" "--update" "--buil if [ "$BUILD_CONFIG" != "Debug" ]; then BUILD_ARGS+=("--enable_lto") fi +if [ "$ENABLE_CACHE" = true ] ; then + BUILD_ARGS+=("--use_cache") +fi ARCH=$(uname -m) From 7683b6ad2d9ea1335fe383335d2f9c00bae426dd Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 04:03:26 +0000 Subject: [PATCH 21/40] update --- .../linux/build_linux_python_package.sh | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index 371bf04df295..e8aeb3b60ada 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -33,30 +33,32 @@ if [ "$BUILD_CONFIG" != "Debug" ]; then fi if [ "$ENABLE_CACHE" = true ] ; then BUILD_ARGS+=("--use_cache") + # No release binary for ccache aarch64, so we need to build it from source. + if ! [ -x "$(command -v ccache)" ]; then + ccache_url="https://github.com/ccache/ccache/archive/refs/tags/v4.8.tar.gz" + pushd . + curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o ccache_src.tar.gz $ccache_url + mkdir ccache_main + cd ccache_main + tar -zxf ../ccache_src.tar.gz --strip=1 + + mkdir build + cd build + cmake -DCMAKE_INSTALL_PREFIX=/build -DCMAKE_BUILD_TYPE=Release .. + make -j$(nproc) + make install + export PATH=/build/bin:$PATH + which ccache + popd + rm -f ccache_src.tar.gz + rm -rf ccache_src + fi + ccache -s; fi ARCH=$(uname -m) -# No release binary for ccache aarch64, so we need to build it from source. -if ! [ -x "$(command -v ccache)" ]; then - ccache_url="https://github.com/ccache/ccache/archive/refs/tags/v4.8.tar.gz" - pushd . - curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o ccache_src.tar.gz $ccache_url - mkdir ccache_main - cd ccache_main - tar -zxf ../ccache_src.tar.gz --strip=1 - - mkdir build - cd build - cmake -DCMAKE_INSTALL_PREFIX=/build -DCMAKE_BUILD_TYPE=Release .. - make -j$(nproc) - make install - export PATH=/build/bin:$PATH - which ccache - popd - rm -f ccache_src.tar.gz - rm -rf ccache_src -fi + echo "EXTRA_ARG:" @@ -94,4 +96,6 @@ do cp /build/"$BUILD_CONFIG"/dist/*.whl /build/dist done -which ccache && ccache -sv && ccache -z +if [ "$ENABLE_CACHE" = true ] ; then + which ccache && ccache -sv && ccache -z +fi From 56808652f629fb29893b3a2b1743cc98fb86151d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 04:11:13 +0000 Subject: [PATCH 22/40] update --- .../azure-pipelines/orttraining-linux-ci-pipeline.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index 3dac8a7598d1..cc39409a60ad 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -83,12 +83,7 @@ jobs: -e BUILD_BUILDNUMBER \ -e CCACHE_DIR=/cache \ onnxruntimecpubuildpythonx86_64 \ - /bin/bash -c " - set -ex; \ - ccache -s; \ - tools/ci_build/github/linux/build_linux_python_package.sh -p /opt/python/cp312-cp312/bin/python3 -e -x "--enable_training" \ - ccache -sv; \ - ccache -z" + tools/ci_build/github/linux/build_linux_python_package.sh -p /opt/python/cp312-cp312/bin/python3 -e -x "--enable_training" workingDirectory: $(Build.SourcesDirectory) - task: PublishTestResults@2 From acdc25063fbb56d7f175f2e6694139a349940fc9 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 04:16:45 +0000 Subject: [PATCH 23/40] update --- .../github/azure-pipelines/orttraining-linux-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index cc39409a60ad..137d62a137df 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -83,7 +83,7 @@ jobs: -e BUILD_BUILDNUMBER \ -e CCACHE_DIR=/cache \ onnxruntimecpubuildpythonx86_64 \ - tools/ci_build/github/linux/build_linux_python_package.sh -p /opt/python/cp312-cp312/bin/python3 -e -x "--enable_training" + /onnxruntime_src/tools/ci_build/github/linux/build_linux_python_package.sh -p /opt/python/cp312-cp312/bin/python3 -e -x "--enable_training" workingDirectory: $(Build.SourcesDirectory) - task: PublishTestResults@2 From 38b966e1643c1d4e7a2ad5c79606e6ff4adf848d Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 04:23:26 +0000 Subject: [PATCH 24/40] update --- tools/ci_build/github/linux/build_linux_python_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index e8aeb3b60ada..9365c73ef341 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -18,7 +18,7 @@ d) BUILD_DEVICE=${OPTARG};; p) PYTHON_EXES=${OPTARG};; x) EXTRA_ARG=${OPTARG};; c) BUILD_CONFIG=${OPTARG};; -e) ENABLE_CACHE=true +e) ENABLE_CACHE=true;; *) echo "Usage: $0 -d [-p ] [-x ] [-c ]" exit 1;; esac From ee18bf73a9d88027951bb2b2533bae81011d8e81 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 04:30:24 +0000 Subject: [PATCH 25/40] update --- tools/ci_build/github/linux/build_linux_python_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index 9365c73ef341..0b89d70615d2 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -10,7 +10,7 @@ ENABLE_CACHE=false # Put 3.10 at the last because Ubuntu 22.04 use python 3.10 and we will upload the intermediate build files of this # config to Azure DevOps Artifacts and download them to a Ubuntu 22.04 machine to run the tests. PYTHON_EXES=("/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12" "/opt/python/cp313-cp313/bin/python3.13" "/opt/python/cp310-cp310/bin/python3.10") -while getopts "d:p:x:c:" parameter_Option +while getopts "d:p:x:c:e" parameter_Option do case "${parameter_Option}" in #GPU|CPU|NPU. From a30a8441869a639cf75f41b78de3631fc91955d7 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 04:38:55 +0000 Subject: [PATCH 26/40] update --- .../github/azure-pipelines/orttraining-linux-ci-pipeline.yml | 2 +- tools/ci_build/github/linux/build_linux_python_package.sh | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index 137d62a137df..bdbfca3cfa23 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -83,7 +83,7 @@ jobs: -e BUILD_BUILDNUMBER \ -e CCACHE_DIR=/cache \ onnxruntimecpubuildpythonx86_64 \ - /onnxruntime_src/tools/ci_build/github/linux/build_linux_python_package.sh -p /opt/python/cp312-cp312/bin/python3 -e -x "--enable_training" + /onnxruntime_src/tools/ci_build/github/linux/build_linux_python_package.sh -p /opt/python/cp312-cp312/bin/python3 -e -x "--enable_training" -c Release -d CPU workingDirectory: $(Build.SourcesDirectory) - task: PublishTestResults@2 diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index 0b89d70615d2..b3ba307f72cb 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -36,7 +36,7 @@ if [ "$ENABLE_CACHE" = true ] ; then # No release binary for ccache aarch64, so we need to build it from source. if ! [ -x "$(command -v ccache)" ]; then ccache_url="https://github.com/ccache/ccache/archive/refs/tags/v4.8.tar.gz" - pushd . + cd /build curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o ccache_src.tar.gz $ccache_url mkdir ccache_main cd ccache_main @@ -49,7 +49,6 @@ if [ "$ENABLE_CACHE" = true ] ; then make install export PATH=/build/bin:$PATH which ccache - popd rm -f ccache_src.tar.gz rm -rf ccache_src fi From 84498a94de15e0340783e0a6da517f149ce39148 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 05:06:29 +0000 Subject: [PATCH 27/40] update --- .../orttraining-linux-ci-pipeline.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index bdbfca3cfa23..e6837b26406c 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -47,12 +47,12 @@ jobs: clean: true submodules: none - - template: templates/get-docker-image-steps.yml + - template: get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile - Context: tools/ci_build/github/linux/docker/inference/x86_64/python/cpu - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuildpythonx86_64 + Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=registry.access.redhat.com/ubi8/ubi" + Repository: onnxruntimecpubuildcentos8x64_packaging - task: Cache@2 inputs: @@ -82,8 +82,8 @@ jobs: -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ -e CCACHE_DIR=/cache \ - onnxruntimecpubuildpythonx86_64 \ - /onnxruntime_src/tools/ci_build/github/linux/build_linux_python_package.sh -p /opt/python/cp312-cp312/bin/python3 -e -x "--enable_training" -c Release -d CPU + onnxruntimecpubuildcentos8x64_packaging \ + python3.9 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_training --skip_submodule_sync --parallel workingDirectory: $(Build.SourcesDirectory) - task: PublishTestResults@2 From edbb04ee22628a58f45b47ba73c319e40eafd977 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 05:10:13 +0000 Subject: [PATCH 28/40] update --- .../build_ort_and_check_binary_size.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py b/tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py index df530a7c5e9a..338e86b07e0d 100644 --- a/tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py +++ b/tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py @@ -35,33 +35,6 @@ def main(): arch = config["arch"] build_params = config["build_params"] build_config = "MinSizeRel" # could make this configurable if needed - # Build and install protoc - protobuf_installation_script = ( - REPO_ROOT - / "tools" - / "ci_build" - / "github" - / "linux" - / "docker" - / "inference" - / "x86_64" - / "python" - / "cpu" - / "scripts" - / "install_protobuf.sh" - ) - subprocess.run( - [ - str(protobuf_installation_script), - "-p", - str(pathlib.Path(args.build_dir) / "installed"), - "-d", - str(REPO_ROOT / "cmake" / "deps.txt"), - ], - shell=False, - check=True, - ) - # build ORT build_command = ( [sys.executable, str(REPO_ROOT / "tools/ci_build/build.py"), *build_params] + (["--cmake_extra_defines", "ADD_DEBUG_INFO_TO_MINIMAL_BUILD=ON"] if args.with_debug_info else []) From e4085397e20d8dbd6269900f9da582d546b38607 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 05:11:40 +0000 Subject: [PATCH 29/40] update --- .../github/azure-pipelines/orttraining-linux-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index e6837b26406c..e5dd122bcc5c 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -47,7 +47,7 @@ jobs: clean: true submodules: none - - template: get-docker-image-steps.yml + - template: templates/get-docker-image-steps.yml parameters: Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu From a805aee8ed4ba493c587e75c6964d43a18b01415 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 05:35:30 +0000 Subject: [PATCH 30/40] update --- tools/ci_build/github/linux/build_cuda_c_api_package.sh | 2 +- tools/ci_build/github/linux/build_tensorrt_c_api_package.sh | 2 +- .../inference/x86_64/default/cpu/scripts/install_centos.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/linux/build_cuda_c_api_package.sh b/tools/ci_build/github/linux/build_cuda_c_api_package.sh index 57a3bedc1e8e..9922fc396b3d 100755 --- a/tools/ci_build/github/linux/build_cuda_c_api_package.sh +++ b/tools/ci_build/github/linux/build_cuda_c_api_package.sh @@ -2,4 +2,4 @@ set -e -x docker run --rm --volume \ $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}build \ -/bin/bash -c "/usr/bin/python3.9 /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr/local/cuda-$CUDA_VERSION --skip_tests --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80' && cd /build/Release && make install DESTDIR=/build/installed" +/bin/bash -c "/usr/bin/python3.12 /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr/local/cuda-$CUDA_VERSION --skip_tests --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80' && cd /build/Release && make install DESTDIR=/build/installed" diff --git a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh index f0c9d51a5344..7f18e2f849d2 100755 --- a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh +++ b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh @@ -3,4 +3,4 @@ set -e -x mkdir -p $HOME/.onnx docker run --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}xtrt86build \ -/bin/bash -c "/usr/bin/python3.9 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_tests --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_java --build_nodejs --use_tensorrt --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr --tensorrt_home=/usr --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80' && cd /build/Release && make install DESTDIR=/build/installed" +/bin/bash -c "/usr/bin/python3.12 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_tests --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_java --build_nodejs --use_tensorrt --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr --tensorrt_home=/usr --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80' && cd /build/Release && make install DESTDIR=/build/installed" diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh index 17b80150c848..44fb6f6480aa 100755 --- a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh +++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh @@ -5,5 +5,5 @@ if [ ! -f /etc/yum.repos.d/microsoft-prod.repo ]; then echo "installing for CentOS version : $os_major_version" rpm -Uvh https://packages.microsoft.com/config/centos/$os_major_version/packages-microsoft-prod.rpm fi -dnf install -y python39-devel glibc-langpack-\* glibc-locale-source which redhat-lsb-core expat-devel tar unzip zlib-devel make bzip2 bzip2-devel msopenjdk-11 graphviz gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran gcc-toolset-12-libasan-devel libasan.x86_64 +dnf install -y python3.12-devel glibc-langpack-\* glibc-locale-source which redhat-lsb-core expat-devel tar unzip zlib-devel make bzip2 bzip2-devel msopenjdk-11 graphviz gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran gcc-toolset-12-libasan-devel libasan.x86_64 locale From 53e312f588b8dd6d97be04dfc41c542cc49b2697 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 05:36:45 +0000 Subject: [PATCH 31/40] update --- .../github/azure-pipelines/orttraining-linux-ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index e5dd122bcc5c..5c3273f79bd3 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -83,7 +83,7 @@ jobs: -e BUILD_BUILDNUMBER \ -e CCACHE_DIR=/cache \ onnxruntimecpubuildcentos8x64_packaging \ - python3.9 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_training --skip_submodule_sync --parallel + /onnxruntime_src/tools/ci_build/github/linux/build_training_ci.sh workingDirectory: $(Build.SourcesDirectory) - task: PublishTestResults@2 From c8426fbe9b27037d88146a10a8ac89b47fff7a53 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 05:43:30 +0000 Subject: [PATCH 32/40] update --- tools/ci_build/github/linux/build_training_ci.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 tools/ci_build/github/linux/build_training_ci.sh diff --git a/tools/ci_build/github/linux/build_training_ci.sh b/tools/ci_build/github/linux/build_training_ci.sh new file mode 100755 index 000000000000..82f75a5cbbc5 --- /dev/null +++ b/tools/ci_build/github/linux/build_training_ci.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e -x +python3.12 -m pip install -r /onnxruntime_src/tools/ci_build/github/linux/python/requirements.txt +python3.12 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_training --skip_submodule_sync --parallel From 0e0874fbbda6e408b25a731d1807bed47eb9e000 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 05:51:47 +0000 Subject: [PATCH 33/40] update --- .../inference/x86_64/default/cpu/scripts/install_centos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh index 44fb6f6480aa..0ac0ca625a1a 100755 --- a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh +++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh @@ -5,5 +5,5 @@ if [ ! -f /etc/yum.repos.d/microsoft-prod.repo ]; then echo "installing for CentOS version : $os_major_version" rpm -Uvh https://packages.microsoft.com/config/centos/$os_major_version/packages-microsoft-prod.rpm fi -dnf install -y python3.12-devel glibc-langpack-\* glibc-locale-source which redhat-lsb-core expat-devel tar unzip zlib-devel make bzip2 bzip2-devel msopenjdk-11 graphviz gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran gcc-toolset-12-libasan-devel libasan.x86_64 +dnf install -y python3.12-pip python3.12-devel glibc-langpack-\* glibc-locale-source which redhat-lsb-core expat-devel tar unzip zlib-devel make bzip2 bzip2-devel msopenjdk-11 graphviz gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran gcc-toolset-12-libasan-devel libasan.x86_64 locale From 01d4974b53ca4ce040af577c66a78ff0b76ec840 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 06:37:04 +0000 Subject: [PATCH 34/40] update --- tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 78dafc0cd7da..0d57560251b7 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -1,4 +1,4 @@ -##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py #### +0;10;1c##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py #### ### please do rerun set-trigger-rules.py ### trigger: branches: @@ -82,7 +82,7 @@ stages: onnxruntimecpubuildcentos8x64 \ /bin/bash -c ' set -ex; \ - python3.9 /onnxruntime_src/tools/ci_build/build.py \ + python3.12 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator 'Ninja' \ --config Debug \ --skip_submodule_sync \ @@ -91,7 +91,7 @@ stages: --build_csharp \ --enable_onnx_tests --enable_address_sanitizer \ --update --build; - LD_PRELOAD=/usr/lib64/libasan.so.6 python3.9 /onnxruntime_src/tools/ci_build/build.py \ + LD_PRELOAD=/usr/lib64/libasan.so.8 python3.12 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator 'Ninja' \ --config Debug \ --skip_submodule_sync \ From 2941239632770b89392e1704ec704a4c31ff87e8 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 18:06:19 +0000 Subject: [PATCH 35/40] update --- .../ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml | 2 +- tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml | 2 +- .../ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml | 4 ++-- .../github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml | 4 ++-- .../github/azure-pipelines/py-cuda-package-test-pipeline.yml | 2 +- .../github/azure-pipelines/py-package-test-pipeline.yml | 2 +- .../azure-pipelines/stages/java-cuda-packaging-stage.yml | 4 ++-- .../stages/jobs/py-linux-cuda-package-test-job.yml | 2 +- .../github/azure-pipelines/stages/py-cuda-packaging-stage.yml | 4 ++-- .../github/azure-pipelines/templates/py-packaging-stage.yml | 2 +- tools/ci_build/github/linux/docker/Dockerfile.aten_cpu | 2 +- .../ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu | 2 +- .../linux/docker/Dockerfile.manylinux2_28_training_cuda11_8 | 2 +- .../linux/docker/Dockerfile.manylinux2_28_training_cuda12_2 | 2 +- .../linux/docker/inference/aarch64/python/cpu/Dockerfile | 2 +- .../linux/docker/inference/x86_64/default/cuda11/Dockerfile | 2 +- .../linux/docker/inference/x86_64/default/cuda12/Dockerfile | 2 +- .../linux/docker/inference/x86_64/python/cpu/Dockerfile | 2 +- 18 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml index 6fd02f6b5986..63f5fef8a816 100644 --- a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml @@ -42,7 +42,7 @@ parameters: variables: - template: templates/common-variables.yml - name: docker_base_image - value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241008.1 + value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241010.2 - name: linux_trt_version value: 10.3.0.26-1.cuda11.8 - name: Repository diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 0d57560251b7..2b71e6117638 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -1,4 +1,4 @@ -0;10;1c##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py #### +##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py #### ### please do rerun set-trigger-rules.py ### trigger: branches: diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 295549b647dc..396252b643a3 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -49,9 +49,9 @@ parameters: variables: - name: docker_base_image ${{ if eq(parameters.CudaVersion, '11.8') }}: - value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241008.1 + value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241010.2 ${{ if eq(parameters.CudaVersion, '12.2') }}: - value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241008.1 + value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241010.2 - name: Repository ${{ if eq(parameters.CudaVersion, '11.8') }}: diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index 6717e8b4faaa..1c64253911b1 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -39,9 +39,9 @@ parameters: variables: - name: docker_base_image ${{ if eq(parameters.CudaVersion, '11.8') }}: - value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241008.1 + value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241010.2 ${{ if eq(parameters.CudaVersion, '12.2') }}: - value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241008.1 + value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241010.2 - name: linux_trt_version ${{ if eq(parameters.CudaVersion, '11.8') }}: value: 10.4.0.26-1.cuda11.8 diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml index 062b88a326dc..48107d902771 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml @@ -18,7 +18,7 @@ stages: machine_pool: 'Onnxruntime-Linux-GPU' python_wheel_suffix: '_gpu' timeout: 480 - docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241008.1 + docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241010.2 trt_version: '10.4.0.26-1.cuda12.6' cuda_version: '12.2' diff --git a/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml index fc66cd9f145f..e731ca174ee5 100644 --- a/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml @@ -54,7 +54,7 @@ stages: machine_pool: 'Onnxruntime-Linux-GPU' python_wheel_suffix: '_gpu' timeout: 480 - docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241008.1 + docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241010.2 trt_version: '10.4.0.26-1.cuda11.8' cuda_version: '11.8' diff --git a/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml index 7bc61268805f..cb76e9c6779a 100644 --- a/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml @@ -143,9 +143,9 @@ stages: value: false - name: docker_base_image ${{ if eq(parameters.CudaVersion, '11.8') }}: - value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241008.1 + value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241010.2 ${{ if eq(parameters.CudaVersion, '12.2') }}: - value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241008.1 + value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241010.2 timeoutInMinutes: 60 steps: diff --git a/tools/ci_build/github/azure-pipelines/stages/jobs/py-linux-cuda-package-test-job.yml b/tools/ci_build/github/azure-pipelines/stages/jobs/py-linux-cuda-package-test-job.yml index 545996a4ffc7..7a342d159105 100644 --- a/tools/ci_build/github/azure-pipelines/stages/jobs/py-linux-cuda-package-test-job.yml +++ b/tools/ci_build/github/azure-pipelines/stages/jobs/py-linux-cuda-package-test-job.yml @@ -46,7 +46,7 @@ jobs: ${{ if eq(parameters.CudaVersion, '11.8') }}: value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8 ${{ if eq(parameters.CudaVersion, '12.2') }}: - value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241008.1 + value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241010.2 - name: linux_trt_version ${{ if eq(parameters.CudaVersion, '11.8') }}: value: 10.4.0.26-1.cuda11.8 diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml index 896843cadbbd..8243cc157272 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml @@ -77,8 +77,8 @@ stages: cmake_build_type: ${{ parameters.cmake_build_type }} cuda_version: ${{ parameters.cuda_version }} ${{ if eq(parameters.cuda_version, '11.8') }}: - docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241008.1 + docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241010.2 trt_version: 10.4.0.26-1.cuda11.8 ${{ if eq(parameters.cuda_version, '12.2') }}: - docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241008.1 + docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241010.2 trt_version: 10.4.0.26-1.cuda12.6 diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index d39a6c57d989..64f64d537437 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -470,7 +470,7 @@ stages: parameters: arch: 'x86_64' machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU' - docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241008.1 + docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241010.2 extra_build_arg: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} trt_version: '10.4.0.26-1.cuda11.8' diff --git a/tools/ci_build/github/linux/docker/Dockerfile.aten_cpu b/tools/ci_build/github/linux/docker/Dockerfile.aten_cpu index 16cf0dfa4f77..13b928f3a829 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.aten_cpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.aten_cpu @@ -1,4 +1,4 @@ -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20241008.1 +FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20241010.2 ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps_aten.sh && rm -rf /tmp/scripts diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu index b517af75d2c9..d4f70edd2e00 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cpu @@ -1,4 +1,4 @@ -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20241008.1 +FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20241010.2 ENV JAVA_HOME=/usr/lib/jvm/msopenjdk-11 diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda11_8 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda11_8 index 913410923793..9ade5dbef84b 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda11_8 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda11_8 @@ -1,4 +1,4 @@ -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241008.1 +FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241010.2 ARG PYTHON_VERSION=3.9 ARG TORCH_VERSION=2.0.0 ARG OPSET_VERSION=17 diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda12_2 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda12_2 index 1bea0df1fc2c..eb376d34441b 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda12_2 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda12_2 @@ -1,4 +1,4 @@ -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241008.1 +FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241010.2 ARG PYTHON_VERSION=3.9 ARG TORCH_VERSION=2.1.0 diff --git a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/Dockerfile index 05f290566b56..64bbe8a7c4d8 100644 --- a/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/aarch64/python/cpu/Dockerfile @@ -1,4 +1,4 @@ -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_aarch64_ubi8_gcc12:20241008.1 +FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_aarch64_ubi8_gcc12:20241010.2 ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cuda11/Dockerfile b/tools/ci_build/github/linux/docker/inference/x86_64/default/cuda11/Dockerfile index f48f684d98e8..65a2c2b840a7 100644 --- a/tools/ci_build/github/linux/docker/inference/x86_64/default/cuda11/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cuda11/Dockerfile @@ -2,7 +2,7 @@ # Licensed under the MIT License. # This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11_dotnet:20241008.1 +FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11_dotnet:20241010.2 ARG TRT_VERSION RUN rpm -Uvh https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm && dnf install -y msopenjdk-11 diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cuda12/Dockerfile b/tools/ci_build/github/linux/docker/inference/x86_64/default/cuda12/Dockerfile index 69e24daf2878..e511c97e0a5c 100644 --- a/tools/ci_build/github/linux/docker/inference/x86_64/default/cuda12/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cuda12/Dockerfile @@ -2,7 +2,7 @@ # Licensed under the MIT License. # This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12_dotnet:20241008.1 +FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12_dotnet:20241010.2 ARG TRT_VERSION #Install TensorRT only if TRT_VERSION is not empty diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile index 1fd91f693f51..b5311c562802 100644 --- a/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile +++ b/tools/ci_build/github/linux/docker/inference/x86_64/python/cpu/Dockerfile @@ -1,4 +1,4 @@ -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20241008.1 +FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20241010.2 ADD scripts /tmp/scripts RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && rm -rf /tmp/scripts From f009c4f6cbd6dd0f8ce7248f4c026da59198c0e1 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 18:25:32 +0000 Subject: [PATCH 36/40] update --- .../linux-cpu-aten-pipeline.yml | 132 ----------------- .../linux-cpu-eager-pipeline.yml | 99 ------------- .../nuget/templates/dml-vs-2022.yml | 2 +- ...ortmodule-distributed-test-ci-pipeline.yml | 135 ------------------ ...-linux-nightly-ortmodule-test-pipeline.yml | 33 ----- .../orttraining-mac-ci-pipeline.yml | 37 ----- .../orttraining-py-packaging-pipeline-cpu.yml | 28 ---- ...orttraining-py-packaging-pipeline-cuda.yml | 28 ---- ...ttraining-py-packaging-pipeline-cuda12.yml | 16 --- ...orttraining-py-packaging-pipeline-rocm.yml | 65 --------- .../templates/c-api-linux-cpu.yml | 3 +- .../github/linux/docker/Dockerfile.aten_cpu | 10 -- .../github/linux/docker/Dockerfile.lort_cpu | 10 -- ...Dockerfile.manylinux2_28_training_cuda11_8 | 26 ---- .../scripts/manylinux/install_deps_aten.sh | 49 ------- .../scripts/manylinux/install_deps_lort.sh | 47 ------ 16 files changed, 3 insertions(+), 717 deletions(-) delete mode 100644 tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml delete mode 100644 tools/ci_build/github/azure-pipelines/linux-cpu-eager-pipeline.yml delete mode 100644 tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml delete mode 100644 tools/ci_build/github/azure-pipelines/orttraining-linux-nightly-ortmodule-test-pipeline.yml delete mode 100644 tools/ci_build/github/azure-pipelines/orttraining-mac-ci-pipeline.yml delete mode 100644 tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml delete mode 100644 tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda.yml delete mode 100644 tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda12.yml delete mode 100644 tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.aten_cpu delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.lort_cpu delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda11_8 delete mode 100755 tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh delete mode 100755 tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml deleted file mode 100644 index b88bad2fae8b..000000000000 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml +++ /dev/null @@ -1,132 +0,0 @@ -##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py #### -### please do rerun set-trigger-rules.py ### -trigger: - branches: - include: - - main - - rel-* - paths: - exclude: - - docs/** - - README.md - - CONTRIBUTING.md - - BUILD.md - - 'js/web' - - 'onnxruntime/core/providers/js' -pr: - branches: - include: - - main - - rel-* - paths: - exclude: - - docs/** - - README.md - - CONTRIBUTING.md - - BUILD.md - - 'js/web' - - 'onnxruntime/core/providers/js' -#### end trigger #### - -jobs: -- job: Linux_Build - timeoutInMinutes: 120 - workspace: - clean: all - variables: - CCACHE_DIR: $(Agent.TempDirectory)/ccache - TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] - pool: onnxruntime-Ubuntu2204-AMD-CPU - steps: - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - - - checkout: self - clean: true - submodules: recursive - - - template: templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.aten_cpu - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: 'onnxruntimecpubuildaten' - UseImageCacheContainerRegistry: true - UsePipelineCache: false - - - template: templates/linux-build-step-with-cache.yml - parameters: - WithCache: true - Today: $(TODAY) - AdditionalKey: ort_aten - CacheDir: $(CCACHE_DIR) - ChangeEveryCommit: true - BuildStep: - - task: CmdLine@2 - displayName: 'build' - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm \ - --volume /data/onnx:/data/onnx:ro \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - --volume $(CCACHE_DIR):/cache \ - -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - -e CCACHE_DIR=/cache \ - onnxruntimecpubuildaten \ - /bin/bash -c " - set -ex; \ - ccache -s; \ - /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build --cmake_generator Ninja \ - --config Release \ - --skip_submodule_sync \ - --build_shared_lib \ - --parallel --use_binskim_compliant_compile_flags \ - --build_wheel \ - --skip_tests \ - --cmake_extra_defines onnxruntime_ENABLE_ATEN=ON \ - --use_cache; \ - ccache -sv; \ - ccache -z" - workingDirectory: $(Build.SourcesDirectory) - - - task: CmdLine@2 - displayName: 'install ort_torch_ext and launch test' - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm \ - --volume /data/onnx:/data/onnx:ro \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - onnxruntimecpubuildaten \ - bash -c "rm -rf /build/Release/onnxruntime /build/Release/pybind11 && \ - /opt/python/cp38-cp38/bin/python3 -m pip install /build/Release/dist/*.whl && \ - /opt/python/cp38-cp38/bin/python3 -m pip install /onnxruntime_src/onnxruntime/python/torch_cpp_extensions && \ - /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/onnxruntime/test/python/contrib_ops/aten_op_tests.py && \ - /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build --cmake_generator Ninja \ - --config Release \ - --skip_submodule_sync \ - --build_shared_lib \ - --parallel --use_binskim_compliant_compile_flags \ - --build_wheel \ - --test \ - --cmake_extra_defines onnxruntime_ENABLE_ATEN=ON" - workingDirectory: $(Build.SourcesDirectory) - - - template: templates/explicitly-defined-final-tasks.yml - - - script: | - df -h - displayName: check disk space diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-eager-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-eager-pipeline.yml deleted file mode 100644 index 72ef660d4b34..000000000000 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-eager-pipeline.yml +++ /dev/null @@ -1,99 +0,0 @@ -##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py #### -### please do rerun set-trigger-rules.py ### -trigger: - branches: - include: - - main - - rel-* - paths: - exclude: - - docs/** - - README.md - - CONTRIBUTING.md - - BUILD.md - - 'js/web' - - 'onnxruntime/core/providers/js' -pr: - branches: - include: - - main - - rel-* - paths: - exclude: - - docs/** - - README.md - - CONTRIBUTING.md - - BUILD.md - - 'js/web' - - 'onnxruntime/core/providers/js' -#### end trigger #### - -jobs: -# This pipeline builds the latest PyTorch commit from source -# and use it in ORT tests. See Dockerfile.manylinux2014_lort_cpu -# for the installation steps. Idally, we should only use one pipeline -# for eager mode and LazyTensor, but we split them due to recent -# PyTorch's breaking changes. -# -# TODO: once ORT eager mode can run with latest PyTorch commit, we -# should -# 1. Set --build_eager_mode when running build.py in the -# first "task" below. -# 2. Copy the second "task" above as the third task below. -- job: BuildAndTestLazyTensor - timeoutInMinutes: 120 - workspace: - clean: all - pool: onnxruntime-Ubuntu2204-AMD-CPU - steps: - - checkout: self - clean: true - submodules: recursive - - - template: templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.lort_cpu - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuildlort - - - task: CmdLine@2 - displayName: 'Build ORT for Python 3.9' - inputs: - script: | - docker run --rm \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - onnxruntimecpubuildlort \ - python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build --cmake_generator Ninja \ - --config Release \ - --skip_submodule_sync \ - --build_shared_lib \ - --parallel --use_binskim_compliant_compile_flags \ - --enable_lazy_tensor --enable_training --build_wheel --skip_test \ - workingDirectory: $(Build.SourcesDirectory) - - - task: CmdLine@2 - displayName: 'Test DORT with Python 3.9' - inputs: - script: | - docker run --rm \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - onnxruntimecpubuildlort \ - bash -c " - export PYTHONPATH=/build/Release && \ - python3 -m pip install /build/Release/dist/*.whl && \ - python3 /onnxruntime_src/orttraining/orttraining/test/python/orttraining_test_dort.py && \ - cd /build/Release && python3 /onnxruntime_src/orttraining/orttraining/test/python/orttraining_test_dort_custom_ops.py" - workingDirectory: $(Build.SourcesDirectory) - condition: succeededOrFailed() - - - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml index c73cbb102a64..4acd97ff66c3 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml @@ -85,7 +85,7 @@ stages: - task: UsePythonVersion@0 inputs: - versionSpec: '3.8' + versionSpec: '3.12' addToPath: true architecture: ${{ parameters.BuildArch }} diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml deleted file mode 100644 index dcbee286136f..000000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml +++ /dev/null @@ -1,135 +0,0 @@ -##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py #### -### please do rerun set-trigger-rules.py ### -trigger: - branches: - include: - - main - - rel-* - paths: - exclude: - - docs/** - - README.md - - CONTRIBUTING.md - - BUILD.md - - 'js/web' - - 'onnxruntime/core/providers/js' -pr: - branches: - include: - - main - - rel-* - paths: - exclude: - - docs/** - - README.md - - CONTRIBUTING.md - - BUILD.md - - 'js/web' - - 'onnxruntime/core/providers/js' -#### end trigger #### - -stages: -- stage: ORTModuleDistributedTest - dependsOn: [] - jobs: - - job: Onnxruntime_Linux_GPU_ORTModule_Distributed_Test - - timeoutInMinutes: 120 - pool: 'Onnxruntime-Linux-GPU-NC24sv3' - - steps: - - checkout: self - clean: true - submodules: recursive - - - template: templates/jobs/download_training_test_data.yml - - - template: templates/run-docker-build-steps.yml - parameters: - RunDockerBuildArgs: | - -o ubuntu20.04 -d gpu \ - -t onnxruntime_ortmodule_distributed_tests_image \ - -x " \ - --config RelWithDebInfo \ - --use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \ - --enable_training \ - --update --build \ - --build_wheel \ - " \ - -m \ - -u \ - -e - DisplayName: 'Build' - - # Entry point for all ORTModule distributed tests - # Refer to orttraining/orttraining/test/python/how_to_add_ortmodule_distributed_ci_pipeline_tests.md for guidelines on how to add new tests to this pipeline. - - script: | - docker run \ - --gpus all \ - --shm-size=1024m \ - --rm \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume $(Agent.TempDirectory)/mnist:/mnist \ - onnxruntime_ortmodule_distributed_tests_image \ - bash -c "rm -rf /build/RelWithDebInfo/onnxruntime/ && python3 -m pip install /build/RelWithDebInfo/dist/onnxruntime*.whl && python3 -m pip install torch==2.3.1+cu118 --index-url https://download.pytorch.org/whl/cu118 && python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install && echo temporarily skip /build/RelWithDebInfo/launch_test.py --cmd_line_with_args 'python orttraining_ortmodule_distributed_tests.py --mnist /mnist' --cwd /build/RelWithDebInfo" \ - displayName: 'Run orttraining_ortmodule_distributed_tests.py' - condition: succeededOrFailed() - timeoutInMinutes: 30 - - - template: templates/component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - - - template: templates/clean-agent-build-directory-step.yml - -- stage: DistributedInferenceTest - dependsOn: [] - jobs: - - job: Onnxruntime_Linux_GPU_Inference_Distributed_Test - - timeoutInMinutes: 120 - pool: 'Onnxruntime-Linux-GPU-NC24sv3' - - steps: - - checkout: self - clean: true - submodules: recursive - - - template: templates/run-docker-build-steps.yml - parameters: - RunDockerBuildArgs: | - -o ubuntu20.04 -d gpu \ - -t onnxruntime_ortmodule_distributed_tests_image \ - -x " \ - --config RelWithDebInfo \ - --use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \ - --update --build \ - --build_wheel \ - --use_mpi \ - --enable_nccl \ - " \ - -m \ - -u \ - -e - DisplayName: 'Build' - - - script: | - docker run \ - --gpus all \ - --shm-size=1024m \ - --rm \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume /mnist:/mnist \ - onnxruntime_ortmodule_distributed_tests_image \ - bash -c "rm -rf /build/RelWithDebInfo/onnxruntime/ && python3 -m pip install mpi4py onnxscript && python3 -m pip install /build/RelWithDebInfo/dist/onnxruntime*.whl && mpirun -n 4 -x NCCL_DEBUG=INFO python /onnxruntime_src/onnxruntime/test/python/onnxruntime_test_collective.py && mpirun -n 2 -x NCCL_DEBUG=INFO python /onnxruntime_src/onnxruntime/test/python/onnxruntime_test_distributed.py && mpirun -n 2 -x NCCL_DEBUG=INFO python /onnxruntime_src/onnxruntime/test/python/transformers/sharded_moe/test_sharded_moe.py" \ - displayName: 'Run onnxruntime_test_collective.py, onnxruntime_test_distributed.py and test_sharded_moe.py' - condition: succeededOrFailed() - timeoutInMinutes: 30 - - - template: templates/component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - - - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-nightly-ortmodule-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-nightly-ortmodule-test-pipeline.yml deleted file mode 100644 index e13ef9160bed..000000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-nightly-ortmodule-test-pipeline.yml +++ /dev/null @@ -1,33 +0,0 @@ -trigger: none - -jobs: -- job: Onnxruntime_Linux_Nightly_ORTModule_tests - - timeoutInMinutes: 120 - pool: 'Onnxruntime-Linux-GPU-NC6sv3' - - steps: - - checkout: self - clean: true - submodules: recursive - - # Entry point for all ortmodule training tests - - script: | - COMMIT_ID=$(python3 -c "import onnxruntime; print(onnxruntime.get_build_info().split('git-commit-id=')[1].split(',')[0])") - cd $(Build.SourcesDirectory) - git checkout $COMMIT_ID - git branch - echo "Retrieved ONNX Runtime Commit ID: $COMMIT_ID" - docker run \ - --gpus all \ - --rm \ - --volume $(Build.SourcesDirectory)/orttraining/orttraining/test/python:/onnxruntime_src \ - --volume $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_nightly:/requirements_torch_nightly \ - ptebic.azurecr.io/internal/aifx/acpt/nightly-ubuntu-cuda-torch-dev \ - bash -c "python3 -m pip install -r /requirements_torch_nightly/requirements.txt && ORTMODULE_ENABLE_MEM_EFFICIENT_GRAD_MGMT=0 python3 -m pytest -sv /onnxruntime_src/orttraining_test_ortmodule_api.py && ORTMODULE_ENABLE_MEM_EFFICIENT_GRAD_MGMT=1 python3 -m pytest -sv /onnxruntime_src/orttraining_test_ortmodule_api.py" - displayName: 'Run ORTModule Tests' - condition: succeededOrFailed() - timeoutInMinutes: 120 - - template: templates/component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/orttraining-mac-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-mac-ci-pipeline.yml deleted file mode 100644 index ec5c30787b61..000000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-mac-ci-pipeline.yml +++ /dev/null @@ -1,37 +0,0 @@ -##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py #### -### please do rerun set-trigger-rules.py ### -trigger: - branches: - include: - - main - - rel-* - paths: - exclude: - - docs/** - - README.md - - CONTRIBUTING.md - - BUILD.md - - 'js/web' - - 'onnxruntime/core/providers/js' -pr: - branches: - include: - - main - - rel-* - paths: - exclude: - - docs/** - - README.md - - CONTRIBUTING.md - - BUILD.md - - 'js/web' - - 'onnxruntime/core/providers/js' -#### end trigger #### - -stages: -- template: templates/mac-cpu-packaging-pipeline.yml - parameters: - AllowReleasedOpsetOnly: 0 - BuildForAllArchs: false - AdditionalBuildFlags: --enable_training --build_objc - WithCache: true diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml deleted file mode 100644 index a71e10f95f3e..000000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml +++ /dev/null @@ -1,28 +0,0 @@ -parameters: -- name: cmake_build_type - type: string - displayName: 'Linux/Windows/iOS packages cmake build type.' - default: 'Release' - values: - - Debug - - Release - - RelWithDebInfo - - MinSizeRel - -trigger: none - -stages: -- template: templates/py-packaging-stage.yml - parameters: - build_py_parameters: --enable_training - cmake_build_type: ${{ parameters.cmake_build_type }} - enable_linux_gpu: false - enable_linux_cpu: true - enable_windows_cpu: true - enable_windows_gpu: false - enable_mac_cpu: true - enable_linux_arm: false - enable_windows_arm64_qnn: false - enable_windows_arm64ec_qnn: false - enable_windows_x64_qnn: false - enable_linux_x64_qnn: false diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda.yml deleted file mode 100644 index be3f67ba450b..000000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda.yml +++ /dev/null @@ -1,28 +0,0 @@ -trigger: none - -parameters: - - name: SpecificArtifact - displayName: Use Specific Artifact - type: boolean - default: false - - - name: BuildId - displayName: Specific Artifact's BuildId - type: string - default: '0' - -stages: -- template: templates/py-packaging-training-cuda-stage.yml - parameters: - build_py_parameters: --enable_training --update --build - torch_version: '2.0.0' - opset_version: '17' - cuda_version: '11.8' - cmake_cuda_architectures: 60;61;70;75;80;86 - docker_file: Dockerfile.manylinux2_28_training_cuda11_8 - agent_pool: Onnxruntime-Linux-GPU - upload_wheel: 'yes' - debug_build: false - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - build_pool_name: 'onnxruntime-Ubuntu2204-AMD-CPU' diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda12.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda12.yml deleted file mode 100644 index 74d299c72891..000000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda12.yml +++ /dev/null @@ -1,16 +0,0 @@ -trigger: none - -stages: -- template: templates/py-packaging-training-cuda-stage.yml - parameters: - # set the paralle count to reduce memory/build_threads to avoid OOM - build_py_parameters: --enable_training --update --build --parallel 8 - torch_version: '2.1.0' - opset_version: '17' - cuda_version: '12.2' - cmake_cuda_architectures: 70;75;80;86;90 - docker_file: Dockerfile.manylinux2_28_training_cuda12_2 - agent_pool: Onnxruntime-Linux-GPU - upload_wheel: 'yes' - debug_build: false - build_pool_name: 'onnxruntime-Ubuntu-2204-Training-CPU' diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml deleted file mode 100644 index a53d110a20a7..000000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml +++ /dev/null @@ -1,65 +0,0 @@ -trigger: none - -resources: - repositories: - - repository: manylinux - type: Github - endpoint: Microsoft - name: pypa/manylinux - ref: 5eda9aded5462201e6310105728d33016e637ea7 - -stages: -- stage: "Python_Packaging_ROCm60_Release" - jobs: - - template: templates/rocm.yml - parameters: - PythonVersion: '3.8' - RocmVersion: '6.0' - RocmVersionPatchSuffix: ".3" - - template: templates/rocm.yml - parameters: - PythonVersion: '3.9' - RocmVersion: '6.0' - RocmVersionPatchSuffix: ".3" - - template: templates/rocm.yml - parameters: - PythonVersion: '3.10' - RocmVersion: '6.0' - RocmVersionPatchSuffix: ".3" - -- stage: "Python_Packaging_ROCm60_Debug" - jobs: - - template: templates/rocm.yml - parameters: - PythonVersion: '3.8' - RocmVersion: '6.0' - RocmVersionPatchSuffix: ".3" - BuildConfig: 'Debug' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.9' - RocmVersion: '6.0' - RocmVersionPatchSuffix: ".3" - BuildConfig: 'Debug' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.10' - RocmVersion: '6.0' - RocmVersionPatchSuffix: ".3" - BuildConfig: 'Debug' - -- stage: "Python_Packaging_ROCm57_Release" - condition: ne(variables['ORT_DISABLE_PYTHON_PACKAGE_LOCAL_VERSION'], 'true') - jobs: - - template: templates/rocm.yml - parameters: - PythonVersion: '3.8' - RocmVersion: '5.7' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.9' - RocmVersion: '5.7' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.10' - RocmVersion: '5.7' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml index 0f4328f75e1b..c5bd4b93db94 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -65,9 +65,10 @@ jobs: - task: CmdLine@2 inputs: script: | + set -e -x mkdir -p $HOME/.onnx docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3.9 \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3.12 \ /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed" workingDirectory: $(Build.SourcesDirectory) diff --git a/tools/ci_build/github/linux/docker/Dockerfile.aten_cpu b/tools/ci_build/github/linux/docker/Dockerfile.aten_cpu deleted file mode 100644 index 13b928f3a829..000000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.aten_cpu +++ /dev/null @@ -1,10 +0,0 @@ -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20241010.2 - -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps_aten.sh && rm -rf /tmp/scripts - -ARG BUILD_UID=1001 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER \ No newline at end of file diff --git a/tools/ci_build/github/linux/docker/Dockerfile.lort_cpu b/tools/ci_build/github/linux/docker/Dockerfile.lort_cpu deleted file mode 100644 index 04b535e49548..000000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.lort_cpu +++ /dev/null @@ -1,10 +0,0 @@ -FROM registry.access.redhat.com/ubi8/ubi - -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps_lort.sh && rm -rf /tmp/scripts -ENV PATH /opt/rh/gcc-toolset-13/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -ARG BUILD_UID=1002 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda11_8 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda11_8 deleted file mode 100644 index 9ade5dbef84b..000000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda11_8 +++ /dev/null @@ -1,26 +0,0 @@ -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241010.2 -ARG PYTHON_VERSION=3.9 -ARG TORCH_VERSION=2.0.0 -ARG OPSET_VERSION=17 -ARG INSTALL_DEPS_EXTRA_ARGS - -#Add our own dependencies -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && \ - /tmp/scripts/manylinux/install_centos.sh && \ - /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_rust.sh - -ENV PATH="/root/.cargo/bin/:$PATH" - -RUN /tmp/scripts/install_ninja.sh && \ - /tmp/scripts/install_python_deps.sh -d gpu -v 11.8 -p $PYTHON_VERSION -h $TORCH_VERSION $INSTALL_DEPS_EXTRA_ARGS && \ - rm -rf /tmp/scripts - -ARG BUILD_UID=1001 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER -ENV PATH /usr/local/dotnet:$PATH -ENV ORTMODULE_ONNX_OPSET_VERSION=$OPSET_VERSION diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh deleted file mode 100755 index 0d1b7049df7e..000000000000 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -set -e -x - -# Development tools and libraries -dnf -y install \ - graphviz - -if [ ! -d "/opt/conda/bin" ]; then - PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11") -else - PYTHON_EXES=("/opt/conda/bin/python") -fi - - -SYS_LONG_BIT=$(getconf LONG_BIT) -mkdir -p /tmp/src - -DISTRIBUTOR=$(lsb_release -i -s) - -if [[ ("$DISTRIBUTOR" = "CentOS" || "$DISTRIBUTOR" = "RedHatEnterprise") && $SYS_LONG_BIT = "64" ]]; then - LIBDIR="lib64" -else - LIBDIR="lib" -fi - -cd /tmp/src -source $(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/install_shared_deps.sh - -cd /tmp/src - -if ! [ -x "$(command -v protoc)" ]; then - source ${0/%install_deps_aten\.sh/..\/install_protobuf.sh} -fi - -export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" - -for PYTHON_EXE in "${PYTHON_EXES[@]}" -do - ${PYTHON_EXE} -m pip install -r ${0/%install_deps_aten\.sh/requirements\.txt} - if ! [[ ${PYTHON_EXE} = "/opt/python/cp310-cp310/bin/python3.10" ]]; then - ${PYTHON_EXE} -m pip install -r ${0/%install_deps_aten\.sh/..\/training\/ortmodule\/stage1\/requirements_torch_cpu\/requirements.txt} - else - ${PYTHON_EXE} -m pip install torch==2.3.0+cpu -f https://download.pytorch.org/whl/torch_stable.html - fi -done - -cd / -rm -rf /tmp/src diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh deleted file mode 100755 index 06a117098d3f..000000000000 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -e -x - -# Development tools and libraries -dnf -y install \ - graphviz xz gcc-toolset-13-gcc-c++ gcc-toolset-13-gcc gcc-toolset-13-libstdc++-devel cmake python39-devel git -source /opt/rh/gcc-toolset-13/enable -mkdir -p /tmp/src - -cd /tmp/src -source $(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/install_shared_deps.sh - -cd /tmp/src - -if ! [ -x "$(command -v protoc)" ]; then - source ${0/%install_deps_lort\.sh/..\/install_protobuf.sh} -fi - -export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" -PYTHON_EXE=/usr/bin/python3.9 - -echo "Installing Pytorch requirements" -# This may install PyTorch, which will be overrided by the PyTorch local build below. -# beartype is installed here so that onnxscript installation step won't -# install a version PyTorch doesn't like. Once beartype fixes this problem. -# We can remove this line. -$PYTHON_EXE -m pip install -r /tmp/scripts/lort/requirements.txt - -cd /usr/local/ -echo "Cloning ONNX Script" -git clone --recursive https://github.com/microsoft/onnxscript.git -cd onnxscript -$PYTHON_EXE -m pip install . -cd ~ && $PYTHON_EXE -c "import onnxscript; print(f'Installed ONNX Script: {onnxscript.__version__}')" - -cd /usr/local -echo "Cloning Pytorch" -git clone --recursive https://github.com/pytorch/pytorch.git -cd pytorch - -echo "Building and installing Pytorch" -VERBOSE=1 BUILD_LAZY_TS_BACKEND=1 $PYTHON_EXE setup.py install -cd ~ && $PYTHON_EXE -c "import torch; print(f'Installed Pytorch: {torch.__version__}')" - -cd / -rm -rf /tmp/src From 281511acf84f27948967c21fd860727ca757b5d3 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 18:33:15 +0000 Subject: [PATCH 37/40] update --- .../orttraining-pai-ci-pipeline.yml | 405 ------------------ .../scripts/manylinux/install_deps_eager.sh | 55 --- .../pai/rocm-ci-pipeline-env.Dockerfile | 143 ------- 3 files changed, 603 deletions(-) delete mode 100644 tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml delete mode 100755 tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh delete mode 100644 tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile diff --git a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml deleted file mode 100644 index 2c520a25cb39..000000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml +++ /dev/null @@ -1,405 +0,0 @@ -trigger: none - -pr: - branches: - include: - - main - - rel-* - paths: - exclude: - - 'js/web' - - 'onnxruntime/core/providers/js' -name: 'orttraining_ci_$(Date:yyyyMMdd)_$(Rev:r)' - -resources: - repositories: - - repository: manylinux - type: Github - endpoint: Microsoft - name: pypa/manylinux - ref: 5eda9aded5462201e6310105728d33016e637ea7 - -variables: - - name: video - value: 44 - - name: render - value: 109 - - name: RocmVersion - value: 6.1 - - name: RocmVersionPatchSuffix - value: ".3" - - name: BuildConfig - value: Release - -jobs: -- job: Linux_Build_manylinux - variables: - skipComponentGovernanceDetection: true - CCACHE_DIR: $(Pipeline.Workspace)/ccache - TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] - workspace: - clean: all - pool: onnxruntime-Ubuntu2204-AMD-CPU - timeoutInMinutes: 240 - - steps: - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - - - checkout: self - clean: true - submodules: recursive - - - template: templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: >- - --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur - --network=host --build-arg POLICY=manylinux_2_28 --build-arg PLATFORM=x86_64 - --build-arg BUILD_UID=$(id -u) - --build-arg ROCM_VERSION=$(RocmVersion)$(RocmVersionPatchSuffix) - --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-12/root - --build-arg PREPEND_PATH=/opt/rh/gcc-toolset-12/root/usr/bin: - --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib - Repository: onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-manylinux-build - CheckOutManyLinux: true - - - task: Cache@2 - inputs: - key: '"manylinux" | "$(TODAY)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"' - path: $(CCACHE_DIR) - cacheHitVar: CACHE_RESTORED - restoreKeys: | - "manylinux" | "$(TODAY)" | "$(Build.SourceBranch)" - "manylinux" | "$(TODAY)" | - displayName: Cache Task - - - script: mkdir -p $(CCACHE_DIR) - condition: ne(variables.CACHE_RESTORED, 'true') - displayName: Create Cache Dir - - - task: CmdLine@2 - inputs: - script: |- - export ROCM_HOME=/opt/rocm - docker run --rm \ - --ipc=host \ - --network=host \ - --cap-add=SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --shm-size=1024m \ - --user $UID:$(id -g $USER) \ - -e CC=/opt/rh/gcc-toolset-12/root/usr/bin/cc -e CXX=/opt/rh/gcc-toolset-12/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \ - -e CCACHE_DIR=/cache \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume $(CCACHE_DIR):/cache \ - --workdir /onnxruntime_src \ - onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-manylinux-build \ - /bin/bash -c " - set -ex; \ - ccache -s; \ - /opt/python/cp39-cp39/bin/python3 tools/ci_build/build.py \ - --config $(BuildConfig) \ - --enable_training \ - --mpi_home /opt/ompi \ - --cmake_extra_defines \ - CMAKE_HIP_COMPILER=${ROCM_HOME}/llvm/bin/clang++ \ - onnxruntime_BUILD_UNIT_TESTS=OFF \ - FETCHCONTENT_TRY_FIND_PACKAGE_MODE=NEVER \ - --use_cache \ - --use_rocm \ - --use_migraphx \ - --rocm_version=$(RocmVersion) \ - --rocm_home ${ROCM_HOME} \ - --nccl_home ${ROCM_HOME}\ - --update \ - --build_dir /build \ - --build \ - --parallel \ - --build_wheel \ - --skip_submodule_sync \ - --skip_tests; \ - ccache -sv; \ - ccache -z" - displayName: 'Build onnxruntime' - - - template: templates/explicitly-defined-final-tasks.yml - -- job: Linux_Build_ubuntu - variables: - skipComponentGovernanceDetection: true - CCACHE_DIR: $(Pipeline.Workspace)/ccache - TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] - workspace: - clean: all - pool: onnxruntime-Ubuntu2204-AMD-CPU - timeoutInMinutes: 240 - - steps: - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - - - checkout: self - clean: true - submodules: recursive - - - template: templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg ROCM_VERSION=$(RocmVersion)$(RocmVersionPatchSuffix)" - Repository: onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-build - - #- script: |- - # sed -i 's|session_options.use_deterministic_compute = False|session_options.use_deterministic_compute = True|g' \ - # orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py - # displayName: 'Toggle ON deterministic compute mode for ORTModule' - - - task: Cache@2 - inputs: - key: '"$(TODAY)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"' - path: $(CCACHE_DIR) - cacheHitVar: CACHE_RESTORED - restoreKeys: | - "$(TODAY)" | "$(Build.SourceBranch)" - "$(TODAY)" | - displayName: Cache Task - - - script: mkdir -p $(CCACHE_DIR) - condition: ne(variables.CACHE_RESTORED, 'true') - displayName: Create Cache Dir - - - task: CmdLine@2 - inputs: - script: |- - export ROCM_HOME=/opt/rocm - docker run --rm \ - --security-opt seccomp=unconfined \ - --shm-size=1024m \ - --user $UID:$(id -g $USER) \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume $(CCACHE_DIR):/cache \ - -e CCACHE_DIR=/cache \ - --workdir /onnxruntime_src \ - onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-build \ - /bin/bash -c " - set -ex; \ - ccache -s; \ - python tools/ci_build/build.py \ - --config $(BuildConfig) \ - --enable_training \ - --mpi_home /opt/ompi \ - --cmake_extra_defines \ - CMAKE_HIP_COMPILER=${ROCM_HOME}/llvm/bin/clang++ \ - onnxruntime_BUILD_KERNEL_EXPLORER=ON \ - --use_cache \ - --use_rocm \ - --rocm_version=$(RocmVersion) \ - --rocm_home ${ROCM_HOME} \ - --nccl_home ${ROCM_HOME}\ - --update \ - --build_dir /build \ - --build \ - --parallel \ - --build_wheel \ - --skip_submodule_sync \ - --skip_tests; \ - ccache -sv; \ - ccache -z" - displayName: 'Build onnxruntime' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/Release' - - - template: templates/explicitly-defined-final-tasks.yml - -- job: Linux_Test_ubuntu - workspace: - clean: all - pool: AMD-GPU - dependsOn: - - Linux_Build_ubuntu - timeoutInMinutes: 240 - - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact' - inputs: - buildType: 'current' - artifactName: 'drop-linux' - targetPath: '$(Build.BinariesDirectory)/Release' - - - checkout: self - clean: true - submodules: recursive - - - template: templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg ROCM_VERSION=$(RocmVersion)$(RocmVersionPatchSuffix)" - Repository: onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-test - - - task: Bash@3 - inputs: - targetType: filePath - filePath: $(Build.SourcesDirectory)/tools/ci_build/github/pai/pai_clean_device.sh - arguments: -n $(Agent.Name) -d $HIP_VISIBLE_DEVICES -r $DRIVER_RENDER - displayName: 'Check ROCm Environment' - - # TODO: move to use ci_build/build.py driven tests - - task: CmdLine@2 - inputs: - script: |- - docker run --rm \ - --security-opt seccomp=unconfined \ - --shm-size=1024m \ - --device=/dev/kfd \ - --device=/dev/dri/renderD$DRIVER_RENDER \ - --group-add $(video) \ - --group-add $(render) \ - --user onnxruntimedev \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - -e OPENBLAS_NUM_THREADS=1 \ - -e OPENMP_NUM_THREADS=1 \ - -e MKL_NUM_THREADS=1 \ - -e PYTHONPATH=/build/$(BuildConfig) \ - onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-test \ - /bin/bash -c " - set -ex; \ - pip install -r /onnxruntime_src/tools/ci_build/requirements/transformers-test/requirements.txt; \ - pytest /onnxruntime_src/onnxruntime/test/python/transformers/test_flash_attn_rocm.py -v -n 4 --reruns 1" - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Run tranformers tests' - condition: succeededOrFailed() - - - task: CmdLine@2 - inputs: - script: |- - docker run --rm \ - --security-opt seccomp=unconfined \ - --shm-size=1024m \ - --device=/dev/kfd \ - --device=/dev/dri/renderD$DRIVER_RENDER \ - --group-add $(video) \ - --group-add $(render) \ - --user onnxruntimedev \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --workdir /build/$(BuildConfig) \ - onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-test \ - /bin/bash -c " - set -ex; \ - chmod a+x /build/Release/onnxruntime_test_all; \ - /onnxruntime_src/tools/ci_build/github/pai/pai_test_launcher.sh" - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Run onnxruntime unit tests' - condition: succeeded() - - - task: CmdLine@2 - inputs: - script: |- - docker run --rm \ - --security-opt seccomp=unconfined \ - --shm-size=1024m \ - --device=/dev/kfd \ - --device=/dev/dri/renderD$DRIVER_RENDER \ - --group-add $(video) \ - --group-add $(render) \ - --user onnxruntimedev \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - -e OPENBLAS_NUM_THREADS=1 \ - -e OPENMP_NUM_THREADS=1 \ - -e MKL_NUM_THREADS=1 \ - -e KERNEL_EXPLORER_BUILD_DIR=/build/$(BuildConfig) \ - -e KERNEL_EXPLORER_BATCHED_GEMM_MAX_BATCH_SIZE=8 \ - -e KERNEL_EXPLORER_TEST_USE_CUPY=1 \ - onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-test \ - pytest /onnxruntime_src/onnxruntime/python/tools/kernel_explorer/ -n 4 --reruns 1 --durations=100 - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Run kernel explorer tests' - condition: succeededOrFailed() - - - task: CmdLine@2 - inputs: - script: |- - docker run --rm \ - --security-opt seccomp=unconfined \ - --shm-size=1024m \ - --device=/dev/kfd \ - --device=/dev/dri/renderD$DRIVER_RENDER \ - --group-add $(video) \ - --group-add $(render) \ - --user onnxruntimedev \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --workdir /build/$(BuildConfig) \ - onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-test \ - /bin/bash -c " - set -ex; \ - export PYTHONPATH=/build/$(BuildConfig); \ - python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install; \ - bash /onnxruntime_src/tools/ci_build/github/pai/pai_huggingface_bert_large_test.sh -v $(RocmVersion)" - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Run Python Hugging-Face BERT-L test' - condition: succeededOrFailed() - - - # Entry point for all ORTModule tests - # The onnxruntime folder is deleted in the build directory - # to enforce use of the onnxruntime wheel - - task: CmdLine@2 - inputs: - script: |- - rm -rf $(Build.BinariesDirectory)/$(BuildConfig)/onnxruntime/ - files=($(Build.BinariesDirectory)/$(BuildConfig)/dist/*.whl) - echo ${files[0]} - whlfilename=$(basename ${files[0]}) - echo $whlfilename - docker run --rm \ - --security-opt seccomp=unconfined \ - --shm-size=1024m \ - --device=/dev/kfd \ - --device=/dev/dri/renderD$DRIVER_RENDER \ - --group-add $(video) \ - --group-add $(render) \ - --user onnxruntimedev \ - --volume $(Build.BinariesDirectory):/build \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --workdir /build/$(BuildConfig) \ - onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)-test \ - /bin/bash -c " - set -ex; \ - unset PYTHONPATH; \ - pip install /build/$(BuildConfig)/dist/$whlfilename; \ - python -m onnxruntime.training.ortmodule.torch_cpp_extensions.install; \ - mkdir /home/onnxruntimedev/mnist /home/onnxruntimedev/bert_data; \ - export ORTMODULE_DISABLE_CPU_TRAINING_TEST=1; \ - export ORTMODULE_ROCM_TEST=1; \ - python orttraining_ortmodule_tests.py \ - --mnist /home/onnxruntimedev/mnist \ - --bert_data /home/onnxruntimedev/bert_data/hf_data/glue_data/CoLA/original/raw" - workingDirectory: $(Build.SourcesDirectory) - displayName: 'Run orttraining_ortmodule_tests.py' - condition: succeededOrFailed() - - - - task: Bash@3 - inputs: - targetType: filePath - filePath: $(Build.SourcesDirectory)/tools/ci_build/github/pai/pai_clean_device.sh - arguments: -n $(Agent.Name) -d $HIP_VISIBLE_DEVICES -r $DRIVER_RENDER - displayName: 'Clean ROCm Environment' - condition: always() - - - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh deleted file mode 100755 index d8d2fbc06a00..000000000000 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_eager.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -e -x - -# Development tools and libraries -yum -y install \ - graphviz - -if [ ! -d "/opt/conda/bin" ]; then - PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp312-cp312/bin/python3.12") -else - PYTHON_EXES=("/opt/conda/bin/python") -fi - -os_major_version=$(tr -dc '0-9.' < /etc/redhat-release |cut -d \. -f1) - -SYS_LONG_BIT=$(getconf LONG_BIT) -mkdir -p /tmp/src -GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.) - -DISTRIBUTOR=$(lsb_release -i -s) - -if [[ ("$DISTRIBUTOR" = "CentOS" || "$DISTRIBUTOR" = "RedHatEnterprise") && $SYS_LONG_BIT = "64" ]]; then - LIBDIR="lib64" -else - LIBDIR="lib" -fi - -cd /tmp/src -source $(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/install_shared_deps.sh - -cd /tmp/src - -if ! [ -x "$(command -v protoc)" ]; then - source ${0/%install_deps_eager\.sh/..\/install_protobuf.sh} -fi - -export ONNX_ML=1 -export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" - -for PYTHON_EXE in "${PYTHON_EXES[@]}" -do - ${PYTHON_EXE} -m pip install -r ${0/%install_deps_eager\.sh/requirements\.txt} - ${PYTHON_EXE} -m pip install -r ${0/%install_deps_eager\.sh/..\/training\/ortmodule\/stage1\/torch_eager_cpu\/requirements.txt} -done - -cd /tmp/src -GetFile 'https://sourceware.org/pub/valgrind/valgrind-3.16.1.tar.bz2' /tmp/src/valgrind-3.16.1.tar.bz2 -tar -jxvf valgrind-3.16.1.tar.bz2 -cd valgrind-3.16.1 -./configure --prefix=/usr --libdir=/usr/lib64 --enable-only64bit --enable-tls -make -j$(getconf _NPROCESSORS_ONLN) -make install - -cd / -rm -rf /tmp/src diff --git a/tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile b/tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile deleted file mode 100644 index 4e55ce29f46f..000000000000 --- a/tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile +++ /dev/null @@ -1,143 +0,0 @@ -# Refer to https://github.com/RadeonOpenCompute/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-22.04-complete -FROM ubuntu:22.04 - -ARG ROCM_VERSION=6.1 -ARG AMDGPU_VERSION=${ROCM_VERSION} -ARG APT_PREF='Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' - -CMD ["/bin/bash"] - -RUN echo "$APT_PREF" > /etc/apt/preferences.d/rocm-pin-600 - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates curl libnuma-dev gnupg && \ - curl -sL https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - &&\ - printf "deb [arch=amd64] https://repo.radeon.com/rocm/apt/$ROCM_VERSION/ jammy main" | tee /etc/apt/sources.list.d/rocm.list && \ - printf "deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu jammy main" | tee /etc/apt/sources.list.d/amdgpu.list && \ - apt-get update && apt-get install -y --no-install-recommends \ - sudo \ - libelf1 \ - kmod \ - file \ - python3 \ - python3-pip \ - rocm-dev \ - rocm-libs \ - build-essential && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN groupadd -g 109 render - -RUN apt-get update -y && apt-get upgrade -y && apt-get autoremove -y libprotobuf\* protobuf-compiler\* && \ - rm -f /usr/local/bin/protoc && apt-get install -y locales unzip wget git && apt-get clean -y -RUN locale-gen en_US.UTF-8 -RUN update-locale LANG=en_US.UTF-8 -ENV LC_ALL C.UTF-8 -ENV LANG C.UTF-8 - -WORKDIR /stage - -# CMake -ENV CMAKE_VERSION=3.30.1 -RUN cd /usr/local && \ - wget -q -O - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz | tar zxf - -ENV PATH=/usr/local/cmake-${CMAKE_VERSION}-linux-x86_64/bin:${PATH} - -# ccache -RUN mkdir -p /tmp/ccache && \ - cd /tmp/ccache && \ - wget -q -O - https://github.com/ccache/ccache/releases/download/v4.7.4/ccache-4.7.4-linux-x86_64.tar.xz | tar --strip 1 -J -xf - && \ - cp /tmp/ccache/ccache /usr/bin && \ - rm -rf /tmp/ccache - -# Install Conda -ENV PATH /opt/miniconda/bin:${PATH} -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh --no-check-certificate && /bin/bash ~/miniconda.sh -b -p /opt/miniconda && \ - conda init bash && \ - conda config --set auto_activate_base false && \ - conda update --all && \ - rm ~/miniconda.sh && conda clean -ya - -# Create rocm-ci environment -ENV CONDA_ENVIRONMENT_PATH /opt/miniconda/envs/rocm-ci -ENV CONDA_DEFAULT_ENV rocm-ci -RUN conda create -y -n ${CONDA_DEFAULT_ENV} python=3.9 -ENV PATH ${CONDA_ENVIRONMENT_PATH}/bin:${PATH} - -# Enable rocm-ci environment -SHELL ["conda", "run", "-n", "rocm-ci", "/bin/bash", "-c"] - -# ln -sf is needed to make sure that version `GLIBCXX_3.4.30' is found -RUN ln -sf /usr/lib/x86_64-linux-gnu/libstdc++.so.6 ${CONDA_ENVIRONMENT_PATH}/bin/../lib/libstdc++.so.6 - -# Install Pytorch -RUN export MAJOR=$(cut -d '.' -f 1 <<< "$ROCM_VERSION") && \ - export MINOR=$(cut -d '.' -f 2 <<< "$ROCM_VERSION") && \ - export PATCH=$(cut -d '.' -f 3 <<< "$ROCM_VERSION") && \ - pip install torch==2.1.2 torchvision==0.16.1 -f https://repo.radeon.com/rocm/manylinux/rocm-rel-${MAJOR}.${MINOR}/ && \ - pip install torch-ort --no-dependencies - -##### Install Cupy to decrease CPU utilization -# Install non dev openmpi -RUN wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.5.tar.bz2 && \ - tar -jxf openmpi-4.1.5.tar.bz2 && \ - cd openmpi-4.1.5 && \ - ./configure --prefix=/opt/ompi && \ - make -j4 all && \ - make install && \ - cd ../ && \ - rm -r openmpi-4.1.5 && \ - rm openmpi-4.1.5.tar.bz2 - -# Install CuPy, No stable version is available -RUN git clone https://github.com/ROCmSoftwarePlatform/cupy && cd cupy && \ - git checkout 432a8683351d681e00903640489cb2f4055d2e09 && \ - export CUPY_INSTALL_USE_HIP=1 && \ - export ROCM_HOME=/opt/rocm && \ - export HCC_AMDGPU_TARGET=gfx906,gfx908,gfx90a && \ - git submodule update --init && \ - pip install -e . --no-cache-dir -vvvv - -##### Install transformers to run tests -# rocm-ci branch contains instrumentation needed for loss curves and perf -RUN git clone https://github.com/microsoft/huggingface-transformers.git &&\ - cd huggingface-transformers &&\ - git checkout rocm-ci &&\ - pip install -e . - -RUN pip install \ - flatbuffers==2.0 \ - numpy==1.24.1 \ - onnx \ - cerberus \ - sympy \ - h5py \ - datasets==2.17.0 \ - requests \ - sacrebleu==1.5.1 \ - sacremoses \ - scipy==1.10.0 \ - scikit-learn \ - tokenizers \ - sentencepiece \ - wget \ - dill==0.3.4 \ - pytorch_lightning==2.3.3 \ - tensorboard \ - pytest-xdist \ - pytest-rerunfailures \ - ml_dtypes==0.3.0 \ - pytest==7.4.4 - -# Install migraphx -RUN apt update && apt install -y migraphx - -ENV ORTMODULE_ONNX_OPSET_VERSION=17 - -ARG BUILD_UID=1001 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER From 70a1a48cced4e51788479d59b32bb5fe21b25c59 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 19:06:57 +0000 Subject: [PATCH 38/40] update --- .../inference/aarch64/default/cpu/scripts/install_centos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/scripts/install_centos.sh index adb0464d6496..596a5ce436c5 100755 --- a/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/scripts/install_centos.sh +++ b/tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/scripts/install_centos.sh @@ -4,5 +4,5 @@ set -e -x os_major_version=$(tr -dc '0-9.' < /etc/redhat-release |cut -d \. -f1) echo "installing for CentOS version : $os_major_version" -dnf install -y python39-devel glibc-langpack-\* glibc-locale-source which redhat-lsb-core expat-devel tar unzip zlib-devel make bzip2 bzip2-devel java-11-openjdk-devel graphviz gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran +dnf install -y python3.12-pip python3.12-devel glibc-langpack-\* glibc-locale-source which redhat-lsb-core expat-devel tar unzip zlib-devel make bzip2 bzip2-devel java-11-openjdk-devel graphviz gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran locale From be597a5484c2811f31b728182bdae80e75677a67 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 19:09:28 +0000 Subject: [PATCH 39/40] update --- ...Dockerfile.manylinux2_28_training_cuda12_2 | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda12_2 diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda12_2 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda12_2 deleted file mode 100644 index eb376d34441b..000000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_training_cuda12_2 +++ /dev/null @@ -1,27 +0,0 @@ -FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241010.2 - -ARG PYTHON_VERSION=3.9 -ARG TORCH_VERSION=2.1.0 -ARG OPSET_VERSION=17 -ARG INSTALL_DEPS_EXTRA_ARGS - -#Add our own dependencies -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && \ - /tmp/scripts/manylinux/install_centos.sh && \ - /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_rust.sh - -ENV PATH="/root/.cargo/bin/:$PATH" - -RUN /tmp/scripts/install_ninja.sh && \ - /tmp/scripts/install_python_deps.sh -d gpu -v 12.2 -p $PYTHON_VERSION -h $TORCH_VERSION $INSTALL_DEPS_EXTRA_ARGS && \ - rm -rf /tmp/scripts - -ARG BUILD_UID=1001 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER -ENV PATH /usr/local/dotnet:$PATH -ENV ORTMODULE_ONNX_OPSET_VERSION=$OPSET_VERSION From 0f007383a027f30b85a8e48c1e1febd0b601eb0e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 11 Oct 2024 19:11:20 +0000 Subject: [PATCH 40/40] revert --- .../inference/x86_64/default/cpu/scripts/install_centos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh index 0ac0ca625a1a..03534d8a2f44 100755 --- a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh +++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh @@ -1,4 +1,4 @@ -!/bin/bash +#!/bin/bash set -e -x if [ ! -f /etc/yum.repos.d/microsoft-prod.repo ]; then os_major_version=$(tr -dc '0-9.' < /etc/redhat-release |cut -d \. -f1)