From 87673d9250f1ed41b05ffa0fabe4d77800779bd5 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Fri, 7 Jul 2023 15:26:08 -0400 Subject: [PATCH 1/2] Remove special cases for torchaudio FFmpeg https://github.com/pytorch/audio/pull/3460 changed the way torchaudio is built, and there is no longer need to supply FFmepg externally. test-infra no longer needs to build FFmpeg and use pre/post-scripts for torchaudio. The only issue remaining for now is smoke test. Currently torchaudio's smoke test script by default expects that FFmpeg libraries are available. This can be disabled with `--no-ffmpeg`, but the way YML files are written in test-infra repo does not allow to pass a flag to smoke test. We can switch the behavior, or use other smoke test. --- .github/workflows/build_conda_linux.yml | 3 --- .github/workflows/build_conda_macos.yml | 2 -- .github/workflows/build_conda_windows.yml | 3 --- .github/workflows/build_wheels_linux.yml | 3 --- .github/workflows/build_wheels_macos.yml | 7 +------ .github/workflows/build_wheels_windows.yml | 3 --- .github/workflows/test_build_conda_linux_with_cuda.yml | 8 ++++---- .github/workflows/test_build_conda_m1.yml | 8 ++++---- .github/workflows/test_build_conda_macos.yml | 8 ++++---- .github/workflows/test_build_conda_windows_with_cuda.yml | 6 +++--- .github/workflows/test_build_wheels_linux_with_cuda.yml | 8 ++++---- .github/workflows/test_build_wheels_m1.yml | 8 ++++---- .github/workflows/test_build_wheels_macos.yml | 8 ++++---- .github/workflows/test_build_wheels_windows_with_cuda.yml | 6 +++--- 14 files changed, 31 insertions(+), 50 deletions(-) diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml index 2eac370c65..931e87bcfd 100644 --- a/.github/workflows/build_conda_linux.yml +++ b/.github/workflows/build_conda_linux.yml @@ -126,8 +126,6 @@ jobs: run: | set -euxo pipefail source "${BUILD_ENV_FILE}" - export FFMPEG_ROOT="${PWD}"/third_party/ffmpeg - export USE_FFMPEG="1" ${CONDA_RUN} conda build \ -c defaults \ @@ -178,7 +176,6 @@ jobs: ${CONDA_RUN_SMOKE} python3 -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)" else echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" - export LD_LIBRARY_PATH="{LD_LIBRARY_PATH}:${{ inputs.repository }}/third_party/ffmpeg/lib" conda install --yes --quiet 'ffmpeg<5' ${CONDA_RUN_SMOKE} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi diff --git a/.github/workflows/build_conda_macos.yml b/.github/workflows/build_conda_macos.yml index 42970fbac6..6471966134 100644 --- a/.github/workflows/build_conda_macos.yml +++ b/.github/workflows/build_conda_macos.yml @@ -133,10 +133,8 @@ jobs: source "${BUILD_ENV_FILE}" if [[ "${{ inputs.package-name }}" = "torchaudio" ]]; then - export USE_FFMPEG="1" export USE_OPENMP="0" fi - export FFMPEG_ROOT="${PWD}/third_party/ffmpeg" ${CONDA_RUN} conda build \ -c defaults \ diff --git a/.github/workflows/build_conda_windows.yml b/.github/workflows/build_conda_windows.yml index 56386b26f4..974fcfd06c 100644 --- a/.github/workflows/build_conda_windows.yml +++ b/.github/workflows/build_conda_windows.yml @@ -150,8 +150,6 @@ jobs: source "${BUILD_ENV_FILE}" export VSTOOLCHAIN_PACKAGE=vs2019 export VSDEVCMD_ARGS='' - export FFMPEG_ROOT="${PWD}"/third_party/ffmpeg - export USE_FFMPEG="1" #conda install -yq conda-build "conda-package-handling!=1.5.0" #bash packaging/build_conda.sh @@ -208,7 +206,6 @@ jobs: ${CONDA_RUN_SMOKE} python -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)" else echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" - export LD_LIBRARY_PATH="{LD_LIBRARY_PATH}:${{ inputs.repository }}/third_party/ffmpeg/lib" conda install --yes --quiet 'ffmpeg<5' ${CONDA_RUN_SMOKE} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index 8bd086e8bf..093e590958 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -141,8 +141,6 @@ jobs: run: | set -euxo pipefail source "${BUILD_ENV_FILE}" - export FFMPEG_ROOT="${PWD}"/third_party/ffmpeg - export USE_FFMPEG="1" export PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')" ${CONDA_RUN} python setup.py bdist_wheel - name: Upload wheel to GitHub @@ -187,7 +185,6 @@ jobs: fi else echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" - export LD_LIBRARY_PATH="{LD_LIBRARY_PATH}:${{ inputs.repository }}/third_party/ffmpeg/lib" ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi - name: Upload package to pytorch.org diff --git a/.github/workflows/build_wheels_macos.yml b/.github/workflows/build_wheels_macos.yml index 5f83397987..772aba6ae6 100644 --- a/.github/workflows/build_wheels_macos.yml +++ b/.github/workflows/build_wheels_macos.yml @@ -139,12 +139,10 @@ jobs: source "${BUILD_ENV_FILE}" if [[ "${{ inputs.package-name }}" = "torchaudio" ]]; then - export USE_FFMPEG="1" export USE_OPENMP="0" fi PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version: *//')" export PYTORCH_VERSION - export FFMPEG_ROOT="${PWD}/third_party/ffmpeg" ${CONDA_RUN} python3 setup.py bdist_wheel - name: Delocate wheel @@ -193,10 +191,7 @@ jobs: else echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" - export DYLD_FALLBACK_LIBRARY_PATH="${PWD}/${{ inputs.repository }}/third_party/ffmpeg/lib" - echo $DYLD_FALLBACK_LIBRARY_PATH - - ${CONDA_RUN} conda install --yes --quiet ffmpeg + ${CONDA_RUN} conda install --yes --quiet 'ffmpeg<5' ${CONDA_RUN} python3 "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi export PATH=${OLD_PATH} diff --git a/.github/workflows/build_wheels_windows.yml b/.github/workflows/build_wheels_windows.yml index fc3d1bea0f..493e065191 100644 --- a/.github/workflows/build_wheels_windows.yml +++ b/.github/workflows/build_wheels_windows.yml @@ -136,8 +136,6 @@ jobs: BUILD_PARAMS: ${{ inputs.wheel-build-params }} run: | source "${BUILD_ENV_FILE}" - export FFMPEG_ROOT="${PWD}"/third_party/ffmpeg - export USE_FFMPEG="1" if [[ "$CU_VERSION" == "cpu" ]]; then # CUDA and CPU are ABI compatible on the CPU-only parts, so strip @@ -187,7 +185,6 @@ jobs: ${CONDA_RUN} python -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)" else echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" - export LD_LIBRARY_PATH="{LD_LIBRARY_PATH}:${{ inputs.repository }}/third_party/ffmpeg/lib" ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi - name: Upload package to pytorch.org diff --git a/.github/workflows/test_build_conda_linux_with_cuda.yml b/.github/workflows/test_build_conda_linux_with_cuda.yml index 229b4f0cfa..7f6ed41e20 100644 --- a/.github/workflows/test_build_conda_linux_with_cuda.yml +++ b/.github/workflows/test_build_conda_linux_with_cuda.yml @@ -25,10 +25,10 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: /third_party/ffmpeg - cache-key: linux-ffmpeg - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh + cache-path: "" + cache-key: "" + pre-script: "" + post-script: "" smoke-test-script: test/smoke_test/smoke_test.py conda-package-directory: packaging/torchaudio package-name: torchaudio diff --git a/.github/workflows/test_build_conda_m1.yml b/.github/workflows/test_build_conda_m1.yml index 219ed4b770..6af0ff3d85 100644 --- a/.github/workflows/test_build_conda_m1.yml +++ b/.github/workflows/test_build_conda_m1.yml @@ -25,10 +25,10 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: /third_party/ffmpeg - cache-key: macos-m1-ffmpeg - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh + cache-path: "" + cache-key: "" + pre-script: "" + post-script: "" conda-package-directory: packaging/torchaudio smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio diff --git a/.github/workflows/test_build_conda_macos.yml b/.github/workflows/test_build_conda_macos.yml index a2724595b0..e71eb0ff7c 100644 --- a/.github/workflows/test_build_conda_macos.yml +++ b/.github/workflows/test_build_conda_macos.yml @@ -24,10 +24,10 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: /third_party/ffmpeg - cache-key: macos-ffmpeg - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh + cache-path: "" + cache-key: "" + pre-script: "" + post-script: "" conda-package-directory: packaging/torchaudio smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio diff --git a/.github/workflows/test_build_conda_windows_with_cuda.yml b/.github/workflows/test_build_conda_windows_with_cuda.yml index 9b60a3ae04..3b71551163 100644 --- a/.github/workflows/test_build_conda_windows_with_cuda.yml +++ b/.github/workflows/test_build_conda_windows_with_cuda.yml @@ -25,9 +25,9 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: /third_party/ffmpeg - cache-key: windows-ffmpeg - pre-script: packaging/ffmpeg/build.bat + cache-path: "" + cache-key: "" + pre-script: "" post-script: "" smoke-test-script: test/smoke_test/smoke_test.py conda-package-directory: packaging/torchaudio diff --git a/.github/workflows/test_build_wheels_linux_with_cuda.yml b/.github/workflows/test_build_wheels_linux_with_cuda.yml index fa06d2cc6e..1419ef2ed6 100644 --- a/.github/workflows/test_build_wheels_linux_with_cuda.yml +++ b/.github/workflows/test_build_wheels_linux_with_cuda.yml @@ -26,10 +26,10 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: /third_party/ffmpeg - cache-key: linux-ffmpeg - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh + cache-path: "" + cache-key: "" + pre-script: "" + post-script: "" smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio - repository: pytorch/vision diff --git a/.github/workflows/test_build_wheels_m1.yml b/.github/workflows/test_build_wheels_m1.yml index 9cb4039384..3024742e03 100644 --- a/.github/workflows/test_build_wheels_m1.yml +++ b/.github/workflows/test_build_wheels_m1.yml @@ -25,10 +25,10 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: /third_party/ffmpeg - cache-key: macos-m1-ffmpeg - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh + cache-path: "" + cache-key: "" + pre-script: "" + post-script: "" conda-package-directory: packaging/torchaudio smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio diff --git a/.github/workflows/test_build_wheels_macos.yml b/.github/workflows/test_build_wheels_macos.yml index 5271c6c9c2..e8cd644067 100644 --- a/.github/workflows/test_build_wheels_macos.yml +++ b/.github/workflows/test_build_wheels_macos.yml @@ -25,10 +25,10 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: /third_party/ffmpeg - cache-key: macos-ffmpeg - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh + cache-path: "" + cache-key: "" + pre-script: "" + post-script: "" conda-package-directory: packaging/torchaudio smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio diff --git a/.github/workflows/test_build_wheels_windows_with_cuda.yml b/.github/workflows/test_build_wheels_windows_with_cuda.yml index 19c01ce08f..57cc0503f6 100644 --- a/.github/workflows/test_build_wheels_windows_with_cuda.yml +++ b/.github/workflows/test_build_wheels_windows_with_cuda.yml @@ -25,10 +25,10 @@ jobs: matrix: include: - repository: pytorch/audio - pre-script: packaging/ffmpeg/build.bat + pre-script: "" env-script: packaging/vc_env_helper.bat - cache-path: /third_party/ffmpeg - cache-key: windows-ffmpeg + cache-path: "" + cache-key: "" wheel-build-params: "--plat-name win_amd64" post-script: "" smoke-test-script: test/smoke_test/smoke_test.py From 87c25ccc08600d3b5246b2deabb7cdf6b59925bd Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Mon, 10 Jul 2023 17:34:24 -0400 Subject: [PATCH 2/2] Remove unused options --- .github/workflows/test_build_conda_linux_with_cuda.yml | 4 ---- .github/workflows/test_build_conda_m1.yml | 4 ---- .github/workflows/test_build_conda_macos.yml | 4 ---- .github/workflows/test_build_conda_windows_with_cuda.yml | 4 ---- .github/workflows/test_build_wheels_linux_with_cuda.yml | 4 ---- .github/workflows/test_build_wheels_m1.yml | 4 ---- .github/workflows/test_build_wheels_macos.yml | 4 ---- .github/workflows/test_build_wheels_windows_with_cuda.yml | 4 ---- 8 files changed, 32 deletions(-) diff --git a/.github/workflows/test_build_conda_linux_with_cuda.yml b/.github/workflows/test_build_conda_linux_with_cuda.yml index 7f6ed41e20..e14c81cffd 100644 --- a/.github/workflows/test_build_conda_linux_with_cuda.yml +++ b/.github/workflows/test_build_conda_linux_with_cuda.yml @@ -25,10 +25,6 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: "" - cache-key: "" - pre-script: "" - post-script: "" smoke-test-script: test/smoke_test/smoke_test.py conda-package-directory: packaging/torchaudio package-name: torchaudio diff --git a/.github/workflows/test_build_conda_m1.yml b/.github/workflows/test_build_conda_m1.yml index 6af0ff3d85..cdbc036847 100644 --- a/.github/workflows/test_build_conda_m1.yml +++ b/.github/workflows/test_build_conda_m1.yml @@ -25,10 +25,6 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: "" - cache-key: "" - pre-script: "" - post-script: "" conda-package-directory: packaging/torchaudio smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio diff --git a/.github/workflows/test_build_conda_macos.yml b/.github/workflows/test_build_conda_macos.yml index e71eb0ff7c..d63802e22c 100644 --- a/.github/workflows/test_build_conda_macos.yml +++ b/.github/workflows/test_build_conda_macos.yml @@ -24,10 +24,6 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: "" - cache-key: "" - pre-script: "" - post-script: "" conda-package-directory: packaging/torchaudio smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio diff --git a/.github/workflows/test_build_conda_windows_with_cuda.yml b/.github/workflows/test_build_conda_windows_with_cuda.yml index 3b71551163..8c68c06858 100644 --- a/.github/workflows/test_build_conda_windows_with_cuda.yml +++ b/.github/workflows/test_build_conda_windows_with_cuda.yml @@ -25,10 +25,6 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: "" - cache-key: "" - pre-script: "" - post-script: "" smoke-test-script: test/smoke_test/smoke_test.py conda-package-directory: packaging/torchaudio package-name: torchaudio diff --git a/.github/workflows/test_build_wheels_linux_with_cuda.yml b/.github/workflows/test_build_wheels_linux_with_cuda.yml index 1419ef2ed6..07d9fe07f1 100644 --- a/.github/workflows/test_build_wheels_linux_with_cuda.yml +++ b/.github/workflows/test_build_wheels_linux_with_cuda.yml @@ -26,10 +26,6 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: "" - cache-key: "" - pre-script: "" - post-script: "" smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio - repository: pytorch/vision diff --git a/.github/workflows/test_build_wheels_m1.yml b/.github/workflows/test_build_wheels_m1.yml index 3024742e03..2b90e9d53b 100644 --- a/.github/workflows/test_build_wheels_m1.yml +++ b/.github/workflows/test_build_wheels_m1.yml @@ -25,10 +25,6 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: "" - cache-key: "" - pre-script: "" - post-script: "" conda-package-directory: packaging/torchaudio smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio diff --git a/.github/workflows/test_build_wheels_macos.yml b/.github/workflows/test_build_wheels_macos.yml index e8cd644067..737ee4bfb1 100644 --- a/.github/workflows/test_build_wheels_macos.yml +++ b/.github/workflows/test_build_wheels_macos.yml @@ -25,10 +25,6 @@ jobs: matrix: include: - repository: pytorch/audio - cache-path: "" - cache-key: "" - pre-script: "" - post-script: "" conda-package-directory: packaging/torchaudio smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio diff --git a/.github/workflows/test_build_wheels_windows_with_cuda.yml b/.github/workflows/test_build_wheels_windows_with_cuda.yml index 57cc0503f6..bf18572489 100644 --- a/.github/workflows/test_build_wheels_windows_with_cuda.yml +++ b/.github/workflows/test_build_wheels_windows_with_cuda.yml @@ -25,12 +25,8 @@ jobs: matrix: include: - repository: pytorch/audio - pre-script: "" env-script: packaging/vc_env_helper.bat - cache-path: "" - cache-key: "" wheel-build-params: "--plat-name win_amd64" - post-script: "" smoke-test-script: test/smoke_test/smoke_test.py package-name: torchaudio - repository: pytorch/vision