diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml
index 2eac370c65..931e87bcfd 100644
--- a/.github/workflows/build_conda_linux.yml
+++ b/.github/workflows/build_conda_linux.yml
@@ -126,8 +126,6 @@ jobs:
         run: |
           set -euxo pipefail
           source "${BUILD_ENV_FILE}"
-          export FFMPEG_ROOT="${PWD}"/third_party/ffmpeg
-          export USE_FFMPEG="1"
 
           ${CONDA_RUN} conda build \
             -c defaults \
@@ -178,7 +176,6 @@ jobs:
             ${CONDA_RUN_SMOKE} python3 -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)"
           else
             echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found"
-            export LD_LIBRARY_PATH="{LD_LIBRARY_PATH}:${{ inputs.repository }}/third_party/ffmpeg/lib"
             conda install --yes --quiet 'ffmpeg<5'
             ${CONDA_RUN_SMOKE} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
           fi
diff --git a/.github/workflows/build_conda_macos.yml b/.github/workflows/build_conda_macos.yml
index 42970fbac6..6471966134 100644
--- a/.github/workflows/build_conda_macos.yml
+++ b/.github/workflows/build_conda_macos.yml
@@ -133,10 +133,8 @@ jobs:
           source "${BUILD_ENV_FILE}"
 
           if [[ "${{ inputs.package-name }}" = "torchaudio" ]]; then
-            export USE_FFMPEG="1"
             export USE_OPENMP="0"
           fi
-          export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
 
           ${CONDA_RUN} conda build \
             -c defaults \
diff --git a/.github/workflows/build_conda_windows.yml b/.github/workflows/build_conda_windows.yml
index 56386b26f4..974fcfd06c 100644
--- a/.github/workflows/build_conda_windows.yml
+++ b/.github/workflows/build_conda_windows.yml
@@ -150,8 +150,6 @@ jobs:
           source "${BUILD_ENV_FILE}"
           export VSTOOLCHAIN_PACKAGE=vs2019
           export VSDEVCMD_ARGS=''
-          export FFMPEG_ROOT="${PWD}"/third_party/ffmpeg
-          export USE_FFMPEG="1"
 
           #conda install -yq conda-build "conda-package-handling!=1.5.0"
           #bash packaging/build_conda.sh
@@ -208,7 +206,6 @@ jobs:
             ${CONDA_RUN_SMOKE} python -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)"
           else
             echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found"
-            export LD_LIBRARY_PATH="{LD_LIBRARY_PATH}:${{ inputs.repository }}/third_party/ffmpeg/lib"
             conda install --yes --quiet 'ffmpeg<5'
             ${CONDA_RUN_SMOKE} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
           fi
diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml
index 8bd086e8bf..093e590958 100644
--- a/.github/workflows/build_wheels_linux.yml
+++ b/.github/workflows/build_wheels_linux.yml
@@ -141,8 +141,6 @@ jobs:
         run: |
           set -euxo pipefail
           source "${BUILD_ENV_FILE}"
-          export FFMPEG_ROOT="${PWD}"/third_party/ffmpeg
-          export USE_FFMPEG="1"
           export PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version:  *//' | sed 's/+.\+//')"
           ${CONDA_RUN} python setup.py bdist_wheel
       - name: Upload wheel to GitHub
@@ -187,7 +185,6 @@ jobs:
             fi
           else
             echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found"
-            export LD_LIBRARY_PATH="{LD_LIBRARY_PATH}:${{ inputs.repository }}/third_party/ffmpeg/lib"
             ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
           fi
       - name: Upload package to pytorch.org
diff --git a/.github/workflows/build_wheels_macos.yml b/.github/workflows/build_wheels_macos.yml
index 5f83397987..772aba6ae6 100644
--- a/.github/workflows/build_wheels_macos.yml
+++ b/.github/workflows/build_wheels_macos.yml
@@ -139,12 +139,10 @@ jobs:
           source "${BUILD_ENV_FILE}"
 
           if [[ "${{ inputs.package-name }}" = "torchaudio" ]]; then
-            export USE_FFMPEG="1"
             export USE_OPENMP="0"
           fi
           PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version:  *//')"
           export PYTORCH_VERSION
-          export FFMPEG_ROOT="${PWD}/third_party/ffmpeg"
 
           ${CONDA_RUN} python3 setup.py bdist_wheel
       - name: Delocate wheel
@@ -193,10 +191,7 @@ jobs:
           else
             echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found"
 
-            export DYLD_FALLBACK_LIBRARY_PATH="${PWD}/${{ inputs.repository }}/third_party/ffmpeg/lib"
-            echo $DYLD_FALLBACK_LIBRARY_PATH
-
-            ${CONDA_RUN} conda install --yes --quiet ffmpeg
+            ${CONDA_RUN} conda install --yes --quiet 'ffmpeg<5'
             ${CONDA_RUN} python3 "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
           fi
           export PATH=${OLD_PATH}
diff --git a/.github/workflows/build_wheels_windows.yml b/.github/workflows/build_wheels_windows.yml
index fc3d1bea0f..493e065191 100644
--- a/.github/workflows/build_wheels_windows.yml
+++ b/.github/workflows/build_wheels_windows.yml
@@ -136,8 +136,6 @@ jobs:
           BUILD_PARAMS: ${{ inputs.wheel-build-params }}
         run: |
           source "${BUILD_ENV_FILE}"
-          export FFMPEG_ROOT="${PWD}"/third_party/ffmpeg
-          export USE_FFMPEG="1"
 
           if [[ "$CU_VERSION" == "cpu" ]]; then
           # CUDA and CPU are ABI compatible on the CPU-only parts, so strip
@@ -187,7 +185,6 @@ jobs:
             ${CONDA_RUN} python -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)"
           else
             echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found"
-            export LD_LIBRARY_PATH="{LD_LIBRARY_PATH}:${{ inputs.repository }}/third_party/ffmpeg/lib"
             ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
           fi
       - name: Upload package to pytorch.org
diff --git a/.github/workflows/test_build_conda_linux_with_cuda.yml b/.github/workflows/test_build_conda_linux_with_cuda.yml
index 229b4f0cfa..e14c81cffd 100644
--- a/.github/workflows/test_build_conda_linux_with_cuda.yml
+++ b/.github/workflows/test_build_conda_linux_with_cuda.yml
@@ -25,10 +25,6 @@ jobs:
       matrix:
         include:
           - repository: pytorch/audio
-            cache-path: /third_party/ffmpeg
-            cache-key: linux-ffmpeg
-            pre-script: packaging/pre_build_script.sh
-            post-script: packaging/post_build_script.sh
             smoke-test-script: test/smoke_test/smoke_test.py
             conda-package-directory: packaging/torchaudio
             package-name: torchaudio
diff --git a/.github/workflows/test_build_conda_m1.yml b/.github/workflows/test_build_conda_m1.yml
index 219ed4b770..cdbc036847 100644
--- a/.github/workflows/test_build_conda_m1.yml
+++ b/.github/workflows/test_build_conda_m1.yml
@@ -25,10 +25,6 @@ jobs:
       matrix:
         include:
           - repository: pytorch/audio
-            cache-path: /third_party/ffmpeg
-            cache-key: macos-m1-ffmpeg
-            pre-script: packaging/pre_build_script.sh
-            post-script: packaging/post_build_script.sh
             conda-package-directory: packaging/torchaudio
             smoke-test-script: test/smoke_test/smoke_test.py
             package-name: torchaudio
diff --git a/.github/workflows/test_build_conda_macos.yml b/.github/workflows/test_build_conda_macos.yml
index a2724595b0..d63802e22c 100644
--- a/.github/workflows/test_build_conda_macos.yml
+++ b/.github/workflows/test_build_conda_macos.yml
@@ -24,10 +24,6 @@ jobs:
       matrix:
         include:
           - repository: pytorch/audio
-            cache-path: /third_party/ffmpeg
-            cache-key: macos-ffmpeg
-            pre-script: packaging/pre_build_script.sh
-            post-script: packaging/post_build_script.sh
             conda-package-directory: packaging/torchaudio
             smoke-test-script: test/smoke_test/smoke_test.py
             package-name: torchaudio
diff --git a/.github/workflows/test_build_conda_windows_with_cuda.yml b/.github/workflows/test_build_conda_windows_with_cuda.yml
index 9b60a3ae04..8c68c06858 100644
--- a/.github/workflows/test_build_conda_windows_with_cuda.yml
+++ b/.github/workflows/test_build_conda_windows_with_cuda.yml
@@ -25,10 +25,6 @@ jobs:
       matrix:
         include:
           - repository: pytorch/audio
-            cache-path: /third_party/ffmpeg
-            cache-key: windows-ffmpeg
-            pre-script: packaging/ffmpeg/build.bat
-            post-script: ""
             smoke-test-script: test/smoke_test/smoke_test.py
             conda-package-directory: packaging/torchaudio
             package-name: torchaudio
diff --git a/.github/workflows/test_build_wheels_linux_with_cuda.yml b/.github/workflows/test_build_wheels_linux_with_cuda.yml
index fa06d2cc6e..07d9fe07f1 100644
--- a/.github/workflows/test_build_wheels_linux_with_cuda.yml
+++ b/.github/workflows/test_build_wheels_linux_with_cuda.yml
@@ -26,10 +26,6 @@ jobs:
       matrix:
         include:
           - repository: pytorch/audio
-            cache-path: /third_party/ffmpeg
-            cache-key: linux-ffmpeg
-            pre-script: packaging/pre_build_script.sh
-            post-script: packaging/post_build_script.sh
             smoke-test-script: test/smoke_test/smoke_test.py
             package-name: torchaudio
           - repository: pytorch/vision
diff --git a/.github/workflows/test_build_wheels_m1.yml b/.github/workflows/test_build_wheels_m1.yml
index 9cb4039384..2b90e9d53b 100644
--- a/.github/workflows/test_build_wheels_m1.yml
+++ b/.github/workflows/test_build_wheels_m1.yml
@@ -25,10 +25,6 @@ jobs:
       matrix:
         include:
           - repository: pytorch/audio
-            cache-path: /third_party/ffmpeg
-            cache-key: macos-m1-ffmpeg
-            pre-script: packaging/pre_build_script.sh
-            post-script: packaging/post_build_script.sh
             conda-package-directory: packaging/torchaudio
             smoke-test-script: test/smoke_test/smoke_test.py
             package-name: torchaudio
diff --git a/.github/workflows/test_build_wheels_macos.yml b/.github/workflows/test_build_wheels_macos.yml
index 5271c6c9c2..737ee4bfb1 100644
--- a/.github/workflows/test_build_wheels_macos.yml
+++ b/.github/workflows/test_build_wheels_macos.yml
@@ -25,10 +25,6 @@ jobs:
       matrix:
         include:
           - repository: pytorch/audio
-            cache-path: /third_party/ffmpeg
-            cache-key: macos-ffmpeg
-            pre-script: packaging/pre_build_script.sh
-            post-script: packaging/post_build_script.sh
             conda-package-directory: packaging/torchaudio
             smoke-test-script: test/smoke_test/smoke_test.py
             package-name: torchaudio
diff --git a/.github/workflows/test_build_wheels_windows_with_cuda.yml b/.github/workflows/test_build_wheels_windows_with_cuda.yml
index 19c01ce08f..bf18572489 100644
--- a/.github/workflows/test_build_wheels_windows_with_cuda.yml
+++ b/.github/workflows/test_build_wheels_windows_with_cuda.yml
@@ -25,12 +25,8 @@ jobs:
       matrix:
         include:
         - repository: pytorch/audio
-          pre-script: packaging/ffmpeg/build.bat
           env-script: packaging/vc_env_helper.bat
-          cache-path: /third_party/ffmpeg
-          cache-key: windows-ffmpeg
           wheel-build-params: "--plat-name win_amd64"
-          post-script: ""
           smoke-test-script: test/smoke_test/smoke_test.py
           package-name: torchaudio
         - repository: pytorch/vision