From c859bfd1abe819a447afbef5eefcf263eb8ad408 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 17 Oct 2023 18:12:29 +0900 Subject: [PATCH] GH-38295: [CI][R] Free up disk space for Azure Pipelines jobs (#38302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change test-r-rhub-ubuntu-gcc-release-latest doesn't have enough disk space. ### What changes are included in this PR? Remove pre-installed files on Azure Pipelines too. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * Closes: #38295 Authored-by: Sutou Kouhei Signed-off-by: Raúl Cumplido --- ci/scripts/util_free_space.sh | 154 +++++++++++++++++----------------- dev/tasks/macros.jinja | 5 ++ dev/tasks/r/azure.linux.yml | 103 +++++++++++------------ 3 files changed, 130 insertions(+), 132 deletions(-) diff --git a/ci/scripts/util_free_space.sh b/ci/scripts/util_free_space.sh index acea074feec63..0518869d06993 100755 --- a/ci/scripts/util_free_space.sh +++ b/ci/scripts/util_free_space.sh @@ -19,81 +19,79 @@ set -eux -if [ "${GITHUB_ACTIONS}" = "true" ]; then - df -h - echo "::group::/usr/local/*" - du -hsc /usr/local/* - echo "::endgroup::" - # ~1GB - sudo rm -rf \ - /usr/local/aws-cli \ - /usr/local/aws-sam-cil \ - /usr/local/julia* || : - echo "::group::/usr/local/bin/*" - du -hsc /usr/local/bin/* - echo "::endgroup::" - # ~1GB (From 1.2GB to 214MB) - sudo rm -rf \ - /usr/local/bin/aliyun \ - /usr/local/bin/aws \ - /usr/local/bin/aws_completer \ - /usr/local/bin/azcopy \ - /usr/local/bin/bicep \ - /usr/local/bin/cmake-gui \ - /usr/local/bin/cpack \ - /usr/local/bin/helm \ - /usr/local/bin/hub \ - /usr/local/bin/kubectl \ - /usr/local/bin/minikube \ - /usr/local/bin/node \ - /usr/local/bin/packer \ - /usr/local/bin/pulumi* \ - /usr/local/bin/sam \ - /usr/local/bin/stack \ - /usr/local/bin/terraform || : - # 142M - sudo rm -rf /usr/local/bin/oc || : \ - echo "::group::/usr/local/share/*" - du -hsc /usr/local/share/* - echo "::endgroup::" - # 506MB - sudo rm -rf /usr/local/share/chromium || : - # 1.3GB - sudo rm -rf /usr/local/share/powershell || : - echo "::group::/usr/local/lib/*" - du -hsc /usr/local/lib/* - echo "::endgroup::" - # 15GB - sudo rm -rf /usr/local/lib/android || : - # 341MB - sudo rm -rf /usr/local/lib/heroku || : - # 1.2GB - sudo rm -rf /usr/local/lib/node_modules || : - echo "::group::/opt/*" - du -hsc /opt/* - echo "::endgroup::" - # 679MB - sudo rm -rf /opt/az || : - echo "::group::/opt/microsoft/*" - du -hsc /opt/microsoft/* - echo "::endgroup::" - # 197MB - sudo rm -rf /opt/microsoft/powershell || : - echo "::group::/opt/hostedtoolcache/*" - du -hsc /opt/hostedtoolcache/* - echo "::endgroup::" - # 5.3GB - sudo rm -rf /opt/hostedtoolcache/CodeQL || : - # 1.4GB - sudo rm -rf /opt/hostedtoolcache/go || : - # 489MB - sudo rm -rf /opt/hostedtoolcache/PyPy || : - # 376MB - sudo rm -rf /opt/hostedtoolcache/node || : - # Remove Web browser packages - sudo apt purge -y \ - firefox \ - google-chrome-stable \ - microsoft-edge-stable - df -h -fi +df -h +echo "::group::/usr/local/*" +du -hsc /usr/local/* +echo "::endgroup::" +# ~1GB +sudo rm -rf \ + /usr/local/aws-cli \ + /usr/local/aws-sam-cil \ + /usr/local/julia* || : +echo "::group::/usr/local/bin/*" +du -hsc /usr/local/bin/* +echo "::endgroup::" +# ~1GB (From 1.2GB to 214MB) +sudo rm -rf \ + /usr/local/bin/aliyun \ + /usr/local/bin/aws \ + /usr/local/bin/aws_completer \ + /usr/local/bin/azcopy \ + /usr/local/bin/bicep \ + /usr/local/bin/cmake-gui \ + /usr/local/bin/cpack \ + /usr/local/bin/helm \ + /usr/local/bin/hub \ + /usr/local/bin/kubectl \ + /usr/local/bin/minikube \ + /usr/local/bin/node \ + /usr/local/bin/packer \ + /usr/local/bin/pulumi* \ + /usr/local/bin/sam \ + /usr/local/bin/stack \ + /usr/local/bin/terraform || : +# 142M +sudo rm -rf /usr/local/bin/oc || : \ +echo "::group::/usr/local/share/*" +du -hsc /usr/local/share/* +echo "::endgroup::" +# 506MB +sudo rm -rf /usr/local/share/chromium || : +# 1.3GB +sudo rm -rf /usr/local/share/powershell || : +echo "::group::/usr/local/lib/*" +du -hsc /usr/local/lib/* +echo "::endgroup::" +# 15GB +sudo rm -rf /usr/local/lib/android || : +# 341MB +sudo rm -rf /usr/local/lib/heroku || : +# 1.2GB +sudo rm -rf /usr/local/lib/node_modules || : +echo "::group::/opt/*" +du -hsc /opt/* +echo "::endgroup::" +# 679MB +sudo rm -rf /opt/az || : +echo "::group::/opt/microsoft/*" +du -hsc /opt/microsoft/* +echo "::endgroup::" +# 197MB +sudo rm -rf /opt/microsoft/powershell || : +echo "::group::/opt/hostedtoolcache/*" +du -hsc /opt/hostedtoolcache/* +echo "::endgroup::" +# 5.3GB +sudo rm -rf /opt/hostedtoolcache/CodeQL || : +# 1.4GB +sudo rm -rf /opt/hostedtoolcache/go || : +# 489MB +sudo rm -rf /opt/hostedtoolcache/PyPy || : +# 376MB +sudo rm -rf /opt/hostedtoolcache/node || : +# Remove Web browser packages +sudo apt purge -y \ + firefox \ + google-chrome-stable \ + microsoft-edge-stable +df -h diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 3bcff2f63541f..9ee95dec15a14 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -182,6 +182,11 @@ on: displayName: Clone arrow {% endmacro %} +{%- macro azure_free_space() -%} + - script: arrow/ci/scripts/util_free_space.sh + displayName: Free up disk space +{% endmacro %} + {%- macro azure_upload_releases(pattern) -%} - task: UsePythonVersion@0 inputs: diff --git a/dev/tasks/r/azure.linux.yml b/dev/tasks/r/azure.linux.yml index 0cbc70ed00b52..e26a59629fa1a 100644 --- a/dev/tasks/r/azure.linux.yml +++ b/dev/tasks/r/azure.linux.yml @@ -16,62 +16,57 @@ {% import 'macros.jinja' as macros with context %} jobs: - - job: linux - pool: - vmImage: ubuntu-latest - timeoutInMinutes: 360 - {% if env is defined %} - variables: +- job: linux + pool: + vmImage: ubuntu-latest + timeoutInMinutes: 360 + {% if env is defined %} + variables: {% for key, value in env.items() %} - {{ key }}: {{ value }} + {{ key }}: {{ value }} {% endfor %} - {% endif %} - steps: - - script: | - set -ex - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - displayName: Clone arrow + {% endif %} + steps: + {{ macros.azure_checkout_arrow() }} + {{ macros.azure_free_space() }} - - script: | - set -ex - docker -v - docker-compose -v - cd arrow - export R_ORG={{ r_org }} - export R_IMAGE={{ r_image }} - export R_TAG={{ r_tag }} - export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }} - export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }} - docker-compose pull --ignore-pull-failures r - docker-compose build r - displayName: Docker build - env: - {{ macros.azure_set_sccache_envvars()|indent(8) }} + - script: | + set -ex + docker -v + docker-compose -v + cd arrow + export R_ORG={{ r_org }} + export R_IMAGE={{ r_image }} + export R_TAG={{ r_tag }} + export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }} + export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }} + docker-compose pull --ignore-pull-failures r + docker-compose build r + displayName: Docker build + env: + {{ macros.azure_set_sccache_envvars()|indent(4) }} - - script: | - set -ex - cd arrow - export R_ORG={{ r_org }} - export R_IMAGE={{ r_image }} - export R_TAG={{ r_tag }} - export ARROW_R_DEV={{ not_cran|default("TRUE") }} - # Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true - # if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off. - docker-compose run {{ flags|default("") }} r - displayName: Docker run - env: - {{ macros.azure_set_sccache_envvars()|indent(8) }} + - script: | + set -ex + cd arrow + export R_ORG={{ r_org }} + export R_IMAGE={{ r_image }} + export R_TAG={{ r_tag }} + export ARROW_R_DEV={{ not_cran|default("TRUE") }} + # Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true + # if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off. + docker-compose run {{ flags|default("") }} r + displayName: Docker run + env: + {{ macros.azure_set_sccache_envvars()|indent(4) }} - - script: | - set -ex - cat arrow/r/check/arrow.Rcheck/00install.out - displayName: Dump install logs - condition: succeededOrFailed() - - script: | - set -ex - cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout* - displayName: Dump test logs - condition: succeededOrFailed() + - script: | + set -ex + cat arrow/r/check/arrow.Rcheck/00install.out + displayName: Dump install logs + condition: succeededOrFailed() + - script: | + set -ex + cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout* + displayName: Dump test logs + condition: succeededOrFailed()