Skip to content

Commit

Permalink
GH-38295: [CI][R] Free up disk space for Azure Pipelines jobs (#38302)
Browse files Browse the repository at this point in the history 
### Rationale for this change

test-r-rhub-ubuntu-gcc-release-latest doesn't have enough disk space.

### What changes are included in this PR?

Remove pre-installed files on Azure Pipelines too.

### Are these changes tested?

Yes.

### Are there any user-facing changes?

No.
* Closes: #38295

Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Raúl Cumplido <raulcumplido@gmail.com>
  • Loading branch information
@kou
kou authored Oct 17, 2023
1 parent ecd3871 commit 94ed8aa
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 132 deletions.
154 changes: 76 additions & 78 deletions ci/scripts/util_free_space.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,81 +19,79 @@

set -eux

if [ "${GITHUB_ACTIONS}" = "true" ]; then
df -h
echo "::group::/usr/local/*"
du -hsc /usr/local/*
echo "::endgroup::"
# ~1GB
sudo rm -rf \
/usr/local/aws-cli \
/usr/local/aws-sam-cil \
/usr/local/julia* || :
echo "::group::/usr/local/bin/*"
du -hsc /usr/local/bin/*
echo "::endgroup::"
# ~1GB (From 1.2GB to 214MB)
sudo rm -rf \
/usr/local/bin/aliyun \
/usr/local/bin/aws \
/usr/local/bin/aws_completer \
/usr/local/bin/azcopy \
/usr/local/bin/bicep \
/usr/local/bin/cmake-gui \
/usr/local/bin/cpack \
/usr/local/bin/helm \
/usr/local/bin/hub \
/usr/local/bin/kubectl \
/usr/local/bin/minikube \
/usr/local/bin/node \
/usr/local/bin/packer \
/usr/local/bin/pulumi* \
/usr/local/bin/sam \
/usr/local/bin/stack \
/usr/local/bin/terraform || :
# 142M
sudo rm -rf /usr/local/bin/oc || : \
echo "::group::/usr/local/share/*"
du -hsc /usr/local/share/*
echo "::endgroup::"
# 506MB
sudo rm -rf /usr/local/share/chromium || :
# 1.3GB
sudo rm -rf /usr/local/share/powershell || :
echo "::group::/usr/local/lib/*"
du -hsc /usr/local/lib/*
echo "::endgroup::"
# 15GB
sudo rm -rf /usr/local/lib/android || :
# 341MB
sudo rm -rf /usr/local/lib/heroku || :
# 1.2GB
sudo rm -rf /usr/local/lib/node_modules || :
echo "::group::/opt/*"
du -hsc /opt/*
echo "::endgroup::"
# 679MB
sudo rm -rf /opt/az || :
echo "::group::/opt/microsoft/*"
du -hsc /opt/microsoft/*
echo "::endgroup::"
# 197MB
sudo rm -rf /opt/microsoft/powershell || :
echo "::group::/opt/hostedtoolcache/*"
du -hsc /opt/hostedtoolcache/*
echo "::endgroup::"
# 5.3GB
sudo rm -rf /opt/hostedtoolcache/CodeQL || :
# 1.4GB
sudo rm -rf /opt/hostedtoolcache/go || :
# 489MB
sudo rm -rf /opt/hostedtoolcache/PyPy || :
# 376MB
sudo rm -rf /opt/hostedtoolcache/node || :
# Remove Web browser packages
sudo apt purge -y \
firefox \
google-chrome-stable \
microsoft-edge-stable
df -h
fi
df -h
echo "::group::/usr/local/*"
du -hsc /usr/local/*
echo "::endgroup::"
# ~1GB
sudo rm -rf \
/usr/local/aws-cli \
/usr/local/aws-sam-cil \
/usr/local/julia* || :
echo "::group::/usr/local/bin/*"
du -hsc /usr/local/bin/*
echo "::endgroup::"
# ~1GB (From 1.2GB to 214MB)
sudo rm -rf \
/usr/local/bin/aliyun \
/usr/local/bin/aws \
/usr/local/bin/aws_completer \
/usr/local/bin/azcopy \
/usr/local/bin/bicep \
/usr/local/bin/cmake-gui \
/usr/local/bin/cpack \
/usr/local/bin/helm \
/usr/local/bin/hub \
/usr/local/bin/kubectl \
/usr/local/bin/minikube \
/usr/local/bin/node \
/usr/local/bin/packer \
/usr/local/bin/pulumi* \
/usr/local/bin/sam \
/usr/local/bin/stack \
/usr/local/bin/terraform || :
# 142M
sudo rm -rf /usr/local/bin/oc || : \
echo "::group::/usr/local/share/*"
du -hsc /usr/local/share/*
echo "::endgroup::"
# 506MB
sudo rm -rf /usr/local/share/chromium || :
# 1.3GB
sudo rm -rf /usr/local/share/powershell || :
echo "::group::/usr/local/lib/*"
du -hsc /usr/local/lib/*
echo "::endgroup::"
# 15GB
sudo rm -rf /usr/local/lib/android || :
# 341MB
sudo rm -rf /usr/local/lib/heroku || :
# 1.2GB
sudo rm -rf /usr/local/lib/node_modules || :
echo "::group::/opt/*"
du -hsc /opt/*
echo "::endgroup::"
# 679MB
sudo rm -rf /opt/az || :
echo "::group::/opt/microsoft/*"
du -hsc /opt/microsoft/*
echo "::endgroup::"
# 197MB
sudo rm -rf /opt/microsoft/powershell || :
echo "::group::/opt/hostedtoolcache/*"
du -hsc /opt/hostedtoolcache/*
echo "::endgroup::"
# 5.3GB
sudo rm -rf /opt/hostedtoolcache/CodeQL || :
# 1.4GB
sudo rm -rf /opt/hostedtoolcache/go || :
# 489MB
sudo rm -rf /opt/hostedtoolcache/PyPy || :
# 376MB
sudo rm -rf /opt/hostedtoolcache/node || :
# Remove Web browser packages
sudo apt purge -y \
firefox \
google-chrome-stable \
microsoft-edge-stable
df -h
5 changes: 5 additions & 0 deletions dev/tasks/macros.jinja
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,11 @@ on:
displayName: Clone arrow
{% endmacro %}

{%- macro azure_free_space() -%}
- script: arrow/ci/scripts/util_free_space.sh
displayName: Free up disk space
{% endmacro %}

{%- macro azure_upload_releases(pattern) -%}
- task: UsePythonVersion@0
inputs:
Expand Down
103 changes: 49 additions & 54 deletions dev/tasks/r/azure.linux.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,62 +16,57 @@
{% import 'macros.jinja' as macros with context %}

jobs:
- job: linux
pool:
vmImage: ubuntu-latest
timeoutInMinutes: 360
{% if env is defined %}
variables:
- job: linux
pool:
vmImage: ubuntu-latest
timeoutInMinutes: 360
{% if env is defined %}
variables:
{% for key, value in env.items() %}
{{ key }}: {{ value }}
{{ key }}: {{ value }}
{% endfor %}
{% endif %}
steps:
- script: |
set -ex
git clone --no-checkout {{ arrow.remote }} arrow
git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
git -C arrow checkout FETCH_HEAD
git -C arrow submodule update --init --recursive
displayName: Clone arrow
{% endif %}
steps:
{{ macros.azure_checkout_arrow() }}
{{ macros.azure_free_space() }}

- script: |
set -ex
docker -v
docker-compose -v
cd arrow
export R_ORG={{ r_org }}
export R_IMAGE={{ r_image }}
export R_TAG={{ r_tag }}
export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }}
export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
docker-compose pull --ignore-pull-failures r
docker-compose build r
displayName: Docker build
env:
{{ macros.azure_set_sccache_envvars()|indent(8) }}
- script: |
set -ex
docker -v
docker-compose -v
cd arrow
export R_ORG={{ r_org }}
export R_IMAGE={{ r_image }}
export R_TAG={{ r_tag }}
export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }}
export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
docker-compose pull --ignore-pull-failures r
docker-compose build r
displayName: Docker build
env:
{{ macros.azure_set_sccache_envvars()|indent(4) }}
- script: |
set -ex
cd arrow
export R_ORG={{ r_org }}
export R_IMAGE={{ r_image }}
export R_TAG={{ r_tag }}
export ARROW_R_DEV={{ not_cran|default("TRUE") }}
# Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true
# if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off.
docker-compose run {{ flags|default("") }} r
displayName: Docker run
env:
{{ macros.azure_set_sccache_envvars()|indent(8) }}
- script: |
set -ex
cd arrow
export R_ORG={{ r_org }}
export R_IMAGE={{ r_image }}
export R_TAG={{ r_tag }}
export ARROW_R_DEV={{ not_cran|default("TRUE") }}
# Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true
# if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off.
docker-compose run {{ flags|default("") }} r
displayName: Docker run
env:
{{ macros.azure_set_sccache_envvars()|indent(4) }}
- script: |
set -ex
cat arrow/r/check/arrow.Rcheck/00install.out
displayName: Dump install logs
condition: succeededOrFailed()
- script: |
set -ex
cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout*
displayName: Dump test logs
condition: succeededOrFailed()
- script: |
set -ex
cat arrow/r/check/arrow.Rcheck/00install.out
displayName: Dump install logs
condition: succeededOrFailed()
- script: |
set -ex
cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout*
displayName: Dump test logs
condition: succeededOrFailed()

0 comments on commit 94ed8aa

Please sign in to comment.