From cd978793fdd2922551c08a5dd629cb2987789df5 Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Wed, 29 Jan 2025 15:22:51 +0100 Subject: [PATCH 01/15] [experiment] ci: free more space in ubuntu 24 free runners --- src/ci/github-actions/jobs.yml | 6 +- src/ci/scripts/free-disk-space.sh | 97 +++++++++++++++++++------------ 2 files changed, 62 insertions(+), 41 deletions(-) diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 4c4863e2b4b3e..379e41e35b544 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -5,7 +5,7 @@ runners: env: { } - &job-linux-4c - os: ubuntu-22.04 + os: ubuntu-24.04 # Free some disk space to avoid running out of space during the build. free_disk: true <<: *base-job @@ -50,7 +50,7 @@ runners: - &job-aarch64-linux # Free some disk space to avoid running out of space during the build. free_disk: true - os: ubuntu-22.04-arm + os: ubuntu-24.04-arm - &job-aarch64-linux-8c os: ubuntu-22.04-arm64-8core-32gb @@ -295,7 +295,7 @@ auto: - name: x86_64-gnu-debug # This seems to be needed because a full stage 2 build + run-make tests # overwhelms the storage capacity of the standard 4c runner. - <<: *job-linux-4c-largedisk + <<: *job-linux-4c - name: x86_64-gnu-distcheck <<: *job-linux-8c diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index 4a7dad0090b2e..b5477dd04fb99 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -1,8 +1,13 @@ #!/bin/bash +set -euo pipefail # Free disk space on Linux GitHub action runners # Script inspired by https://github.com/jlumbroso/free-disk-space +# When updating to a new ubuntu version: +# - Check that there are no docker images preinstalled with `docker image ls` +# - Check that there are no big packages preinstalled that we aren't using + # print a line of the specified character printSeparationLine() { for ((i = 0; i < 80; i++)); do @@ -14,11 +19,15 @@ printSeparationLine() { # compute available space # REF: https://unix.stackexchange.com/a/42049/60849 # REF: https://stackoverflow.com/a/450821/408734 -getAvailableSpace() { echo $(df -a | awk 'NR > 1 {avail+=$4} END {print avail}'); } +getAvailableSpace() { + df -a | awk 'NR > 1 {avail+=$4} END {print avail}' +} # make Kb human readable (assume the input is Kb) # REF: https://unix.stackexchange.com/a/44087/60849 -formatByteCount() { echo $(numfmt --to=iec-i --suffix=B --padding=7 $1'000'); } +formatByteCount() { + numfmt --to=iec-i --suffix=B --padding=7 "$1"'000' +} # macro to output saved space printSavedSpace() { @@ -58,11 +67,30 @@ removeDir() { dir=${1} local before - before=$(getAvailableSpace) - - sudo rm -rf "$dir" || true + if [ ! -d "$dir" ]; then + echo "::warning::Directory $dir does not exist, skipping." + else + before=$(getAvailableSpace) + sudo rm -rf "$dir" + printSavedSpace "$before" "Removed $dir" + fi +} - printSavedSpace "$before" "$dir" +removeUnusedDirectories() { + local dirs_to_remove=( + "/usr/local/lib/android" + # Haskell runtime + "/usr/local/.ghcup" + # Azure + "/opt/az" + "/etc/mysql" + "/usr/share/php" + "/etc/php/" + ) + + for dir in "${dirs_to_remove[@]}"; do + removeDir "$dir" + done } execAndMeasureSpaceChange() { @@ -79,34 +107,33 @@ execAndMeasureSpaceChange() { # Remove large packages # REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh cleanPackages() { - sudo apt-get -qq remove -y --fix-missing \ - '^aspnetcore-.*' \ - '^dotnet-.*' \ - '^llvm-.*' \ - 'php.*' \ - '^mongodb-.*' \ - '^mysql-.*' \ - 'azure-cli' \ - 'google-chrome-stable' \ - 'firefox' \ - 'powershell' \ - 'mono-devel' \ - 'libgl1-mesa-dri' \ - 'google-cloud-sdk' \ - 'google-cloud-cli' + sudo apt-get purge -y --fix-missing \ + '^aspnetcore-.*' \ + '^dotnet-.*' \ + '^java-*' \ + '^libllvm.*' \ + '^llvm-.*' \ + '^mysql-.*' \ + '^vim.*' \ + 'azure-cli' \ + 'firefox' \ + 'gcc' \ + 'gcc-12' \ + 'gcc-13' \ + 'google-chrome-stable' \ + 'google-cloud-cli' \ + 'groff-base' \ + 'kubectl' \ + 'libgl1-mesa-dri' \ + 'microsoft-edge-stable' \ + 'php.*' \ + 'powershell' \ + 'snapd' sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed" sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed" } -# Remove Docker images -cleanDocker() { - echo "Removing the following docker images:" - sudo docker image ls - echo "Removing docker images..." - sudo docker image prune --all --force || true -} - # Remove Swap storage cleanSwap() { sudo swapoff -a || true @@ -121,17 +148,11 @@ AVAILABLE_INITIAL=$(getAvailableSpace) printDF "BEFORE CLEAN-UP:" echo "" -removeDir /usr/local/lib/android -removeDir /usr/share/dotnet - -# Haskell runtime -removeDir /opt/ghc -removeDir /usr/local/.ghcup - -execAndMeasureSpaceChange cleanPackages "Large misc. packages" -execAndMeasureSpaceChange cleanDocker "Docker images" +execAndMeasureSpaceChange cleanPackages "Unused packages" execAndMeasureSpaceChange cleanSwap "Swap storage" +removeUnusedDirectories + # Output saved space statistic echo "" printDF "AFTER CLEAN-UP:" From 64ed17ef033dacf4ccc491255d2c42fff3cc0d7c Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Wed, 29 Jan 2025 17:34:01 +0100 Subject: [PATCH 02/15] remove more stuff --- src/ci/scripts/free-disk-space.sh | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index b5477dd04fb99..5ff6899f1c924 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -79,13 +79,18 @@ removeDir() { removeUnusedDirectories() { local dirs_to_remove=( "/usr/local/lib/android" + "/etc/mysql" + "/usr/share/php" + # Haskell runtime "/usr/local/.ghcup" + # Azure "/opt/az" - "/etc/mysql" - "/usr/share/php" - "/etc/php/" + "/usr/share/az_"* + + # Environemnt variable set by GitHub Actions + "$AGENT_TOOLSDIRECTORY" ) for dir in "${dirs_to_remove[@]}"; do @@ -107,19 +112,27 @@ execAndMeasureSpaceChange() { # Remove large packages # REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh cleanPackages() { - sudo apt-get purge -y --fix-missing \ + sudo apt-get purge -y --autoremove --fix-missing \ + '.*-icon-theme$' \ '^aspnetcore-.*' \ '^dotnet-.*' \ '^java-*' \ '^libllvm.*' \ '^llvm-.*' \ + '^mercurial.*' \ '^mysql-.*' \ '^vim.*' \ + '^fonts-.*' \ 'azure-cli' \ + 'buildah' \ + 'cpp-13' \ 'firefox' \ - 'gcc' \ 'gcc-12' \ 'gcc-13' \ + 'gcc-14' \ + 'gcc' \ + 'g++-14' \ + 'gfortran-14' \ 'google-chrome-stable' \ 'google-cloud-cli' \ 'groff-base' \ @@ -127,10 +140,15 @@ cleanPackages() { 'libgl1-mesa-dri' \ 'microsoft-edge-stable' \ 'php.*' \ + 'podman' \ 'powershell' \ - 'snapd' + 'skopeo' \ + 'snapd' \ + 'tmux' + echo "=> apt-get autoremove" sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed" + echo "=> apt-get clean" sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed" } From 08e10c4fc0c2894387d1192016552b3215609dfa Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Thu, 30 Jan 2025 10:00:41 +0100 Subject: [PATCH 03/15] use free runner --- src/ci/github-actions/jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 379e41e35b544..f415cd2b6634e 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -182,7 +182,7 @@ auto: <<: *job-linux-4c - name: dist-powerpc64le-linux - <<: *job-linux-4c-largedisk + <<: *job-linux-4c - name: dist-riscv64-linux <<: *job-linux-4c From 6bfbed78b1825ba0af9c9a7ff1cf259affdb0b6d Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:34:14 +0100 Subject: [PATCH 04/15] delete more --- src/ci/scripts/free-disk-space.sh | 87 ++++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 13 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index 5ff6899f1c924..67f9a198d4e91 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -4,9 +4,10 @@ set -euo pipefail # Free disk space on Linux GitHub action runners # Script inspired by https://github.com/jlumbroso/free-disk-space -# When updating to a new ubuntu version: +# When updating to a new ubuntu version (e.g. from ubuntu-24.04): # - Check that there are no docker images preinstalled with `docker image ls` # - Check that there are no big packages preinstalled that we aren't using +# - Check that all directores we are removing are still present (look at the warnings) # print a line of the specified character printSeparationLine() { @@ -63,24 +64,62 @@ printDF() { printSeparationLine "=" } -removeDir() { - dir=${1} +removeRecursive() { + element=${1} local before - if [ ! -d "$dir" ]; then - echo "::warning::Directory $dir does not exist, skipping." + if [ ! -e "$element" ]; then + echo "::warning::Directory or file $element does not exist, skipping." else before=$(getAvailableSpace) - sudo rm -rf "$dir" - printSavedSpace "$before" "Removed $dir" + sudo rm -rf "$element" + printSavedSpace "$before" "Removed $element" fi } -removeUnusedDirectories() { - local dirs_to_remove=( - "/usr/local/lib/android" +removeUnusedDirsAndFiles() { + local to_remove=( "/etc/mysql" + "/usr/local/aws-sam-cli" + "/usr/local/doc/cmake" + "/usr/local/julia"* + "/usr/local/lib/android" + "/usr/local/share/chromedriver-"* + "/usr/local/share/chromium" + "/usr/local/share/cmake-"* + "/usr/local/share/edge_driver" + "/usr/local/share/gecko_driver" + "/usr/local/share/icons" + "/usr/local/share/vim" + "/usr/local/share/emacs" + "/usr/local/share/powershell" + "/usr/local/share/vcpkg" + "/usr/share/apache-maven-"* + "/usr/share/gradle-"* + "/usr/share/java" + "/usr/share/kotlinc" + "/usr/share/miniconda" "/usr/share/php" + "/usr/share/ri" + "/usr/share/swift" + + # binaries + "/usr/local/bin/azcopy" + "/usr/local/bin/bicep" + "/usr/local/bin/ccmake" + "/usr/local/bin/cmake-"* + "/usr/local/bin/cmake" + "/usr/local/bin/cpack" + "/usr/local/bin/ctest" + "/usr/local/bin/helm" + "/usr/local/bin/kind" + "/usr/local/bin/kustomize" + "/usr/local/bin/minikube" + "/usr/local/bin/packer" + "/usr/local/bin/phpunit" + "/usr/local/bin/pulumi-"* + "/usr/local/bin/pulumi" + "/usr/local/bin/stack" # Haskell runtime "/usr/local/.ghcup" @@ -93,11 +132,27 @@ removeUnusedDirectories() { "$AGENT_TOOLSDIRECTORY" ) - for dir in "${dirs_to_remove[@]}"; do - removeDir "$dir" + for element in "${to_remove[@]}"; do + removeRecursive "$element" done } +removeNodeModules() { + sudo npm uninstall -g \ + "@bazel/bazelisk" \ + "bazel" \ + "grunt" \ + "gulp" \ + "lerna" \ + "n" \ + "newman" \ + "parcel" \ + "typescript" \ + "webpack-cli" \ + "webpack" \ + "yarn" +} + execAndMeasureSpaceChange() { local operation=${1} # Function to execute local title=${2} @@ -159,6 +214,10 @@ cleanSwap() { free -h } +removePythonPackages() { + sudo pipx uninstall ansible-core +} + # Display initial disk space stats AVAILABLE_INITIAL=$(getAvailableSpace) @@ -168,8 +227,10 @@ echo "" execAndMeasureSpaceChange cleanPackages "Unused packages" execAndMeasureSpaceChange cleanSwap "Swap storage" +execAndMeasureSpaceChange removeNodeModules "Node modules" +execAndMeasureSpaceChange removePythonPackages "Python Packages" -removeUnusedDirectories +removeUnusedDirsAndFiles # Output saved space statistic echo "" From 75802a998316db2ab18573b8c0e307f6ed132193 Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Thu, 30 Jan 2025 15:23:38 +0100 Subject: [PATCH 05/15] debug --- src/ci/scripts/free-disk-space.sh | 186 ++++++++++++++++-------------- 1 file changed, 97 insertions(+), 89 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index 67f9a198d4e91..e7b9dc7e3b887 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -30,25 +30,38 @@ formatByteCount() { numfmt --to=iec-i --suffix=B --padding=7 "$1"'000' } -# macro to output saved space -printSavedSpace() { - # Disk space before the operation - local before=${1} - local title=${2:-} +isX86() { + local arch + arch=$(uname -m) + if [ "$arch" = "x86_64" ]; then + return 0 + else + return 1 + fi +} + +# Measure saved space and how long it took to run the task +execAndMeasure() { + local task_name=${1} + + local start + start=$(date +%s) + local before + before=$(getAvailableSpace) + + # Run the task. Skip the first argument because it's the task name. + "${@:2}" + + local end + end=$(date +%s) local after after=$(getAvailableSpace) + local saved=$((after - before)) + local seconds_taken=$((end - start)) - echo "" - printSeparationLine "*" - if [ -n "${title}" ]; then - echo "=> ${title}: Saved $(formatByteCount "$saved")" - else - echo "=> Saved $(formatByteCount "$saved")" - fi - printSeparationLine "*" - echo "" + echo "==> ${task_name}: Saved $(formatByteCount "$saved") in $seconds_taken seconds" } # macro to print output of df with caption @@ -64,19 +77,6 @@ printDF() { printSeparationLine "=" } -removeRecursive() { - element=${1} - - local before - if [ ! -e "$element" ]; then - echo "::warning::Directory or file $element does not exist, skipping." - else - before=$(getAvailableSpace) - sudo rm -rf "$element" - printSavedSpace "$before" "Removed $element" - fi -} - removeUnusedDirsAndFiles() { local to_remove=( "/etc/mysql" @@ -133,11 +133,17 @@ removeUnusedDirsAndFiles() { ) for element in "${to_remove[@]}"; do - removeRecursive "$element" + if [ ! -e "$element" ]; then + echo "::warning::Directory or file $element does not exist, skipping." + else + execAndMeasure "Removed $element" sudo rm -rf "$element" + fi done } removeNodeModules() { + npm list -g --depth=0 + sudo npm uninstall -g \ "@bazel/bazelisk" \ "bazel" \ @@ -153,53 +159,50 @@ removeNodeModules() { "yarn" } -execAndMeasureSpaceChange() { - local operation=${1} # Function to execute - local title=${2} - - local before - before=$(getAvailableSpace) - $operation - - printSavedSpace "$before" "$title" -} - # Remove large packages # REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh cleanPackages() { - sudo apt-get purge -y --autoremove --fix-missing \ - '.*-icon-theme$' \ - '^aspnetcore-.*' \ - '^dotnet-.*' \ - '^java-*' \ - '^libllvm.*' \ - '^llvm-.*' \ - '^mercurial.*' \ - '^mysql-.*' \ - '^vim.*' \ - '^fonts-.*' \ - 'azure-cli' \ - 'buildah' \ - 'cpp-13' \ - 'firefox' \ - 'gcc-12' \ - 'gcc-13' \ - 'gcc-14' \ - 'gcc' \ - 'g++-14' \ - 'gfortran-14' \ - 'google-chrome-stable' \ - 'google-cloud-cli' \ - 'groff-base' \ - 'kubectl' \ - 'libgl1-mesa-dri' \ - 'microsoft-edge-stable' \ - 'php.*' \ - 'podman' \ - 'powershell' \ - 'skopeo' \ - 'snapd' \ + local packages=( + '.*-icon-theme$' + '^aspnetcore-.*' + '^dotnet-.*' + '^java-*' + '^libllvm.*' + '^llvm-.*' + '^mercurial.*' + '^mysql-.*' + '^vim.*' + '^fonts-.*' + 'azure-cli' + 'buildah' + 'cpp-13' + 'firefox' + 'gcc-12' + 'gcc-13' + 'gcc-14' + 'gcc' + 'g++-14' + 'gfortran-14' + 'groff-base' + 'kubectl' + 'libgl1-mesa-dri' + 'microsoft-edge-stable' + 'php.*' + 'podman' + 'powershell' + 'skopeo' + 'snapd' 'tmux' + ) + + if isX86; then + packages+=( + 'google-chrome-stable' + 'google-cloud-cli' + ) + fi + + sudo apt-get purge -y --autoremove --fix-missing "${packages[@]}" echo "=> apt-get autoremove" sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed" @@ -215,28 +218,33 @@ cleanSwap() { } removePythonPackages() { - sudo pipx uninstall ansible-core -} - -# Display initial disk space stats + local packages=( + ) -AVAILABLE_INITIAL=$(getAvailableSpace) + if isX86; then + packages+=( + 'ansible-core' + ) + fi -printDF "BEFORE CLEAN-UP:" -echo "" + for p in "${packages[@]}"; do + sudo pipx uninstall "$p" + done +} -execAndMeasureSpaceChange cleanPackages "Unused packages" -execAndMeasureSpaceChange cleanSwap "Swap storage" -execAndMeasureSpaceChange removeNodeModules "Node modules" -execAndMeasureSpaceChange removePythonPackages "Python Packages" +main() { + printDF "BEFORE CLEAN-UP:" + echo "" -removeUnusedDirsAndFiles + execAndMeasure "Unused packages" cleanPackages + execAndMeasure "Swap storage" cleanSwap + execAndMeasure "Node modules" removeNodeModules + execAndMeasure "Python Packages" removePythonPackages -# Output saved space statistic -echo "" -printDF "AFTER CLEAN-UP:" + removeUnusedDirsAndFiles -echo "" -echo "" + printDF "AFTER CLEAN-UP:" + echo "" +} -printSavedSpace "$AVAILABLE_INITIAL" "Total saved" +execAndMeasure "Total" main From c0cf64052118da505ab65038a4e64d403a3a007e Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:22:01 +0100 Subject: [PATCH 06/15] wip --- src/ci/scripts/free-disk-space.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index e7b9dc7e3b887..c5730182f3553 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -142,8 +142,6 @@ removeUnusedDirsAndFiles() { } removeNodeModules() { - npm list -g --depth=0 - sudo npm uninstall -g \ "@bazel/bazelisk" \ "bazel" \ @@ -206,7 +204,6 @@ cleanPackages() { echo "=> apt-get autoremove" sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed" - echo "=> apt-get clean" sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed" } From 25853a473f19a54ff598807e06c1ed3e541faa13 Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:24:08 +0100 Subject: [PATCH 07/15] comment --- src/ci/scripts/free-disk-space.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index c5730182f3553..e55d6c99aea69 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -40,7 +40,7 @@ isX86() { fi } -# Measure saved space and how long it took to run the task +# Execute a task, printing how much space was saved and how long it took to run the task execAndMeasure() { local task_name=${1} @@ -75,6 +75,7 @@ printDF() { echo "" df -h printSeparationLine "=" + echo "" } removeUnusedDirsAndFiles() { @@ -231,7 +232,6 @@ removePythonPackages() { main() { printDF "BEFORE CLEAN-UP:" - echo "" execAndMeasure "Unused packages" cleanPackages execAndMeasure "Swap storage" cleanSwap @@ -241,7 +241,6 @@ main() { removeUnusedDirsAndFiles printDF "AFTER CLEAN-UP:" - echo "" } execAndMeasure "Total" main From 215178cf165908e46197b47c2abac48c1cdeff2c Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:29:21 +0100 Subject: [PATCH 08/15] comments --- src/ci/scripts/free-disk-space.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index e55d6c99aea69..c826d6dac4e25 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -# Free disk space on Linux GitHub action runners +# Free disk space on Linux GitHub action runners. # Script inspired by https://github.com/jlumbroso/free-disk-space # When updating to a new ubuntu version (e.g. from ubuntu-24.04): @@ -9,7 +9,7 @@ set -euo pipefail # - Check that there are no big packages preinstalled that we aren't using # - Check that all directores we are removing are still present (look at the warnings) -# print a line of the specified character +# Print a line of the specified character. printSeparationLine() { for ((i = 0; i < 80; i++)); do printf "%s" "$1" @@ -17,19 +17,20 @@ printSeparationLine() { printf "\n" } -# compute available space +# Compute available space. # REF: https://unix.stackexchange.com/a/42049/60849 # REF: https://stackoverflow.com/a/450821/408734 getAvailableSpace() { df -a | awk 'NR > 1 {avail+=$4} END {print avail}' } -# make Kb human readable (assume the input is Kb) +# Make Kb human readable (assume the input is Kb). # REF: https://unix.stackexchange.com/a/44087/60849 formatByteCount() { numfmt --to=iec-i --suffix=B --padding=7 "$1"'000' } +# Check if the architecture is x86. isX86() { local arch arch=$(uname -m) @@ -55,16 +56,19 @@ execAndMeasure() { local end end=$(date +%s) + local after after=$(getAvailableSpace) + # How much space was saved. local saved=$((after - before)) + # How long the task took. local seconds_taken=$((end - start)) echo "==> ${task_name}: Saved $(formatByteCount "$saved") in $seconds_taken seconds" } -# macro to print output of df with caption +# Print output of df with caption. It shows information about disk space. printDF() { local caption=${1} @@ -158,8 +162,7 @@ removeNodeModules() { "yarn" } -# Remove large packages -# REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh +# Remove unused packages. cleanPackages() { local packages=( '.*-icon-theme$' From 1b360c83c0ee94554ebf088bfcb4228a63117ce8 Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:30:05 +0100 Subject: [PATCH 09/15] remove large runner --- src/ci/github-actions/jobs.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 4c94b05450c1d..5c7a06667744a 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -10,11 +10,6 @@ runners: free_disk: true <<: *base-job - # Large runner used mainly for its bigger disk capacity - - &job-linux-4c-largedisk - os: ubuntu-22.04-4core-16gb - <<: *base-job - - &job-linux-8c os: ubuntu-22.04-8core-32gb <<: *base-job From 85edc28e4cdcab247bf8009aff7a697b124418dc Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Fri, 31 Jan 2025 11:05:28 +0100 Subject: [PATCH 10/15] remove directories at once --- src/ci/scripts/free-disk-space.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index c826d6dac4e25..dbd81b394a44c 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -137,19 +137,21 @@ removeUnusedDirsAndFiles() { "$AGENT_TOOLSDIRECTORY" ) + local existing=() for element in "${to_remove[@]}"; do if [ ! -e "$element" ]; then echo "::warning::Directory or file $element does not exist, skipping." else - execAndMeasure "Removed $element" sudo rm -rf "$element" + existing+=("$element") fi done + + execAndMeasure "Removed unused directories" sudo rm -rf "${existing[@]}" } removeNodeModules() { sudo npm uninstall -g \ "@bazel/bazelisk" \ - "bazel" \ "grunt" \ "gulp" \ "lerna" \ @@ -237,11 +239,10 @@ main() { printDF "BEFORE CLEAN-UP:" execAndMeasure "Unused packages" cleanPackages - execAndMeasure "Swap storage" cleanSwap execAndMeasure "Node modules" removeNodeModules execAndMeasure "Python Packages" removePythonPackages - removeUnusedDirsAndFiles + execAndMeasure "Swap storage" cleanSwap printDF "AFTER CLEAN-UP:" } From 883074bb2e94ea05d89ed659c0feb2b29213adcc Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Fri, 31 Jan 2025 11:05:58 +0100 Subject: [PATCH 11/15] don't remove swap --- src/ci/scripts/free-disk-space.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index dbd81b394a44c..d728813225fa5 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -213,13 +213,6 @@ cleanPackages() { sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed" } -# Remove Swap storage -cleanSwap() { - sudo swapoff -a || true - sudo rm -rf /mnt/swapfile || true - free -h -} - removePythonPackages() { local packages=( ) @@ -242,7 +235,6 @@ main() { execAndMeasure "Node modules" removeNodeModules execAndMeasure "Python Packages" removePythonPackages removeUnusedDirsAndFiles - execAndMeasure "Swap storage" cleanSwap printDF "AFTER CLEAN-UP:" } From 7f61b4fbfb23af951ae688efc7b313e4e3369639 Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Fri, 31 Jan 2025 11:43:57 +0100 Subject: [PATCH 12/15] ci: free more space in ubuntu 24 free runners --- src/ci/scripts/free-disk-space.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index 1d413a608061a..507720eeb9a93 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -9,6 +9,7 @@ set -euo pipefail # - Check that there are no big packages preinstalled that we aren't using # - Check that all directores we are removing are still present (look at the warnings) +# Print a line of the specified character. printSeparationLine() { for ((i = 0; i < 80; i++)); do printf "%s" "$1" @@ -231,8 +232,11 @@ main() { printDF "BEFORE CLEAN-UP:" execAndMeasure "Unused packages" cleanPackages - execAndMeasure "Node modules" removeNodeModules - execAndMeasure "Python Packages" removePythonPackages + execAndMeasure "Unused Python Packages" removePythonPackages + if isX86; then + # On ARM, `npm uninstall` fails with a segmentation fault. + execAndMeasure "Unused Node modules" removeNodeModules + fi removeUnusedDirsAndFiles # Output saved space statistic From a9a83e015a5c2bfc2ed14b3ed5b3f8237bc06233 Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Tue, 4 Feb 2025 11:19:29 +0100 Subject: [PATCH 13/15] move function to minimize diff --- src/ci/scripts/free-disk-space.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index 507720eeb9a93..b20175144855c 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -9,6 +9,17 @@ set -euo pipefail # - Check that there are no big packages preinstalled that we aren't using # - Check that all directores we are removing are still present (look at the warnings) +# Check if the architecture is x86. +isX86() { + local arch + arch=$(uname -m) + if [ "$arch" = "x86_64" ]; then + return 0 + else + return 1 + fi +} + # Print a line of the specified character. printSeparationLine() { for ((i = 0; i < 80; i++)); do @@ -30,17 +41,6 @@ formatByteCount() { numfmt --to=iec-i --suffix=B --padding=7 "$1"'000' } -# Check if the architecture is x86. -isX86() { - local arch - arch=$(uname -m) - if [ "$arch" = "x86_64" ]; then - return 0 - else - return 1 - fi -} - # Execute a task, printing how much space was saved and how long it took to run the task execAndMeasure() { local task_name=${1} From f64595571eb808085300386b42478754f9778c3b Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Wed, 5 Feb 2025 18:37:38 +0100 Subject: [PATCH 14/15] setup latest node --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a46d123c5d94..c49f012456513 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,10 @@ jobs: - name: disable git crlf conversion run: git config --global core.autocrlf false + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: checkout the source code uses: actions/checkout@v4 with: From c27d28e5e6b4412b8a48845623ecb5f88448a1a8 Mon Sep 17 00:00:00 2001 From: MarcoIeni <11428655+MarcoIeni@users.noreply.github.com> Date: Wed, 5 Feb 2025 18:38:40 +0100 Subject: [PATCH 15/15] npm on arm --- src/ci/scripts/free-disk-space.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index b20175144855c..c6fd1e704ecf6 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -233,10 +233,8 @@ main() { execAndMeasure "Unused packages" cleanPackages execAndMeasure "Unused Python Packages" removePythonPackages - if isX86; then - # On ARM, `npm uninstall` fails with a segmentation fault. - execAndMeasure "Unused Node modules" removeNodeModules - fi + execAndMeasure "Unused Node modules" removeNodeModules + removeUnusedDirsAndFiles # Output saved space statistic