From 48291f11214c0ff665aa4c041fff59b99895d939 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 10 Jan 2023 14:57:29 +0100 Subject: [PATCH] include extraction in the retrying for submodule download The code to download larger submodules previously used retries around the `curl` invocation to handle network failures, but we saw in recent build failures that failures can also happen during extraction, for example if a response got terminated early. This commit moves the retry outwards, wrapping the whole download+extraction function in the retrying code. This means, if the extraction fails the tarball will be re-downloaded. --- src/ci/scripts/checkout-submodules.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ci/scripts/checkout-submodules.sh b/src/ci/scripts/checkout-submodules.sh index f6cb8f8a6da65..9de8d3825f515 100755 --- a/src/ci/scripts/checkout-submodules.sh +++ b/src/ci/scripts/checkout-submodules.sh @@ -23,8 +23,9 @@ fi function fetch_github_commit_archive { local module=$1 local cached="download-${module//\//-}.tar.gz" - retry sh -c "rm -f $cached && \ - curl -f -sSL -o $cached $2" + rm -f "${cached}" + rm -rf "${module}" + curl -f -sSL -o "${cached}" "$2" mkdir $module touch "$module/.git" # On Windows, the default behavior is to emulate symlinks by copying @@ -32,6 +33,7 @@ function fetch_github_commit_archive { # which can cause a failure if the symlink comes first. This env var # causes tar to use real symlinks instead, which are allowed to dangle. export MSYS=winsymlinks:nativestrict + mkdir -p "${module}" tar -C $module --strip-components=1 -xf $cached rm $cached } @@ -50,7 +52,7 @@ for i in ${!modules[@]}; do git rm $module url=${urls[$i]} url=${url/\.git/} - fetch_github_commit_archive $module "$url/archive/$commit.tar.gz" & + retry fetch_github_commit_archive $module "$url/archive/$commit.tar.gz" & bg_pids[${i}]=$! continue else