Skip to content

Commit a152784

Browse files
geyslangeyslan
committed
feat: enhance sparse checkout tool with incremental, ...
... exclusion and no-apply mode and usage instructions. Add comprehensive support for building sparse-checkout patterns incrementally: - --incremental: Append patterns instead of replacing existing ones - --exclude: Treat DISTRO/ARCH/VERSION as exclusion patterns - --no-apply: Build patterns without downloading files - --apply-only: Apply existing patterns and trigger download This enables efficient workflows where users can build complex include/exclude patterns across multiple calls, then download all matching files in a single operation. Detailed usage instructions are provided in the script header.
1 parent 416176d commit a152784

File tree

1 file changed

+183
-25
lines changed

1 file changed

+183
-25
lines changed

‎tools/archive-sparse-checkout.sh‎

Lines changed: 183 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,112 @@ die() {
55
exit 1
66
}
77

8+
show_usage() {
9+
cat << EOF
10+
Usage: $0 [--incremental|-i] [options]
11+
12+
Configures sparse checkout for the btfhub-archive repository.
13+
14+
Options:
15+
--incremental, -i Append to existing sparse-checkout patterns instead of replacing them
16+
This allows multiple calls to incrementally add more patterns
17+
--exclude, -e Treat DISTRO, ARCH, and VERSION as exclusion patterns instead of inclusion
18+
--no-apply, -n Skip applying sparse-checkout (defer download until final call)
19+
Useful for building patterns incrementally without downloading after each call
20+
--apply-only, -a Only apply existing patterns without adding new ones
21+
Use this for the final call to trigger download after building patterns
22+
23+
Environment Variables:
24+
DISTRO Space-separated list of distributions (default: "*")
25+
ARCH Space-separated list of architectures (default: "*")
26+
VERSION Version pattern (default: "*")
27+
EXCLUDE_PATTERNS Additional custom exclude patterns (optional, only used with --exclude)
28+
29+
Examples:
30+
# Clear existing patterns and set new ones (default behavior)
31+
$0
32+
33+
# Append patterns to existing sparse checkout
34+
$0 --incremental
35+
36+
# Set specific distro and arch, clearing existing patterns
37+
DISTRO="ubuntu" ARCH="x86_64" $0
38+
39+
# Add additional distro to existing patterns
40+
DISTRO="centos" ARCH="arm64" $0 --incremental
41+
42+
# Exclude specific distro (first include everything, then exclude)
43+
$0 # include everything
44+
DISTRO="centos" $0 --exclude --incremental
45+
46+
# Exclude specific version pattern
47+
VERSION="5.4.0*" $0 --exclude --incremental
48+
49+
# Include ubuntu x86_64, then exclude debug versions
50+
DISTRO="ubuntu" ARCH="x86_64" $0
51+
VERSION="*debug*" $0 --exclude --incremental
52+
53+
# Custom exclude patterns
54+
EXCLUDE_PATTERNS="*/debug/* */test/*" $0 --exclude --incremental
55+
56+
# Build patterns incrementally without downloading until the end
57+
$0 --no-apply # include everything without download
58+
EXCLUDE_PATTERNS="*/*/*/3.*.btf.tar.xz */*/*/4.*.btf.tar.xz" $0 --exclude --incremental --no-apply
59+
DISTRO="centos" VERSION="7" $0 --exclude --incremental --no-apply
60+
DISTRO="rhel" VERSION="7" $0 --exclude --incremental --no-apply
61+
# Final call applies all patterns and downloads
62+
$0 --apply-only
63+
EOF
64+
}
65+
66+
# Check for help flag
67+
if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then
68+
show_usage
69+
exit 0
70+
fi
71+
72+
# Parse flags
73+
INCREMENTAL=false
74+
EXCLUDE_MODE=false
75+
NO_APPLY=false
76+
APPLY_ONLY=false
77+
78+
while [[ $# -gt 0 ]]; do
79+
case $1 in
80+
--incremental|-i)
81+
INCREMENTAL=true
82+
shift
83+
;;
84+
--exclude|-e)
85+
EXCLUDE_MODE=true
86+
shift
87+
;;
88+
--no-apply|-n)
89+
NO_APPLY=true
90+
shift
91+
;;
92+
--apply-only|-a)
93+
APPLY_ONLY=true
94+
shift
95+
;;
96+
*)
97+
# Unknown option, stop parsing
98+
break
99+
;;
100+
esac
101+
done
102+
8103
if [ -z "$VERSION" ]; then
9104
VERSION="*"
10105
fi
11106

12107
set -f # disable globbing
13108
DISTROS=${DISTRO:-"*"}
14109
ARCHS=${ARCH:-"*"}
110+
EXCLUDE_CUSTOM=${EXCLUDE_PATTERNS:-""}
15111

16112
ARCHS="${ARCHS//aarch64/arm64}"
113+
17114
for arch in $ARCHS; do
18115
case $arch in
19116
x86_64|arm64|'*')
@@ -27,10 +124,12 @@ set +f # enable globbing
27124

28125
IFS=' ' read -r -a DISTROS_ARRAY <<< "$DISTROS"
29126
IFS=' ' read -r -a ARCHS_ARRAY <<< "$ARCHS"
127+
IFS=' ' read -r -a EXCLUDE_CUSTOM_ARRAY <<< "$EXCLUDE_CUSTOM"
30128

31129
# count the number of elements
32130
NUM_DISTROS=${#DISTROS_ARRAY[@]}
33131
NUM_ARCHS=${#ARCHS_ARRAY[@]}
132+
NUM_EXCLUDE_CUSTOM=${#EXCLUDE_CUSTOM_ARRAY[@]}
34133

35134
if [ "$VERSION" != "*" ] && { [ "$NUM_DISTROS" -gt 1 ] || [ "$NUM_ARCHS" -gt 1 ]; }; then
36135
die "VERSION must be * when DISTRO or ARCH are arrays"
@@ -49,35 +148,94 @@ fi
49148
if [ -d "$ARCHIVE_DIR/.git" ]; then
50149
echo "Directory is already a Git repository."
51150
else
52-
# clone into the directory
53-
git clone --sparse --filter=blob:none "$REPO_URL" "$ARCHIVE_DIR"
151+
# clone into the directory with minimal data
152+
git clone --filter=blob:none --no-checkout --single-branch "$REPO_URL" "$ARCHIVE_DIR"
153+
# Enable sparse-checkout
154+
git -C "$ARCHIVE_DIR" config core.sparseCheckout true
155+
# Set up sparse-checkout immediately to avoid downloading everything
156+
git -C "$ARCHIVE_DIR" sparse-checkout init --no-cone
54157
fi
55158

56159
SPARSE_CHECKOUT_FILE="$ARCHIVE_DIR/.git/info/sparse-checkout"
57-
true > "$SPARSE_CHECKOUT_FILE" # clear existing file
58-
59-
# initialize sparse-checkout
60-
git -C "$ARCHIVE_DIR" sparse-checkout init --no-cone
61-
62-
echo "Settings patterns to $SPARSE_CHECKOUT_FILE file..."
63-
if [ "$NUM_DISTROS" -eq 1 ] && [ "$NUM_ARCHS" -eq 1 ]; then
64-
echo "${DISTROS_ARRAY[0]}/$VERSION/${ARCHS_ARRAY[0]}/*.btf.tar.xz" | tee "$SPARSE_CHECKOUT_FILE"
65-
elif [ "$NUM_DISTROS" -gt 1 ] && [ "$NUM_ARCHS" -eq 1 ]; then
66-
for distro in "${DISTROS_ARRAY[@]}"; do
67-
echo "$distro/*/${ARCHS_ARRAY[0]}/*.btf.tar.xz" | tee -a "$SPARSE_CHECKOUT_FILE"
68-
done
69-
elif [ "$NUM_DISTROS" -eq 1 ] && [ "$NUM_ARCHS" -gt 1 ]; then
70-
for arch in "${ARCHS_ARRAY[@]}"; do
71-
echo "${DISTROS_ARRAY[0]}/*/$arch/*.btf.tar.xz" | tee -a "$SPARSE_CHECKOUT_FILE"
72-
done
160+
161+
# Clear existing file only if not in incremental mode AND not in apply-only mode
162+
if [ "$INCREMENTAL" = false ] && [ "$APPLY_ONLY" = false ]; then
163+
true > "$SPARSE_CHECKOUT_FILE" # clear existing file
164+
echo "Clearing existing sparse-checkout patterns..."
165+
elif [ "$APPLY_ONLY" = false ]; then
166+
echo "Appending to existing sparse-checkout patterns..."
167+
fi
168+
169+
# configure sparse-checkout for exclude patterns
170+
git -C "$ARCHIVE_DIR" config core.sparseCheckout true
171+
git -C "$ARCHIVE_DIR" config core.sparseCheckoutCone false
172+
173+
if [ "$APPLY_ONLY" = true ]; then
174+
echo "Applying existing sparse-checkout patterns..."
73175
else
74-
for distro in "${DISTROS_ARRAY[@]}"; do
75-
for arch in "${ARCHS_ARRAY[@]}"; do
76-
echo "$distro/*/$arch/*.btf.tar.xz" | tee -a "$SPARSE_CHECKOUT_FILE"
77-
done
78-
done
176+
if [ "$EXCLUDE_MODE" = true ]; then
177+
echo "Adding exclude patterns to $SPARSE_CHECKOUT_FILE file..."
178+
else
179+
echo "Adding include patterns to $SPARSE_CHECKOUT_FILE file..."
180+
fi
79181
fi
80182

81-
git -C "$ARCHIVE_DIR" sparse-checkout reapply || die "failed to reapply sparse-checkout"
183+
# Only generate patterns if not in apply-only mode
184+
if [ "$APPLY_ONLY" = false ]; then
185+
# Determine tee flag based on mode
186+
TEE_FLAG=""
187+
if [ "$INCREMENTAL" = true ]; then
188+
TEE_FLAG="-a"
189+
fi
190+
191+
# Helper function to add patterns (include or exclude)
192+
add_pattern() {
193+
local pattern="$1"
194+
if [ "$EXCLUDE_MODE" = true ]; then
195+
pattern="!$pattern"
196+
fi
197+
echo "$pattern" | tee $TEE_FLAG "$SPARSE_CHECKOUT_FILE"
198+
TEE_FLAG="-a" # After first iteration, always append
199+
}
82200

83-
echo "Sparse checkout completed successfully in $ARCHIVE_DIR"
201+
# Generate patterns based on DISTRO, ARCH, and VERSION
202+
# Skip pattern generation in exclude mode if all values are wildcards (default state)
203+
if [ "$EXCLUDE_MODE" = true ] && [ "${DISTROS_ARRAY[0]}" = "*" ] && [ "${ARCHS_ARRAY[0]}" = "*" ] && [ "$VERSION" = "*" ]; then
204+
echo "Skipping wildcard pattern generation in exclude mode (use EXCLUDE_PATTERNS for custom patterns)"
205+
else
206+
if [ "$NUM_DISTROS" -eq 1 ] && [ "$NUM_ARCHS" -eq 1 ]; then
207+
add_pattern "${DISTROS_ARRAY[0]}/$VERSION/${ARCHS_ARRAY[0]}/*.btf.tar.xz"
208+
elif [ "$NUM_DISTROS" -gt 1 ] && [ "$NUM_ARCHS" -eq 1 ]; then
209+
for distro in "${DISTROS_ARRAY[@]}"; do
210+
add_pattern "$distro/$VERSION/${ARCHS_ARRAY[0]}/*.btf.tar.xz"
211+
done
212+
elif [ "$NUM_DISTROS" -eq 1 ] && [ "$NUM_ARCHS" -gt 1 ]; then
213+
for arch in "${ARCHS_ARRAY[@]}"; do
214+
add_pattern "${DISTROS_ARRAY[0]}/$VERSION/$arch/*.btf.tar.xz"
215+
done
216+
else
217+
for distro in "${DISTROS_ARRAY[@]}"; do
218+
for arch in "${ARCHS_ARRAY[@]}"; do
219+
add_pattern "$distro/$VERSION/$arch/*.btf.tar.xz"
220+
done
221+
done
222+
fi
223+
fi
224+
225+
# Add custom exclude patterns (only when in exclude mode)
226+
if [ "$EXCLUDE_MODE" = true ] && [ "$NUM_EXCLUDE_CUSTOM" -gt 0 ] && [ "${EXCLUDE_CUSTOM_ARRAY[0]}" != "" ]; then
227+
for custom_pattern in "${EXCLUDE_CUSTOM_ARRAY[@]}"; do
228+
echo "!$custom_pattern" | tee -a "$SPARSE_CHECKOUT_FILE"
229+
done
230+
fi
231+
fi
232+
233+
if [ "$NO_APPLY" = true ]; then
234+
echo "Patterns updated but not applied (use without --no-apply to download)"
235+
else
236+
echo "Applying sparse checkout and downloading files..."
237+
git -C "$ARCHIVE_DIR" sparse-checkout reapply || die "failed to reapply sparse-checkout"
238+
# Reset and checkout to force download of matching files
239+
git -C "$ARCHIVE_DIR" reset --hard HEAD 2>/dev/null || echo "Reset completed"
240+
echo "Sparse checkout completed successfully in $ARCHIVE_DIR"
241+
fi

0 commit comments

Comments
 (0)