Skip to content

Commit 782bb56

Browse files
committed
ci-automation: Update sdk-bootstrap, packages-tag and upload functions
- Tighten the patterns used for nightly tags detection. - Compare hashes instead of names to figure out if we are on top of a branch (fixes the issue of no nightly tags reachable from the release branches). Jenkins is doing `git fetch origin "${branch}"; git checkout FETCH_HEAD` and this was confusing the `git rev-parse --abbrev-ref HEAD` code (it returned `HEAD` instead of `${branch}`). - Account for possible multiple tags in a single commit. - Made the tagging fail in dubious situations. - Reindent the code, modernize a bit.
1 parent a1047bb commit 782bb56

File tree

3 files changed

+138
-86
lines changed

3 files changed

+138
-86
lines changed

ci-automation/ci_automation_common.sh

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,15 @@ function check_version_string() {
2222
# --
2323

2424
function update_and_push_version() {
25-
local version="$1"
26-
local target_branch="${2:-}"
25+
local version=${1}
26+
local target_branch=${2:-}
2727

2828
# set up author and email so git does not complain when tagging
2929
if ! git config --get user.name >/dev/null 2>&1 ; then
30-
git -C . config user.name "${CI_GIT_AUTHOR}"
30+
git config user.name "${CI_GIT_AUTHOR}"
3131
fi
3232
if ! git config --get user.email >/dev/null 2>&1 ; then
33-
git -C . config user.email "${CI_GIT_EMAIL}"
33+
git config user.email "${CI_GIT_EMAIL}"
3434
fi
3535

3636
# Add and commit local changes
@@ -39,33 +39,34 @@ function update_and_push_version() {
3939
git commit --allow-empty -m "New version: ${version}"
4040

4141
git fetch --all --tags --force
42-
local ret=0
43-
git diff --exit-code "${version}" || ret=$?
42+
local -i ret=0
43+
git diff --quiet --exit-code "${version}" 2>/dev/null || ret=${?}
4444
# This will return != 0 if
4545
# - the remote tag does not exist (rc: 127)
4646
# - the tag does not exist locally (rc: 128)
4747
# - the remote tag has changes compared to the local tree (rc: 1)
48-
if [ "$ret" = "0" ]; then
49-
echo "Reusing existing tag" >&2
50-
git checkout -f "${version}"
51-
return
52-
elif [ "$ret" = "1" ]; then
53-
echo "Remote tag exists already and is not equal" >&2
54-
return 1
55-
elif [ "$ret" != "127" ] && [ "$ret" != "128" ]; then
56-
echo "Error: Unexpected git diff return code ($ret)" >&2
57-
return 1
48+
if [[ ret -eq 0 ]]; then
49+
# this means that we created an empty commit above, reusing the tag gets rid of it
50+
echo "Reusing existing tag" >&2
51+
git checkout --force "${version}"
52+
return
53+
elif [[ ret -eq 1 ]]; then
54+
echo "Remote tag exists already and is not equal" >&2
55+
return 1
56+
elif [[ ret -ne 127 && ret -ne 128 ]]; then
57+
echo "Error: Unexpected git diff return code (${ret})" >&2
58+
return 1
5859
fi
5960

60-
local -a TAG_ARGS
61-
if [ "${SIGN-0}" = 1 ]; then
62-
TAG_ARGS=("-s" "-m" "${version}")
61+
local -a TAG_ARGS=()
62+
if [[ ${SIGN-0} = 1 ]]; then
63+
TAG_ARGS=("--sign" "--message=${version}")
6364
fi
6465

65-
git tag -f "${TAG_ARGS[@]}" "${version}"
66+
git tag --force "${TAG_ARGS[@]}" "${version}"
6667

67-
if [[ -n "${target_branch}" ]]; then
68-
git push origin "HEAD:${target_branch}"
68+
if [[ -n ${target_branch} ]]; then
69+
git push origin "HEAD:${target_branch}"
6970
fi
7071

7172
git push origin "${version}"

ci-automation/packages-tag.sh

Lines changed: 51 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -56,15 +56,15 @@ function packages_tag() {
5656
# --
5757

5858
function _packages_tag_impl() {
59-
local version="$1"
59+
local version=${1}
6060

6161
source ci-automation/ci_automation_common.sh
6262
source ci-automation/gpg_setup.sh
6363

6464
check_version_string "${version}"
6565

6666
source sdk_container/.repo/manifests/version.txt
67-
local sdk_version="${FLATCAR_SDK_VERSION}"
67+
local sdk_version=${FLATCAR_SDK_VERSION}
6868

6969
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
7070
set -x
@@ -77,41 +77,69 @@ function _packages_tag_impl() {
7777
# builds without messing with actual release branches.
7878
local flatcar_branch_prefix=${CIA_DEBUGFLATCARBRANCHPREFIX:-flatcar}
7979
local nightly=${CIA_DEBUGNIGHTLY:-nightly}
80-
# Patterns used below.
81-
local nightly_pattern_1='^(stable|alpha|beta|lts)-[0-9.]+-'"${nightly}"'-[-0-9]+$'
82-
local nightly_pattern_2='^(stable|alpha|beta|lts)-[0-9.]+(|-'"${nightly}"'-[-0-9]+)$'
83-
local flatcar_pattern='^'"${flatcar_branch_prefix}"'-[0-9]+$'
84-
if [[ "${version}" =~ ${nightly_pattern_1} ]] \
85-
&& [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ${flatcar_pattern} ]] ; then
86-
target_branch="$(git rev-parse --abbrev-ref HEAD)"
87-
local existing_tag=""
80+
# Matches the usual nightly tag name (stable-1234.2.3-nightly-yyyymmdd-hhmm)
81+
local nightly_pattern='^(stable|alpha|beta|lts)-([0-9]+)(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$'
82+
local -i major_version=0
83+
local branch_name=''
84+
local branch_hash=''
85+
if [[ ${version} =~ ${nightly_pattern} ]]; then
86+
major_version=${BASH_REMATCH[2]}
87+
branch_name=${flatcar_branch_prefix}-${major_version}
88+
branch_hash=$(git rev-parse "origin/${branch_name}")
89+
fi
90+
local -a existing_tags=()
91+
if [[ -n ${branch_hash} ]]; then
92+
if [[ $(git rev-parse HEAD) != "${branch_hash}" ]]; then
93+
echo "We are doing a nightly build but we are not on top of the ${branch_name} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2
94+
exit 1
95+
fi
96+
target_branch=${branch_name}
8897
# Check for the existing tag only when we allow shortcutting
8998
# the builds. That way we can skip the checks for build
9099
# shortcutting.
91100
if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then
92101
echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2
93102
else
94-
existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
103+
git fetch --all --tags --force
104+
# exit code of git tag is always 0; output may be empty,
105+
# but may also have multiple tags
106+
mapfile -t existing_tags < <(git tag --points-at HEAD)
95107
fi
96-
# If the found tag is a release or nightly tag, we stop this build if there are no changes
97-
if [[ "${existing_tag}" =~ ${nightly_pattern_2} ]]; then
98-
local ret=0
99-
git diff --exit-code "${existing_tag}" || ret=$?
100-
if [[ ret -eq 0 ]]; then
108+
fi
109+
local nightly_or_release_tag=''
110+
if [[ major_version -gt 0 && ${#existing_tags[@]} -gt 0 ]]; then
111+
local nightly_or_release_pattern='^(stable|alpha|beta|lts)-'"${major_version}"'(\.[0-9]+){2}(-'"${nightly}"'-[0-9]{8}-[0-9]{4})?$'
112+
local tag
113+
for tag in "${existing_tags[@]}"; do
114+
if [[ ${tag} =~ ${nightly_or_release_pattern} ]]; then
115+
nightly_or_release_tag=${tag}
116+
break
117+
fi
118+
done
119+
fi
120+
# If the found tag is a release or nightly tag, we stop this build
121+
# if there are no changes and the relevant images can be found in
122+
# bincache.
123+
if [[ -n ${nightly_or_release_tag} ]]; then
124+
local -i ret=0
125+
git diff --exit-code --quiet "${nightly_or_release_tag}" || ret=$?
126+
if [[ ret -eq 0 ]]; then
127+
# no changes in the code, but check if images exist (they
128+
# could be missing if build failed)
101129
if check_bincache_images_existence \
102130
"https://${BUILDCACHE_SERVER}/images/amd64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2" \
103131
"https://${BUILDCACHE_SERVER}/images/arm64/${FLATCAR_VERSION}/flatcar_production_image.bin.bz2"; then
104132
touch ./skip-build
105-
echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${existing_tag} and the Flatcar images exist" >&2
133+
echo "Creating ./skip-build flag file, indicating that the build must not to continue because no new tag got created as there are no changes since tag ${nightly_or_release_tag} and the Flatcar images exist" >&2
106134
return 0
107135
fi
108136
echo "No changes but continuing build because Flatcar images do not exist"
109-
elif [[ ret -eq 1 ]]; then
110-
echo "Found changes since last tag ${existing_tag}" >&2
111-
else
137+
elif [[ ret -eq 1 ]]; then
138+
echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2
139+
exit 1
140+
else
112141
echo "Error: Unexpected git diff return code (${ret})" >&2
113142
return 1
114-
fi
115143
fi
116144
fi
117145
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
@@ -120,8 +148,8 @@ function _packages_tag_impl() {
120148

121149
# Create version file
122150
(
123-
source sdk_lib/sdk_container_common.sh
124-
create_versionfile "$sdk_version" "$version"
151+
source sdk_lib/sdk_container_common.sh
152+
create_versionfile "${sdk_version}" "${version}"
125153
)
126154
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
127155
set -x

ci-automation/sdk_bootstrap.sh

Lines changed: 64 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ function sdk_bootstrap() {
6565
# --
6666

6767
function _sdk_bootstrap_impl() {
68-
local seed_version="$1"
69-
local version="$2"
68+
local seed_version=${1}
69+
local version=${2}
7070
: ${ARCH:="amd64"}
7171

7272
source ci-automation/ci_automation_common.sh
@@ -86,65 +86,86 @@ function _sdk_bootstrap_impl() {
8686
# builds without messing with actual release branches.
8787
local main_branch=${CIA_DEBUGMAINBRANCH:-main}
8888
local nightly=${CIA_DEBUGNIGHTLY:-nightly}
89-
# Patterns used below.
90-
local nightly_pattern_1='^main-[0-9.]+-'"${nightly}"'-[-0-9]+(-INTERMEDIATE)?$'
91-
local nightly_pattern_2='^main-[0-9.]+-'"${nightly}"'-[-0-9]+$'
92-
if [[ "${version}" =~ ${nightly_pattern_1} ]] \
93-
&& [ "$(git rev-parse HEAD)" = "$(git rev-parse "origin/${main_branch}")" ] ; then
89+
# Matches the usual nightly tag name, optionally with an
90+
# intermediate suffix of the build ID too
91+
# (main-1234.2.3-nightly-yyyymmdd-hhmm-INTERMEDIATE).
92+
local nightly_pattern_1='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}(-INTERMEDIATE)?$'
93+
local main_branch_hash=''
94+
if [[ ${version} =~ ${nightly_pattern_1} ]]; then
95+
main_branch_hash=$(git rev-parse "origin/${main_branch}")
96+
fi
97+
local -a existing_tags=()
98+
if [[ -n ${main_branch_hash} ]]; then
99+
if [[ $(git rev-parse HEAD) != "${main_branch_hash}" ]] ; then
100+
echo "We are doing a nightly build but we are not on top of the ${main_branch} branch. This is wrong and would result in the nightly tag not being a part of the branch." >&2
101+
exit 1
102+
fi
94103
target_branch=${main_branch}
95-
local existing_tag=""
96-
# Check for the existing tag only when we allow shortcutting
97-
# the builds. That way we can skip the checks for build
98-
# shortcutting.
104+
# Check for the existing tag only when we allow
105+
# shortcutting the builds. That way we can skip the checks
106+
# for build shortcutting.
99107
if bool_is_true "${AVOID_NIGHTLY_BUILD_SHORTCUTS}"; then
100108
echo "Continuing the build because AVOID_NIGHTLY_BUILD_SHORTCUTS is bool true (${AVOID_NIGHTLY_BUILD_SHORTCUTS})" >&2
101109
else
102-
existing_tag=$(git tag --points-at HEAD) # exit code is always 0, output may be empty
110+
git fetch --all --tags --force
111+
# exit code is always 0, output may be empty
112+
mapfile -t existing_tags < <(git tag --points-at HEAD)
103113
fi
104-
# If the found tag is a nightly tag, we stop this build if there are no changes
105-
if [[ "${existing_tag}" =~ ${nightly_pattern_2} ]]; then
106-
local ret=0
107-
git diff --exit-code "${existing_tag}" || ret=$?
108-
if [ "$ret" = "0" ]; then
109-
local versions=(
110-
$(
111-
source sdk_lib/sdk_container_common.sh
112-
source "${sdk_container_common_versionfile}"
113-
echo "${FLATCAR_SDK_VERSION}"
114-
echo "${FLATCAR_VERSION}"
115-
)
114+
fi
115+
local nightly_pattern_2='^main-[0-9]+(\.[0-9]+){2}-'"${nightly}"'-[0-9]{8}-[0-9]{4}$'
116+
local tag nightly_tag=''
117+
for tag in "${existing_tags[@]}"; do
118+
if [[ ${tag} =~ ${nightly_pattern_2} ]]; then
119+
nightly_tag=${tag}
120+
break
121+
fi
122+
done
123+
# If the found tag is a nightly tag, we stop this build if there
124+
# are no changes and the relevant images can be found in the
125+
# bincache.
126+
if [[ -n ${nightly_tag} ]]; then
127+
local -i ret=0
128+
git diff --exit-code --quiet "${nightly_tag}" || ret=$?
129+
if [[ ret -eq 0 ]]; then
130+
local -a versions=(
131+
$(
132+
source sdk_lib/sdk_container_common.sh
133+
source "${sdk_container_common_versionfile}"
134+
echo "${FLATCAR_SDK_VERSION}"
135+
echo "${FLATCAR_VERSION}"
136+
)
116137
)
117-
local flatcar_sdk_version="${versions[0]}"
118-
local flatcar_version="${versions[1]}"
138+
local flatcar_sdk_version=${versions[0]}
139+
local flatcar_version=${versions[1]}
119140
local sdk_docker_vernum=""
120141
sdk_docker_vernum=$(vernum_to_docker_image_version "${flatcar_sdk_version}")
121142
if check_bincache_images_existence \
122143
"https://${BUILDCACHE_SERVER}/containers/${sdk_docker_vernum}/flatcar-sdk-all-${sdk_docker_vernum}.tar.zst" \
123144
"https://${BUILDCACHE_SERVER}/images/amd64/${flatcar_version}/flatcar_production_image.bin.bz2" \
124145
"https://${BUILDCACHE_SERVER}/images/arm64/${flatcar_version}/flatcar_production_image.bin.bz2"; then
125-
echo "Stopping build because there are no changes since tag ${existing_tag}, the SDK container tar ball and the Flatcar images exist" >&2
146+
echo "Stopping build because there are no changes since tag ${nightly_tag}, the SDK container tar ball and the Flatcar images exist" >&2
126147
return 0
127148
fi
128149
echo "No changes but continuing build because SDK container tar ball and/or the Flatcar images do not exist" >&2
129-
elif [ "$ret" = "1" ]; then
130-
echo "Found changes since last tag ${existing_tag}" >&2
131-
else
150+
elif [[ ret -eq 1 ]]; then
151+
echo "HEAD is tagged with a nightly tag and yet there a differences? This is fishy and needs to be investigated. Maybe you forgot to commit your changes?" >&2
152+
exit 1
153+
else
132154
echo "Error: Unexpected git diff return code (${ret})" >&2
133155
return 1
134-
fi
135156
fi
136157
fi
137158
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
138159
set +x
139160
fi
140161

141-
local vernum="${version#*-}" # remove alpha-,beta-,stable-,lts- version tag
142-
local git_vernum="${vernum}"
162+
local vernum=${version#*-} # remove alpha-,beta-,stable-,lts- version tag
163+
local git_vernum=${vernum}
143164

144165
# Update FLATCAR_VERSION[_ID], BUILD_ID, and SDK in versionfile
145166
(
146-
source sdk_lib/sdk_container_common.sh
147-
create_versionfile "${vernum}"
167+
source sdk_lib/sdk_container_common.sh
168+
create_versionfile "${vernum}"
148169
)
149170
if [[ -n ${CIA_DEBUGTESTRUN:-} ]]; then
150171
set -x
@@ -167,14 +188,16 @@ function _sdk_bootstrap_impl() {
167188
# to ourselves, otherwise we could fail to sign the artifacts as
168189
# we lacked write permissions in the directory of the signed
169190
# artifact
170-
local uid=$(id --user)
171-
local gid=$(id --group)
191+
local uid
192+
local gid
193+
uid=$(id --user)
194+
gid=$(id --group)
172195
sudo chown --recursive "${uid}:${gid}" __build__
173196
(
174-
cd "__build__/images/catalyst/builds/flatcar-sdk"
175-
create_digests "${SIGNER}" "${dest_tarball}"
176-
sign_artifacts "${SIGNER}" "${dest_tarball}"*
177-
copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
197+
cd "__build__/images/catalyst/builds/flatcar-sdk"
198+
create_digests "${SIGNER}" "${dest_tarball}"
199+
sign_artifacts "${SIGNER}" "${dest_tarball}"*
200+
copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"*
178201
)
179202
}
180203
# --

0 commit comments

Comments
 (0)