ci/tuf-repo: Consistent artifact versions (#3437)

Fixes #3430. Changes:
- Changes the prerelease field in our version from `alpha` to `0.ci`,
which comes before alpha, so that we can use `alpha` or other words
found early in the dictionary and have them count as later than a CI
build.
- Stamps a non-placeholder version into the global zone packages
installed in the host and trampoline OS images.
- Adds `caboose-util`, loosely based on the small part of
hubedit I needed, tweaked for usefulness to a shell script.
- Reads the version out of a Hubris archive's caboose and uses that for
the TUF repository, instead of the control plane version we generate
here.

Author: iliana

.github/buildomat/jobs/ci-tools.sh

@@ -6,6 +6,7 @@
 #: rust_toolchain = "1.70.0"
 #: output_rules = [
 #:	"=/work/end-to-end-tests/*.gz",
+#:	"=/work/caboose-util.gz",
 #:	"=/work/tufaceous.gz",
 #: ]
 
@@ -37,6 +38,13 @@ for p in target/debug/bootstrap $(/opt/ooce/bin/jq -r 'select(.profile.test) | .
 	ptime -m gzip < "$p" > /work/end-to-end-tests/"$(basename "$p").gz"
 done
 
+########## caboose-util ##########
+
+banner caboose-util
+
+ptime -m cargo build --locked -p caboose-util --release
+ptime -m gzip < target/release/caboose-util > /work/caboose-util.gz
+
 ########## tufaceous ##########
 
 banner tufaceous

.github/buildomat/jobs/package.sh

@@ -5,6 +5,7 @@
 #: target = "helios-latest"
 #: rust_toolchain = "1.70.0"
 #: output_rules = [
+#:	"=/work/version.txt",
 #:	"=/work/package.tar.gz",
 #:	"=/work/global-zone-packages.tar.gz",
 #:	"=/work/trampoline-global-zone-packages.tar.gz",
@@ -28,6 +29,17 @@ set -o xtrace
 cargo --version
 rustc --version
 
+#
+# Generate the version for control plane artifacts here. We use `0.git` as the
+# prerelease field because it comes before `alpha`.
+#
+# In this job, we stamp the version into packages installed in the host and
+# trampoline global zone images.
+#
+COMMIT=$(git rev-parse HEAD)
+VERSION="1.0.0-0.ci+git${COMMIT:0:11}"
+echo "$VERSION" >/work/version.txt
+
 ptime -m ./tools/install_builder_prerequisites.sh -yp
 ptime -m ./tools/ci_download_softnpu_machinery
 
@@ -37,8 +49,6 @@ ptime -m cargo run --locked --release --bin omicron-package -- \
 ptime -m cargo run --locked --release --bin omicron-package -- \
   -t test package
 
-tarball_src_dir="$(pwd)/out"
-
 # Assemble some utilities into a tarball that can be used by deployment
 # phases of buildomat.
 
@@ -56,14 +66,29 @@ files=(
 pfexec mkdir -p /work && pfexec chown $USER /work
 ptime -m tar cvzf /work/package.tar.gz "${files[@]}"
 
+tarball_src_dir="$(pwd)/out/versioned"
+stamp_packages() {
+	for package in "$@"; do
+		# TODO: remove once https://github.com/oxidecomputer/omicron-package/pull/54 lands
+		if [[ $package == maghemite ]]; then
+			echo "0.0.0" > VERSION
+			tar rvf "out/$package.tar" VERSION
+			rm VERSION
+		fi
+
+		cargo run --locked --release --bin omicron-package -- stamp "$package" "$VERSION"
+	done
+}
+
 # Build necessary for the global zone
 ptime -m cargo run --locked --release --bin omicron-package -- \
   -t host target create -i standard -m gimlet -s asic
 ptime -m cargo run --locked --release --bin omicron-package -- \
   -t host package
+stamp_packages omicron-sled-agent maghemite propolis-server
 
 # Create global zone package @ /work/global-zone-packages.tar.gz
-ptime -m ./tools/build-global-zone-packages.sh $tarball_src_dir /work
+ptime -m ./tools/build-global-zone-packages.sh "$tarball_src_dir" /work
 
 # Non-Global Zones
 
@@ -101,6 +126,7 @@ ptime -m cargo run --locked --release --bin omicron-package -- \
   -t recovery target create -i trampoline
 ptime -m cargo run --locked --release --bin omicron-package -- \
   -t recovery package
+stamp_packages installinator maghemite
 
 # Create trampoline global zone package @ /work/trampoline-global-zone-packages.tar.gz
-ptime -m ./tools/build-trampoline-global-zone-packages.sh $tarball_src_dir /work
+ptime -m ./tools/build-trampoline-global-zone-packages.sh "$tarball_src_dir" /work

.github/buildomat/jobs/tuf-repo.sh

@@ -44,17 +44,12 @@ set -o pipefail
 set -o xtrace
 
 TOP=$PWD
+VERSION=$(< /input/package/work/version.txt)
 
-source "$TOP/tools/dvt_dock_version"
-DVT_DOCK_COMMIT=$COMMIT
-source "$TOP/tools/hubris_version"
-HUBRIS_COMMIT=$COMMIT
-
-COMMIT=$(git rev-parse HEAD)
-VERSION="1.0.0-alpha+git${COMMIT:0:11}"
-
-ptime -m gunzip < /input/ci-tools/work/tufaceous.gz > /work/tufaceous
-chmod a+x /work/tufaceous
+for bin in caboose-util tufaceous; do
+    ptime -m gunzip < /input/ci-tools/work/$bin.gz > /work/$bin
+    chmod a+x /work/$bin
+done
 
 #
 # We do two things here:
@@ -113,31 +108,41 @@ EOF
 done
 
 # Fetch signed ROT images from oxidecomputer/dvt-dock.
+source "$TOP/tools/dvt_dock_version"
 git clone https://github.com/oxidecomputer/dvt-dock.git /work/dvt-dock
-(cd /work/dvt-dock; git checkout "$DVT_DOCK_COMMIT")
-DVT_DOCK_VERSION="1.0.0-alpha+git${DVT_DOCK_COMMIT:0:11}"
+(cd /work/dvt-dock; git checkout "$COMMIT")
 
 for noun in gimlet psc sidecar; do
     tufaceous_kind=${noun//sidecar/switch}_rot
     hubris_kind=${noun}-rot
+    path_a="/work/dvt-dock/staging/build-$hubris_kind-image-a-cert-dev.zip"
+    path_b="/work/dvt-dock/staging/build-$hubris_kind-image-b-cert-dev.zip"
+    version_a=$(/work/caboose-util read-version "$path_a")
+    version_b=$(/work/caboose-util read-version "$path_b")
+    if [[ "$version_a" != "$version_b" ]]; then
+        echo "version mismatch:"
+        echo "  $path_a: $version_a"
+        echo "  $path_b: $version_b"
+        exit 1
+    fi
     cat >>/work/manifest.toml <<EOF
 [artifact.$tufaceous_kind]
 name = "$tufaceous_kind"
-version = "$DVT_DOCK_VERSION"
+version = "$version_a"
 [artifact.$tufaceous_kind.source]
 kind = "composite-rot"
 [artifact.$tufaceous_kind.source.archive_a]
 kind = "file"
-path = "/work/dvt-dock/staging/build-$hubris_kind-image-a-cert-dev.zip"
+path = "$path_a"
 [artifact.$tufaceous_kind.source.archive_b]
 kind = "file"
-path = "/work/dvt-dock/staging/build-$hubris_kind-image-b-cert-dev.zip"
+path = "$path_b"
 EOF
 done
 
 # Fetch SP images from oxidecomputer/hubris GHA artifacts.
-HUBRIS_VERSION="1.0.0-alpha+git${HUBRIS_COMMIT:0:11}"
-run_id=$(curl --netrc -fsS "https://api.github.com/repos/oxidecomputer/hubris/actions/runs?head_sha=$HUBRIS_COMMIT" \
+source "$TOP/tools/hubris_version"
+run_id=$(curl --netrc -fsS "https://api.github.com/repos/oxidecomputer/hubris/actions/runs?head_sha=$COMMIT" \
     | /opt/ooce/bin/jq -r '.workflow_runs[] | select(.path == ".github/workflows/dist.yml") | .id')
 artifacts=$(curl --netrc -fsS "https://api.github.com/repos/oxidecomputer/hubris/actions/runs/$run_id/artifacts")
 for noun in gimlet-c psc-b sidecar-b; do
@@ -147,13 +152,15 @@ for noun in gimlet-c psc-b sidecar-b; do
     url=$(/opt/ooce/bin/jq --arg name "$job_name" -r '.artifacts[] | select(.name == $name) | .archive_download_url' <<<"$artifacts")
     curl --netrc -fsSL -o $job_name.zip "$url"
     unzip $job_name.zip
+    path="$PWD/build-$noun-image-default.zip"
+    version=$(/work/caboose-util read-version "$path")
     cat >>/work/manifest.toml <<EOF
 [artifact.$tufaceous_kind]
 name = "$tufaceous_kind"
-version = "$HUBRIS_VERSION"
+version = "$version"
 [artifact.$tufaceous_kind.source]
 kind = "file"
-path = "$PWD/build-$noun-image-default.zip"
+path = "$path"
 EOF
 done
 

Cargo.lock

@@ -3,6 +3,7 @@ members = [
     "api_identity",
     "bootstore",
     "bootstrap-agent-client",
+    "caboose-util",
     "certificates",
     "common",
     "ddm-admin-client",
@@ -63,6 +64,7 @@ members = [
 
 default-members = [
     "bootstrap-agent-client",
+    "caboose-util",
     "certificates",
     "common",
     "ddm-admin-client",
@@ -182,10 +184,11 @@ hex-literal = "0.3.4"
 hkdf = "0.12.3"
 http = "0.2.9"
 httptest = "0.15.4"
-hyper-rustls = "0.24.0"
+hubtools = { git = "https://github.com/oxidecomputer/hubtools.git" }
+humantime = "2.1.0"
 hyper = "0.14"
+hyper-rustls = "0.24.0"
 hyper-staticfile = "0.9.5"
-humantime = "2.1.0"
 illumos-utils = { path = "illumos-utils" }
 indexmap = "1.9.3"
 indicatif = { version = "0.17.5", features = ["rayon"] }

Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "caboose-util"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]
+anyhow.workspace = true
+hubtools.workspace = true

caboose-util/Cargo.toml

@@ -0,0 +1,23 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Copyright 2023 Oxide Computer Company
+
+use anyhow::{bail, Context, Result};
+use hubtools::RawHubrisArchive;
+
+fn main() -> Result<()> {
+    let mut args = std::env::args().skip(1);
+    match args.next().context("subcommand required")?.as_str() {
+        "read-version" => {
+            let archive = RawHubrisArchive::load(
+                &args.next().context("path to hubris archive required")?,
+            )?;
+            let caboose = archive.read_caboose()?;
+            println!("{}", std::str::from_utf8(caboose.version()?)?);
+            Ok(())
+        }
+        unknown => bail!("unknown command {}", unknown),
+    }
+}