diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c7c037..4a6b7c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - local: + release: name: Local release runs-on: ubuntu-latest @@ -48,7 +48,7 @@ jobs: run: docker compose up -d - name: Run the local release process for channel ${{ matrix.channel }} - run: ./run.sh ${{ matrix.channel }} + run: ./run.sh release ${{ matrix.channel }} - name: Validate the generated signatures run: docker compose exec -T local /src/local/check-signature.sh ${{ matrix.channel }} @@ -61,6 +61,42 @@ jobs: env: RUSTUP_DIST_SERVER: http://localhost:9000/static + rustup: + name: Local rustup + runs-on: ubuntu-latest + + env: + RUSTUP_OVERRIDE_VERSION: 99.0.0 + + strategy: + fail-fast: false + matrix: + channel: [stable, beta] + + steps: + - name: Clone the source code + uses: actions/checkout@v3 + + - name: Ensure Rust Stable is up to date + run: rustup self update && rustup update stable + + - name: Start the local environment + run: docker compose up -d + + - name: Run the local release process for channel ${{ matrix.channel }} + run: ./run.sh rustup ${{ matrix.channel }} + + - name: Update Rustup from the local environment + run: | + if rustup self update | grep -q "rustup updated"; then + echo "Rustup was updated successfully." + else + echo "Rustup was not updated." + exit 1 + fi + env: + RUSTUP_UPDATE_ROOT: http://localhost:9000/static/rustup + docker: name: Build Docker image runs-on: ubuntu-latest @@ -90,7 +126,7 @@ jobs: permissions: id-token: write - needs: [test, local, docker] + needs: [test, release, rustup, docker] if: github.event_name == 'push' && github.repository == 'rust-lang/promote-release' && github.ref == 'refs/heads/master' steps: diff --git a/docker-compose.yml b/docker-compose.yml index 5b99bbc..cfebd46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ --- services: minio: - image: quay.io/minio/minio:RELEASE.2023-04-13T03-08-07Z + image: quay.io/minio/minio:RELEASE.2025-02-28T09-55-16Z command: minio server /data ports: - 9000:9000 diff --git a/local/Dockerfile b/local/Dockerfile index 0087840..55a1f56 100644 --- a/local/Dockerfile +++ b/local/Dockerfile @@ -5,25 +5,32 @@ # regularly takes 2 minutes to download 20MB of binary). The only other way # they distribute the CLI is from Docker, so we load their image as a stage and # then copy the binary from it later in the build. -FROM quay.io/minio/mc:RELEASE.2023-04-12T02-21-51Z AS mc +FROM quay.io/minio/mc:RELEASE.2025-02-21T16-00-46Z AS mc -FROM ubuntu:22.04 +FROM ubuntu:24.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - awscli \ build-essential \ curl \ git \ gnupg \ jq \ + libssl-dev \ + openssl \ + pkg-config \ python3 \ - socat + socat \ + unzip + +# Install the AWS CLI +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(arch).zip" -o "awscliv2.zip"; \ + unzip awscliv2.zip; \ + rm awscliv2.zip; \ + ./aws/install # Install rustup while removing the pre-installed stable toolchain. -RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \ - chmod +x /tmp/rustup-init && \ - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \ - /root/.cargo/bin/rustup toolchain remove stable +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable ENV PATH=/root/.cargo/bin:$PATH COPY --from=mc /usr/bin/mc /usr/local/bin/mc diff --git a/local/run.sh b/local/release.sh similarity index 95% rename from local/run.sh rename to local/release.sh index 45f7b98..7564352 100755 --- a/local/run.sh +++ b/local/release.sh @@ -101,6 +101,14 @@ for file in "${DOWNLOAD_STANDALONE[@]}"; do download "${file}" done +# Build the promote-release binary if it hasn't been pre-built +if [[ ! -f "/src/target/release/promote-release" ]]; then + echo "==> building promote-release" + cd /src + cargo build --release + cd .. +fi + echo "==> configuring the environment" # Point to the right GnuPG environment export GNUPGHOME=/persistent/gpg-home diff --git a/local/rustup.sh b/local/rustup.sh new file mode 100755 index 0000000..038ed11 --- /dev/null +++ b/local/rustup.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# This script is executed at the start of each local release for Rustup, and +# prepares the environment by copying the artifacts built by CI onto the MinIO +# instance. Then, it starts promote-release with the right flags. + +set -euo pipefail +IFS=$'\n\t' + +RUSTUP_REPO="https://github.com/rust-lang/rustup" +RUSTUP_DEFAULT_BRANCH="stable" + +# S3 bucket from which to download the Rustup artifacts +S3_BUCKET="rustup-builds" + +# CDN from which to download the CI artifacts +DOWNLOAD_BASE="https://rustup-builds.rust-lang.org" + +# The artifacts for the following targets will be downloaded and copied during +# the release process. At least one target is required. +DOWNLOAD_TARGETS=( + "aarch64-unknown-linux-gnu" + "x86_64-unknown-linux-gnu" +) + +# The following files will be downloaded and put into the local MinIO instance. +DOWNLOAD_FILES=( + "rustup-init" + "rustup-init.sha256" + "rustup-setup" + "rustup-setup.sha256" +) + +channel="$1" +override_commit="$2" + +if [[ "${override_commit}" = "" ]]; then + echo "==> detecting the last Rustup commit on the default branch" + commit="$(git ls-remote "${RUSTUP_REPO}" | grep "refs/heads/${RUSTUP_DEFAULT_BRANCH}" | awk '{print($1)}')" +else + echo "=>> using overridden commit ${override_commit}" + commit="${override_commit}" +fi + +for target in "${DOWNLOAD_TARGETS[@]}"; do + if ! mc stat "local/rustup-builds/${commit}/dist/${target}" >/dev/null 2>&1; then + echo "==> copying ${target} from S3" + + for file in "${DOWNLOAD_FILES[@]}"; do + echo "==> copying ${file} from S3" + + if curl -Lo /tmp/component "${DOWNLOAD_BASE}/${commit}/dist/${target}/${file}" --fail; then + mc cp /tmp/component "local/rustup-builds/${commit}/dist/${target}/${file}" >/dev/null + fi + done + else + echo "==> reusing cached ${target} target" + fi +done + +if ! mc stat "local/rustup-builds/${commit}/rustup-init.sh" >/dev/null 2>&1; then + echo "==> copying rustup-init.sh from S3" + + if curl -Lo /tmp/component "${DOWNLOAD_BASE}/${commit}/rustup-init.sh" --fail; then + mc cp /tmp/component "local/rustup-builds/${commit}/rustup-init.sh" >/dev/null + fi +else + echo "==> reusing cached rustup-init.sh" +fi + +# Build the promote-release binary if it hasn't been pre-built +if [[ ! -f "/src/target/release/promote-release" ]]; then + echo "==> building promote-release" + cd /src + cargo build --release + cd .. +fi + +echo "==> configuring the environment" + +# Release Rustup +export PROMOTE_RELEASE_ACTION="promote-rustup" + +# Point to the right GnuPG environment +export GNUPGHOME=/persistent/gpg-home + +## Environment variables also used in prod releases +export AWS_ACCESS_KEY_ID="access_key" +export AWS_SECRET_ACCESS_KEY="secret_key" +export PROMOTE_RELEASE_CHANNEL="${channel}" +export PROMOTE_RELEASE_CLOUDFRONT_DOC_ID="" +export PROMOTE_RELEASE_CLOUDFRONT_STATIC_ID="" +export PROMOTE_RELEASE_DOWNLOAD_BUCKET="rustup-builds" +export PROMOTE_RELEASE_DOWNLOAD_DIR="" +export PROMOTE_RELEASE_GPG_KEY_FILE="" +export PROMOTE_RELEASE_GPG_PASSWORD_FILE="" +export PROMOTE_RELEASE_SKIP_CLOUDFRONT_INVALIDATIONS="yes" +export PROMOTE_RELEASE_UPLOAD_ADDR="" +export PROMOTE_RELEASE_UPLOAD_BUCKET="static" +export PROMOTE_RELEASE_UPLOAD_STORAGE_CLASS="STANDARD" +export PROMOTE_RELEASE_UPLOAD_DIR="rustup" + +## Environment variables used only by local releases +export PROMOTE_RELEASE_S3_ENDPOINT_URL="http://minio:9000" + +# Conditional environment variables +if [[ "${override_commit}" != "" ]]; then + export PROMOTE_RELEASE_OVERRIDE_COMMIT="${override_commit}" +fi + +# Conditionally set a version for the next Rustup release +if [[ "${RUSTUP_OVERRIDE_VERSION:-}" != "" ]]; then + export PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION="${RUSTUP_OVERRIDE_VERSION}" +fi + +echo "==> starting promote-release" +/src/target/release/promote-release /persistent/release "${channel}" diff --git a/local/setup.sh b/local/setup.sh index 2e7e1e0..97219b4 100755 --- a/local/setup.sh +++ b/local/setup.sh @@ -11,7 +11,7 @@ MINIO_URL="http://${MINIO_HOST}:${MINIO_PORT}" MINIO_ACCESS_KEY="access_key" MINIO_SECRET_KEY="secret_key" -MINIO_BUCKETS=( "static" "artifacts" ) +MINIO_BUCKETS=( "static" "artifacts" "rustup-builds" ) # Quit immediately when docker-compose receives a Ctrl+C trap exit EXIT @@ -77,9 +77,9 @@ cat < [commit]" +RUSTUP_OVERRIDE_VERSION="${RUSTUP_OVERRIDE_VERSION:-}" + +if [[ "$#" -lt 1 ]]; then + echo "Usage: $0 " + exit 1 +fi +command="$1" + +if [[ "${command}" == "release" ]]; then + if [[ "$#" -lt 2 ]] || [[ "$#" -gt 3 ]]; then + echo "Usage: $0 release [commit]" + exit 1 + fi +fi + +if [[ "${command}" == "rustup" ]]; then + if [[ "$#" -lt 2 ]] || [[ "$#" -gt 3 ]]; then + echo "Usage: $0 rustup [commit]" exit 1 + fi fi -channel="$1" -override_commit="${2-}" + +channel="$2" +override_commit="${3-}" container_id="$(docker compose ps -q local)" if [[ "${container_id}" == "" ]]; then @@ -27,8 +45,16 @@ if [[ "${container_status}" != "running" ]]; then exit 1 fi -# Ensure the release build is done -cargo build --release +# Pre-built the binary if the host and Docker environments match +if [[ "$(uname)" == "Linux" ]]; then + cargo build --release +fi -# Run the command inside the docker environment. -docker compose exec -T local /src/local/run.sh "${channel}" "${override_commit}" +if [[ "$RUSTUP_OVERRIDE_VERSION" != "" ]]; then + # If the RUSTUP_OVERRIDE_VERSION environment variable is set, forward it to the Docker environment. + echo "==> running local release with override version ${RUSTUP_OVERRIDE_VERSION}" + docker compose exec -e "RUSTUP_OVERRIDE_VERSION=${RUSTUP_OVERRIDE_VERSION}" -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" +else + # Run the command inside the docker environment. + docker compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}" +fi diff --git a/src/config.rs b/src/config.rs index 070ce46..fc3f0fb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -48,6 +48,8 @@ impl std::fmt::Display for Channel { } } +// Allow all variant names to start with `Promote` +#[allow(clippy::enum_variant_names)] #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum Action { /// This is the default action, what we'll do if the environment variable @@ -64,6 +66,13 @@ pub(crate) enum Action { /// * Create a rust-lang/cargo branch for the appropriate beta commit. /// * Post a PR against the newly created beta branch bump src/ci/channel to `beta`. PromoteBranches, + + /// This promotes a new rustup release: + /// + /// * Copy binaries into archives + /// * Copy binaries from dev-static to production + /// * Update dev release number + PromoteRustup, } impl FromStr for Action { @@ -73,7 +82,8 @@ impl FromStr for Action { match input { "promote-release" => Ok(Action::PromoteRelease), "promote-branches" => Ok(Action::PromoteBranches), - _ => anyhow::bail!("unknown channel: {}", input), + "promote-rustup" => Ok(Action::PromoteRustup), + _ => anyhow::bail!("unknown action: {}", input), } } } diff --git a/src/main.rs b/src/main.rs index 9bafd1c..ceb8fac 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod discourse; mod fastly; mod github; mod recompress; +mod rustup; mod sign; mod smoke_test; @@ -76,6 +77,7 @@ impl Context { match self.config.action { config::Action::PromoteRelease => self.do_release()?, config::Action::PromoteBranches => self.do_branching()?, + config::Action::PromoteRustup => self.promote_rustup()?, } Ok(()) } diff --git a/src/rustup.rs b/src/rustup.rs new file mode 100644 index 0000000..e3fc567 --- /dev/null +++ b/src/rustup.rs @@ -0,0 +1,297 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Error}; +use curl::easy::Easy; +use serde::Deserialize; + +use crate::config::Channel; +use crate::curl_helper::BodyExt; +use crate::{run, Context}; + +#[derive(Deserialize)] +struct Content { + content: String, +} + +#[derive(Deserialize)] +struct CargoToml { + workspace: Workspace, +} + +#[derive(Deserialize)] +struct Workspace { + package: Package, +} + +#[derive(Deserialize)] +struct Package { + version: String, +} + +impl Context { + /// Promote a `rustup` release + /// + /// The [release process] for `rustup` involves copying existing artifacts from one S3 bucket to + /// another, updating the manifest, and archiving the artifacts for long-term storage. + /// + /// `rustup` uses different branches to manage releases. Whenever a commit is pushed to the + /// `stable` branch in [rust-lang/rustup], GitHub Actions workflows build release artifacts and + /// copy them into `s3://rustup-builds/builds/${commit-sha}/`. + /// + /// When a new release is cut and this method is invoked, it downloads the artifacts from that + /// bucket (which must always be set as the `DOWNLOAD_BUCKET` variable). A copy of the artifacts + /// is archived in `s3://${UPLOAD_BUCKET}/rustup/archive/${version}/`, where `version` is + /// derived from the Cargo.toml file in the `stable` branch. `UPLOAD_BUCKET` can either be the + /// `dev-static` or the `static` bucket. + /// + /// The artifacts are also copied to the `dist/` path in the `UPLOAD_BUCKET` bucket, which is + /// used by the `rustup` installer to download the latest release. + /// + /// Then, the `release-stable.toml` manifest is updated with the new version and copied to + /// `s3://${UPLOAD_BUCKET}/rustup/release-stable.toml`. + /// + /// [release process]: https://rust-lang.github.io/rustup/dev-guide/release-process.html + /// [rust-lang/rustup]: https://github.com/rust-lang/rustup + pub fn promote_rustup(&mut self) -> anyhow::Result<()> { + // Rustup only has beta and stable releases, so we fail fast when trying to promote nightly + self.enforce_rustup_channel()?; + + // Get the latest commit from the `stable` branch or use the user-provided override + let head_sha = self.get_commit_sha_for_rustup_release()?; + + // The commit on the `stable` branch is used to determine the version number + let version = self.get_next_rustup_version(&head_sha)?; + + // Download the Rustup artifacts from S3 + let dist_dir = self.download_rustup_artifacts(&head_sha)?; + + // Archive the artifacts + self.archive_rustup_artifacts(&dist_dir, &version)?; + + // Promote the artifacts to the release bucket + self.promote_rustup_artifacts(&dist_dir)?; + + // Update the `rustup` installer + self.update_rustup_installer(&dist_dir)?; + + // Update the release number + self.update_rustup_release(&version)?; + + // Invalidate the CDN caches + self.invalidate_rustup()?; + + Ok(()) + } + + fn enforce_rustup_channel(&self) -> anyhow::Result<()> { + println!("Checking channel..."); + + if self.config.channel != Channel::Stable && self.config.channel != Channel::Beta { + return Err(anyhow!( + "promoting rustup is only supported for the stable and beta channels" + )); + } + + Ok(()) + } + + fn get_commit_sha_for_rustup_release(&self) -> anyhow::Result { + match &self.config.override_commit { + Some(sha) => Ok(sha.clone()), + None => self.get_head_sha_for_rustup(), + } + } + + fn get_head_sha_for_rustup(&self) -> anyhow::Result { + #[derive(Deserialize)] + struct Commit { + sha: String, + } + + let mut client = Easy::new(); + client.url("https://api.github.com/repos/rust-lang/rustup/commits/stable")?; + client.useragent("rust-lang/promote-release")?; + + let commit: Commit = client.without_body().send_with_response()?; + + Ok(commit.sha) + } + + fn get_next_rustup_version(&self, sha: &str) -> anyhow::Result { + // Allow the version to be overridden manually, for example to test the release process + if let Ok(version) = std::env::var("PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION") { + println!("Using override version: {}", version); + Ok(version) + } else { + self.get_next_rustup_version_from_github(sha) + } + } + + fn get_next_rustup_version_from_github(&self, sha: &str) -> anyhow::Result { + println!("Getting next Rustup version from Cargo.toml..."); + + let url = + format!("https://api.github.com/repos/rust-lang/rustup/contents/Cargo.toml?ref={sha}"); + + let mut client = Easy::new(); + client.url(&url)?; + client.useragent("rust-lang/promote-release")?; + + let content: Content = client.without_body().send_with_response()?; + let toml = decode_and_deserialize_cargo_toml(&content.content)?; + + Ok(toml.workspace.package.version) + } + + fn download_rustup_artifacts(&mut self, sha: &str) -> Result { + println!( + "Downloading artifacts from {}...", + self.config.download_bucket + ); + + let dl = self.dl_dir().join("rustup"); + // Remove the directory if it exists, otherwise just ignore. + let _ = fs::remove_dir_all(&dl); + fs::create_dir_all(&dl)?; + + let artifacts_url = format!("s3://{}/{}", self.config.download_bucket, sha); + + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(artifacts_url) + .arg(format!("{}/", dl.display())))?; + + Ok(dl) + } + + fn archive_rustup_artifacts(&mut self, dist_dir: &Path, version: &str) -> Result<(), Error> { + println!("Archiving artifacts for version {version}..."); + + let path = format!("archive/{}/", version); + + self.upload_rustup_artifacts(&dist_dir.join("dist"), &path) + } + + fn promote_rustup_artifacts(&mut self, dist_dir: &Path) -> Result<(), Error> { + println!("Promoting artifacts to dist/..."); + + let release_bucket_url = format!( + "s3://{}/{}/dist/", + self.config.upload_bucket, self.config.upload_dir, + ); + + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(format!("{}/dist/", dist_dir.display())) + .arg(&release_bucket_url)) + } + + fn update_rustup_installer(&mut self, dist_dir: &Path) -> Result<(), Error> { + println!("Updating the rustup installer..."); + + let release_bucket_url = format!( + "s3://{}/{}", + self.config.upload_bucket, self.config.upload_dir, + ); + + let destinations = ["rustup-init.sh", "rustup.sh"]; + + for destination in destinations { + run(self + .aws_s3() + .arg("cp") + .arg("--only-show-errors") + .arg(format!("{}/rustup-init.sh", dist_dir.display())) + .arg(format!("{}/{}", release_bucket_url, destination)))?; + } + + Ok(()) + } + + fn upload_rustup_artifacts(&mut self, dist_dir: &Path, target_path: &str) -> Result<(), Error> { + run(self + .aws_s3() + .arg("cp") + .arg("--recursive") + .arg("--only-show-errors") + .arg(format!("{}/", dist_dir.display())) + .arg(format!( + "s3://{}/{}/{}", + self.config.upload_bucket, self.config.upload_dir, target_path + ))) + } + + fn update_rustup_release(&mut self, version: &str) -> Result<(), Error> { + println!("Updating version and manifest..."); + + let manifest_path = self.dl_dir().join("release-stable.toml"); + let manifest = format!( + r#" +schema-version = '1' +version = '{}' + "#, + version + ); + + fs::write(&manifest_path, manifest)?; + + run(self + .aws_s3() + .arg("cp") + .arg("--only-show-errors") + .arg(manifest_path) + .arg(format!( + "s3://{}/{}/release-stable.toml", + self.config.upload_bucket, self.config.upload_dir + ))) + } + + fn invalidate_rustup(&mut self) -> Result<(), Error> { + println!("Invalidating CDN caches..."); + + let paths = [ + self.config.upload_dir.clone(), + "rustup-init.sh".into(), + "rustup.sh".into(), + "rustup/release-stable.toml".into(), + ]; + + self.invalidate_cloudfront(&self.config.cloudfront_static_id, &paths)?; + self.invalidate_fastly(&paths)?; + + Ok(()) + } +} + +fn decode_and_deserialize_cargo_toml(base64_encoded_toml: &str) -> Result { + let decoded_content = base64::decode(base64_encoded_toml.replace('\n', ""))?; + let content_as_string = String::from_utf8(decoded_content)?; + + toml::from_str(&content_as_string).map_err(Into::into) +} + +#[cfg(test)] +mod tests { + use crate::rustup::decode_and_deserialize_cargo_toml; + + #[test] + fn decode_cargo_toml() { + let base64_encoded_toml = base64::encode( + r#" + [workspace.package] + version = "1.2.3" + "#, + ); + + let toml = decode_and_deserialize_cargo_toml(&base64_encoded_toml).unwrap(); + + assert_eq!(toml.workspace.package.version, "1.2.3"); + } +}