Start auto builds in merge queue

Author: Sakib25800

.sqlx/query-3ae0ba7ce98f0a68b47df52a9181134df3ac7818af0c085ac432eb52874a62a2.json

@@ -185,3 +185,10 @@ fn list_workflows_status(workflows: &[WorkflowModel]) -> String {
         .collect::<Vec<_>>()
         .join("\n")
 }
+
+pub fn auto_build_started_comment(head_sha: &CommitSha, merge_sha: &CommitSha) -> Comment {
+    Comment::new(format!(
+        ":hourglass: Testing commit {} with merge {}...",
+        head_sha, merge_sha
+    ))
+}

src/bors/comment.rs

@@ -237,9 +237,13 @@ async fn try_complete_build(
 #[cfg(test)]
 mod tests {
     use crate::bors::handlers::trybuild::TRY_BRANCH_NAME;
+    use crate::bors::merge_queue::{AUTO_BUILD_CHECK_RUN_NAME, AUTO_BRANCH_NAME};
     use crate::database::WorkflowStatus;
     use crate::database::operations::get_all_workflows;
-    use crate::tests::mocks::{Branch, CheckSuite, Workflow, WorkflowEvent, run_test};
+    use crate::tests::mocks::{
+        BorsBuilder, Branch, CheckSuite, GitHubState, Workflow, WorkflowEvent, default_pr_number,
+        run_test,
+    };
 
     #[sqlx::test]
     async fn workflow_started_unknown_build(pool: sqlx::PgPool) {

src/bors/handlers/workflow.rs

@@ -1,12 +1,24 @@
+use anyhow::anyhow;
+use octocrab::params::checks::{CheckRunOutput, CheckRunStatus};
 use std::future::Future;
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use tracing::Instrument;
 
 use crate::BorsContext;
 use crate::bors::RepositoryState;
+use crate::bors::comment::auto_build_started_comment;
+use crate::database::{BuildStatus, PullRequestModel};
+use crate::github::api::client::GithubRepositoryClient;
+use crate::github::api::operations::ForcePush;
+use crate::github::{CommitSha, MergeError};
 use crate::utils::sort_queue::sort_queue_prs;
 
+enum MergeResult {
+    Success(CommitSha),
+    Conflict,
+}
+
 #[derive(Debug)]
 enum MergeQueueEvent {
     Trigger,
@@ -31,10 +43,17 @@ impl MergeQueueSender {
     }
 }
 
+/// Branch used for performing merge operations.
+/// This branch should not run CI checks.
+pub(super) const AUTO_MERGE_BRANCH_NAME: &str = "automation/bors/auto-merge";
+
 /// Branch where CI checks run for auto builds.
 /// This branch should run CI checks.
 pub(super) const AUTO_BRANCH_NAME: &str = "automation/bors/auto";
 
+// The name of the check run seen in the GitHub UI.
+pub(super) const AUTO_BUILD_CHECK_RUN_NAME: &str = "Bors auto build";
+
 pub async fn merge_queue_tick(ctx: Arc<BorsContext>) -> anyhow::Result<()> {
     let repos: Vec<Arc<RepositoryState>> =
         ctx.repositories.read().unwrap().values().cloned().collect();
@@ -61,14 +80,173 @@ pub async fn merge_queue_tick(ctx: Arc<BorsContext>) -> anyhow::Result<()> {
         // then pending builds (which block the queue to prevent starting simultaneous auto-builds).
         let prs = sort_queue_prs(prs);
 
-        for _ in prs {
-            // Process PRs...
+        for pr in prs {
+            let pr_num = pr.number;
+
+            if let Some(auto_build) = &pr.auto_build {
+                match auto_build.status {
+                    // Build successful - point the base branch to the merged commit.
+                    BuildStatus::Success => {
+                        // Break to give GitHub time to update the base branch.
+                        break;
+                    }
+                    // Build in progress - stop queue. We can only have one PR being built
+                    // at a time.
+                    BuildStatus::Pending => {
+                        tracing::info!("PR {pr_num} has a pending build - blocking queue");
+                        break;
+                    }
+                    BuildStatus::Failure | BuildStatus::Cancelled | BuildStatus::Timeouted => {
+                        unreachable!("Failed auto builds should be filtered out by SQL query");
+                    }
+                }
+            }
+
+            // No build exists for this PR - start a new auto build.
+            match start_auto_build(&repo, &ctx, pr).await {
+                Ok(true) => {
+                    tracing::info!("Starting auto build for PR {pr_num}");
+                    break;
+                }
+                Ok(false) => {
+                    tracing::debug!("Failed to start auto build for PR {pr_num}");
+                    continue;
+                }
+                Err(error) => {
+                    // Unexpected error - the PR will remain in the "queue" for a retry.
+                    tracing::error!("Error starting auto build for PR {pr_num}: {:?}", error);
+                    continue;
+                }
+            }
         }
     }
 
+    #[cfg(test)]
+    crate::bors::WAIT_FOR_MERGE_QUEUE.mark();
+
     Ok(())
 }
 
+/// Starts a new auto build for a pull request.
+async fn start_auto_build(
+    repo: &Arc<RepositoryState>,
+    ctx: &Arc<BorsContext>,
+    pr: PullRequestModel,
+) -> anyhow::Result<bool> {
+    let client = &repo.client;
+
+    let gh_pr = client.get_pull_request(pr.number).await?;
+    let base_sha = client.get_branch_sha(&pr.base_branch).await?;
+
+    let auto_merge_commit_message = format!(
+        "Auto merge of #{} - {}, r={}\n\n{}\n\n{}",
+        pr.number,
+        gh_pr.head_label,
+        pr.approver().unwrap_or("<unknown>"),
+        pr.title,
+        gh_pr.message
+    );
+
+    // 1. Merge PR head with base branch on `AUTO_MERGE_BRANCH_NAME`
+    match attempt_merge(
+        &repo.client,
+        &gh_pr.head.sha,
+        &base_sha,
+        &auto_merge_commit_message,
+    )
+    .await?
+    {
+        MergeResult::Success(merge_sha) => {
+            // 2. Push merge commit to `AUTO_BRANCH_NAME` where CI runs
+            client
+                .set_branch_to_sha(AUTO_BRANCH_NAME, &merge_sha, ForcePush::Yes)
+                .await?;
+
+            // 3. Record the build in the database
+            let build_id = ctx
+                .db
+                .attach_auto_build(
+                    &pr,
+                    AUTO_BRANCH_NAME.to_string(),
+                    merge_sha.clone(),
+                    base_sha,
+                )
+                .await?;
+
+            // 4. Set GitHub check run to pending on PR head
+            match client
+                .create_check_run(
+                    AUTO_BUILD_CHECK_RUN_NAME,
+                    &gh_pr.head.sha,
+                    CheckRunStatus::InProgress,
+                    CheckRunOutput {
+                        title: AUTO_BUILD_CHECK_RUN_NAME.to_string(),
+                        summary: "".to_string(),
+                        text: None,
+                        annotations: vec![],
+                        images: vec![],
+                    },
+                    &build_id.to_string(),
+                )
+                .await
+            {
+                Ok(check_run) => {
+                    tracing::info!(
+                        "Created check run {} for build {build_id}",
+                        check_run.id.into_inner()
+                    );
+                    ctx.db
+                        .update_build_check_run_id(build_id, check_run.id.into_inner() as i64)
+                        .await?;
+                }
+                Err(error) => {
+                    // Check runs aren't critical, don't block progress if they fail
+                    tracing::error!("Cannot create check run: {error:?}");
+                }
+            }
+
+            // 5. Post status comment
+            let comment = auto_build_started_comment(&gh_pr.head.sha, &merge_sha);
+            client.post_comment(pr.number, comment).await?;
+
+            Ok(true)
+        }
+        MergeResult::Conflict => Ok(false),
+    }
+}
+
+/// Attempts to merge the given head SHA with base SHA via `AUTO_MERGE_BRANCH_NAME`.
+async fn attempt_merge(
+    client: &GithubRepositoryClient,
+    head_sha: &CommitSha,
+    base_sha: &CommitSha,
+    merge_message: &str,
+) -> anyhow::Result<MergeResult> {
+    tracing::debug!("Attempting to merge with base SHA {base_sha}");
+
+    // Reset auto merge branch to point to base branch
+    client
+        .set_branch_to_sha(AUTO_MERGE_BRANCH_NAME, base_sha, ForcePush::Yes)
+        .await
+        .map_err(|error| anyhow!("Cannot set auto merge branch to {}: {error:?}", base_sha.0))?;
+
+    // then merge PR head commit into auto merge branch.
+    match client
+        .merge_branches(AUTO_MERGE_BRANCH_NAME, head_sha, merge_message)
+        .await
+    {
+        Ok(merge_sha) => {
+            tracing::debug!("Merge successful, SHA: {merge_sha}");
+            Ok(MergeResult::Success(merge_sha))
+        }
+        Err(MergeError::Conflict) => {
+            tracing::warn!("Merge conflict");
+            Ok(MergeResult::Conflict)
+        }
+        Err(error) => Err(error.into()),
+    }
+}
+
 pub fn start_merge_queue(ctx: Arc<BorsContext>) -> (MergeQueueSender, impl Future<Output = ()>) {
     let (tx, mut rx) = mpsc::channel::<MergeQueueEvent>(10);
     let sender = MergeQueueSender { inner: tx };
@@ -103,3 +281,76 @@ pub fn start_merge_queue(ctx: Arc<BorsContext>) -> (MergeQueueSender, impl Futur
 
     (sender, fut)
 }
+
+#[cfg(test)]
+mod tests {
+
+    use octocrab::params::checks::CheckRunStatus;
+
+    use crate::{
+        bors::merge_queue::{AUTO_BRANCH_NAME, AUTO_BUILD_CHECK_RUN_NAME},
+        database::{WorkflowStatus, operations::get_all_workflows},
+        tests::mocks::{WorkflowEvent, run_test},
+    };
+
+    #[sqlx::test]
+    async fn auto_workflow_started(pool: sqlx::PgPool) {
+        run_test(pool.clone(), |mut tester| async {
+            tester.create_branch(AUTO_BRANCH_NAME).expect_suites(1);
+
+            tester.post_comment("@bors r+").await?;
+            tester.expect_comments(1).await;
+
+            tester.process_merge_queue().await;
+            tester.expect_comments(1).await;
+
+            tester
+                .workflow_event(WorkflowEvent::started(tester.auto_branch()))
+                .await?;
+            Ok(tester)
+        })
+        .await;
+        let suite = get_all_workflows(&pool).await.unwrap().pop().unwrap();
+        assert_eq!(suite.status, WorkflowStatus::Pending);
+    }
+
+    #[sqlx::test]
+    async fn auto_workflow_check_run_created(pool: sqlx::PgPool) {
+        run_test(pool, |mut tester| async {
+            tester.create_branch(AUTO_BRANCH_NAME).expect_suites(1);
+
+            tester.post_comment("@bors r+").await?;
+            tester.expect_comments(1).await;
+
+            tester.process_merge_queue().await;
+            tester.expect_comments(1).await;
+            tester.expect_check_run(
+                &tester.default_pr().await.get_gh_pr().head_sha,
+                AUTO_BUILD_CHECK_RUN_NAME,
+                AUTO_BUILD_CHECK_RUN_NAME,
+                CheckRunStatus::InProgress,
+                None,
+            );
+            Ok(tester)
+        })
+        .await;
+    }
+
+    #[sqlx::test]
+    async fn auto_workflow_started_comment(pool: sqlx::PgPool) {
+        run_test(pool, |mut tester| async {
+            tester.create_branch(AUTO_BRANCH_NAME).expect_suites(1);
+
+            tester.post_comment("@bors r+").await?;
+            tester.expect_comments(1).await;
+
+            tester.process_merge_queue().await;
+            insta::assert_snapshot!(
+                tester.get_comment().await?,
+                @":hourglass: Testing commit pr-1-sha with merge merge-main-sha1-pr-1-sha-0..."
+            );
+            Ok(tester)
+        })
+        .await;
+    }
+}

src/bors/merge_queue.rs

@@ -37,6 +37,9 @@ pub static WAIT_FOR_PR_STATUS_REFRESH: TestSyncMarker = TestSyncMarker::new();
 #[cfg(test)]
 pub static WAIT_FOR_WORKFLOW_STARTED: TestSyncMarker = TestSyncMarker::new();
 
+#[cfg(test)]
+pub static WAIT_FOR_MERGE_QUEUE: TestSyncMarker = TestSyncMarker::new();
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum CheckSuiteStatus {
     Pending,

src/bors/mod.rs

@@ -17,8 +17,9 @@ use super::operations::{
     get_running_builds, get_workflow_urls_for_build, get_workflows_for_build,
     insert_repo_if_not_exists, set_pr_assignees, set_pr_priority, set_pr_rollup, set_pr_status,
     unapprove_pull_request, undelegate_pull_request, update_build_check_run_id,
-    update_build_status, update_mergeable_states_by_base_branch, update_pr_mergeable_state,
-    update_pr_try_build_id, update_workflow_status, upsert_pull_request, upsert_repository,
+    update_build_status, update_mergeable_states_by_base_branch, update_pr_auto_build_id,
+    update_pr_mergeable_state, update_pr_try_build_id, update_workflow_status, upsert_pull_request,
+    upsert_repository,
 };
 use super::{ApprovalInfo, DelegatedPermission, MergeableState, RunId, UpsertPullRequestParams};
 
@@ -191,6 +192,21 @@ impl PgDbClient {
         Ok(build_id)
     }
 
+    pub async fn attach_auto_build(
+        &self,
+        pr: &PullRequestModel,
+        branch: String,
+        commit_sha: CommitSha,
+        parent: CommitSha,
+    ) -> anyhow::Result<i32> {
+        let mut tx = self.pool.begin().await?;
+        let build_id =
+            create_build(&mut *tx, &pr.repository, &branch, &commit_sha, &parent).await?;
+        update_pr_auto_build_id(&mut *tx, pr.id, build_id).await?;
+        tx.commit().await?;
+        Ok(build_id)
+    }
+
     pub async fn find_build(
         &self,
         repo: &GithubRepoName,