Feat; enqueue master commits, cron written but not wired up

Author: Jamesbarford

database/src/lib.rs

@@ -798,3 +798,154 @@ pub struct ArtifactCollection {
     pub duration: Duration,
     pub end_time: DateTime<Utc>,
 }
+
+#[derive(Debug)]
+pub enum BenchmarkRequestCommitType {
+    Try { pr: u32 },
+    Master { pr: u32 },
+    Release { tag: String },
+}
+
+impl fmt::Display for BenchmarkRequestCommitType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            BenchmarkRequestCommitType::Try { pr: _ } => write!(f, "try"),
+            BenchmarkRequestCommitType::Master { pr: _ } => write!(f, "master"),
+            BenchmarkRequestCommitType::Release { tag: _ } => write!(f, "release"),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum BenchmarkRequestStatus {
+    WaitingForArtifacts,
+    WaitingForParent,
+    Queued,
+    InProgress,
+    Completed,
+}
+
+impl fmt::Display for BenchmarkRequestStatus {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            BenchmarkRequestStatus::WaitingForArtifacts => write!(f, "waiting_for_artifacts"),
+            BenchmarkRequestStatus::WaitingForParent => write!(f, "waiting_for_parent"),
+            BenchmarkRequestStatus::Queued => write!(f, "queued"),
+            BenchmarkRequestStatus::InProgress => write!(f, "in_progress"),
+            BenchmarkRequestStatus::Completed => write!(f, "completed"),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum BenchmarkRequestType {
+    /// A Try commit
+    Try {
+        sha: String,
+        parent_sha: Option<String>,
+        pr: u32,
+    },
+    /// A Master commit
+    Master {
+        sha: String,
+        parent_sha: Option<String>,
+        pr: u32,
+    },
+    /// A release only has a tag
+    Release { tag: String },
+}
+
+impl fmt::Display for BenchmarkRequestType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            BenchmarkRequestType::Try {
+                sha: _,
+                parent_sha: _,
+                pr: _,
+            } => write!(f, "try"),
+            BenchmarkRequestType::Master {
+                sha: _,
+                parent_sha: _,
+                pr: _,
+            } => write!(f, "master"),
+            BenchmarkRequestType::Release { tag: _ } => write!(f, "release"),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct BenchmarkRequest {
+    pub commit_type: BenchmarkRequestType,
+    pub created_at: DateTime<Utc>,
+    pub completed_at: Option<DateTime<Utc>>,
+    pub status: BenchmarkRequestStatus,
+    pub backends: String,
+    pub profiles: String,
+}
+
+impl BenchmarkRequest {
+    pub fn create_release(
+        tag: &str,
+        created_at: DateTime<Utc>,
+        status: BenchmarkRequestStatus,
+        backends: &str,
+        profiles: &str,
+    ) -> Self {
+        Self {
+            commit_type: BenchmarkRequestType::Release {
+                tag: tag.to_string(),
+            },
+            created_at,
+            completed_at: None,
+            status,
+            backends: backends.to_string(),
+            profiles: profiles.to_string(),
+        }
+    }
+
+    pub fn create_try(
+        sha: &str,
+        parent_sha: Option<&str>,
+        pr: u32,
+        created_at: DateTime<Utc>,
+        status: BenchmarkRequestStatus,
+        backends: &str,
+        profiles: &str,
+    ) -> Self {
+        Self {
+            commit_type: BenchmarkRequestType::Try {
+                pr,
+                sha: sha.to_string(),
+                parent_sha: parent_sha.map(|it| it.to_string()),
+            },
+            created_at,
+            completed_at: None,
+            status,
+            backends: backends.to_string(),
+            profiles: profiles.to_string(),
+        }
+    }
+
+    pub fn create_master(
+        sha: &str,
+        parent_sha: Option<&str>,
+        pr: u32,
+        created_at: DateTime<Utc>,
+        status: BenchmarkRequestStatus,
+        backends: &str,
+        profiles: &str,
+    ) -> Self {
+        Self {
+            commit_type: BenchmarkRequestType::Master {
+                pr,
+                sha: sha.to_string(),
+                parent_sha: parent_sha.map(|it| it.to_string()),
+            },
+            created_at,
+            completed_at: None,
+            status,
+            backends: backends.to_string(),
+            profiles: profiles.to_string(),
+        }
+    }
+}

database/src/pool.rs

@@ -1,5 +1,6 @@
 use crate::{
-    ArtifactCollection, ArtifactId, ArtifactIdNumber, CodegenBackend, CompileBenchmark, Target,
+    ArtifactCollection, ArtifactId, ArtifactIdNumber, BenchmarkRequest, CodegenBackend,
+    CompileBenchmark, Target,
 };
 use crate::{CollectionId, Index, Profile, QueuedCommit, Scenario, Step};
 use chrono::{DateTime, Utc};
@@ -178,6 +179,10 @@ pub trait Connection: Send + Sync {
 
     /// Removes all data associated with the given artifact.
     async fn purge_artifact(&self, aid: &ArtifactId);
+
+    /// Add an item to the `benchmark_requests`, if the `benchmark_request`
+    /// extists it will be ignored
+    async fn insert_benchmark_request(&self, benchmark_request: &BenchmarkRequest);
 }
 
 #[async_trait::async_trait]

database/src/pool/postgres.rs

@@ -1,7 +1,8 @@
 use crate::pool::{Connection, ConnectionManager, ManagedConnection, Transaction};
 use crate::{
-    ArtifactCollection, ArtifactId, ArtifactIdNumber, Benchmark, CodegenBackend, CollectionId,
-    Commit, CommitType, CompileBenchmark, Date, Index, Profile, QueuedCommit, Scenario, Target,
+    ArtifactCollection, ArtifactId, ArtifactIdNumber, Benchmark, BenchmarkRequest,
+    BenchmarkRequestType, CodegenBackend, CollectionId, Commit, CommitType, CompileBenchmark, Date,
+    Index, Profile, QueuedCommit, Scenario, Target,
 };
 use anyhow::Context as _;
 use chrono::{DateTime, TimeZone, Utc};
@@ -285,6 +286,22 @@ static MIGRATIONS: &[&str] = &[
     alter table pstat_series drop constraint test_case;
     alter table pstat_series add constraint test_case UNIQUE(crate, profile, scenario, backend, target, metric);
     "#,
+    r#"
+    CREATE TABLE IF NOT EXISTS benchmark_requests (
+        id           SERIAL PRIMARY KEY,
+        tag          TEXT NOT NULL UNIQUE,
+        parent_sha   TEXT,
+        commit_type  TEXT,
+        pr           INTEGER,
+        created_at   TIMESTAMPTZ NOT NULL,
+        completed_at TIMESTAMPTZ,
+        status       TEXT NOT NULL,
+        backends     TEXT,
+        profiles     TEXT
+    );
+    CREATE INDEX IF NOT EXISTS benchmark_requests_status_idx on benchmark_requests (status);
+    CREATE INDEX IF NOT EXISTS benchmark_requests_commit_type on benchmark_requests (commit_type);
+    "#,
 ];
 
 #[async_trait::async_trait]
@@ -1365,6 +1382,66 @@ where
             .await
             .unwrap();
     }
+
+    async fn insert_benchmark_request(&self, benchmark_request: &BenchmarkRequest) {
+        match &benchmark_request.commit_type {
+            BenchmarkRequestType::Try {
+                sha,
+                parent_sha,
+                pr,
+            }
+            | BenchmarkRequestType::Master {
+                sha,
+                parent_sha,
+                pr,
+            } => {
+                self.conn()
+                    .execute(
+                        r#"
+                        INSERT INTO
+                           benchmark_requests(parent_sha, pr, tag, created_at,
+                                              commit_type, status, backends, profiles)
+                        VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+                        ON CONFLICT DO NOTHING;
+                        "#,
+                        &[
+                            &parent_sha,
+                            pr,
+                            sha,
+                            &benchmark_request.created_at,
+                            &benchmark_request.commit_type.to_string(),
+                            &benchmark_request.status.to_string(),
+                            &benchmark_request.backends,
+                            &benchmark_request.profiles,
+                        ],
+                    )
+                    .await
+                    .unwrap();
+            }
+            BenchmarkRequestType::Release { tag } => {
+                self.conn()
+                    .execute(
+                        r#"
+                        INSERT INTO
+                            benchmark_requests(tag, created_at, commit_type,
+                                               status, backends, profiles)
+                        VALUES ($1, $2, $3, $4, $5, $6);
+                        ON CONFLICT DO NOTHING;
+                        "#,
+                        &[
+                            tag,
+                            &benchmark_request.created_at,
+                            &benchmark_request.commit_type.to_string(),
+                            &benchmark_request.status.to_string(),
+                            &benchmark_request.backends,
+                            &benchmark_request.profiles,
+                        ],
+                    )
+                    .await
+                    .unwrap();
+            }
+        }
+    }
 }
 
 fn parse_artifact_id(ty: &str, sha: &str, date: Option<DateTime<Utc>>) -> ArtifactId {

database/src/pool/sqlite.rs

@@ -1,7 +1,7 @@
 use crate::pool::{Connection, ConnectionManager, ManagedConnection, Transaction};
 use crate::{
-    ArtifactCollection, ArtifactId, Benchmark, CodegenBackend, CollectionId, Commit, CommitType,
-    CompileBenchmark, Date, Profile, Target,
+    ArtifactCollection, ArtifactId, Benchmark, BenchmarkRequest, CodegenBackend, CollectionId,
+    Commit, CommitType, CompileBenchmark, Date, Profile, Target,
 };
 use crate::{ArtifactIdNumber, Index, QueuedCommit};
 use chrono::{DateTime, TimeZone, Utc};
@@ -1252,6 +1252,10 @@ impl Connection for SqliteConnection {
             )
             .unwrap();
     }
+
+    async fn insert_benchmark_request(&self, _benchmark_request: &BenchmarkRequest) {
+        panic!("Queueing for SQLite has not been implemented, if you are wanting to test the queueing functionality please use postgres. Presuming you have docker installed, at the root of the repo you can run `make start-postgres` to spin up a postrgres database.");
+    }
 }
 
 fn parse_artifact_id(ty: &str, sha: &str, date: Option<i64>) -> ArtifactId {

site/src/lib.rs

@@ -9,6 +9,7 @@ pub mod server;
 mod average;
 mod benchmark_metadata;
 mod comparison;
+mod queue_jobs;
 mod request_handlers;
 mod resources;
 mod selector;

site/src/load.rs

@@ -14,8 +14,8 @@ use serde::{Deserialize, Serialize};
 use crate::self_profile::SelfProfileCache;
 use collector::compile::benchmark::category::Category;
 use collector::{Bound, MasterCommit};
-use database::Pool;
 pub use database::{ArtifactId, Benchmark, Commit};
+use database::{BenchmarkRequest, BenchmarkRequestStatus, Pool};
 use database::{CommitType, Date};
 
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
@@ -282,6 +282,27 @@ impl SiteCtxt {
 
         commits
     }
+
+    /// Store the latest master commits or do nothing if all of them are
+    /// already in the database
+    pub async fn enqueue_master_commits(&self) {
+        let conn = self.conn().await;
+        let master_commits = &self.get_master_commits().commits;
+
+        for master_commit in master_commits {
+            let pr = master_commit.pr.unwrap_or(0);
+            let benchmark = BenchmarkRequest::create_master(
+                &master_commit.sha,
+                Some(&master_commit.parent_sha),
+                pr,
+                master_commit.time,
+                BenchmarkRequestStatus::Queued,
+                "",
+                "",
+            );
+            conn.insert_benchmark_request(&benchmark).await;
+        }
+    }
 }
 
 /// Parses an artifact tag like `1.63.0` or `beta-2022-08-19` from a line taken from