[nexus] Implement on-boot DB Schema Migration (#3760)

This PR does the following:

- Adds a `schema/crdb` directory, containing `dbinit.sql`, `dbwipe.sql`,
and a collection of versions.
- When new DB versions are added, we can update `dbinit.sql` (as we have
been doing) but also add a `up.sql` file to a directory named with the
new DB version.
- On boot, nexus acquires a lease for the ability to perform schema
changes, and it executes them. Nexus refuses to boot until the on-disk
schema version matches the version embedded in the Nexus binary.

Furthermore, we provide an example "schema migration" from version 1.0.0
to version 1.0.1.

Some implementation details:
- This PR adds several tests to confirm this behavior works as expected,
across all current and future updates
- This PR makes `db_metadata` more strongly-typed, so make it easier to
access the underlying data
- This PR modified `dbinit.sql` to be idempotent, to be in-line with the
recommended pattern for all subsequent schema migrations too
- We embed the schema changes into the Nexus binary, so they can be
parsed and selectively applied. Along these lines, it might make sense
to *stop* embedding a schema into the CRDB zone, and leave the
initialization entirely under the control of Nexus.

Author: smklein

Cargo.lock

@@ -118,7 +118,7 @@ pub struct UpdateArtifactId {
 //    must be left as a `todo!()` for now; `types::UpdateArtifactKind` will not
 //    be updated with the new variant until step 5 below.
 //
-// 3. Add it to <repo root>/common/src/sql/dbinit.sql under (CREATE TYPE
+// 3. Add it to the sql database schema under (CREATE TYPE
 //    omicron.public.update_artifact_kind).
 //
 //    TODO: After omicron ships this would likely involve a DB migration.

common/src/api/internal/nexus.rs

@@ -204,6 +204,12 @@ pub struct UpdatesConfig {
     pub default_base_url: String,
 }
 
+/// Options to tweak database schema changes.
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+pub struct SchemaConfig {
+    pub schema_dir: PathBuf,
+}
+
 /// Optional configuration for the timeseries database.
 #[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
 pub struct TimeseriesDbConfig {
@@ -355,6 +361,9 @@ pub struct PackageConfig {
     /// this is unconfigured.
     #[serde(default)]
     pub updates: Option<UpdatesConfig>,
+    /// Describes how to handle and perform schema changes.
+    #[serde(default)]
+    pub schema: Option<SchemaConfig>,
     /// Tunable configuration for testing and experimentation
     #[serde(default)]
     pub tunables: Tunables,
@@ -642,6 +651,7 @@ mod test {
                         trusted_root: PathBuf::from("/path/to/root.json"),
                         default_base_url: "http://example.invalid/".into(),
                     }),
+                    schema: None,
                     tunables: Tunables { max_vpc_ipv4_subnet_prefix: 27 },
                     dendrite: HashMap::from([(
                         SwitchLocation::Switch0,

common/src/nexus_config.rs

@@ -303,8 +303,7 @@ async fn test_db_run() {
 
         anyhow::ensure!(has_omicron_schema(&client).await);
 
-        // Now run db-populate.  It should fail because the database is already
-        // populated.
+        // Now run db-populate.
         eprintln!("running db-populate");
         let populate_result = Exec::cmd(&cmd_path)
             .arg("db-populate")
@@ -317,13 +316,6 @@ async fn test_db_run() {
         eprintln!("exit status: {:?}", populate_result.exit_status);
         eprintln!("stdout: {:?}", populate_result.stdout_str());
         eprintln!("stdout: {:?}", populate_result.stderr_str());
-        anyhow::ensure!(matches!(
-            populate_result.exit_status,
-            ExitStatus::Exited(1)
-        ));
-        anyhow::ensure!(populate_result
-            .stderr_str()
-            .contains("database \"omicron\" already exists"),);
         anyhow::ensure!(has_omicron_schema(&client).await);
 
         // Try again, but with the --wipe flag.

dev-tools/tests/test_omicron_dev.rs

@@ -38,7 +38,7 @@ this document should act as a jumping-off point.
 * If your application logic involes any multi-step operations which would be interrupted by Nexus stopping mid-execution (due to reboot, crash, failure, etc), it is recommended to use a https://github.com/oxidecomputer/omicron/tree/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/src/app/sagas[saga] to define the operations durably.
 
 === **Database**
-* `CREATE TABLE` for the resource in xref:../common/src/sql/dbinit.sql[dbinit.sql] (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/common/src/sql/dbinit.sql#L1103-L1129[Example])
+* `CREATE TABLE` for the resource in xref:../schema/crdb/dbinit.sql[dbinit.sql] (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/common/src/sql/dbinit.sql#L1103-L1129[Example])
 * Add an equivalent schema for the resource in xref:../nexus/db-model/src/schema.rs[schema.rs], which allows https://docs.diesel.rs/master/diesel/index.html[Diesel] to translate raw SQL to rust queries (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/db-model/src/schema.rs#L144-L155[Example])
 * Add a Rust representation of the database object to xref:../nexus/db-model/src[the DB model] (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/db-model/src/ip_pool.rs#L24-L40[Example])
 ** Remember to make the object visible; this is usually defined in xref:../nexus/db-model/src/lib.rs[lib.rs] (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/db-model/src/lib.rs#L102[Example]).

docs/adding-an-endpoint.adoc

@@ -70,7 +70,9 @@ serde_urlencoded.workspace = true
 serde_with.workspace = true
 sled-agent-client.workspace = true
 slog.workspace = true
+slog-async.workspace = true
 slog-dtrace.workspace = true
+slog-term.workspace = true
 steno.workspace = true
 tempfile.workspace = true
 thiserror.workspace = true
@@ -112,6 +114,7 @@ openapiv3.workspace = true
 oxide-client.workspace = true
 pem.workspace = true
 petgraph.workspace = true
+pretty_assertions.workspace = true
 rcgen.workspace = true
 regex.workspace = true
 rustls = { workspace = true }

nexus/Cargo.toml

@@ -3,25 +3,33 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use crate::schema::db_metadata;
+use crate::SemverVersion;
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
 
 /// Internal database metadata
-#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[derive(
+    Queryable, Insertable, Debug, Clone, Selectable, Serialize, Deserialize,
+)]
 #[diesel(table_name = db_metadata)]
 pub struct DbMetadata {
-    name: String,
-    value: String,
+    singleton: bool,
+    time_created: DateTime<Utc>,
+    time_modified: DateTime<Utc>,
+    version: SemverVersion,
+    target_version: Option<SemverVersion>,
 }
 
 impl DbMetadata {
-    pub fn new(name: String, value: String) -> Self {
-        Self { name, value }
+    pub fn time_created(&self) -> &DateTime<Utc> {
+        &self.time_created
     }
 
-    pub fn name(&self) -> &str {
-        &self.name
+    pub fn time_modified(&self) -> &DateTime<Utc> {
+        &self.time_modified
     }
 
-    pub fn value(&self) -> &str {
-        &self.value
+    pub fn version(&self) -> &SemverVersion {
+        &self.version
     }
 }

nexus/db-model/src/db_metadata.rs

@@ -174,6 +174,8 @@ pub struct InstanceRuntimeState {
     // TODO-cleanup: Different type?
     #[diesel(column_name = hostname)]
     pub hostname: String,
+    #[diesel(column_name = boot_on_fault)]
+    pub boot_on_fault: bool,
 }
 
 impl From<InstanceRuntimeState>
@@ -225,6 +227,7 @@ impl From<internal::nexus::InstanceRuntimeState> for InstanceRuntimeState {
             hostname: state.hostname,
             gen: state.gen.into(),
             time_updated: state.time_updated,
+            boot_on_fault: false,
         }
     }
 }

nexus/db-model/src/instance.rs

@@ -356,6 +356,7 @@ table! {
         ncpus -> Int8,
         memory -> Int8,
         hostname -> Text,
+        boot_on_fault -> Bool,
     }
 }
 
@@ -1114,15 +1115,21 @@ table! {
 }
 
 table! {
-    db_metadata (name) {
-        name -> Text,
-        value -> Text,
+    db_metadata (singleton) {
+        singleton -> Bool,
+        time_created -> Timestamptz,
+        time_modified -> Timestamptz,
+        version -> Text,
+        target_version -> Nullable<Text>,
     }
 }
 
 /// The version of the database schema this particular version of Nexus was
 /// built against.
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(1, 0, 0);
+///
+/// This should be updated whenever the schema is changed. For more details,
+/// refer to: schema/crdb/README.adoc
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(2, 0, 0);
 
 allow_tables_to_appear_in_same_query!(
     system_update,