Proxy cockroach HTTP requests through the admin server (#8377)

- Start a CockroachDB admin server in the integration test environment
- Add endpoints to the admin server to proxy to the CockroachDB HTTP
server

This is an alternative to
#8351

Author: smklein

Cargo.lock

@@ -23,6 +23,7 @@ omicron-common.workspace = true
 omicron-uuid-kinds.workspace = true
 # See omicron-rpaths for more about the "pq-sys" dependency.
 pq-sys = "*"
+reqwest.workspace = true
 schemars.workspace = true
 slog.workspace = true
 slog-async.workspace = true

cockroach-admin/Cargo.toml

@@ -10,6 +10,7 @@ workspace = true
 [dependencies]
 cockroach-admin-types.workspace = true
 dropshot.workspace = true
+http.workspace = true
 omicron-common.workspace = true
 omicron-uuid-kinds.workspace = true
 omicron-workspace-hack.workspace = true

cockroach-admin/api/Cargo.toml

@@ -61,6 +61,29 @@ pub trait CockroachAdminApi {
         rqctx: RequestContext<Self::Context>,
         body: TypedBody<NodeId>,
     ) -> Result<HttpResponseOk<NodeDecommission>, HttpError>;
+
+    /// Proxy to CockroachDB's /_status/vars endpoint
+    //
+    // Dropshot isn't happy about the "_status" portion of the path; it fails
+    // the linter. Instead, I'm adding the prefix "proxy" to make it clear these
+    // are "intended-to-be-proxied" endpoints, rather than exact matches for the
+    // CRDB HTTP interface paths.
+    #[endpoint {
+        method = GET,
+        path = "/proxy/status/vars",
+    }]
+    async fn status_vars(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<String>, HttpError>;
+
+    /// Proxy to CockroachDB's /_status/nodes endpoint
+    #[endpoint {
+        method = GET,
+        path = "/proxy/status/nodes",
+    }]
+    async fn status_nodes(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<String>, HttpError>;
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]

cockroach-admin/api/src/lib.rs

@@ -29,6 +29,13 @@ enum Args {
         #[clap(long, action)]
         cockroach_address: SocketAddrV6,
 
+        /// Socket address for a running cockroach HTTP server instance
+        ///
+        /// Although this may be on an arbitrary address, it should point
+        /// to the same address as the "cockroach_address" server.
+        #[clap(long, action)]
+        cockroach_http_address: SocketAddr,
+
         /// Address on which this server should run
         #[clap(long, action)]
         http_address: SocketAddrV6,
@@ -57,12 +64,16 @@ async fn main_impl() -> Result<(), CmdError> {
         Args::Run {
             path_to_cockroach_binary,
             cockroach_address,
+            cockroach_http_address,
             http_address,
             config_file_path,
             zone_id,
         } => {
-            let cockroach_cli =
-                CockroachCli::new(path_to_cockroach_binary, cockroach_address);
+            let cockroach_cli = CockroachCli::new(
+                path_to_cockroach_binary,
+                cockroach_address,
+                cockroach_http_address,
+            );
             let mut config = Config::from_file(&config_file_path)
                 .map_err(|err| CmdError::Failure(anyhow!(err)))?;
             config.dropshot.bind_address = SocketAddr::V6(http_address);

cockroach-admin/src/bin/cockroach-admin.rs

@@ -12,7 +12,7 @@ use illumos_utils::output_to_exec_error;
 use slog_error_chain::InlineErrorChain;
 use slog_error_chain::SlogInlineError;
 use std::io;
-use std::net::SocketAddrV6;
+use std::net::{SocketAddr, SocketAddrV6};
 use std::process::Output;
 use tokio::process::Command;
 
@@ -93,20 +93,30 @@ impl From<CockroachCliError> for HttpError {
 pub struct CockroachCli {
     path_to_cockroach_binary: Utf8PathBuf,
     cockroach_address: SocketAddrV6,
+    cockroach_http_address: SocketAddr,
 }
 
 impl CockroachCli {
     pub fn new(
         path_to_cockroach_binary: Utf8PathBuf,
         cockroach_address: SocketAddrV6,
+        cockroach_http_address: SocketAddr,
     ) -> Self {
-        Self { path_to_cockroach_binary, cockroach_address }
+        Self {
+            path_to_cockroach_binary,
+            cockroach_address,
+            cockroach_http_address,
+        }
     }
 
     pub fn cockroach_address(&self) -> SocketAddrV6 {
         self.cockroach_address
     }
 
+    pub fn cockroach_http_address(&self) -> SocketAddr {
+        self.cockroach_http_address
+    }
+
     pub async fn cluster_init(&self) -> Result<(), CockroachCliError> {
         self.invoke_cli_raw(
             ["init"],
@@ -443,7 +453,11 @@ mod tests {
         )
         .parse()
         .expect("valid SocketAddrV6");
-        let cli = CockroachCli::new("cockroach".into(), cockroach_address);
+        let cli = CockroachCli::new(
+            "cockroach".into(),
+            cockroach_address,
+            SocketAddr::V6(cockroach_address),
+        );
         let status = cli.node_status().await.expect("got node status");
 
         // We can't check all the fields exactly, but some we know based on the
@@ -505,7 +519,11 @@ mod tests {
         )
         .parse()
         .expect("valid SocketAddrV6");
-        let cli = CockroachCli::new("cockroach".into(), cockroach_address);
+        let cli = CockroachCli::new(
+            "cockroach".into(),
+            cockroach_address,
+            SocketAddr::V6(cockroach_address),
+        );
         let result = cli
             .invoke_node_decommission("1")
             .await
@@ -652,7 +670,11 @@ mod tests {
         // Repeat the above test but using our wrapper.
         {
             let db = UninitializedCockroach::start().await;
-            let cli = CockroachCli::new("cockroach".into(), db.listen_addr);
+            let cli = CockroachCli::new(
+                "cockroach".into(),
+                db.listen_addr,
+                SocketAddr::V6(db.listen_addr),
+            );
 
             cli.cluster_init().await.expect("cluster initialized");
             cli.cluster_init().await.expect("cluster still initialized");
@@ -675,7 +697,11 @@ mod tests {
         .parse()
         .expect("valid SocketAddrV6");
 
-        let cli = CockroachCli::new("cockroach".into(), cockroach_address);
+        let cli = CockroachCli::new(
+            "cockroach".into(),
+            cockroach_address,
+            SocketAddr::V6(cockroach_address),
+        );
         cli.schema_init_impl(DBINIT_RELATIVE_PATH)
             .await
             .expect("initialized schema");
@@ -692,7 +718,11 @@ mod tests {
         let logctx =
             dev::test_setup_log("test_cluster_schema_init_interleaved");
         let db = UninitializedCockroach::start().await;
-        let cli = CockroachCli::new("cockroach".into(), db.listen_addr);
+        let cli = CockroachCli::new(
+            "cockroach".into(),
+            db.listen_addr,
+            SocketAddr::V6(db.listen_addr),
+        );
 
         // We should be able to initialize the cluster, then install the schema,
         // then do both of those things again.
@@ -778,6 +808,7 @@ mod tests {
         let cli = CockroachCli::new(
             "never-called".into(),
             "[::1]:0".parse().unwrap(),
+            SocketAddr::V6("[::1]:0".parse().unwrap()),
         );
 
         match cli.validate_node_decommissionable(

cockroach-admin/src/cockroach_cli.rs

@@ -164,7 +164,11 @@ mod tests {
         )
         .parse()
         .expect("valid SocketAddrV6");
-        let cli = CockroachCli::new("cockroach".into(), cockroach_address);
+        let cli = CockroachCli::new(
+            "cockroach".into(),
+            cockroach_address,
+            SocketAddr::V6(cockroach_address),
+        );
         let context = ServerContext::new(
             OmicronZoneUuid::new_v4(),
             cli,

cockroach-admin/src/context.rs

@@ -74,4 +74,88 @@ impl CockroachAdminApi for CockroachAdminImpl {
         );
         Ok(HttpResponseOk(decommission_status))
     }
+
+    async fn status_vars(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<String>, HttpError> {
+        let ctx = rqctx.context();
+        let cockroach_http_address =
+            ctx.cockroach_cli().cockroach_http_address();
+        let url = format!("http://{}/_status/vars", cockroach_http_address);
+
+        let client = reqwest::Client::new();
+        let response = client.get(&url).send().await.map_err(|e| {
+            HttpError::for_internal_error(format!(
+                "Failed to proxy to CockroachDB: {}",
+                e
+            ))
+        })?;
+
+        if !response.status().is_success() {
+            let status = response.status();
+            let body = response.text().await.unwrap_or_else(|_| {
+                "Failed to read error response".to_string()
+            });
+            let status_code = dropshot::ErrorStatusCode::from_status(status)
+                .unwrap_or(dropshot::ErrorStatusCode::INTERNAL_SERVER_ERROR);
+            return Err(HttpError {
+                status_code,
+                error_code: None,
+                external_message: body.clone(),
+                internal_message: body,
+                headers: None,
+            });
+        }
+
+        let body = response.text().await.map_err(|e| {
+            HttpError::for_internal_error(format!(
+                "Failed to read response body: {}",
+                e
+            ))
+        })?;
+
+        Ok(HttpResponseOk(body))
+    }
+
+    async fn status_nodes(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseOk<String>, HttpError> {
+        let ctx = rqctx.context();
+        let cockroach_http_address =
+            ctx.cockroach_cli().cockroach_http_address();
+        let url = format!("http://{}/_status/nodes", cockroach_http_address);
+
+        let client = reqwest::Client::new();
+        let response = client.get(&url).send().await.map_err(|e| {
+            HttpError::for_internal_error(format!(
+                "Failed to proxy to CockroachDB: {}",
+                e
+            ))
+        })?;
+
+        if !response.status().is_success() {
+            let status = response.status();
+            let body = response.text().await.unwrap_or_else(|_| {
+                "Failed to read error response".to_string()
+            });
+            let status_code = dropshot::ErrorStatusCode::from_status(status)
+                .unwrap_or(dropshot::ErrorStatusCode::INTERNAL_SERVER_ERROR);
+            return Err(HttpError {
+                status_code,
+                error_code: None,
+                external_message: body.clone(),
+                internal_message: body,
+                headers: None,
+            });
+        }
+
+        let body = response.text().await.map_err(|e| {
+            HttpError::for_internal_error(format!(
+                "Failed to read response body: {}",
+                e
+            ))
+        })?;
+
+        Ok(HttpResponseOk(body))
+    }
 }

cockroach-admin/src/http_entrypoints.rs

@@ -34,6 +34,7 @@ nexus-db-queries = { workspace = true, features = [ "testing" ] }
 nexus-sled-agent-shared.workspace = true
 nexus-test-interface.workspace = true
 nexus-types.workspace = true
+omicron-cockroach-admin.workspace = true
 omicron-common.workspace = true
 omicron-passwords.workspace = true
 omicron-sled-agent.workspace = true

nexus/test-utils/Cargo.toml

@@ -166,6 +166,7 @@ pub struct ControlPlaneTestContext<N> {
     pub internal_client: ClientTestContext,
     pub server: N,
     pub database: dev::db::CockroachInstance,
+    pub database_admin: omicron_cockroach_admin::Server,
     pub clickhouse: dev::clickhouse::ClickHouseDeployment,
     pub logctx: LogContext,
     pub sled_agents: Vec<ControlPlaneTestContextSledAgent>,
@@ -377,6 +378,7 @@ pub struct ControlPlaneTestContextBuilder<'a, N: NexusServer> {
 
     pub server: Option<N>,
     pub database: Option<dev::db::CockroachInstance>,
+    pub database_admin: Option<omicron_cockroach_admin::Server>,
     pub clickhouse: Option<dev::clickhouse::ClickHouseDeployment>,
     pub sled_agents: Vec<ControlPlaneTestContextSledAgent>,
     pub oximeter: Option<Oximeter>,
@@ -429,6 +431,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             internal_client: None,
             server: None,
             database: None,
+            database_admin: None,
             clickhouse: None,
             sled_agents: vec![],
             oximeter: None,
@@ -541,7 +544,28 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             ),
             image_source: BlueprintZoneImageSource::InstallDataset,
         });
+        let http_address = database.http_addr();
         self.database = Some(database);
+
+        let cli = omicron_cockroach_admin::CockroachCli::new(
+            omicron_test_utils::dev::db::COCKROACHDB_BIN.into(),
+            address,
+            http_address,
+        );
+        let server = omicron_cockroach_admin::start_server(
+            zone_id,
+            cli,
+            omicron_cockroach_admin::Config {
+                dropshot: dropshot::ConfigDropshot::default(),
+                log: ConfigLogging::StderrTerminal {
+                    level: ConfigLoggingLevel::Error,
+                },
+            },
+        )
+        .await
+        .expect("Failed to start CRDB admin server");
+
+        self.database_admin = Some(server);
     }
 
     // Start ClickHouse database server.
@@ -1409,6 +1433,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             techport_client: self.techport_client.unwrap(),
             internal_client: self.internal_client.unwrap(),
             database: self.database.unwrap(),
+            database_admin: self.database_admin.unwrap(),
             clickhouse: self.clickhouse.unwrap(),
             sled_agents: self.sled_agents,
             oximeter: self.oximeter.unwrap(),