deploy 5 CockroachDB nodes from RSS (#3450)

Author: davepacheco

common/src/sql/dbinit.sql

@@ -34,6 +34,12 @@ CREATE DATABASE omicron;
 CREATE USER omicron;
 ALTER DEFAULT PRIVILEGES GRANT INSERT, SELECT, UPDATE, DELETE ON TABLES to omicron;
 
+/*
+ * Configure a replication factor of 5 to ensure that the system can maintain
+ * availability in the face of any two node failures.
+ */
+ALTER RANGE default CONFIGURE ZONE USING num_replicas = 5;
+
 /*
  * Racks
  */

end-to-end-tests/src/helpers/ctx.rs

@@ -127,7 +127,7 @@ pub async fn nexus_addr() -> Result<IpAddr> {
         &resolver,
         &dns_name,
         Duration::from_secs(1),
-        Duration::from_secs(300),
+        Duration::from_secs(600),
     )
     .await
 }
@@ -240,7 +240,7 @@ pub async fn build_client() -> Result<oxide_client::Client> {
             })
         },
         &Duration::from_secs(1),
-        &Duration::from_secs(300),
+        &Duration::from_secs(600),
     )
     .await
     .context("logging in")?;

internal-dns-cli/src/bin/dnswait.rs

@@ -90,8 +90,8 @@ async fn main() -> Result<()> {
     .await
     .context("unexpectedly gave up")?;
 
-    for ip in result {
-        println!("{}", ip)
+    for (target, port) in result {
+        println!("{}:{}", target, port)
     }
 
     Ok(())

internal-dns/src/resolver.rs

@@ -137,7 +137,7 @@ impl Resolver {
     pub async fn lookup_srv(
         &self,
         srv: crate::ServiceName,
-    ) -> Result<Vec<String>, ResolveError> {
+    ) -> Result<Vec<(String, u16)>, ResolveError> {
         let name = format!("{}.{}", srv.dns_name(), DNS_ZONE);
         trace!(self.log, "lookup_srv"; "dns_name" => &name);
         let response = self.inner.srv_lookup(&name).await?;
@@ -148,7 +148,10 @@ impl Resolver {
             "response" => ?response
         );
 
-        Ok(response.into_iter().map(|srv| srv.target().to_string()).collect())
+        Ok(response
+            .into_iter()
+            .map(|srv| (srv.target().to_string(), srv.port()))
+            .collect())
     }
 
     pub async fn lookup_all_ipv6(

sled-agent/src/rack_setup/plan/service.rs

@@ -47,7 +47,7 @@ const BOUNDARY_NTP_COUNT: usize = 2;
 const NEXUS_COUNT: usize = 1;
 
 // The number of CRDB instances to create from RSS.
-const CRDB_COUNT: usize = 1;
+const CRDB_COUNT: usize = 5;
 
 // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove
 // when Nexus provisions Oximeter.

smf/sled-agent/non-gimlet/config.toml

@@ -18,18 +18,23 @@ sidecar_revision.soft = { front_port_count = 1, rear_port_count = 1 }
 # in-sync, rather than querying its NTP zone.
 skip_timesync = true
 
-# A file-backed zpool can be manually created with the following:
+# For testing purposes, A file-backed zpool can be manually created with the
+# following:
+#
 # # truncate -s 10GB testpool.vdev
 # # zpool create oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b "$PWD/testpool.vdev"
 #
 # Note that you'll need to create one such zpool for each below, with a
-# different vdev for each.
+# different vdev for each.  The `create_virtual_hardware.sh` script does this
+# for you.
 zpools = [
   "oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b",
   "oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b",
   "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b",
   "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
   "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
+  "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03",
+  "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03",
 ]
 
 # Percentage of usable physical DRAM to use for the VMM reservoir, which

tools/create_virtual_hardware.sh

@@ -47,7 +47,7 @@ function ensure_zpools {
             echo "Zpool: [$ZPOOL]"
             VDEV_PATH="$OMICRON_TOP/$ZPOOL.vdev"
             if ! [[ -f "$VDEV_PATH" ]]; then
-                dd if=/dev/zero of="$VDEV_PATH" bs=1 count=0 seek=10G
+                dd if=/dev/zero of="$VDEV_PATH" bs=1 count=0 seek=6G
             fi
             success "ZFS vdev $VDEV_PATH exists"
             if [[ -z "$(zpool list -o name | grep $ZPOOL)" ]]; then