Zero touch switch config for dev environments (#3576)

Author: rcgoodfellow

.github/buildomat/jobs/deploy.sh

@@ -39,9 +39,9 @@ _exit_trap() {
 	pfexec netstat -rncva
 	pfexec netstat -anu
 	pfexec arp -an
-	pfexec ./out/softnpu/scadm \
-		--server /opt/oxide/softnpu/stuff/server \
-		--client /opt/oxide/softnpu/stuff/client \
+	pfexec zlogin sidecar_softnpu /softnpu/scadm \
+		--server /softnpu/server \
+		--client /softnpu/client \
 		standalone \
 		dump-state
 	pfexec /opt/oxide/opte/bin/opteadm list-ports
@@ -73,7 +73,7 @@ _exit_trap() {
 		pfexec zlogin "$z" arp -an
 	done
 
-	pfexec zlogin softnpu cat /softnpu.log
+	pfexec zlogin sidecar_softnpu cat /var/log/softnpu.log
 
 	exit $status
 }
@@ -135,6 +135,20 @@ for p in /input/ci-tools/work/end-to-end-tests/*.gz; do
 	chmod a+x "tests/$(basename "${p%.gz}")"
 done
 
+# Lab gateway ip
+export GATEWAY_IP=192.168.1.199
+# Proxy arp settings.
+export PXA_START="192.168.1.50"
+export PXA_END="192.168.1.90"
+
+# Nexus (and any instances using the above IP pool) are configured to use external
+# IPs from a fixed subnet (192.168.1.0/24). OPTE/SoftNPU/Boundary Services take care
+# of NATing between the private VPC networks and this "external network".
+# We create a static IP in this subnet in the global zone and configure the switch
+# to use it as the default gateway.
+# NOTE: Keep in sync with $[SERVICE_]IP_POOL_{START,END}
+pfexec ipadm create-addr -T static -a $GATEWAY_IP/24 igb0/sidehatch
+
 pfexec zpool create -f scratch c1t1d0 c2t1d0
 ZPOOL_VDEV_DIR=/scratch ptime -m pfexec ./tools/create_virtual_hardware.sh
 
@@ -186,7 +200,6 @@ rmdir pkg
 # will end up once installed.
 E2E_TLS_CERT="/opt/oxide/sled-agent/pkg/initial-tls-cert.pem"
 
-
 #
 # Image-related tests use images served by catacomb. The lab network is
 # IPv4-only; the propolis zones are IPv6-only. These steps set up tcpproxy
@@ -222,46 +235,23 @@ do
 	retry=$((retry + 1))
 done
 
-# Nexus (and any instances using the above IP pool) are configured to use external
-# IPs from a fixed subnet (192.168.1.0/24). OPTE/SoftNPU/Boundary Services take care
-# of NATing between the private VPC networks and this "external network".
-# We create a static IP in this subnet in the global zone and configure the switch
-# to use it as the default gateway.
-# NOTE: Keep in sync with $[SERVICE_]IP_POOL_{START,END}
-export GATEWAY_IP=192.168.1.199
-pfexec ipadm create-addr -T static -a $GATEWAY_IP/24 igb0/sidehatch
-
-# NOTE: this script configures softnpu's "rack network" settings using swadm
-./tools/scrimlet/softnpu-init.sh
-
-# NOTE: this command configures proxy arp for softnpu. This is needed if you want to be
-# able to reach instances from the same L2 network segment.
-# Keep consistent with `get_system_ip_pool` in `end-to-end-tests`.
-IP_POOL_START="192.168.1.50"
-IP_POOL_END="192.168.1.90"
-# `dladm` won't return leading zeroes but `scadm` expects them, use sed to add any missing zeroes
-SOFTNPU_MAC=$(dladm show-vnic sc0_1 -p -o macaddress | sed -E 's/[ :]/&0/g; s/0([^:]{2}(:|$))/\1/g')
-pfexec ./out/softnpu/scadm \
-	--server /opt/oxide/softnpu/stuff/server \
-	--client /opt/oxide/softnpu/stuff/client \
-	standalone \
-	add-proxy-arp $IP_POOL_START $IP_POOL_END $SOFTNPU_MAC
 
 # We also need to configure proxy arp for any services which use OPTE for external connectivity (e.g. Nexus)
 tar xf out/omicron-sled-agent.tar pkg/config-rss.toml
+SOFTNPU_MAC=$(dladm show-vnic sc0_1 -p -o macaddress | sed -E 's/[ :]/&0/g; s/0([^:]{2}(:|$))/\1/g')
 SERVICE_IP_POOL_START="$(sed -n 's/^first = "\(.*\)"/\1/p' pkg/config-rss.toml)"
 SERVICE_IP_POOL_END="$(sed -n 's/^last = "\(.*\)"/\1/p' pkg/config-rss.toml)"
 rm -r pkg
 
-pfexec ./out/softnpu/scadm \
-	--server /opt/oxide/softnpu/stuff/server \
-	--client /opt/oxide/softnpu/stuff/client \
+pfexec zlogin sidecar_softnpu /softnpu/scadm \
+	--server /softnpu/server \
+	--client /softnpu/client \
 	standalone \
 	add-proxy-arp $SERVICE_IP_POOL_START $SERVICE_IP_POOL_END $SOFTNPU_MAC
 
-pfexec ./out/softnpu/scadm \
-	--server /opt/oxide/softnpu/stuff/server \
-	--client /opt/oxide/softnpu/stuff/client \
+pfexec zlogin sidecar_softnpu /softnpu/scadm \
+	--server /softnpu/server \
+	--client /softnpu/client \
 	standalone \
 	dump-state
 

.github/buildomat/jobs/package.sh

@@ -55,11 +55,12 @@ ptime -m cargo run --locked --release --bin omicron-package -- \
 files=(
 	out/*.tar
 	out/target/test
-	out/softnpu/*
+	out/npuzone/*
 	package-manifest.toml
 	smf/sled-agent/non-gimlet/config.toml
 	target/release/omicron-package
 	tools/create_virtual_hardware.sh
+    tools/virtual_hardware.sh
 	tools/scrimlet/*
 )
 

docs/how-to-run.adoc

@@ -176,13 +176,19 @@ The Sled Agent supports operation on both:
 
 This script also sets up a "softnpu" zone to implement Boundary Services.  SoftNPU simulates the Tofino device that's used in real systems.  Just like Tofino, it can implement sled-to-sled networking, but that's beyond the scope of this doc.
 
-If you're running on a PC and using an existing network as your external network, you can usually just run:
+If you're running on a PC and using an existing network as your external network, you can usually just run this script with a few environment vaiables set. These environment varaibles tell SoftNPU about your local network. You'll need to carve out part of your network for the Oxide platform, making sure that the range you specify below is not occupied by other hosts on your network.
 
+[source,bash]
 ----
+export PHYSICAL_LINK=igb0           # The physical link for your local network.
+export GATEWAY_IP=192.168.1.199     # The gateway IP address for your local network.
+export PXA_START=192.168.1.2        # The first IP address your Oxide cluster can use.
+export PXA_END=192.168.1.100        # The last IP address your Oxide cluster can use.
+
 $ pfexec ./tools/create_virtual_hardware.sh
 ----
 
-If above you made up a new "external" network only accessible within the Sled, then you'll want to override PHYSICAL_LINK:
+If above you made up a new "external" network only accessible within the Sled, then you'll want to override PHYSICAL_LINK as follows:
 
 ----
 $ PHYSICAL_LINK=fake_external_stub0 pfexec ./tools/create_virtual_hardware.sh
@@ -350,46 +356,6 @@ $ zoneadm list -cnv
 $ pfexec tail -f $(pfexec svcs -z oxz_nexus_<UUID> -L nexus)
 ----
 
-=== SoftNPU Configuration
-
-After installing omicron with `omicron-package install`, you can run the `softnpu-init.sh` script to configure SoftNPU.  If your external network is the one that your global zone is already on (i.e., an existing network), then you can likely just run:
-
-[source,console]
-----
-$ ./tools/scrimlet/softnpu-init.sh
-----
-
-In this case, `softnpu-init.sh` determines a network gateway by looking at your system's default gateway.  Again, the assumption for this use case is that your rack gets external connectivity the same way your system does because it's part of the same network.
-
-If your external network is one you made up above that's local to your own system, this won't be right.  In that case, you'll want to override the gateway IP to be the address of the global zone on your made-up network.  In our example:
-
-[source,console]
-----
-$ GATEWAY_IP=192.168.1.199 ./tools/scrimlet/softnpu-init.sh
-----
-
-In other configurations, you might use a different GATEWAY_IP.  You can also override GATEWAY_MAC if needed, but that shouldn't be necessary for the configurations described here.
-
-=== SoftNPU Proxy ARP Setup
-
-Services that require external connectivity (e.g. Nexus, Boundary NTP, External DNS) do so via OPTE using addresses from the services IP pool.  When using SoftNPU, we'll need to configure Proxy ARP for those addresses.
-
-In this snippet, `$SERVICE_IP_POOL_START` and `$SERVICE_IP_POOL_END` are the addresses you put into `config-rss.toml` above for `internal_services_ip_pool_ranges`:
-
-[source,console]
-----
-# dladm won't return leading zeroes but `scadm` expects them
-$ SOFTNPU_MAC=$(dladm show-vnic sc0_1 -p -o macaddress | gsed 's/\b\(\w\)\b/0\1/g')
-$ pfexec /opt/oxide/softnpu/stuff/scadm \
-  --server /opt/oxide/softnpu/stuff/server \
-  --client /opt/oxide/softnpu/stuff/client \
-  standalone \
-  add-proxy-arp \
-  $SERVICE_IP_POOL_START \
-  $SERVICE_IP_POOL_END \
-  $SOFTNPU_MAC
-----
-
 == Using Omicron
 
 At this point, the system should be up and running!  You should be able to reach the external API and web console from your external network.  But how?  The URL for the API and console will be:
@@ -440,9 +406,9 @@ With SoftNPU you will generally also need to configure Proxy ARP.  Below, `IP_PO
 ----
 # dladm won't return leading zeroes but `scadm` expects them
 $ SOFTNPU_MAC=$(dladm show-vnic sc0_1 -p -o macaddress | gsed 's/\b\(\w\)\b/0\1/g')
-$ pfexec /opt/oxide/softnpu/stuff/scadm \
-  --server /opt/oxide/softnpu/stuff/server \
-  --client /opt/oxide/softnpu/stuff/client \
+$ pfexec zlogin sidecar_softnpu /softnpu/scadm \
+  --server /softnpu/server \
+  --client /softnpu/client \
   standalone \
   add-proxy-arp \
   $IP_POOL_START \

illumos-utils/src/dladm.rs

@@ -86,6 +86,14 @@ pub struct GetVnicError {
     err: ExecutionError,
 }
 
+/// Errors returned from [`Dladm::get_simulated_tfports`].
+#[derive(thiserror::Error, Debug)]
+#[error("Failed to get simnets: {err}")]
+pub struct GetSimnetError {
+    #[source]
+    err: ExecutionError,
+}
+
 /// Errors returned from [`Dladm::delete_vnic`].
 #[derive(thiserror::Error, Debug)]
 #[error("Failed to delete vnic {name}: {err}")]
@@ -416,6 +424,25 @@ impl Dladm {
         Ok(vnics)
     }
 
+    /// Returns simnet links masquerading as tfport devices
+    pub fn get_simulated_tfports() -> Result<Vec<String>, GetSimnetError> {
+        let mut command = std::process::Command::new(PFEXEC);
+        let cmd = command.args(&[DLADM, "show-simnet", "-p", "-o", "LINK"]);
+        let output = execute(cmd).map_err(|err| GetSimnetError { err })?;
+
+        let tfports = String::from_utf8_lossy(&output.stdout)
+            .lines()
+            .filter_map(|name| {
+                if name.starts_with("tfport") {
+                    Some(name.to_owned())
+                } else {
+                    None
+                }
+            })
+            .collect();
+        Ok(tfports)
+    }
+
     /// Remove a vnic from the sled.
     pub fn delete_vnic(name: &str) -> Result<(), DeleteVnicError> {
         let mut command = std::process::Command::new(PFEXEC);

illumos-utils/src/running_zone.rs

@@ -1104,6 +1104,7 @@ impl InstalledZone {
         unique_name: Option<Uuid>,
         datasets: &[zone::Dataset],
         filesystems: &[zone::Fs],
+        data_links: &[String],
         devices: &[zone::Device],
         opte_ports: Vec<(Port, PortTicket)>,
         bootstrap_vnic: Option<Link>,
@@ -1140,14 +1141,21 @@ impl InstalledZone {
                     .collect(),
             })?;
 
-        let net_device_names: Vec<String> = opte_ports
+        let mut net_device_names: Vec<String> = opte_ports
             .iter()
             .map(|(port, _)| port.vnic_name().to_string())
             .chain(std::iter::once(control_vnic.name().to_string()))
             .chain(bootstrap_vnic.as_ref().map(|vnic| vnic.name().to_string()))
             .chain(links.iter().map(|nic| nic.name().to_string()))
+            .chain(data_links.iter().map(|x| x.to_string()))
             .collect();
 
+        // There are many sources for device names. In some cases they can
+        // overlap, depending on the contents of user defined config files. This
+        // can cause zones to fail to start if duplicate data links are given.
+        net_device_names.sort();
+        net_device_names.dedup();
+
         Zones::install_omicron_zone(
             log,
             &zone_root_path,

installinator/src/bootstrap.rs

@@ -26,9 +26,12 @@ const MG_DDM_MANIFEST_PATH: &str = "/opt/oxide/mg-ddm/pkg/ddm/manifest.xml";
 // TODO-cleanup The implementation of this function is heavily derived from
 // `sled_agent::bootstrap::server::Server::start()`; consider whether we could
 // find a way for them to share it.
-pub(crate) async fn bootstrap_sled(log: Logger) -> Result<()> {
+pub(crate) async fn bootstrap_sled(
+    data_links: &[String; 2],
+    log: Logger,
+) -> Result<()> {
     // Find address objects to pass to maghemite.
-    let links = underlay::find_chelsio_links()
+    let links = underlay::find_chelsio_links(data_links)
         .context("failed to find chelsio links")?;
     ensure!(
         !links.is_empty(),

installinator/src/dispatch.rs

@@ -151,6 +151,18 @@ struct InstallOpts {
     #[clap(long)]
     install_on_gimlet: bool,
 
+    //TODO(ry) this probably needs to get plumbed somewhere instead of relying
+    //on a default.
+    /// The first gimlet data link to use.
+    #[clap(long, default_value = "cxgbe0")]
+    data_link0: String,
+
+    //TODO(ry) this probably needs to get plumbed somewhere instead of relying
+    //on a default.
+    /// The second gimlet data link to use.
+    #[clap(long, default_value = "cxgbe1")]
+    data_link1: String,
+
     // TODO: checksum?
 
     // The destination to write to.
@@ -164,7 +176,8 @@ struct InstallOpts {
 impl InstallOpts {
     async fn exec(self, log: &slog::Logger) -> Result<()> {
         if self.bootstrap_sled {
-            crate::bootstrap::bootstrap_sled(log.clone()).await?;
+            let data_links = [self.data_link0.clone(), self.data_link1.clone()];
+            crate::bootstrap::bootstrap_sled(&data_links, log.clone()).await?;
         }
 
         let image_id = self.artifact_ids.resolve()?;

internal-dns/src/resolver.rs

@@ -76,6 +76,7 @@ impl Resolver {
         // The underlay is IPv6 only, so this helps avoid needless lookups of
         // the IPv4 variant.
         opts.ip_strategy = LookupIpStrategy::Ipv6Only;
+        opts.negative_max_ttl = Some(std::time::Duration::from_secs(15));
         let resolver = TokioAsyncResolver::tokio(rc, opts)?;
 
         Ok(Self { log, resolver })

sled-agent/src/bootstrap/server.rs

@@ -55,9 +55,10 @@ impl Server {
         }
 
         // Find address objects to pass to maghemite.
-        let mg_addr_objs = underlay::find_nics().map_err(|err| {
-            format!("Failed to find address objects for maghemite: {err}")
-        })?;
+        let mg_addr_objs = underlay::find_nics(&sled_config.data_links)
+            .map_err(|err| {
+                format!("Failed to find address objects for maghemite: {err}")
+            })?;
         if mg_addr_objs.is_empty() {
             return Err(
                 "underlay::find_nics() returned 0 address objects".to_string()

sled-agent/src/config.rs

@@ -71,6 +71,10 @@ pub struct Config {
     /// systems.
     pub data_link: Option<PhysicalLink>,
 
+    /// The data links that sled-agent will treat as a real gimlet cxgbe0/cxgbe1
+    /// links.
+    pub data_links: [String; 2],
+
     #[serde(default)]
     pub updates: ConfigUpdates,