[feature] ASIC-focused multicast replication and dendrite API

.github/buildomat/jobs/image.sh

@@ -101,11 +101,11 @@ pfexec chown "$UID" /out
 
 banner "P4 Codegen"
 # Add gcc-12 so the p4 compiler can find cpp
-# The tofino2 has 20 stages, but the current sidecar.p4 will fit into 14.  We
+# The tofino2 has 20 stages, but the current sidecar.p4 will fit into 19.  We
 # add the "--stages 14" here to detect if/when the program grows beyond that
 # limit.  It's not necessarily a problem if we grow, but given the limited space
 # on the ASIC, we want to grow deliberatately and thoughtfully.
-PATH=/opt/gcc-12/bin:$PATH cargo xtask codegen --stages 14
+PATH=/opt/gcc-12/bin:$PATH cargo xtask codegen --stages 19
 
 # Preserve all the diagnostics spit out by the compiler
 mkdir -p /out/p4c-diags

Cargo.toml

@@ -39,7 +39,7 @@ omicron-common = { git = "https://github.com/oxidecomputer/omicron", branch= "ma
 oximeter = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 oximeter-producer = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 oximeter-instruments = { git = "https://github.com/oxidecomputer/omicron", branch = "main", default-features = false, features = ["kstat"] }
-oxnet = { version = "0.1.0", default-features = false, features = ["schemars", "serde"] }
+oxnet = { version = "0.1.1", default-features = false, features = ["schemars", "serde"] }
 propolis = { git = "https://github.com/oxidecomputer/propolis" }
 sled-agent-client = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 smf = { git = "https://github.com/illumos/smf-rs" }

aal/src/lib.rs

@@ -200,6 +200,21 @@ pub trait AsicOps {
     /// Destroy a multicast group.
     fn mc_group_destroy(&self, group_id: u16) -> AsicResult<()>;
 
+    /// Check if a multicast group exists.
+    fn mc_group_exists(&self, group_id: u16) -> bool {
+        self.mc_domains().contains(&group_id)
+    }
+
+    /// Get the total number of multicast groups.
+    fn mc_groups_count(&self) -> AsicResult<usize>;
+
+    /// Set the maximum number of multicast nodes.
+    fn mc_set_max_nodes(
+        &self,
+        max_nodes: u32,
+        max_link_aggregated_nodes: u32,
+    ) -> AsicResult<()>;
+
     /// Get sidecar identifiers of the device being managed.
     fn get_sidecar_identifiers(&self) -> AsicResult<impl SidecarIdentifiers>;
 

aal/src/match_action.rs

@@ -72,7 +72,7 @@ impl MatchData {
 /// The MatchParse trait defines the behavior needed to convert a high-level
 /// Match field into our intermediate representation.
 pub trait MatchParse {
-    /// Return all the name sand values of the key fields as strings
+    /// Return all the names and values of the key fields as strings
     fn key_values(&self) -> BTreeMap<String, String>;
     /// Convert the key Struct to a MatchData struct
     fn key_to_ir(&self) -> AsicResult<MatchData>;

asic/src/chaos/mod.rs

@@ -94,7 +94,9 @@ impl TableChaos {
             (table::SWITCH_IPV4_ADDR, v),
             (table::SWITCH_IPV6_ADDR, v),
             (table::NAT_INGRESS_IPV4, v),
-            (table::NAT_INGRESS_IPV6, v)
+            (table::NAT_INGRESS_IPV6, v),
+            (table::MCAST_IPV4_TABLE_NAME, v),
+            (table::MCAST_IPV6_TABLE_NAME, v)
         )
     }
 
@@ -141,6 +143,8 @@ pub struct AsicConfig {
     pub mc_port_remove: Chaos,
     pub mc_group_create: Chaos,
     pub mc_group_destroy: Chaos,
+    pub mc_groups_count: Chaos,
+    pub mc_set_max_nodes: Chaos,
     pub get_sidecar_identifiers: Chaos,
     pub table_new: TableChaos,
     pub table_clear: TableChaos,
@@ -177,6 +181,8 @@ impl AsicConfig {
             mc_port_remove: Chaos::new(v),
             mc_group_create: Chaos::new(v),
             mc_group_destroy: Chaos::new(v),
+            mc_groups_count: Chaos::new(v),
+            mc_set_max_nodes: Chaos::new(v),
             get_sidecar_identifiers: Chaos::new(v),
             table_new: TableChaos::uniform(v),
             table_clear: TableChaos::uniform(v),
@@ -203,6 +209,7 @@ impl AsicConfig {
             port_enable_get: Chaos::new(v),
             connector_avail_channels: Chaos::new(v),
             mc_port_count: Chaos::new(v),
+            mc_groups_count: Chaos::new(v),
             get_sidecar_identifiers: Chaos::new(v),
             ..Default::default()
         }
@@ -224,6 +231,7 @@ impl AsicConfig {
             mc_port_remove: Chaos::new(v),
             mc_group_create: Chaos::new(v),
             mc_group_destroy: Chaos::new(v),
+            mc_set_max_nodes: Chaos::new(v),
             // TODO this can cause dpd to fail to start
             //table_clear: TableChaos::uniform(v),
             table_default_set: TableChaos::uniform(v),
@@ -496,6 +504,20 @@ impl AsicOps for Handle {
         Ok(())
     }
 
+    fn mc_groups_count(&self) -> AsicResult<usize> {
+        unfurl!(self, mc_groups_count);
+        Ok(self.ports.lock().unwrap().len())
+    }
+
+    fn mc_set_max_nodes(
+        &self,
+        _max_nodes: u32,
+        _max_link_aggregated_nodes: u32,
+    ) -> AsicResult<()> {
+        unfurl!(self, mc_set_max_nodes);
+        Ok(())
+    }
+
     fn get_sidecar_identifiers(&self) -> AsicResult<impl SidecarIdentifiers> {
         unfurl!(self, get_sidecar_identifiers);
         Ok(Identifiers::default())

asic/src/chaos/table.rs

@@ -25,6 +25,10 @@ pub const SWITCH_IPV4_ADDR: &str = "pipe.Ingress.filter.switch_ipv4_addr";
 pub const SWITCH_IPV6_ADDR: &str = "pipe.Ingress.filter.switch_ipv6_addr";
 pub const NAT_INGRESS_IPV4: &str = "pipe.Ingress.nat_ingress.ingress_ipv4";
 pub const NAT_INGRESS_IPV6: &str = "pipe.Ingress.nat_ingress.ingress_ipv6";
+pub const MCAST_IPV4_TABLE_NAME: &str =
+    "pipe.Ingress.multicast.mcast_route_ipv4";
+pub const MCAST_IPV6_TABLE_NAME: &str =
+    "pipe.Ingress.multicast.mcast_route_ipv6";
 
 pub struct Table {
     name: String,

asic/src/softnpu/mod.rs

@@ -365,6 +365,18 @@ impl AsicOps for Handle {
         Ok(())
     }
 
+    fn mc_groups_count(&self) -> AsicResult<usize> {
+        Ok(self.ports.lock().unwrap().len())
+    }
+
+    fn mc_set_max_nodes(
+        &self,
+        _max_nodes: u32,
+        _max_link_aggregated_nodes: u32,
+    ) -> AsicResult<()> {
+        Ok(())
+    }
+
     fn get_sidecar_identifiers(&self) -> AsicResult<impl SidecarIdentifiers> {
         Ok(Identifiers {
             id: Uuid::new_v4(),

asic/src/tofino_asic/imported_bf_functions

@@ -22,11 +22,13 @@ bf_mc_create_session
 bf_mc_destroy_session
 bf_mc_mgrp_create
 bf_mc_mgrp_destroy
+bf_mc_mgrp_get_count
 bf_mc_node_create
 bf_mc_node_destroy
 bf_mc_node_update
 bf_mc_associate_node
 bf_mc_dissociate_node
+bf_mc_set_max_node_threshold
 
 # bf_rt calls
 bf_rt_table_from_name_get

asic/src/tofino_asic/multicast.rs → asic/src/tofino_asic/mcast.rs

@@ -80,6 +80,19 @@ fn mgrp_destroy(
     Ok(())
 }
 
+fn mgrp_get_count(
+    mcast_hdl: &Handle,
+    dev_id: bf_dev_id_t,
+    mut count: u32,
+) -> AsicResult<usize> {
+    unsafe {
+        bf_mc_mgrp_get_count(mcast_hdl.bf_get().mcast_hdl, dev_id, &mut count)
+            .check_error("getting total count of multicast groups")?;
+    }
+
+    Ok(count as usize)
+}
+
 fn associate_node(
     mcast_hdl: bf_mc_session_hdl_t,
     dev_id: bf_dev_id_t,
@@ -166,6 +179,26 @@ fn cleanup_node(
     node_destroy(bf.mcast_hdl, bf.dev_id, port_state.node_hdl)
 }
 
+fn set_max_node_threshold(
+    mcast_hdl: bf_mc_session_hdl_t,
+    dev_id: bf_dev_id_t,
+    node_count: i32,
+    node_port_lag_count: i32,
+) -> AsicResult<()> {
+    unsafe {
+        bf_mc_set_max_node_threshold(
+            mcast_hdl,
+            dev_id,
+            node_count,
+            node_port_lag_count,
+        )
+        .check_error("setting max node threshold")?;
+    }
+
+    Ok(())
+}
+
+/// All multicast domains.
 pub fn domains(hdl: &Handle) -> Vec<u16> {
     let mut list = Vec::new();
     let domains = hdl.domains.lock().unwrap();
@@ -190,6 +223,7 @@ fn domain_ports(domain: &DomainState) -> Vec<u16> {
     list
 }
 
+/// Get the number of ports in a multicast domain.
 pub fn domain_port_count(hdl: &Handle, group_id: u16) -> AsicResult<usize> {
     let mut domains = hdl.domains.lock().unwrap();
     match domains.get_mut(&group_id) {
@@ -198,6 +232,7 @@ pub fn domain_port_count(hdl: &Handle, group_id: u16) -> AsicResult<usize> {
     }
 }
 
+/// Add a port to a multicast domain.
 pub fn domain_add_port(
     hdl: &Handle,
     group_id: u16,
@@ -259,6 +294,7 @@ pub fn domain_add_port(
     }
 }
 
+/// Remove a port from a multicast domain.
 pub fn domain_remove_port(
     hdl: &Handle,
     group_id: u16,
@@ -287,6 +323,7 @@ pub fn domain_remove_port(
     Ok(())
 }
 
+/// Create a multicast domain.
 pub fn domain_create(hdl: &Handle, group_id: u16) -> AsicResult<()> {
     info!(hdl.log, "creating multicast domain {}", group_id);
     let mut domains = hdl.domains.lock().unwrap();
@@ -308,6 +345,7 @@ pub fn domain_create(hdl: &Handle, group_id: u16) -> AsicResult<()> {
     Ok(())
 }
 
+/// Destroy a multicast domain.
 pub fn domain_destroy(hdl: &Handle, group_id: u16) -> AsicResult<()> {
     info!(hdl.log, "destroying multicast domain {}", group_id);
     let mut domains = hdl.domains.lock().unwrap();
@@ -333,3 +371,30 @@ pub fn domain_destroy(hdl: &Handle, group_id: u16) -> AsicResult<()> {
 
     mgrp_destroy(bf.mcast_hdl, bf.dev_id, domain.mgrp_hdl)
 }
+
+/// Domain exists.
+pub fn domain_exists(hdl: &Handle, group_id: u16) -> bool {
+    let domains = hdl.domains.lock().unwrap();
+    domains.contains_key(&group_id)
+}
+
+/// Get the total number of multicast domains.
+pub fn domains_count(hdl: &Handle) -> AsicResult<usize> {
+    let bf = hdl.bf_get();
+    mgrp_get_count(hdl, bf.dev_id, 0)
+}
+
+/// Set the maximum number of multicast nodes.
+pub fn set_max_nodes(
+    hdl: &Handle,
+    node_count: u32,
+    node_port_lag_count: u32,
+) -> AsicResult<()> {
+    let bf = hdl.bf_get();
+    set_max_node_threshold(
+        bf.mcast_hdl,
+        bf.dev_id,
+        node_count as i32,
+        node_port_lag_count as i32,
+    )
+}

asic/src/tofino_asic/mod.rs

@@ -19,7 +19,7 @@ mod bf_wrapper;
 mod genpd;
 
 mod link_fsm;
-pub mod multicast;
+pub mod mcast;
 pub mod ports;
 pub mod qsfp;
 mod sde_log;
@@ -150,27 +150,43 @@ impl AsicOps for Handle {
     }
 
     fn mc_domains(&self) -> Vec<u16> {
-        multicast::domains(self)
+        mcast::domains(self)
     }
 
     fn mc_port_count(&self, group_id: u16) -> AsicResult<usize> {
-        multicast::domain_port_count(self, group_id)
+        mcast::domain_port_count(self, group_id)
     }
 
     fn mc_port_add(&self, group_id: u16, port: u16) -> AsicResult<()> {
-        multicast::domain_add_port(self, group_id, port)
+        mcast::domain_add_port(self, group_id, port)
     }
 
     fn mc_port_remove(&self, group_id: u16, port: u16) -> AsicResult<()> {
-        multicast::domain_remove_port(self, group_id, port)
+        mcast::domain_remove_port(self, group_id, port)
     }
 
     fn mc_group_create(&self, group_id: u16) -> AsicResult<()> {
-        multicast::domain_create(self, group_id)
+        mcast::domain_create(self, group_id)
     }
 
     fn mc_group_destroy(&self, group_id: u16) -> AsicResult<()> {
-        multicast::domain_destroy(self, group_id)
+        mcast::domain_destroy(self, group_id)
+    }
+
+    fn mc_group_exists(&self, group_id: u16) -> bool {
+        mcast::domain_exists(self, group_id)
+    }
+
+    fn mc_groups_count(&self) -> AsicResult<usize> {
+        mcast::domains_count(self)
+    }
+
+    fn mc_set_max_nodes(
+        &self,
+        max_nodes: u32,
+        max_link_aggregated_nodes: u32,
+    ) -> AsicResult<()> {
+        mcast::set_max_nodes(self, max_nodes, max_link_aggregated_nodes)
     }
 
     // Ideally we would get some sort of sidecar-level ID from the FRUID.
@@ -238,7 +254,7 @@ pub struct Handle {
     rt: tofino_common::BfRt,
     log: slog::Logger,
     phys_ports: Mutex<ports::PortData>,
-    domains: Mutex<HashMap<u16, multicast::DomainState>>,
+    domains: Mutex<HashMap<u16, mcast::DomainState>>,
     eth_connector_id: Option<u32>,
 }
 
@@ -322,7 +338,7 @@ impl Handle {
 
         // Note: we assume that bf_mc_init() has been called as part of the
         // bf_switch_init() operation.
-        bf.mcast_hdl = multicast::create_session()?;
+        bf.mcast_hdl = mcast::create_session()?;
 
         Ok(Handle {
             dev_id,