[feature] ASIC-focused multicast replication and dendrite API (#14)

This brings multicast group management and hardware-accelerated forwarding to Dendrite, providing the foundation for our multicast networking stack. 

The PR includes:

**API Layer:**
  - Multicast group operations and lifecycle management
  - Integrated Source-Specific Multicast (SSM) support
  - Bulk reset functionality for clearing multicast state
  - Endpoints designed for integration with higher-level orchestration

**Hardware Integration:**
  - Updated sidecar.p4 for native Tofino ASIC multicast replication
  - ASIC table management for efficient multicast forwarding
  - Geneve option parsing for multicast tagging
  - Egress table handling for decapped packets and MAC rewriting
  - Range-based port bitmap matching for multicast egress (for decap)

**Network Processing:**
  - Leverages the Tofino packet replication engine for replication
  - Handles proper encapsulation/decapsulation flows
  - *Bifurcated multicast handling*: Separate processing paths for internal admin-scoped IPv6 groups vs external IPv4/IPv6 groups
  - *Multiple replication scenarios*:
    - Unencapped external multicast → encapsulated for underlay transport
    - Encapped IPv6 multicast → underlay-only replication (infrastructure traffic)
    - Encapped IPv6 multicast → external-only replication with decapsulation
    - Bifurcated replication: Both external members (decapped) and underlay members (encapped) from a group
  - *Admin-scoped NAT target validation*: External groups properly reference internal admin-scoped groups

**Validation:**
  - Extensive iperf3 testing shows no performance regression between baseline and multicast-enhanced versions
  - *Comprehensive integration test suite* covering:
    - All replication scenarios (external-only, underlay-only, bifurcated)
    - Encapsulation/decapsulation flows with Geneve multicast tagging
    - Source-Specific Multicast (SSM) filtering with exact and prefix matching
    - Admin-scoped NAT target validation and lifecycle management
    - Performance validation with concurrent packet replication
    - Edge cases including TTL handling, MAC derivation, and VLAN propagation

**Context:**
  - Implements the dendrite side of the bifurcated multicast design, handling both **internal admin-scoped IPv6  groups** (admin-local, site-local, and organization-local scopes) and external IPv4/IPv6 groups with NAT encapsulation
  - External multicast traffic gets encapsulated for underlay transport, then properly decapped and MAC-rewritten for local delivery
  - Internal groups stay within the admin scope for rack-local multicast communication
  - Supports complex replication patterns where single groups can serve both customer traffic (external, decapped) and infrastructure traffic (underlay, encapsulated)

**Associated PRs**
  - [X] oxidecomputer/omicron#8576

Author: zeeshanlakhani

.github/buildomat/jobs/image.sh

@@ -101,11 +101,11 @@ pfexec chown "$UID" /out
 
 banner "P4 Codegen"
 # Add gcc-12 so the p4 compiler can find cpp
-# The tofino2 has 20 stages, but the current sidecar.p4 will fit into 14.  We
-# add the "--stages 14" here to detect if/when the program grows beyond that
+# The tofino2 has 20 stages, but the current sidecar.p4 will fit into 18.  We
+# add the "--stages 18" here to detect if/when the program grows beyond that
 # limit.  It's not necessarily a problem if we grow, but given the limited space
 # on the ASIC, we want to grow deliberatately and thoughtfully.
-PATH=/opt/gcc-12/bin:$PATH cargo xtask codegen --stages 14
+PATH=/opt/gcc-12/bin:$PATH cargo xtask codegen --stages 18
 
 # Preserve all the diagnostics spit out by the compiler
 mkdir -p /out/p4c-diags

.gitignore

@@ -17,6 +17,7 @@ p4_artifacts*
 # Editor config
 .vscode
 .dir-locals.el
+bacon.toml
 
 # OS artifacts
 .DS_Store

Cargo.lock

@@ -44,7 +44,7 @@ omicron-common = { git = "https://github.com/oxidecomputer/omicron", branch= "ma
 oximeter = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 oximeter-producer = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 oximeter-instruments = { git = "https://github.com/oxidecomputer/omicron", branch = "main", default-features = false, features = ["kstat"] }
-oxnet = { version = "0.1.1", default-features = false, features = ["schemars", "serde"] }
+oxnet = { version = "0.1.2", default-features = false, features = ["schemars", "serde"] }
 propolis = { git = "https://github.com/oxidecomputer/propolis" }
 sled-agent-client = { git = "https://github.com/oxidecomputer/omicron", branch = "main" }
 smf = { git = "https://github.com/illumos/smf-rs" }

Cargo.toml

@@ -202,9 +202,15 @@ pub trait AsicOps {
     /// For a given multicast group, return the number of ports assigned to it.
     fn mc_port_count(&self, group_id: u16) -> AsicResult<usize>;
 
-    /// Add a port to a multicast group.  The port is identified using its ASIC
+    /// Add a port to a multicast group. The port is identified using its ASIC
     /// identifier.
-    fn mc_port_add(&self, group_id: u16, port: AsicId) -> AsicResult<()>;
+    fn mc_port_add(
+        &self,
+        group_id: u16,
+        port: AsicId,
+        rid: u16,
+        level_1_excl_id: u16,
+    ) -> AsicResult<()>;
 
     /// Remove a port from a multicast group.  The port is identified using its ASIC
     /// identifier.
@@ -216,6 +222,21 @@ pub trait AsicOps {
     /// Destroy a multicast group.
     fn mc_group_destroy(&self, group_id: u16) -> AsicResult<()>;
 
+    /// Check if a multicast group exists.
+    fn mc_group_exists(&self, group_id: u16) -> bool {
+        self.mc_domains().contains(&group_id)
+    }
+
+    /// Get the total number of multicast groups.
+    fn mc_groups_count(&self) -> AsicResult<usize>;
+
+    /// Set the maximum number of multicast nodes.
+    fn mc_set_max_nodes(
+        &self,
+        max_nodes: u32,
+        max_link_aggregated_nodes: u32,
+    ) -> AsicResult<()>;
+
     /// Get sidecar identifiers of the device being managed.
     fn get_sidecar_identifiers(&self) -> AsicResult<impl SidecarIdentifiers>;
 

aal/src/lib.rs

@@ -72,7 +72,7 @@ impl MatchData {
 /// The MatchParse trait defines the behavior needed to convert a high-level
 /// Match field into our intermediate representation.
 pub trait MatchParse {
-    /// Return all the name sand values of the key fields as strings
+    /// Return all the names and values of the key fields as strings
     fn key_values(&self) -> BTreeMap<String, String>;
     /// Convert the key Struct to a MatchData struct
     fn key_to_ir(&self) -> AsicResult<MatchData>;
@@ -452,6 +452,27 @@ impl From<bool> for ValueTypes {
     }
 }
 
+impl TryFrom<&ValueTypes> for bool {
+    type Error = &'static str;
+
+    fn try_from(v: &ValueTypes) -> Result<Self, Self::Error> {
+        match v {
+            ValueTypes::U64(v) => {
+                if *v == 0 {
+                    Ok(false)
+                } else if *v == 1 {
+                    Ok(true)
+                } else {
+                    Err("value not a boolean")
+                }
+            }
+            _ => Err("value not a boolean"),
+        }
+    }
+}
+
+unwrap_value_entry!(bool);
+
 #[derive(Debug, Hash, Clone)]
 pub enum ValueTypes {
     U64(u64),

aal/src/match_action.rs

@@ -94,7 +94,18 @@ impl TableChaos {
             (table::SWITCH_IPV4_ADDR, v),
             (table::SWITCH_IPV6_ADDR, v),
             (table::NAT_INGRESS_IPV4, v),
-            (table::NAT_INGRESS_IPV6, v)
+            (table::NAT_INGRESS_IPV6, v),
+            (table::MCAST_NAT_INGRESS_IPV4, v),
+            (table::MCAST_NAT_INGRESS_IPV6, v),
+            (table::MCAST_REPLICATION_IPV4, v),
+            (table::MCAST_REPLICATION_IPV6, v),
+            (table::MCAST_SRC_FILTER_IPV4, v),
+            (table::MCAST_SRC_FILTER_IPV6, v),
+            (table::MCAST_ROUTE_IPV4, v),
+            (table::MCAST_ROUTE_IPV6, v),
+            (table::MCAST_MAC_REWRITE, v),
+            (table::MCAST_DECAP_PORTS, v),
+            (table::MCAST_PORT_ID_MAPPING, v)
         )
     }
 
@@ -141,6 +152,8 @@ pub struct AsicConfig {
     pub mc_port_remove: Chaos,
     pub mc_group_create: Chaos,
     pub mc_group_destroy: Chaos,
+    pub mc_groups_count: Chaos,
+    pub mc_set_max_nodes: Chaos,
     pub get_sidecar_identifiers: Chaos,
     pub table_new: TableChaos,
     pub table_clear: TableChaos,
@@ -177,6 +190,8 @@ impl AsicConfig {
             mc_port_remove: Chaos::new(v),
             mc_group_create: Chaos::new(v),
             mc_group_destroy: Chaos::new(v),
+            mc_groups_count: Chaos::new(v),
+            mc_set_max_nodes: Chaos::new(v),
             get_sidecar_identifiers: Chaos::new(v),
             table_new: TableChaos::uniform(v),
             table_clear: TableChaos::uniform(v),
@@ -203,6 +218,7 @@ impl AsicConfig {
             port_enable_get: Chaos::new(v),
             connector_avail_channels: Chaos::new(v),
             mc_port_count: Chaos::new(v),
+            mc_groups_count: Chaos::new(v),
             get_sidecar_identifiers: Chaos::new(v),
             ..Default::default()
         }
@@ -224,6 +240,7 @@ impl AsicConfig {
             mc_port_remove: Chaos::new(v),
             mc_group_create: Chaos::new(v),
             mc_group_destroy: Chaos::new(v),
+            mc_set_max_nodes: Chaos::new(v),
             // TODO this can cause dpd to fail to start
             //table_clear: TableChaos::uniform(v),
             table_default_set: TableChaos::uniform(v),
@@ -476,7 +493,13 @@ impl AsicOps for Handle {
         Ok(self.ports.lock().unwrap().len())
     }
 
-    fn mc_port_add(&self, _group_id: u16, _port: u16) -> AsicResult<()> {
+    fn mc_port_add(
+        &self,
+        _group_id: u16,
+        _port: u16,
+        _rid: u16,
+        _level1_excl_id: u16,
+    ) -> AsicResult<()> {
         unfurl!(self, mc_port_add);
         Err(AsicError::OperationUnsupported)
     }
@@ -496,6 +519,20 @@ impl AsicOps for Handle {
         Ok(())
     }
 
+    fn mc_groups_count(&self) -> AsicResult<usize> {
+        unfurl!(self, mc_groups_count);
+        Ok(self.ports.lock().unwrap().len())
+    }
+
+    fn mc_set_max_nodes(
+        &self,
+        _max_nodes: u32,
+        _max_link_aggregated_nodes: u32,
+    ) -> AsicResult<()> {
+        unfurl!(self, mc_set_max_nodes);
+        Ok(())
+    }
+
     fn get_sidecar_identifiers(&self) -> AsicResult<impl SidecarIdentifiers> {
         unfurl!(self, get_sidecar_identifiers);
         Ok(Identifiers::default())

asic/src/chaos/mod.rs

@@ -25,6 +25,28 @@ pub const SWITCH_IPV4_ADDR: &str = "pipe.Ingress.filter.switch_ipv4_addr";
 pub const SWITCH_IPV6_ADDR: &str = "pipe.Ingress.filter.switch_ipv6_addr";
 pub const NAT_INGRESS_IPV4: &str = "pipe.Ingress.nat_ingress.ingress_ipv4";
 pub const NAT_INGRESS_IPV6: &str = "pipe.Ingress.nat_ingress.ingress_ipv6";
+pub(crate) const MCAST_NAT_INGRESS_IPV4: &str =
+    "pipe.Ingress.nat_ingress.ingress_ipv4_mcast";
+pub(crate) const MCAST_NAT_INGRESS_IPV6: &str =
+    "pipe.Ingress.nat_ingress.ingress_ipv6_mcast";
+pub(crate) const MCAST_REPLICATION_IPV4: &str =
+    "pipe.Ingress.mcast_ingress.mcast_replication_ipv4";
+pub(crate) const MCAST_REPLICATION_IPV6: &str =
+    "pipe.Ingress.mcast_ingress.mcast_replication_ipv6";
+pub(crate) const MCAST_SRC_FILTER_IPV4: &str =
+    "pipe.Ingress.mcast_ingress.mcast_source_filter_ipv4";
+pub(crate) const MCAST_SRC_FILTER_IPV6: &str =
+    "pipe.Ingress.mcast_ingress.mcast_source_filter_ipv6";
+pub(crate) const MCAST_ROUTE_IPV4: &str =
+    "pipe.Ingress.l3_router.MulticastRouter4.tbl";
+pub(crate) const MCAST_ROUTE_IPV6: &str =
+    "pipe.Ingress.l3_router.MulticastRouter6.tbl";
+pub(crate) const MCAST_MAC_REWRITE: &str =
+    "pipe.Egress.mac_rewrite.mac_rewrite";
+pub(crate) const MCAST_DECAP_PORTS: &str =
+    "pipe.Egress.mcast_egress.tbl_decap_ports";
+pub(crate) const MCAST_PORT_ID_MAPPING: &str =
+    "pipe.Egress.mcast_egress.asic_id_to_port";
 
 pub struct Table {
     name: String,

asic/src/chaos/table.rs

@@ -349,7 +349,13 @@ impl AsicOps for Handle {
         Ok(self.ports.lock().unwrap().len())
     }
 
-    fn mc_port_add(&self, _group_id: u16, _port: u16) -> AsicResult<()> {
+    fn mc_port_add(
+        &self,
+        _group_id: u16,
+        _port: u16,
+        _rid: u16,
+        _level1_excl_id: u16,
+    ) -> AsicResult<()> {
         Err(AsicError::OperationUnsupported)
     }
 
@@ -365,6 +371,18 @@ impl AsicOps for Handle {
         Ok(())
     }
 
+    fn mc_groups_count(&self) -> AsicResult<usize> {
+        Ok(self.ports.lock().unwrap().len())
+    }
+
+    fn mc_set_max_nodes(
+        &self,
+        _max_nodes: u32,
+        _max_link_aggregated_nodes: u32,
+    ) -> AsicResult<()> {
+        Ok(())
+    }
+
     fn get_sidecar_identifiers(&self) -> AsicResult<impl SidecarIdentifiers> {
         Ok(Identifiers {
             id: Uuid::new_v4(),

asic/src/softnpu/mod.rs

@@ -22,11 +22,13 @@ bf_mc_create_session
 bf_mc_destroy_session
 bf_mc_mgrp_create
 bf_mc_mgrp_destroy
+bf_mc_mgrp_get_count
 bf_mc_node_create
 bf_mc_node_destroy
 bf_mc_node_update
 bf_mc_associate_node
 bf_mc_dissociate_node
+bf_mc_set_max_node_threshold
 
 # bf_rt calls
 bf_rt_table_from_name_get