Refactor heartbeat + Concentratord ZMQ interface.

This refactors the heartbeat payload into an event payload, of which
a heartbeat is one of the possible options. This also refactors the
Concentratord ZMQ interface, moving away from multipart messages in
favor of a single message containing an Event / Command Protobuf
message.

Author: brocaar

Cargo.lock

@@ -16,7 +16,7 @@
     "usage",
     "derive",
   ] }
-  chirpstack_api = { version = "4.12", default-features = false }
+  chirpstack_api = { version = "4.12.0", default-features = false, path = "../chirpstack/api/rust" }
   lrwn_filters = { version = "4.12", features = ["serde"] }
   log = "0.4"
   simple_logger = "5.0"

Cargo.toml

@@ -16,8 +16,7 @@ static RELAY_ID: OnceLock<Mutex<[u8; 4]>> = OnceLock::new();
 static CONCENTRATORD_CMD_CHAN: OnceLock<CommandChannel> = OnceLock::new();
 static MESH_CONCENTRATORD_CMD_CHAN: OnceLock<CommandChannel> = OnceLock::new();
 
-type Event = (String, Vec<u8>);
-type Command = ((String, Vec<u8>), oneshot::Sender<Result<Vec<u8>>>);
+type Command = (gw::Command, oneshot::Sender<Result<Vec<u8>>>);
 type CommandChannel = mpsc::UnboundedSender<Command>;
 
 pub async fn setup(conf: &Configuration) -> Result<()> {
@@ -47,7 +46,7 @@ async fn setup_concentratord(conf: &Configuration) -> Result<()> {
             sock.connect(&command_url).unwrap();
 
             while let Some(cmd) = cmd_rx.blocking_recv() {
-                let resp = send_zmq_command(&mut sock, &cmd);
+                let resp = send_zmq_command(&mut sock, &cmd.0);
                 cmd.1.send(resp).unwrap();
             }
 
@@ -60,23 +59,30 @@ async fn setup_concentratord(conf: &Configuration) -> Result<()> {
     trace!("Reading Gateway ID");
     let mut gateway_id: [u8; 8] = [0; 8];
     let (gateway_id_tx, gateway_id_rx) = oneshot::channel::<Result<Vec<u8>>>();
-    cmd_tx.send((("gateway_id".to_string(), vec![]), gateway_id_tx))?;
+    cmd_tx.send((
+        gw::Command {
+            command: Some(gw::command::Command::GetGatewayId(
+                gw::GetGatewayIdRequest {},
+            )),
+        },
+        gateway_id_tx,
+    ))?;
     let resp = gateway_id_rx.await??;
-    gateway_id.copy_from_slice(&resp);
-    info!("Retrieved Gateway ID: {}", hex::encode(gateway_id));
+
+    let resp = gw::GetGatewayIdResponse::decode(resp.as_slice())?;
+    gateway_id.copy_from_slice(&hex::decode(&resp.gateway_id)?);
+    info!("Retrieved Gateway ID: {}", resp.gateway_id);
     GATEWAY_ID
         .set(Mutex::new(gateway_id))
         .map_err(|e| anyhow!("OnceLock error: {:?}", e))?;
 
     // Set CMD channel.
-
     CONCENTRATORD_CMD_CHAN
         .set(cmd_tx)
         .map_err(|e| anyhow!("OnceLock error: {:?}", e))?;
 
     // Setup ZMQ event.
-
-    let (event_tx, event_rx) = mpsc::unbounded_channel::<Event>();
+    let (event_tx, event_rx) = mpsc::unbounded_channel::<gw::Event>();
 
     // Spawn the zmq event handler to a dedicated thread.
     thread::spawn({
@@ -143,7 +149,7 @@ async fn setup_mesh_conncentratord(conf: &Configuration) -> Result<()> {
             sock.connect(&command_url).unwrap();
 
             while let Some(cmd) = cmd_rx.blocking_recv() {
-                let resp = send_zmq_command(&mut sock, &cmd);
+                let resp = send_zmq_command(&mut sock, &cmd.0);
                 cmd.1.send(resp).unwrap();
             }
 
@@ -155,12 +161,20 @@ async fn setup_mesh_conncentratord(conf: &Configuration) -> Result<()> {
     trace!("Reading Gateway ID");
 
     let (gateway_id_tx, gateway_id_rx) = oneshot::channel::<Result<Vec<u8>>>();
-    cmd_tx.send((("gateway_id".to_string(), vec![]), gateway_id_tx))?;
+    cmd_tx.send((
+        gw::Command {
+            command: Some(gw::command::Command::GetGatewayId(
+                gw::GetGatewayIdRequest {},
+            )),
+        },
+        gateway_id_tx,
+    ))?;
     let resp = gateway_id_rx.await??;
-    info!("Retrieved Gateway ID: {}", hex::encode(&resp));
+    let resp = gw::GetGatewayIdResponse::decode(resp.as_slice())?;
+    info!("Retrieved Gateway ID: {}", resp.gateway_id);
 
     let mut relay_id: [u8; 4] = [0; 4];
-    relay_id.copy_from_slice(&resp[4..]);
+    relay_id.copy_from_slice(&hex::decode(&resp.gateway_id)?[4..]);
     RELAY_ID
         .set(Mutex::new(relay_id))
         .map_err(|e| anyhow!("OnceLock error: {:?}", e))?;
@@ -173,7 +187,7 @@ async fn setup_mesh_conncentratord(conf: &Configuration) -> Result<()> {
 
     // Setup ZMQ event.
 
-    let (event_tx, event_rx) = mpsc::unbounded_channel::<Event>();
+    let (event_tx, event_rx) = mpsc::unbounded_channel::<gw::Event>();
 
     // Spawn the zmq event handler to a dedicated thread;
     thread::spawn({
@@ -211,15 +225,15 @@ async fn setup_mesh_conncentratord(conf: &Configuration) -> Result<()> {
 async fn event_loop(
     border_gateway: bool,
     border_gateway_ignore_direct_uplinks: bool,
-    mut event_rx: mpsc::UnboundedReceiver<Event>,
+    mut event_rx: mpsc::UnboundedReceiver<gw::Event>,
     filters: lrwn_filters::Filters,
 ) {
     trace!("Starting event loop");
     while let Some(event) = event_rx.recv().await {
         if let Err(e) = handle_event_msg(
             border_gateway,
             border_gateway_ignore_direct_uplinks,
-            &event,
+            event,
             &filters,
         )
         .await
@@ -230,10 +244,10 @@ async fn event_loop(
     }
 }
 
-async fn mesh_event_loop(border_gateway: bool, mut event_rx: mpsc::UnboundedReceiver<Event>) {
+async fn mesh_event_loop(border_gateway: bool, mut event_rx: mpsc::UnboundedReceiver<gw::Event>) {
     trace!("Starting mesh event loop");
     while let Some(event) = event_rx.recv().await {
-        if let Err(e) = handle_mesh_event_msg(border_gateway, &event).await {
+        if let Err(e) = handle_mesh_event_msg(border_gateway, event).await {
             error!("Handle mesh event error: {}", e);
             continue;
         }
@@ -243,20 +257,14 @@ async fn mesh_event_loop(border_gateway: bool, mut event_rx: mpsc::UnboundedRece
 async fn handle_event_msg(
     border_gateway: bool,
     border_gateway_ignore_direct_uplinks: bool,
-    event: &Event,
+    event: gw::Event,
     filters: &lrwn_filters::Filters,
 ) -> Result<()> {
-    trace!(
-        "Handling event, event: {}, data: {}",
-        event.0,
-        hex::encode(&event.1)
-    );
-
-    match event.0.as_str() {
-        "up" => {
-            let pl = gw::UplinkFrame::decode(event.1.as_slice())?;
+    trace!("Handling event, event: {:?}", event,);
 
-            if let Some(rx_info) = &pl.rx_info {
+    match &event.event {
+        Some(gw::event::Event::UplinkFrame(v)) => {
+            if let Some(rx_info) = &v.rx_info {
                 // Filter out frames with invalid CRC.
                 if rx_info.crc_status() != gw::CrcStatus::CrcOk {
                     debug!(
@@ -267,7 +275,7 @@ async fn handle_event_msg(
                 }
 
                 // Filter out proprietary payloads.
-                if pl.phy_payload.first().cloned().unwrap_or_default() & 0xe0 == 0xe0 {
+                if v.phy_payload.first().cloned().unwrap_or_default() & 0xe0 == 0xe0 {
                     debug!(
                         "Discarding proprietary uplink, uplink_id: {}",
                         rx_info.uplink_id
@@ -282,44 +290,34 @@ async fn handle_event_msg(
                 }
 
                 // Filter uplinks based on DevAddr and JoinEUI filters.
-                if !lrwn_filters::matches(&pl.phy_payload, filters) {
+                if !lrwn_filters::matches(&v.phy_payload, filters) {
                     debug!(
                         "Discarding uplink because of dev_addr and join_eui filters, uplink_id: {}",
                         rx_info.uplink_id
                     )
                 }
 
-                info!("Frame received - {}", helpers::format_uplink(&pl)?);
-                mesh::handle_uplink(border_gateway, pl).await?;
+                info!("Frame received - {}", helpers::format_uplink(&v)?);
+                mesh::handle_uplink(border_gateway, v).await?;
             }
         }
-        "stats" => {
+        Some(gw::event::Event::GatewayStats(_)) => {
             if border_gateway {
-                let pl = gw::GatewayStats::decode(event.1.as_slice())?;
-                info!("Gateway stats received, gateway_id: {}", pl.gateway_id);
-                proxy::send_stats(&pl).await?;
+                proxy::send_event(event).await?;
             }
         }
-        _ => {
-            return Ok(());
-        }
+        _ => {}
     }
 
     Ok(())
 }
 
-async fn handle_mesh_event_msg(border_gateway: bool, event: &Event) -> Result<()> {
-    trace!(
-        "Handling mesh event, event: {}, data: {}",
-        event.0,
-        hex::encode(&event.1)
-    );
-
-    match event.0.as_str() {
-        "up" => {
-            let pl = gw::UplinkFrame::decode(event.1.as_slice())?;
+async fn handle_mesh_event_msg(border_gateway: bool, event: gw::Event) -> Result<()> {
+    trace!("Handling mesh event, event: {:?}", event);
 
-            if let Some(rx_info) = &pl.rx_info {
+    match &event.event {
+        Some(gw::event::Event::UplinkFrame(v)) => {
+            if let Some(rx_info) = &v.rx_info {
                 // Filter out frames with invalid CRC.
                 if rx_info.crc_status() != gw::CrcStatus::CrcOk {
                     debug!(
@@ -331,79 +329,79 @@ async fn handle_mesh_event_msg(border_gateway: bool, event: &Event) -> Result<()
             }
 
             // The mesh event msg must always be a proprietary payload.
-            if pl.phy_payload.first().cloned().unwrap_or_default() & 0xe0 == 0xe0 {
-                info!("Mesh frame received - {}", helpers::format_uplink(&pl)?);
-                mesh::handle_mesh(border_gateway, pl).await?;
+            if v.phy_payload.first().cloned().unwrap_or_default() & 0xe0 == 0xe0 {
+                info!("Mesh frame received - {}", helpers::format_uplink(v)?);
+                mesh::handle_mesh(border_gateway, v).await?;
             }
         }
-        _ => {
-            return Ok(());
-        }
+        _ => {}
     }
 
     Ok(())
 }
 
-async fn send_command(cmd: &str, b: &[u8]) -> Result<Vec<u8>> {
-    trace!(
-        "Sending command, command: {}, data: {}",
-        cmd,
-        hex::encode(b)
-    );
+async fn send_command(cmd: gw::Command) -> Result<Vec<u8>> {
+    trace!("Sending command, command: {:?}", cmd,);
 
     let cmd_chan = CONCENTRATORD_CMD_CHAN
         .get()
         .ok_or_else(|| anyhow!("CONCENTRATORD_CMD_CHAN is not set"))?;
 
     let (cmd_tx, cmd_rx) = oneshot::channel::<Result<Vec<u8>>>();
-    cmd_chan.send(((cmd.to_string(), b.to_vec()), cmd_tx))?;
+    cmd_chan.send((cmd, cmd_tx))?;
     cmd_rx.await?
 }
 
-async fn send_mesh_command(cmd: &str, b: &[u8]) -> Result<Vec<u8>> {
-    trace!(
-        "Sending mesh command, command: {}, data: {}",
-        cmd,
-        hex::encode(b)
-    );
+async fn send_mesh_command(cmd: gw::Command) -> Result<Vec<u8>> {
+    trace!("Sending mesh command, command: {:?}", cmd);
 
     let cmd_chan = MESH_CONCENTRATORD_CMD_CHAN
         .get()
         .ok_or_else(|| anyhow!("MESH_CONCENTRATORD_CMD_CHAN is not set"))?;
 
     let (cmd_tx, cmd_rx) = oneshot::channel::<Result<Vec<u8>>>();
-    cmd_chan.send(((cmd.to_string(), b.to_vec()), cmd_tx))?;
+    cmd_chan.send((cmd, cmd_tx))?;
     cmd_rx.await?
 }
 
-pub async fn mesh(pl: &gw::DownlinkFrame) -> Result<()> {
-    info!("Sending mesh frame - {}", helpers::format_downlink(pl)?);
+pub async fn mesh(pl: gw::DownlinkFrame) -> Result<()> {
+    info!("Sending mesh frame - {}", helpers::format_downlink(&pl)?);
+    let downlink_id = pl.downlink_id;
 
     let tx_ack = {
-        let b = pl.encode_to_vec();
-        let resp_b = send_mesh_command("down", &b).await?;
+        let pl = gw::Command {
+            command: Some(gw::command::Command::SendDownlinkFrame(pl)),
+        };
+        let resp_b = send_mesh_command(pl).await?;
         gw::DownlinkTxAck::decode(resp_b.as_slice())?
     };
     helpers::tx_ack_to_err(&tx_ack)?;
-    info!("Enqueue acknowledged, downlink_id: {}", pl.downlink_id);
+    info!("Enqueue acknowledged, downlink_id: {}", downlink_id);
     Ok(())
 }
 
-pub async fn send_downlink(pl: &gw::DownlinkFrame) -> Result<gw::DownlinkTxAck> {
-    info!("Sending downlink frame - {}", helpers::format_downlink(pl)?);
+pub async fn send_downlink(pl: gw::DownlinkFrame) -> Result<gw::DownlinkTxAck> {
+    info!(
+        "Sending downlink frame - {}",
+        helpers::format_downlink(&pl)?
+    );
 
-    let b = pl.encode_to_vec();
-    let resp_b = send_command("down", &b).await?;
+    let pl = gw::Command {
+        command: Some(gw::command::Command::SendDownlinkFrame(pl)),
+    };
+    let resp_b = send_command(pl).await?;
     let tx_ack = gw::DownlinkTxAck::decode(resp_b.as_slice())?;
 
     Ok(tx_ack)
 }
 
-pub async fn send_gateway_configuration(pl: &gw::GatewayConfiguration) -> Result<()> {
+pub async fn send_gateway_configuration(pl: gw::GatewayConfiguration) -> Result<()> {
     info!("Sending gateway configuration, version: {}", pl.version);
 
-    let b = pl.encode_to_vec();
-    let _ = send_command("config", &b).await?;
+    let pl = gw::Command {
+        command: Some(gw::command::Command::SetGatewayConfiguration(pl)),
+    };
+    let _ = send_command(pl).await?;
 
     Ok(())
 }
@@ -428,15 +426,9 @@ pub async fn get_gateway_id() -> Result<[u8; 8]> {
         .await)
 }
 
-fn send_zmq_command(sock: &mut zmq::Socket, cmd: &Command) -> Result<Vec<u8>> {
-    debug!(
-        "Sending command to socket, command: {}, payload: {}",
-        &cmd.0 .0,
-        hex::encode(&cmd.0 .1)
-    );
-
-    sock.send(&cmd.0 .0, zmq::SNDMORE)?;
-    sock.send(&cmd.0 .1, 0)?;
+fn send_zmq_command(sock: &mut zmq::Socket, cmd: &gw::Command) -> Result<Vec<u8>> {
+    debug!("Sending command to socket, command: {:?}", &cmd,);
+    sock.send(cmd.encode_to_vec(), 0)?;
 
     // set poller so that we can timeout after 100ms
     let mut items = [sock.as_poll_item(zmq::POLLIN)];
@@ -445,19 +437,13 @@ fn send_zmq_command(sock: &mut zmq::Socket, cmd: &Command) -> Result<Vec<u8>> {
         return Err(anyhow!("Could not read down response"));
     }
 
-    // red tx ack response
+    // read tx ack response
     let resp_b: &[u8] = &sock.recv_bytes(0)?;
     Ok(resp_b.to_vec())
 }
 
-fn receive_zmq_event(sock: &mut zmq::Socket) -> Result<Event> {
-    let msg = sock.recv_multipart(0)?;
-    if msg.len() != 2 {
-        return Err(anyhow!("Event must have 2 frames"));
-    }
-
-    let event = String::from_utf8(msg[0].to_vec())?;
-    let b = msg[1].to_vec();
-
-    Ok((event, b))
+fn receive_zmq_event(sock: &mut zmq::Socket) -> Result<gw::Event> {
+    let b = sock.recv_bytes(0)?;
+    let event = gw::Event::decode(b.as_slice())?;
+    Ok(event)
 }