From 2d5770ff4bcda86ac2f3089fa11b382f6df8420b Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 24 Mar 2025 11:30:00 +0100 Subject: [PATCH 01/53] Add a new mapper: tedge-gen-mapper This commit is a very first step, scaffolding a generic mapper. The aim is to let users define their own mapping rules to tranform, filter or enrich data received from various sources. The idea is to form pipelines of user-provided transformation functions that: - consume messages from MQTT, - stream these messages along the transformation stages, - publish back to MQTT the resulting messages. Signed-off-by: Didier Wenzek --- Cargo.lock | 10 ++++ Cargo.toml | 2 + crates/core/tedge_mapper/Cargo.toml | 1 + crates/core/tedge_mapper/src/gen/mod.rs | 26 ++++++++ crates/core/tedge_mapper/src/lib.rs | 5 ++ crates/extensions/tedge_gen_mapper/Cargo.toml | 17 ++++++ .../extensions/tedge_gen_mapper/src/actor.rs | 32 ++++++++++ crates/extensions/tedge_gen_mapper/src/lib.rs | 59 +++++++++++++++++++ 8 files changed, 152 insertions(+) create mode 100644 crates/core/tedge_mapper/src/gen/mod.rs create mode 100644 crates/extensions/tedge_gen_mapper/Cargo.toml create mode 100644 crates/extensions/tedge_gen_mapper/src/actor.rs create mode 100644 crates/extensions/tedge_gen_mapper/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 25ab287722e..a235188c22e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4553,6 +4553,7 @@ dependencies = [ "tedge_config", "tedge_downloader_ext", "tedge_file_system_ext", + "tedge_gen_mapper", "tedge_health_ext", "tedge_http_ext", "tedge_mqtt_bridge", @@ -4802,6 +4803,15 @@ dependencies = [ "try-traits", ] +[[package]] +name = "tedge_gen_mapper" +version = "1.5.1" +dependencies = [ + "async-trait", + "tedge_actors", + "tedge_mqtt_ext", +] + [[package]] name = "tedge_health_ext" version = "1.6.0" diff --git a/Cargo.toml b/Cargo.toml index 8c30ffe1f0d..70db5731565 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ tedge_config_macros-impl = { path = "crates/common/tedge_config_macros/impl" } tedge_config_manager = { path = "crates/extensions/tedge_config_manager" } tedge_downloader_ext = { path = "crates/extensions/tedge_downloader_ext" } tedge_file_system_ext = { path = "crates/extensions/tedge_file_system_ext" } +tedge_gen_mapper = { path = "crates/extensions/tedge_gen_mapper" } tedge_health_ext = { path = "crates/extensions/tedge_health_ext" } tedge_http_ext = { path = "crates/extensions/tedge_http_ext" } tedge_log_manager = { path = "crates/extensions/tedge_log_manager" } @@ -164,6 +165,7 @@ regex = "1.4" reqwest = { version = "0.12", default-features = false } ron = "0.8" rpassword = "5.0" +rquickjs = { version = "0.9", default-features = false} rstest = "0.16.0" rumqttc = { git = "https://github.com/jarhodes314/rumqtt", rev = "8c489faf6af910956c97b55587ff3ecb2ac4e96f" } rumqttd = "0.19" diff --git a/crates/core/tedge_mapper/Cargo.toml b/crates/core/tedge_mapper/Cargo.toml index c9f02766e49..908c84dd9f0 100644 --- a/crates/core/tedge_mapper/Cargo.toml +++ b/crates/core/tedge_mapper/Cargo.toml @@ -28,6 +28,7 @@ tedge_api = { workspace = true } tedge_config = { workspace = true } tedge_downloader_ext = { workspace = true } tedge_file_system_ext = { workspace = true } +tedge_gen_mapper = { workspace = true } tedge_health_ext = { workspace = true } tedge_http_ext = { workspace = true } tedge_mqtt_bridge = { workspace = true } diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs new file mode 100644 index 00000000000..50e80ad6b92 --- /dev/null +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -0,0 +1,26 @@ +use crate::core::mapper::start_basic_actors; +use crate::TEdgeComponent; +use tedge_config::TEdgeConfig; +use tedge_gen_mapper::GenMapperBuilder; + +pub struct GenMapper; + +#[async_trait::async_trait] +impl TEdgeComponent for GenMapper { + async fn start( + &self, + tedge_config: TEdgeConfig, + _config_dir: &tedge_config::Path, + ) -> Result<(), anyhow::Error> { + let (mut runtime, mut mqtt_actor) = + start_basic_actors("tedge-gen-mapper", &tedge_config).await?; + + let mut wasm_mapper = GenMapperBuilder::new("/etc/tedge/gen-mapper"); + wasm_mapper.connect(&mut mqtt_actor); + + runtime.spawn(wasm_mapper).await?; + runtime.spawn(mqtt_actor).await?; + runtime.run_to_completion().await?; + Ok(()) + } +} diff --git a/crates/core/tedge_mapper/src/lib.rs b/crates/core/tedge_mapper/src/lib.rs index ca3eb34867b..7ef0cc3ad4e 100644 --- a/crates/core/tedge_mapper/src/lib.rs +++ b/crates/core/tedge_mapper/src/lib.rs @@ -8,6 +8,7 @@ use crate::az::mapper::AzureMapper; use crate::c8y::mapper::CumulocityMapper; use crate::collectd::mapper::CollectdMapper; use crate::core::component::TEdgeComponent; +use crate::gen::GenMapper; use anyhow::Context; use clap::Parser; use flockfile::check_another_instance_is_not_running; @@ -25,6 +26,7 @@ mod az; mod c8y; mod collectd; mod core; +mod gen; /// Set the cloud profile either from the CLI argument or env variable, /// then set the environment variable so child processes automatically @@ -60,6 +62,7 @@ fn lookup_component(component_name: MapperName) -> Box { MapperName::C8y { profile } => Box::new(CumulocityMapper { profile: read_and_set_var!(profile, "TEDGE_CLOUD_PROFILE"), }), + MapperName::Gen => Box::new(GenMapper), } } @@ -109,6 +112,7 @@ pub enum MapperName { profile: Option, }, Collectd, + Gen, } impl fmt::Display for MapperName { @@ -133,6 +137,7 @@ impl fmt::Display for MapperName { profile: Some(profile), } => write!(f, "tedge-mapper-c8y@{profile}"), MapperName::Collectd => write!(f, "tedge-mapper-collectd"), + MapperName::Gen => write!(f, "tedge-mapper-gen"), } } } diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml new file mode 100644 index 00000000000..bcc8196a3e9 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "tedge_gen_mapper" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true + +[dependencies] +async-trait = { workspace = true } +tedge_actors = { workspace = true } +tedge_mqtt_ext = { workspace = true } + +[lints] +workspace = true diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs new file mode 100644 index 00000000000..e9afb2a5d7b --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -0,0 +1,32 @@ +use async_trait::async_trait; +use tedge_actors::Actor; +use tedge_actors::MessageReceiver; +use tedge_actors::RuntimeError; +use tedge_actors::SimpleMessageBox; +use tedge_mqtt_ext::MqttMessage; + +pub struct GenMapper { + messages: SimpleMessageBox, +} + +#[async_trait] +impl Actor for GenMapper { + fn name(&self) -> &str { + "GenMapper" + } + + async fn run(mut self) -> Result<(), RuntimeError> { + while let Some(message) = self.messages.recv().await { + self.filter(message).await; + } + Ok(()) + } +} + +impl GenMapper { + pub fn new(messages: SimpleMessageBox) -> Self { + Self { messages } + } + + async fn filter(&mut self, _message: MqttMessage) {} +} diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs new file mode 100644 index 00000000000..9b71871925d --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -0,0 +1,59 @@ +mod actor; + +use crate::actor::GenMapper; +use std::convert::Infallible; +use std::path::Path; +use tedge_actors::Builder; +use tedge_actors::DynSender; +use tedge_actors::MessageSink; +use tedge_actors::MessageSource; +use tedge_actors::NoConfig; +use tedge_actors::RuntimeRequest; +use tedge_actors::RuntimeRequestSink; +use tedge_actors::SimpleMessageBoxBuilder; +use tedge_mqtt_ext::MqttMessage; +use tedge_mqtt_ext::TopicFilter; + +pub struct GenMapperBuilder { + message_box: SimpleMessageBoxBuilder, +} + +impl GenMapperBuilder { + pub fn new(config_dir: impl AsRef) -> Self { + let _config_dir = config_dir.as_ref(); + let messages = SimpleMessageBoxBuilder::new("WasmMapper", 16); + GenMapperBuilder { + message_box: messages, + } + } + + pub fn connect( + &mut self, + mqtt: &mut (impl MessageSource + MessageSink), + ) { + mqtt.connect_sink(self.topics(), &self.message_box); + self.message_box.connect_sink(NoConfig, mqtt); + } + + fn topics(&self) -> TopicFilter { + TopicFilter::empty() + } +} + +impl RuntimeRequestSink for GenMapperBuilder { + fn get_signal_sender(&self) -> DynSender { + self.message_box.get_signal_sender() + } +} + +impl Builder for GenMapperBuilder { + type Error = Infallible; + + fn try_build(self) -> Result { + Ok(self.build()) + } + + fn build(self) -> GenMapper { + GenMapper::new(self.message_box.build()) + } +} From cf18156378fa0ba3699b25416822dada9856d989 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 25 Mar 2025 22:29:24 +0100 Subject: [PATCH 02/53] Sketch message transformation pipelines Signed-off-by: Didier Wenzek --- Cargo.lock | 7 ++ crates/core/tedge_mapper/src/gen/mod.rs | 7 +- crates/extensions/tedge_gen_mapper/Cargo.toml | 7 ++ .../extensions/tedge_gen_mapper/src/actor.rs | 62 ++++++++- .../extensions/tedge_gen_mapper/src/config.rs | 71 +++++++++++ .../tedge_gen_mapper/src/gen_filter.rs | 38 ++++++ crates/extensions/tedge_gen_mapper/src/lib.rs | 76 +++++++++-- .../tedge_gen_mapper/src/pipeline.rs | 118 ++++++++++++++++++ 8 files changed, 370 insertions(+), 16 deletions(-) create mode 100644 crates/extensions/tedge_gen_mapper/src/config.rs create mode 100644 crates/extensions/tedge_gen_mapper/src/gen_filter.rs create mode 100644 crates/extensions/tedge_gen_mapper/src/pipeline.rs diff --git a/Cargo.lock b/Cargo.lock index a235188c22e..493a515401c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4808,8 +4808,15 @@ name = "tedge_gen_mapper" version = "1.5.1" dependencies = [ "async-trait", + "camino", + "serde", "tedge_actors", "tedge_mqtt_ext", + "thiserror 1.0.69", + "time", + "tokio", + "toml 0.8.8", + "tracing", ] [[package]] diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs index 50e80ad6b92..ceb8b1acb6c 100644 --- a/crates/core/tedge_mapper/src/gen/mod.rs +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -15,10 +15,11 @@ impl TEdgeComponent for GenMapper { let (mut runtime, mut mqtt_actor) = start_basic_actors("tedge-gen-mapper", &tedge_config).await?; - let mut wasm_mapper = GenMapperBuilder::new("/etc/tedge/gen-mapper"); - wasm_mapper.connect(&mut mqtt_actor); + let mut gen_mapper = GenMapperBuilder::default(); + gen_mapper.load("/etc/tedge/gen-mapper").await; + gen_mapper.connect(&mut mqtt_actor); - runtime.spawn(wasm_mapper).await?; + runtime.spawn(gen_mapper).await?; runtime.spawn(mqtt_actor).await?; runtime.run_to_completion().await?; Ok(()) diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index bcc8196a3e9..ff51e439ed7 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -10,8 +10,15 @@ repository.workspace = true [dependencies] async-trait = { workspace = true } +camino = { workspace = true } +serde = { workspace = true, features = ["derive"] } tedge_actors = { workspace = true } tedge_mqtt_ext = { workspace = true } +thiserror = { workspace = true } +time = { workspace = true } +tokio = { workspace = true, features = ["fs", "macros", "time"] } +toml = { workspace = true, features = ["parse"] } +tracing = { workspace = true } [lints] workspace = true diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index e9afb2a5d7b..ac72cd562d7 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,12 +1,20 @@ +use crate::pipeline::Pipeline; use async_trait::async_trait; +use std::collections::HashMap; use tedge_actors::Actor; use tedge_actors::MessageReceiver; use tedge_actors::RuntimeError; +use tedge_actors::Sender; use tedge_actors::SimpleMessageBox; use tedge_mqtt_ext::MqttMessage; +use time::OffsetDateTime; +use tokio::time::interval; +use tokio::time::Duration; +use tracing::error; pub struct GenMapper { - messages: SimpleMessageBox, + pub(super) mqtt: SimpleMessageBox, + pub(super) pipelines: HashMap, } #[async_trait] @@ -16,17 +24,59 @@ impl Actor for GenMapper { } async fn run(mut self) -> Result<(), RuntimeError> { - while let Some(message) = self.messages.recv().await { - self.filter(message).await; + let mut interval = interval(Duration::from_secs(5)); + + loop { + tokio::select! { + _ = interval.tick() => { + self.tick().await?; + } + message = self.mqtt.recv() => { + match message { + Some(message) => self.filter(message).await?, + None => break, + } + } + } } Ok(()) } } impl GenMapper { - pub fn new(messages: SimpleMessageBox) -> Self { - Self { messages } + async fn filter(&mut self, message: MqttMessage) -> Result<(), RuntimeError> { + let timestamp = OffsetDateTime::now_utc(); + for (pipeline_id, pipeline) in self.pipelines.iter_mut() { + match pipeline.process(timestamp, &message) { + Ok(messages) => { + for message in messages { + self.mqtt.send(message).await?; + } + } + Err(err) => { + error!(target: "gen-mapper", "{pipeline_id}: {err}"); + } + } + } + + Ok(()) } - async fn filter(&mut self, _message: MqttMessage) {} + async fn tick(&mut self) -> Result<(), RuntimeError> { + let timestamp = OffsetDateTime::now_utc(); + for (pipeline_id, pipeline) in self.pipelines.iter_mut() { + match pipeline.tick(timestamp) { + Ok(messages) => { + for message in messages { + self.mqtt.send(message).await?; + } + } + Err(err) => { + error!(target: "gen-mapper", "{pipeline_id}: {err}"); + } + } + } + + Ok(()) + } } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs new file mode 100644 index 00000000000..a445729f86f --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -0,0 +1,71 @@ +use crate::pipeline::Pipeline; +use crate::pipeline::Stage; +use crate::gen_filter::GenFilter; +use serde::Deserialize; +use std::path::PathBuf; +use tedge_mqtt_ext::TopicFilter; + +#[derive(Deserialize)] +pub struct PipelineConfig { + input_topics: Vec, + stages: Vec, +} + +#[derive(Deserialize)] +pub struct StageConfig { + filter: FilterSpec, + + #[serde(default)] + config_topics: Vec, +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum FilterSpec { + JavaScript(PathBuf), +} + +#[derive(thiserror::Error, Debug)] +pub enum ConfigError { + #[error("Not a valid MQTT topic filter: {0}")] + IncorrectTopicFilter(String), +} + +impl TryFrom for Pipeline { + type Error = ConfigError; + + fn try_from(config: PipelineConfig) -> Result { + let input = topic_filters(&config.input_topics)?; + let stages = config + .stages + .into_iter() + .map(Stage::try_from) + .collect::, _>>()?; + Ok(Pipeline { input_topics: input, stages }) + } +} + +impl TryFrom for Stage { + type Error = ConfigError; + + fn try_from(config: StageConfig) -> Result { + let filter = match config.filter { + FilterSpec::JavaScript(path) => GenFilter::new(path), + }; + let config = topic_filters(&config.config_topics)?; + Ok(Stage { + filter: Box::new(filter), + config_topics: config, + }) + } +} + +fn topic_filters(patterns: &Vec) -> Result { + let mut topics = TopicFilter::empty(); + for pattern in patterns { + topics + .add(pattern.as_str()) + .map_err(|_| ConfigError::IncorrectTopicFilter(pattern.clone()))?; + } + Ok(topics) +} diff --git a/crates/extensions/tedge_gen_mapper/src/gen_filter.rs b/crates/extensions/tedge_gen_mapper/src/gen_filter.rs new file mode 100644 index 00000000000..038543c8340 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/gen_filter.rs @@ -0,0 +1,38 @@ +use crate::pipeline::Filter; +use crate::pipeline::FilterError; +use std::path::PathBuf; +use tedge_mqtt_ext::MqttMessage; +use time::OffsetDateTime; +use tracing::debug; + +/// User-defined filter +pub struct GenFilter {} + +impl GenFilter { + pub fn new(path: impl Into) -> Self { + let path = path.into(); + debug!(target: "MAPPING", "new({path:?})"); + GenFilter {} + } +} + +impl Filter for GenFilter { + fn process( + &mut self, + timestamp: OffsetDateTime, + message: &MqttMessage, + ) -> Result, FilterError> { + debug!(target: "MAPPING", "process({timestamp}, {message:?})"); + Ok(vec![message.clone()]) + } + + fn update_config(&mut self, config: &MqttMessage) -> Result<(), FilterError> { + debug!(target: "MAPPING", "update_config({config:?})"); + Ok(()) + } + + fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError> { + debug!(target: "MAPPING", "tick({timestamp})"); + Ok(vec![]) + } +} diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 9b71871925d..b5aee325715 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -1,6 +1,12 @@ mod actor; +mod config; +mod pipeline; +mod gen_filter; use crate::actor::GenMapper; +use crate::pipeline::Pipeline; +use camino::Utf8Path; +use std::collections::HashMap; use std::convert::Infallible; use std::path::Path; use tedge_actors::Builder; @@ -13,18 +19,58 @@ use tedge_actors::RuntimeRequestSink; use tedge_actors::SimpleMessageBoxBuilder; use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::TopicFilter; +use tokio::fs::read_dir; +use tokio::fs::read_to_string; +use tracing::error; +use tracing::info; pub struct GenMapperBuilder { message_box: SimpleMessageBoxBuilder, + pipelines: HashMap, } -impl GenMapperBuilder { - pub fn new(config_dir: impl AsRef) -> Self { - let _config_dir = config_dir.as_ref(); - let messages = SimpleMessageBoxBuilder::new("WasmMapper", 16); +impl Default for GenMapperBuilder { + fn default() -> Self { GenMapperBuilder { - message_box: messages, + message_box: SimpleMessageBoxBuilder::new("GenMapper", 16), + pipelines: HashMap::default(), + } + } +} + +impl GenMapperBuilder { + pub async fn load(&mut self, config_dir: impl AsRef) { + let config_dir = config_dir.as_ref(); + let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| + error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) + ) else { + return; + }; + + while let Ok(Some(entry)) = entries.next_entry().await { + let Some(path) = Utf8Path::from_path(&entry.path()).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", entry.path().display()); + continue; + }; + if let Ok(file_type) = entry.file_type().await { + if file_type.is_file() && path.extension() == Some("toml") { + info!(target: "MAPPING", "Loading pipeline: {path}"); + if let Err(err) = self.load_pipeline(path).await { + error!(target: "MAPPING", "Failed to load pipeline: {err}"); + } + } + } + } + } + + async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { + if let Some(name) = file.as_ref().file_name() { + let specs = read_to_string(file.as_ref()).await?; + let pipeline: Pipeline = toml::from_str(&specs)?; + self.pipelines.insert(name.to_string(), pipeline); } + + Ok(()) } pub fn connect( @@ -36,7 +82,11 @@ impl GenMapperBuilder { } fn topics(&self) -> TopicFilter { - TopicFilter::empty() + let mut topics = TopicFilter::empty(); + for pipeline in self.pipelines.values() { + topics.add_all(pipeline.topics()) + } + topics } } @@ -54,6 +104,18 @@ impl Builder for GenMapperBuilder { } fn build(self) -> GenMapper { - GenMapper::new(self.message_box.build()) + GenMapper { + mqtt: self.message_box.build(), + pipelines: self.pipelines, + } } } + +#[derive(thiserror::Error, Debug)] +pub enum LoadError { + #[error(transparent)] + IoError(#[from] std::io::Error), + + #[error(transparent)] + TomlError(#[from] toml::de::Error), +} diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs new file mode 100644 index 00000000000..03adc975276 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -0,0 +1,118 @@ +use serde::Deserialize; +use tedge_mqtt_ext::MqttMessage; +use tedge_mqtt_ext::TopicFilter; +use time::OffsetDateTime; + +/// A chain of transformation of MQTT messages +#[derive(Deserialize)] +#[serde(try_from = "crate::config::PipelineConfig")] +pub struct Pipeline { + /// The source topics + pub input_topics: TopicFilter, + + /// Transformation stages to apply in order to the messages + pub stages: Vec, +} + +/// A message transformation stage +pub struct Stage { + pub filter: Box, + pub config_topics: TopicFilter, +} + +/// A filter process a stream of messages, producing a stream of transformed messages +/// +/// Filters are chained along pipelines, consuming MQTT messages as input +/// and producing MQTT messages as output. +/// +/// The behavior of a filter can be time related and +/// +/// Filters are dynamically configured. New partial configuration updates are sent overtime, +/// giving the opportunity for a filter to adapt its behavior. +pub trait Filter: 'static + Send + Sync { + /// Process a single message; producing zero, one or more transformed messages + fn process( + &mut self, + timestamp: OffsetDateTime, + message: &MqttMessage, + ) -> Result, FilterError>; + + /// Update the filter configuration + fn update_config(&mut self, config: &MqttMessage) -> Result<(), FilterError>; + + /// Close the current time-window; producing zero, one or more accumulated messages + fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError>; +} + +#[derive(thiserror::Error, Debug)] +pub enum FilterError { + #[error("Input message cannot be processed: {0}")] + UnsupportedMessage(String), + + #[error("No messages can be processed due to an incorrect setting: {0}")] + IncorrectSetting(String), +} + +impl Pipeline { + pub fn topics(&self) -> TopicFilter { + let mut topics = self.input_topics.clone(); + for stage in self.stages.iter() { + topics.add_all(stage.config_topics.clone()) + } + topics + } + + pub fn update_config(&mut self, message: &MqttMessage) -> Result<(), FilterError> { + for stage in self.stages.iter_mut() { + if stage.config_topics.accept(message) { + stage.filter.update_config(message)? + } + } + Ok(()) + } + + pub fn process( + &mut self, + timestamp: OffsetDateTime, + message: &MqttMessage, + ) -> Result, FilterError> { + self.update_config(message)?; + if !self.input_topics.accept(message) { + return Ok(vec![]); + } + + let mut messages = vec![message.clone()]; + for stage in self.stages.iter_mut() { + let mut transformed_messages = vec![]; + for filter_output in messages + .iter() + .map(|message| stage.filter.process(timestamp, message)) + { + transformed_messages.extend(filter_output?); + } + messages = transformed_messages; + } + Ok(messages) + } + + pub fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError> { + let mut messages = vec![]; + for stage in self.stages.iter_mut() { + // Process first the messages triggered upstream by the tick + let mut transformed_messages = vec![]; + for filter_output in messages + .iter() + .map(|message| stage.filter.process(timestamp, message)) + { + transformed_messages.extend(filter_output?); + } + + // Only then process the tick + transformed_messages.extend(stage.filter.tick(timestamp)?); + + // Iterate with all the messages collected at this stage + messages = transformed_messages; + } + Ok(messages) + } +} From 469b0f3b0fc501b41f2e2655a0ca0a7c6365f068 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 14 May 2025 19:23:51 +0200 Subject: [PATCH 03/53] Run JS transformation stages Signed-off-by: Didier Wenzek --- Cargo.lock | 1642 ++++++++++------- Cargo.toml | 2 +- crates/core/tedge_mapper/src/gen/mod.rs | 4 +- crates/extensions/tedge_gen_mapper/Cargo.toml | 4 +- .../pipelines/add_timestamp.js | 9 + .../tedge_gen_mapper/pipelines/collectd.toml | 6 + .../pipelines/measurements.toml | 6 + .../tedge_gen_mapper/pipelines/te_to_c8y.js | 6 + .../extensions/tedge_gen_mapper/src/actor.rs | 9 +- .../extensions/tedge_gen_mapper/src/config.rs | 50 +- .../tedge_gen_mapper/src/gen_filter.rs | 38 - .../tedge_gen_mapper/src/js_filter.rs | 250 +++ crates/extensions/tedge_gen_mapper/src/lib.rs | 84 +- .../tedge_gen_mapper/src/pipeline.rs | 64 +- 14 files changed, 1337 insertions(+), 837 deletions(-) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/collectd.toml create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/measurements.toml create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js delete mode 100644 crates/extensions/tedge_gen_mapper/src/gen_filter.rs create mode 100644 crates/extensions/tedge_gen_mapper/src/js_filter.rs diff --git a/Cargo.lock b/Cargo.lock index 493a515401c..fda02b4ff11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,13 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "adler2" version = "2.0.0" @@ -29,16 +23,16 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.16", "once_cell", "version_check", ] [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "once_cell", @@ -48,24 +42,25 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "anstream" -version = "0.6.11" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] @@ -77,46 +72,47 @@ checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "once_cell", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.79" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" dependencies = [ "backtrace", ] [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "arrayvec" @@ -158,9 +154,9 @@ dependencies = [ [[package]] name = "asn1-rs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "607495ec7113b178fbba7a6166a27f99e774359ef4823adbefd756b5b81d7970" +checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" dependencies = [ "asn1-rs-derive 0.6.0", "asn1-rs-impl 0.2.0", @@ -169,7 +165,7 @@ dependencies = [ "num-bigint", "num-traits", "rusticata-macros", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] @@ -192,8 +188,8 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] @@ -204,8 +200,8 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] @@ -227,7 +223,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -242,14 +238,15 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.13" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00ad3f3a942eee60335ab4342358c161ee296829e0d16ff42fc1d6cb07815467" +checksum = "2bd389a4b2970a01282ee455294913c0a43724daedcd1a24c3eb0ec1c1320b66" dependencies = [ "anstyle", "bstr", "doc-comment", - "predicates 3.1.0", + "libc", + "predicates 3.1.3", "predicates-core", "predicates-tree", "wait-timeout", @@ -263,9 +260,9 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-compat" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f68a707c1feb095d8c07f8a65b9f506b117d30af431cab89374357de7c11461b" +checksum = "7bab94bde396a3f7b4962e396fdad640e241ed797d4d8d77fc8c237d14c58fc0" dependencies = [ "futures-core", "futures-io", @@ -286,6 +283,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-tempfile" version = "0.7.0" @@ -297,13 +305,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.77" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -322,18 +330,21 @@ dependencies = [ [[package]] name = "async-tungstenite" -version = "0.28.0" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e661b6cb0a6eb34d02c520b052daa3aa9ac0cc02495c9d066bbce13ead132b" +checksum = "1c348fb0b6d132c596eca3dcd941df48fb597aafcb07a738ec41c004b087dc99" dependencies = [ + "atomic-waker", + "futures-core", "futures-io", + "futures-task", "futures-util", "log", "pin-project-lite", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tungstenite 0.24.0", ] @@ -375,9 +386,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws_mapper_ext" @@ -409,9 +420,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.32", "itoa", "matchit 0.7.3", "memchr", @@ -432,17 +443,17 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.1" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" +checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" dependencies = [ - "axum-core 0.5.0", + "axum-core 0.5.2", "axum-macros", "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper 1.6.0", @@ -477,7 +488,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "mime", "rustversion", @@ -487,13 +498,13 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" +checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ "bytes", - "futures-util", - "http 1.2.0", + "futures-core", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "mime", @@ -507,20 +518,21 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b" +checksum = "45bf463831f5131b7d3c756525b305d40f1185b688565648a92e1392ca35713d" dependencies = [ - "axum 0.8.1", - "axum-core 0.5.0", + "axum 0.8.4", + "axum-core 0.5.2", "bytes", "futures-util", "headers", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", + "rustversion", "serde", "tower 0.5.2", "tower-layer", @@ -535,30 +547,28 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "axum-server" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56bac90848f6a9393ac03c63c640925c4b7c8ca21654de40d53f55964667c7d8" +checksum = "495c05f60d6df0093e8fb6e74aa5846a0ad06abaf96d76166283720bf740f8ab" dependencies = [ "arc-swap", "bytes", - "futures-util", - "http 1.2.0", + "fs-err", + "http 1.3.1", "http-body 1.0.1", - "http-body-util", "hyper 1.6.0", "hyper-util", "pin-project-lite", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.1", - "tower 0.4.13", + "tokio-rustls 0.26.2", "tower-service", ] @@ -568,7 +578,7 @@ version = "1.6.0" dependencies = [ "anyhow", "assert_matches", - "axum 0.8.1", + "axum 0.8.4", "axum-server", "camino", "futures", @@ -577,12 +587,12 @@ dependencies = [ "pin-project", "rcgen", "reqwest", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-pemfile 2.2.0", "tedge_config", "tempfile", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tower 0.4.13", "tracing", "x509-parser 0.16.0", @@ -615,26 +625,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ "futures-core", - "getrandom 0.2.12", + "getrandom 0.2.16", "instant", "pin-project-lite", - "rand", + "rand 0.8.5", "tokio", ] [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", - "miniz_oxide 0.7.1", + "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -676,18 +686,18 @@ dependencies = [ [[package]] name = "bit-set" -version = "0.5.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ "bit-vec", ] [[package]] name = "bit-vec" -version = "0.6.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" @@ -697,9 +707,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" dependencies = [ "serde", ] @@ -737,26 +747,26 @@ dependencies = [ [[package]] name = "bstr" -version = "1.9.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", - "regex-automata 0.4.5", + "regex-automata 0.4.9", "serde", ] [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytemuck" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" [[package]] name = "byteorder" @@ -766,9 +776,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" dependencies = [ "serde", ] @@ -796,8 +806,8 @@ version = "1.6.0" dependencies = [ "async-compat", "async-http-proxy", - "async-tungstenite 0.28.0", - "axum 0.8.1", + "async-tungstenite 0.28.2", + "axum 0.8.4", "base64 0.22.1", "bytes", "c8y_api", @@ -806,11 +816,11 @@ dependencies = [ "csv", "futures", "futures-util", - "http 1.2.0", + "http 1.3.1", "miette", - "rand", + "rand 0.8.5", "rstest", - "rustls 0.23.22", + "rustls 0.23.27", "serde", "sha1", "tedge_config", @@ -818,7 +828,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "url", "ws_stream_tungstenite 0.14.0", ] @@ -845,7 +855,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", "url", ] @@ -856,7 +866,7 @@ version = "1.6.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.1", + "axum 0.8.4", "axum-extra", "axum-server", "axum_tls", @@ -872,7 +882,7 @@ dependencies = [ "pin-project", "rcgen", "reqwest", - "rustls 0.23.22", + "rustls 0.23.27", "tedge_actors", "tedge_config", "tedge_config_macros", @@ -912,7 +922,7 @@ version = "1.6.0" dependencies = [ "anyhow", "c8y_api", - "http 1.2.0", + "http 1.3.1", "mockito", "reqwest", "serde", @@ -961,16 +971,16 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", "url", ] [[package]] name = "camino" -version = "1.1.6" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" dependencies = [ "serde", ] @@ -983,9 +993,9 @@ checksum = "6f125eb85b84a24c36b02ed1d22c9dd8632f53b3cde6e4d23512f94021030003" [[package]] name = "cc" -version = "1.2.10" +version = "1.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +checksum = "32db95edf998450acc7881c932f94cd9b05c87b4b2599e8bab064753da4acfd1" dependencies = [ "shlex", ] @@ -995,13 +1005,13 @@ name = "certificate" version = "1.6.0" dependencies = [ "anyhow", - "asn1-rs 0.7.0", + "asn1-rs 0.7.1", "assert_matches", "base64 0.22.1", "camino", "rcgen", "reqwest", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-native-certs", "rustls-pemfile 2.2.0", "sha1", @@ -1028,18 +1038,18 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "num-traits", ] [[package]] name = "clap" -version = "4.5.26" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" +checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" dependencies = [ "clap_builder", "clap_derive", @@ -1047,9 +1057,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.26" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" +checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" dependencies = [ "anstream", "anstyle", @@ -1059,9 +1069,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.42" +version = "4.5.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33a7e468e750fa4b6be660e8b5651ad47372e8fb114030b594c2d75d48c5ffd0" +checksum = "c91d3baa3bcd889d60e6ef28874126a0b384fd225ab83aa6d8a801c519194ce1" dependencies = [ "clap", "clap_lex", @@ -1071,14 +1081,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.24" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -1121,18 +1131,26 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "colored" -version = "2.1.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "lazy_static", - "windows-sys 0.48.0", + "windows-sys 0.59.0", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", ] [[package]] @@ -1178,9 +1196,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -1230,9 +1248,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" @@ -1269,9 +1287,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" dependencies = [ "csv-core", "itoa", @@ -1281,9 +1299,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" dependencies = [ "memchr", ] @@ -1300,12 +1318,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core 0.20.10", - "darling_macro 0.20.10", + "darling_core 0.20.11", + "darling_macro 0.20.11", ] [[package]] @@ -1324,16 +1342,16 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -1349,20 +1367,20 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core 0.20.10", + "darling_core 0.20.11", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "data-encoding" -version = "2.5.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "der-parser" @@ -1394,9 +1412,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", "serde", @@ -1426,13 +1444,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -1481,7 +1499,7 @@ name = "download" version = "1.6.0" dependencies = [ "anyhow", - "axum 0.8.1", + "axum 0.8.4", "axum_tls", "backoff", "certificate", @@ -1491,7 +1509,7 @@ dependencies = [ "nix", "rcgen", "reqwest", - "rustls 0.23.22", + "rustls 0.23.27", "serde", "tedge_utils", "tempfile", @@ -1507,14 +1525,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f5ce6d7f6b0c1a6330fb8450f49a8423b78e30d04132146938c35baab3877eb" dependencies = [ "fnv", - "rand", + "rand 0.8.5", ] [[package]] name = "either" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "embedded-io" @@ -1543,9 +1561,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -1557,47 +1575,68 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fastrand" -version = "2.0.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "figment" -version = "0.10.14" +version = "0.10.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b6e5bc7bd59d60d0d45a6ccab6cf0f4ce28698fb4e81e750ddf229c9b824026" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" dependencies = [ "atomic", "parking_lot", "pear", "serde", "tempfile", - "toml 0.8.8", + "toml 0.8.22", "uncased", "version_check", ] [[package]] name = "file-id" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6584280525fb2059cba3db2c04abf947a1a29a45ddae89f3870f8281704fafc9" +checksum = "6bc904b9bbefcadbd8e3a9fb0d464a9b979de6324c03b3c663e8994f46a5be36" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "filetime" -version = "0.2.23" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", - "redox_syscall", - "windows-sys 0.52.0", + "libredox", + "windows-sys 0.59.0", ] [[package]] @@ -1613,7 +1652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", - "miniz_oxide 0.8.8", + "miniz_oxide", ] [[package]] @@ -1638,9 +1677,9 @@ dependencies = [ [[package]] name = "flume" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", @@ -1664,9 +1703,9 @@ dependencies = [ [[package]] name = "fragile" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" +checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" [[package]] name = "freedesktop_entry_parser" @@ -1678,6 +1717,16 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "fs-err" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f89bda4c2a21204059a977ed3bfe746677dfd137b83c339e702b0ac91d482aa" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -1695,9 +1744,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -1726,9 +1775,9 @@ checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1749,7 +1798,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -1766,9 +1815,9 @@ checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" @@ -1800,9 +1849,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", @@ -1813,41 +1862,43 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", - "wasi 0.13.3+wasi-0.2.2", - "windows-targets 0.52.6", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] name = "gimli" -version = "0.28.1" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" -version = "0.4.6" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.2.0", - "indexmap 2.2.1", + "http 1.3.1", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", @@ -1878,7 +1929,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.12", ] [[package]] @@ -1887,9 +1938,15 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.12", ] +[[package]] +name = "hashbrown" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" + [[package]] name = "headers" version = "0.4.0" @@ -1899,7 +1956,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http 1.2.0", + "http 1.3.1", "httpdate", "mime", "sha1", @@ -1911,7 +1968,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.2.0", + "http 1.3.1", ] [[package]] @@ -1942,9 +1999,15 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" [[package]] name = "hex" @@ -1954,18 +2017,18 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -1974,9 +2037,9 @@ dependencies = [ [[package]] name = "http" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -1990,7 +2053,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http 0.2.11", + "http 0.2.12", "pin-project-lite", ] @@ -2001,27 +2064,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.2.0", + "http 1.3.1", ] [[package]] name = "http-body-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "futures-util", - "http 1.2.0", + "futures-core", + "http 1.3.1", "http-body 1.0.1", "pin-project-lite", ] [[package]] name = "httparse" -version = "1.9.4" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" @@ -2031,21 +2094,21 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "http 0.2.11", + "http 0.2.12", "http-body 0.4.6", "httparse", "httpdate", @@ -2068,7 +2131,7 @@ dependencies = [ "futures-channel", "futures-util", "h2", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "httparse", "httpdate", @@ -2086,29 +2149,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http 1.2.0", + "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tower-service", ] [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "hyper 1.6.0", + "libc", "pin-project-lite", "socket2", "tokio", @@ -2118,21 +2182,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", + "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -2141,31 +2206,11 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ "displaydoc", "icu_collections", @@ -2173,67 +2218,54 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "2549ca8c7241c82f59c80ba2a6f415d931c5b58d24fb8412caa1a1f02c49139a" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "potential_utf", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "8197e866e47b68f8f7d95249e172903bec06004b18b2937f1095d40a0c57de04" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", + "icu_locale_core", "stable_deref_trait", "tinystr", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.96", -] - [[package]] name = "ident_case" version = "1.0.1" @@ -2253,9 +2285,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -2273,12 +2305,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433de089bd45971eecf4668ee0ee8f4cec17db4f8bd8f7bc3197a6ce37aa7d9b" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.3", ] [[package]] @@ -2309,35 +2341,35 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] [[package]] name = "ipnet" -version = "2.9.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is-terminal" -version = "0.4.10" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ - "hermit-abi", - "rustix 0.38.34", - "windows-sys 0.52.0", + "hermit-abi 0.5.1", + "libc", + "windows-sys 0.59.0", ] [[package]] name = "is_ci" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" [[package]] name = "is_executable" @@ -2348,6 +2380,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -2368,16 +2406,17 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -2423,9 +2462,9 @@ dependencies = [ [[package]] name = "kqueue" -version = "1.0.8" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" dependencies = [ "kqueue-sys", "libc", @@ -2443,15 +2482,15 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.169" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libloading" @@ -2463,10 +2502,15 @@ dependencies = [ ] [[package]] -name = "libm" -version = "0.2.8" +name = "libredox" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.9.0", + "libc", + "redox_syscall", +] [[package]] name = "linked-hash-map" @@ -2476,9 +2520,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" @@ -2488,15 +2532,15 @@ checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -2504,9 +2548,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "mach2" @@ -2546,9 +2596,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -2565,7 +2615,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.12", "metrics-macros", "portable-atomic", ] @@ -2577,7 +2627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" dependencies = [ "base64 0.21.7", - "hyper 0.14.28", + "hyper 0.14.32", "indexmap 1.9.3", "ipnet", "metrics", @@ -2596,7 +2646,7 @@ checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -2643,7 +2693,7 @@ checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -2654,9 +2704,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" dependencies = [ "mime", "unicase", @@ -2668,15 +2718,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.8.8" @@ -2738,21 +2779,21 @@ dependencies = [ [[package]] name = "mockito" -version = "1.5.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b34bd91b9e5c5b06338d392463e1318d683cf82ec3d3af4014609be6e2108d" +checksum = "7760e0e418d9b7e5777c0374009ca4c93861b9066f18cb334a20ce50ab63aa48" dependencies = [ "assert-json-diff", "bytes", "colored", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper 1.6.0", "hyper-util", "log", - "rand", + "rand 0.9.1", "regex", "serde_json", "serde_urlencoded", @@ -2815,7 +2856,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8" dependencies = [ - "rand", + "rand 0.8.5", ] [[package]] @@ -2853,7 +2894,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "crossbeam-channel", "filetime", "fsevent-sys", @@ -2921,7 +2962,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -2930,24 +2970,24 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] [[package]] name = "object" -version = "0.32.2" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -2972,15 +3012,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.3" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "ordered-multimap" @@ -3013,11 +3053,17 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -3025,15 +3071,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -3050,15 +3096,15 @@ checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" [[package]] name = "pathdiff" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pear" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ccca0f6c17acc81df8e242ed473ec144cbf5c98037e69aa6d144780aad103c8" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" dependencies = [ "inlinable_string", "pear_codegen", @@ -3067,14 +3113,14 @@ dependencies = [ [[package]] name = "pear_codegen" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e22670e8eb757cff11d6c199ca7b987f352f0346e0be4dd23869ec72cb53c77" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -3095,20 +3141,20 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" +checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6" dependencies = [ "memchr", - "thiserror 1.0.69", + "thiserror 2.0.12", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcd6ab1236bbdb3a49027e920e693192ebfe8913f6d60e294de57463a493cfde" +checksum = "d725d9cfd79e87dccc9341a2ef39d1b6f6353d68c4b33c177febbe1a402c97c5" dependencies = [ "pest", "pest_generator", @@ -3116,22 +3162,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a31940305ffc96863a735bef7c7994a00b325a7138fdbc5bda0f1a0476d3275" +checksum = "db7d01726be8ab66ab32f9df467ae8b1148906685bbe75c82d1e65d7f5b3f841" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "pest_meta" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ff62f5259e53b78d1af898941cdcdccfae7385cf7d793a6e55de5d05bb4b7d" +checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0" dependencies = [ "once_cell", "pest", @@ -3150,29 +3196,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.4" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.4" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -3206,9 +3252,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" [[package]] name = "postcard" @@ -3223,6 +3269,15 @@ dependencies = [ "serde", ] +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3231,9 +3286,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] [[package]] name = "predicates" @@ -3251,9 +3309,9 @@ dependencies = [ [[package]] name = "predicates" -version = "3.1.0" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" +checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" dependencies = [ "anstyle", "difflib", @@ -3262,15 +3320,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" +checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" [[package]] name = "predicates-tree" -version = "1.0.9" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" dependencies = [ "predicates-core", "termtree", @@ -3288,19 +3346,19 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.22" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" dependencies = [ "proc-macro2", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "proc-macro2" -version = "1.0.93" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -3313,26 +3371,26 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", "version_check", "yansi", ] [[package]] name = "proptest" -version = "1.4.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.8.0", + "bitflags 2.9.0", "lazy_static", "num-traits", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.8.2", + "regex-syntax 0.8.5", "rusty-fork", "tempfile", "unarray", @@ -3362,37 +3420,40 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quinn" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" dependencies = [ "bytes", + "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.22", + "rustls 0.23.27", "socket2", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", "tracing", + "web-time", ] [[package]] name = "quinn-proto" -version = "0.11.9" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" dependencies = [ "bytes", - "getrandom 0.2.12", - "rand", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.1", "ring", "rustc-hash", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-pki-types", "slab", - "thiserror 2.0.11", + "thiserror 2.0.12", "tinyvec", "tracing", "web-time", @@ -3400,9 +3461,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.9" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" +checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" dependencies = [ "cfg_aliases", "libc", @@ -3414,13 +3475,19 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "radium" version = "0.7.0" @@ -3434,8 +3501,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -3445,7 +3522,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -3454,7 +3541,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.16", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", ] [[package]] @@ -3463,7 +3559,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3568,23 +3664,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", ] [[package]] name = "regex" -version = "1.10.3" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.5", - "regex-syntax 0.8.2", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", ] [[package]] @@ -3598,13 +3694,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.5" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.5", ] [[package]] @@ -3615,21 +3711,21 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ "base64 0.22.1", "bytes", "futures-core", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper 1.6.0", @@ -3644,7 +3740,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-native-certs", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -3653,8 +3749,9 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.2", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tokio-util", + "tower 0.5.2", "tower-service", "url", "wasm-bindgen", @@ -3666,13 +3763,13 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.13" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.12", + "getrandom 0.2.16", "libc", "untrusted", "windows-sys 0.52.0", @@ -3696,7 +3793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64 0.21.7", - "bitflags 2.8.0", + "bitflags 2.9.0", "serde", "serde_derive", ] @@ -3711,6 +3808,34 @@ dependencies = [ "winapi", ] +[[package]] +name = "rquickjs" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5227859c4dfc83f428e58f9569bf439e628c8d139020e7faff437e6f5abaa0" +dependencies = [ + "rquickjs-core", +] + +[[package]] +name = "rquickjs-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82e0ca83028ad5b533b53b96c395bbaab905a5774de4aaf1004eeacafa3d85d" +dependencies = [ + "async-lock", + "rquickjs-sys", +] + +[[package]] +name = "rquickjs-sys" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fed0097b0b4fbb2a87f6dd3b995a7c64ca56de30007eb7e867dfdfc78324ba5" +dependencies = [ + "cc", +] + [[package]] name = "rstest" version = "0.16.0" @@ -3751,9 +3876,9 @@ dependencies = [ "rustls-native-certs", "rustls-pemfile 2.2.0", "rustls-webpki 0.102.8", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tokio-stream", "tokio-util", ] @@ -3774,7 +3899,7 @@ dependencies = [ "metrics", "metrics-exporter-prometheus", "parking_lot", - "rand", + "rand 0.8.5", "rustls-pemfile 1.0.4", "rustls-webpki 0.101.7", "serde", @@ -3802,21 +3927,21 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] @@ -3832,15 +3957,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys 0.4.13", - "windows-sys 0.52.0", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", ] [[package]] @@ -3849,7 +3974,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "errno", "libc", "linux-raw-sys 0.9.4", @@ -3858,9 +3983,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.11" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", @@ -3870,14 +3995,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.22" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.3", "subtle", "zeroize", ] @@ -3943,11 +4068,22 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.103.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] name = "rusty-fork" @@ -3963,9 +4099,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -3978,11 +4114,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4022,7 +4158,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "core-foundation", "core-foundation-sys", "libc", @@ -4041,46 +4177,47 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.21" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "serde" -version = "1.0.196" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.196" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "serde_json" -version = "1.0.113" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] [[package]] name = "serde_path_to_error" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" dependencies = [ "itoa", "serde", @@ -4088,9 +4225,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -4120,9 +4257,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -4131,9 +4268,9 @@ dependencies = [ [[package]] name = "sha256" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18278f6a914fa3070aa316493f7d2ddfb9ac86ebc06fa3b83bffda487e9065b0" +checksum = "f880fc8562bdeb709793f00eb42a2ad0e672c4f883bbe59122b926eca935c8f6" dependencies = [ "async-trait", "bytes", @@ -4165,24 +4302,24 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] [[package]] name = "similar" -version = "2.4.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "sketches-ddsketch" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a406c1882ed7f29cd5e248c9848a80e7cb6ae0fea82346d2746f2f941c07e1" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" [[package]] name = "slab" @@ -4195,9 +4332,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "smawk" @@ -4229,12 +4366,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4333,9 +4470,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.96" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -4371,13 +4508,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -4452,7 +4589,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", "tracing-subscriber", "url", @@ -4469,7 +4606,7 @@ dependencies = [ "anyhow", "assert-json-diff", "async-trait", - "axum 0.8.1", + "axum 0.8.4", "axum-server", "axum_tls", "camino", @@ -4487,7 +4624,7 @@ dependencies = [ "rcgen", "reqwest", "ron 0.8.1", - "rustls 0.23.22", + "rustls 0.23.27", "serde", "serde_json", "sha256", @@ -4511,7 +4648,7 @@ dependencies = [ "time", "tokio", "tokio-util", - "toml 0.8.8", + "toml 0.8.22", "tower 0.4.13", "tower-http", "tracing", @@ -4570,18 +4707,18 @@ name = "tedge-p11-server" version = "1.6.0" dependencies = [ "anyhow", - "asn1-rs 0.7.0", + "asn1-rs 0.7.1", "camino", "clap", "cryptoki", "percent-encoding", "postcard", - "rustls 0.23.22", + "rustls 0.23.27", "sd-listen-fds", "serde", "tempfile", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", "tracing-subscriber", ] @@ -4660,7 +4797,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", "walkdir", ] @@ -4682,7 +4819,7 @@ dependencies = [ "path-clean", "regex", "reqwest", - "rustls 0.23.22", + "rustls 0.23.27", "serde", "strum", "strum_macros", @@ -4693,7 +4830,7 @@ dependencies = [ "test-case", "thiserror 1.0.69", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", "tracing-subscriber", "url", @@ -4716,7 +4853,7 @@ dependencies = [ "serde_json", "tedge_config_macros-macro", "thiserror 1.0.69", - "toml 0.8.8", + "toml 0.8.22", "tracing", "url", ] @@ -4725,7 +4862,7 @@ dependencies = [ name = "tedge_config_macros-impl" version = "1.6.0" dependencies = [ - "darling 0.20.10", + "darling 0.20.11", "heck 0.4.1", "itertools 0.13.0", "pretty_assertions", @@ -4733,7 +4870,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.96", + "syn 2.0.101", "test-case", ] @@ -4743,7 +4880,7 @@ version = "1.6.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", "tedge_config_macros-impl", ] @@ -4770,7 +4907,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", - "toml 0.8.8", + "toml 0.8.22", "uzers", ] @@ -4809,13 +4946,15 @@ version = "1.5.1" dependencies = [ "async-trait", "camino", + "rquickjs", "serde", + "serde_json", "tedge_actors", "tedge_mqtt_ext", "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", "tracing", ] @@ -4839,13 +4978,13 @@ name = "tedge_http_ext" version = "1.6.0" dependencies = [ "async-trait", - "http 1.2.0", + "http 1.3.1", "http-body-util", "hyper 1.6.0", "hyper-rustls", "hyper-util", "mockito", - "rustls 0.23.22", + "rustls 0.23.27", "serde", "serde_json", "tedge_actors", @@ -4864,7 +5003,7 @@ dependencies = [ "filetime", "glob", "log", - "rand", + "rand 0.8.5", "regex", "serde", "serde_json", @@ -4879,7 +5018,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml 0.8.8", + "toml 0.8.22", ] [[package]] @@ -4954,7 +5093,7 @@ dependencies = [ "anyhow", "camino", "tempfile", - "toml 0.8.8", + "toml 0.8.22", ] [[package]] @@ -5012,14 +5151,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.12.0" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ - "cfg-if", "fastrand", + "getrandom 0.3.3", "once_cell", - "rustix 0.38.34", + "rustix 1.0.7", "windows-sys 0.59.0", ] @@ -5044,9 +5183,9 @@ dependencies = [ [[package]] name = "termtree" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "test-case" @@ -5066,7 +5205,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -5077,7 +5216,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", "test-case-core", ] @@ -5103,11 +5242,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.11", + "thiserror-impl 2.0.12", ] [[package]] @@ -5118,25 +5257,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "thiserror-impl" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", @@ -5144,9 +5283,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", "itoa", @@ -5161,15 +5300,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -5177,9 +5316,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -5187,9 +5326,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -5202,9 +5341,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.2" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" dependencies = [ "backtrace", "bytes", @@ -5226,7 +5365,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] @@ -5235,17 +5374,17 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.11", + "rustls 0.21.12", "tokio", ] [[package]] name = "tokio-rustls" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.22", + "rustls 0.23.27", "tokio", ] @@ -5262,32 +5401,31 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4bf6fecd69fcdede0ec680aaf474cdab988f9de6bc73d3758f0160e3b7025a" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" dependencies = [ "futures-util", "log", - "rustls 0.23.22", + "rustls 0.23.27", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.1", - "tungstenite 0.26.1", + "tokio-rustls 0.26.2", + "tungstenite 0.26.2", ] [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -5301,9 +5439,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.8" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" dependencies = [ "serde", "serde_spanned", @@ -5313,26 +5451,33 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.21.0" +version = "0.22.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" +checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.9.0", "serde", "serde_spanned", "toml_datetime", + "toml_write", "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" + [[package]] name = "tower" version = "0.4.13" @@ -5371,9 +5516,9 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "bytes", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "pin-project-lite", @@ -5395,9 +5540,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -5407,20 +5552,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -5439,9 +5584,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -5477,10 +5622,10 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 0.2.11", + "http 0.2.12", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror 1.0.69", "url", @@ -5496,11 +5641,11 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.2.0", + "http 1.3.1", "httparse", "log", - "rand", - "rustls 0.23.22", + "rand 0.8.5", + "rustls 0.23.27", "rustls-pki-types", "sha1", "thiserror 1.0.69", @@ -5509,29 +5654,28 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413083a99c579593656008130e29255e54dcaae495be556cc26888f211648c24" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ - "byteorder", "bytes", "data-encoding", - "http 1.2.0", + "http 1.3.1", "httparse", "log", - "rand", - "rustls 0.23.22", + "rand 0.9.1", + "rustls 0.23.27", "rustls-pki-types", "sha1", - "thiserror 2.0.11", + "thiserror 2.0.12", "utf-8", ] [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "typewit" @@ -5550,9 +5694,9 @@ checksum = "e36a83ea2b3c704935a01b4642946aadd445cea40b10935e3f8bd8052b8193d6" [[package]] name = "ucd-trie" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unarray" @@ -5571,18 +5715,15 @@ dependencies = [ [[package]] name = "unicase" -version = "2.7.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-linebreak" @@ -5592,15 +5733,15 @@ checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" @@ -5613,7 +5754,7 @@ name = "upload" version = "1.6.0" dependencies = [ "anyhow", - "axum 0.8.1", + "axum 0.8.4", "axum_tls", "backoff", "camino", @@ -5648,12 +5789,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -5662,17 +5797,17 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.15.1" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom 0.3.1", + "getrandom 0.3.3", ] [[package]] @@ -5687,30 +5822,30 @@ dependencies = [ [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wait-timeout" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" dependencies = [ "libc", ] [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -5733,9 +5868,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" -version = "0.13.3+wasi-0.2.2" +version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" dependencies = [ "wit-bindgen-rt", ] @@ -5748,46 +5883,48 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5795,28 +5932,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" dependencies = [ "futures-util", "js-sys", @@ -5827,9 +5967,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -5854,14 +5994,14 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.34", + "rustix 0.38.44", ] [[package]] name = "whoami" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fec781d48b41f8163426ed18e8fc2864c12937df9ce54c88ede7bd47270893e" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" dependencies = [ "redox_syscall", "wasite", @@ -5886,11 +6026,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -5899,34 +6039,39 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + [[package]] name = "windows-registry" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", "windows-strings", - "windows-targets 0.52.6", + "windows-targets 0.53.0", ] [[package]] name = "windows-result" -version = "0.2.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" dependencies = [ - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "windows-strings" -version = "0.1.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-result", - "windows-targets 0.52.6", + "windows-link", ] [[package]] @@ -5980,13 +6125,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5999,6 +6160,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -6011,6 +6178,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -6023,12 +6196,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -6041,6 +6226,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -6053,6 +6244,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -6065,6 +6262,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -6077,35 +6280,35 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" -version = "0.5.35" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1931d78a9c73861da0134f453bb1f790ce49b2e30eba8410b4b79bac72b46a2d" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" dependencies = [ "memchr", ] [[package]] name = "wit-bindgen-rt" -version = "0.33.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "ws_stream_tungstenite" @@ -6115,7 +6318,7 @@ checksum = "e283cc794a890f5bdc01e358ad7c34535025f79ba83c1b5c7e01e5d6c60b336d" dependencies = [ "async-tungstenite 0.23.0", "async_io_stream", - "bitflags 2.8.0", + "bitflags 2.9.0", "futures-core", "futures-io", "futures-sink", @@ -6133,9 +6336,9 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed39ff9f8b2eda91bf6390f9f49eee93d655489e15708e3bb638c1c4f07cecb4" dependencies = [ - "async-tungstenite 0.28.0", + "async-tungstenite 0.28.2", "async_io_stream", - "bitflags 2.8.0", + "bitflags 2.9.0", "futures-core", "futures-io", "futures-sink", @@ -6225,9 +6428,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", @@ -6237,55 +6440,55 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] [[package]] name = "zerofrom" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", - "synstructure 0.13.1", + "syn 2.0.101", + "synstructure 0.13.2", ] [[package]] @@ -6294,11 +6497,22 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ "yoke", "zerofrom", @@ -6307,11 +6521,11 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.101", ] diff --git a/Cargo.toml b/Cargo.toml index 70db5731565..789b922128d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,7 +165,7 @@ regex = "1.4" reqwest = { version = "0.12", default-features = false } ron = "0.8" rpassword = "5.0" -rquickjs = { version = "0.9", default-features = false} +rquickjs = { version = "0.9", default-features = false } rstest = "0.16.0" rumqttc = { git = "https://github.com/jarhodes314/rumqtt", rev = "8c489faf6af910956c97b55587ff3ecb2ac4e96f" } rumqttd = "0.19" diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs index ceb8b1acb6c..091fb47542b 100644 --- a/crates/core/tedge_mapper/src/gen/mod.rs +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -15,8 +15,8 @@ impl TEdgeComponent for GenMapper { let (mut runtime, mut mqtt_actor) = start_basic_actors("tedge-gen-mapper", &tedge_config).await?; - let mut gen_mapper = GenMapperBuilder::default(); - gen_mapper.load("/etc/tedge/gen-mapper").await; + let mut gen_mapper = GenMapperBuilder::try_new("/etc/tedge/gen-mapper").await?; + gen_mapper.load().await; gen_mapper.connect(&mut mqtt_actor); runtime.spawn(gen_mapper).await?; diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index ff51e439ed7..32b302357e3 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -10,8 +10,10 @@ repository.workspace = true [dependencies] async-trait = { workspace = true } -camino = { workspace = true } +camino = { workspace = true, features = ["serde1"] } +rquickjs = { workspace = true, features = ["futures","parallel"] } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } tedge_actors = { workspace = true } tedge_mqtt_ext = { workspace = true } thiserror = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js b/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js new file mode 100644 index 00000000000..2c58927b104 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js @@ -0,0 +1,9 @@ +export function process (timestamp, message) { + let payload = JSON.parse(message.payload) + payload.time = Number(timestamp.seconds) + (timestamp.nanoseconds / 1e9) + + return [{ + topic: message.topic, + payload: JSON.stringify(payload) + }] +} diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml new file mode 100644 index 00000000000..2501140d5e0 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml @@ -0,0 +1,6 @@ +input_topics = ["collectd/+/+/+"] + +stages = [ + { filter = "collectd-to-te.js" }, + { filter = "group_by_timestamp.js" } +] diff --git a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml new file mode 100644 index 00000000000..0d5e06d55a0 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml @@ -0,0 +1,6 @@ +input_topics = ["te/+/+/+/+/m/+"] + +stages = [ + { filter = "add_timestamp.js" }, + { filter = "te_to_c8y.js" } +] diff --git a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js new file mode 100644 index 00000000000..9c7583a8bd3 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -0,0 +1,6 @@ +export function process(t,msg) { + msg.topic = "te/error" + return [msg] +} + + diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index ac72cd562d7..69cc6c9b77f 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,3 +1,4 @@ +use crate::js_filter::JsRuntime; use crate::pipeline::Pipeline; use async_trait::async_trait; use std::collections::HashMap; @@ -15,6 +16,7 @@ use tracing::error; pub struct GenMapper { pub(super) mqtt: SimpleMessageBox, pub(super) pipelines: HashMap, + pub(super) js_runtime: JsRuntime, } #[async_trait] @@ -47,7 +49,10 @@ impl GenMapper { async fn filter(&mut self, message: MqttMessage) -> Result<(), RuntimeError> { let timestamp = OffsetDateTime::now_utc(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - match pipeline.process(timestamp, &message) { + match pipeline + .process(&self.js_runtime, timestamp, &message) + .await + { Ok(messages) => { for message in messages { self.mqtt.send(message).await?; @@ -65,7 +70,7 @@ impl GenMapper { async fn tick(&mut self) -> Result<(), RuntimeError> { let timestamp = OffsetDateTime::now_utc(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - match pipeline.tick(timestamp) { + match pipeline.tick(&self.js_runtime, timestamp).await { Ok(messages) => { for message in messages { self.mqtt.send(message).await?; diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index a445729f86f..694fb901451 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -1,8 +1,10 @@ +use crate::js_filter::JsRuntime; use crate::pipeline::Pipeline; use crate::pipeline::Stage; -use crate::gen_filter::GenFilter; +use crate::LoadError; +use camino::Utf8PathBuf; use serde::Deserialize; -use std::path::PathBuf; +use std::path::Path; use tedge_mqtt_ext::TopicFilter; #[derive(Deserialize)] @@ -22,40 +24,48 @@ pub struct StageConfig { #[derive(Deserialize)] #[serde(untagged)] pub enum FilterSpec { - JavaScript(PathBuf), + JavaScript(Utf8PathBuf), } #[derive(thiserror::Error, Debug)] pub enum ConfigError { #[error("Not a valid MQTT topic filter: {0}")] IncorrectTopicFilter(String), -} -impl TryFrom for Pipeline { - type Error = ConfigError; + #[error(transparent)] + LoadError(#[from] LoadError), +} - fn try_from(config: PipelineConfig) -> Result { - let input = topic_filters(&config.input_topics)?; - let stages = config +impl PipelineConfig { + pub fn compile( + self, + js_runtime: &JsRuntime, + config_dir: &Path, + ) -> Result { + let input = topic_filters(&self.input_topics)?; + let stages = self .stages .into_iter() - .map(Stage::try_from) + .map(|stage| stage.compile(js_runtime, config_dir)) .collect::, _>>()?; - Ok(Pipeline { input_topics: input, stages }) + Ok(Pipeline { + input_topics: input, + stages, + }) } } -impl TryFrom for Stage { - type Error = ConfigError; - - fn try_from(config: StageConfig) -> Result { - let filter = match config.filter { - FilterSpec::JavaScript(path) => GenFilter::new(path), +impl StageConfig { + pub fn compile(self, js_runtime: &JsRuntime, config_dir: &Path) -> Result { + let path = match self.filter { + FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), + FilterSpec::JavaScript(path) => config_dir.join(path), }; - let config = topic_filters(&config.config_topics)?; + let filter = js_runtime.loaded_module(path)?; + let config_topics = topic_filters(&self.config_topics)?; Ok(Stage { - filter: Box::new(filter), - config_topics: config, + filter, + config_topics, }) } } diff --git a/crates/extensions/tedge_gen_mapper/src/gen_filter.rs b/crates/extensions/tedge_gen_mapper/src/gen_filter.rs deleted file mode 100644 index 038543c8340..00000000000 --- a/crates/extensions/tedge_gen_mapper/src/gen_filter.rs +++ /dev/null @@ -1,38 +0,0 @@ -use crate::pipeline::Filter; -use crate::pipeline::FilterError; -use std::path::PathBuf; -use tedge_mqtt_ext::MqttMessage; -use time::OffsetDateTime; -use tracing::debug; - -/// User-defined filter -pub struct GenFilter {} - -impl GenFilter { - pub fn new(path: impl Into) -> Self { - let path = path.into(); - debug!(target: "MAPPING", "new({path:?})"); - GenFilter {} - } -} - -impl Filter for GenFilter { - fn process( - &mut self, - timestamp: OffsetDateTime, - message: &MqttMessage, - ) -> Result, FilterError> { - debug!(target: "MAPPING", "process({timestamp}, {message:?})"); - Ok(vec![message.clone()]) - } - - fn update_config(&mut self, config: &MqttMessage) -> Result<(), FilterError> { - debug!(target: "MAPPING", "update_config({config:?})"); - Ok(()) - } - - fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError> { - debug!(target: "MAPPING", "tick({timestamp})"); - Ok(vec![]) - } -} diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs new file mode 100644 index 00000000000..906f3287068 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -0,0 +1,250 @@ +use crate::pipeline; +use crate::pipeline::FilterError; +use crate::LoadError; +use rquickjs::Ctx; +use rquickjs::FromJs; +use rquickjs::IntoJs; +use rquickjs::Object; +use rquickjs::Value; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use tedge_mqtt_ext::MqttMessage; +use time::OffsetDateTime; +use tracing::debug; + +#[derive(serde::Deserialize, serde::Serialize)] +pub struct DateTime { + seconds: u64, + nanoseconds: u32, +} + +#[derive(serde::Deserialize, serde::Serialize)] +pub struct Message { + topic: String, + payload: String, +} + +#[derive(Clone)] +pub struct JsFilter { + path: PathBuf, +} + +impl JsFilter { + pub async fn process( + &self, + js: &JsRuntime, + timestamp: OffsetDateTime, + message: &MqttMessage, + ) -> Result, FilterError> { + debug!(target: "MAPPING", "{}: process({timestamp}, {message:?})", self.path.display()); + let timestamp = DateTime::try_from(timestamp)?; + let message = Message::try_from(message)?; + let input = (timestamp, message); + let output: Vec = js + .call_function(&self, "process", input) + .await + .map_err(error_from_js)?; + output.into_iter().map(MqttMessage::try_from).collect() + } + + pub fn update_config(&self, _js: &JsRuntime, config: &MqttMessage) -> Result<(), FilterError> { + debug!(target: "MAPPING", "{}: update_config({config:?})", self.path.display()); + Ok(()) + } + + pub fn tick( + &self, + _js: &JsRuntime, + timestamp: OffsetDateTime, + ) -> Result, FilterError> { + debug!(target: "MAPPING", "{}: tick({timestamp})", self.path.display()); + Ok(vec![]) + } +} + +pub struct JsRuntime { + context: rquickjs::AsyncContext, + modules: HashMap>, +} + +impl JsRuntime { + pub async fn try_new() -> Result { + let runtime = rquickjs::AsyncRuntime::new()?; + let context = rquickjs::AsyncContext::full(&runtime).await?; + let modules = HashMap::new(); + Ok(JsRuntime { context, modules }) + } + + pub async fn load_file(&mut self, path: impl AsRef) -> Result { + let path = path.as_ref(); + let source = tokio::fs::read_to_string(path).await?; + self.load_js(path, source) + } + + pub fn load_js( + &mut self, + path: impl AsRef, + source: impl Into>, + ) -> Result { + let path = path.as_ref().to_path_buf(); + self.modules.insert(path.clone(), source.into()); + Ok(JsFilter { path }) + } + + pub fn loaded_module(&self, path: PathBuf) -> Result { + match self.modules.get(&path) { + None => Err(LoadError::ScriptNotLoaded { path }), + Some(_) => Ok(JsFilter { path }), + } + } + + pub async fn call_function( + &self, + module: &JsFilter, + function: &str, + args: Args, + ) -> Result + where + for<'a> Args: rquickjs::function::IntoArgs<'a> + Send + 'a, + for<'a> Ret: rquickjs::FromJs<'a> + Send + 'a, + { + let Some(source) = self.modules.get(&module.path) else { + return Err(LoadError::ScriptNotLoaded { + path: module.path.clone(), + }); + }; + + let name = module.path.display().to_string(); + + rquickjs::async_with!(self.context => |ctx| { + let m = rquickjs::Module::declare(ctx, name, source.clone())?; + let (m,p) = m.eval()?; + let () = p.finish()?; + + let f: rquickjs::Value = m.get(function)?; + let f = rquickjs::Function::from_value(f)?; + let r = f.call(args)?; + Ok(r) + }) + .await + } +} + +impl TryFrom for DateTime { + type Error = FilterError; + + fn try_from(value: OffsetDateTime) -> Result { + let seconds = u64::try_from(value.unix_timestamp()).map_err(|err| { + FilterError::UnsupportedMessage(format!("failed to convert timestamp: {}", err)) + })?; + + Ok(DateTime { + seconds, + nanoseconds: value.nanosecond(), + }) + } +} + +impl TryFrom<&MqttMessage> for Message { + type Error = FilterError; + + fn try_from(message: &MqttMessage) -> Result { + let topic = message.topic.to_string(); + let payload = message + .payload_str() + .map_err(|_| { + pipeline::FilterError::UnsupportedMessage("Not an UTF8 payload".to_string()) + })? + .to_string(); + Ok(Message { topic, payload }) + } +} + +impl TryFrom for MqttMessage { + type Error = FilterError; + + fn try_from(message: Message) -> Result { + let topic = message.topic.as_str().try_into().map_err(|_| { + FilterError::UnsupportedMessage(format!("invalid topic {}", message.topic)) + })?; + Ok(MqttMessage::new(&topic, message.payload)) + } +} + +impl<'js> FromJs<'js> for Message { + fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { + match value.as_object() { + None => Ok(Message { + topic: "".to_string(), + payload: "".to_string(), + }), + Some(object) => Ok(Message { + topic: object.get("topic")?, + payload: object.get("payload")?, + }), + } + } +} + +impl<'js> IntoJs<'js> for Message { + fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { + let msg = Object::new(ctx.clone())?; + msg.set("topic", self.topic)?; + msg.set("payload", self.payload)?; + Ok(Value::from_object(msg)) + } +} + +impl<'js> IntoJs<'js> for DateTime { + fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { + let msg = Object::new(ctx.clone())?; + msg.set("topic", self.seconds)?; + msg.set("payload", self.nanoseconds)?; + Ok(Value::from_object(msg)) + } +} + +fn error_from_js(err: LoadError) -> FilterError { + FilterError::IncorrectSetting(format!("{}", err)) +} + +#[cfg(test)] +mod tests { + use super::*; + use tedge_mqtt_ext::Topic; + + #[tokio::test] + async fn identity_filter() { + let script = "export function process(t,msg) { return [msg]; };"; + let mut runtime = JsRuntime::try_new().await.unwrap(); + let filter = runtime.load_js("id.js", script).unwrap(); + + let topic = Topic::new_unchecked("te/main/device///m/"); + let input = MqttMessage::new(&topic, "hello world"); + let output = input.clone(); + assert_eq!( + filter + .process(&runtime, OffsetDateTime::now_utc(), &input) + .await + .unwrap(), + vec![output] + ); + } + + #[tokio::test] + async fn error_filter() { + let script = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; + let mut runtime = JsRuntime::try_new().await.unwrap(); + let filter = runtime.load_js("err.js", script).unwrap(); + + let topic = Topic::new_unchecked("te/main/device///m/"); + let input = MqttMessage::new(&topic, "hello world"); + let error = filter + .process(&runtime, OffsetDateTime::now_utc(), &input) + .await + .unwrap_err(); + eprintln!("{:?}", error); + assert!(error.to_string().contains("Exception generated by QuickJS")); + } +} diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index b5aee325715..8019c489ae2 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -1,14 +1,17 @@ mod actor; mod config; +mod js_filter; mod pipeline; -mod gen_filter; use crate::actor::GenMapper; +use crate::config::PipelineConfig; +use crate::js_filter::JsRuntime; use crate::pipeline::Pipeline; use camino::Utf8Path; use std::collections::HashMap; use std::convert::Infallible; use std::path::Path; +use std::path::PathBuf; use tedge_actors::Builder; use tedge_actors::DynSender; use tedge_actors::MessageSink; @@ -25,24 +28,29 @@ use tracing::error; use tracing::info; pub struct GenMapperBuilder { + config_dir: PathBuf, message_box: SimpleMessageBoxBuilder, pipelines: HashMap, + pipeline_specs: HashMap, + js_runtime: JsRuntime, } -impl Default for GenMapperBuilder { - fn default() -> Self { - GenMapperBuilder { +impl GenMapperBuilder { + pub async fn try_new(config_dir: impl AsRef) -> Result { + let config_dir = config_dir.as_ref().to_owned(); + let js_runtime = JsRuntime::try_new().await?; + Ok(GenMapperBuilder { + config_dir, message_box: SimpleMessageBoxBuilder::new("GenMapper", 16), pipelines: HashMap::default(), - } + pipeline_specs: HashMap::default(), + js_runtime, + }) } -} -impl GenMapperBuilder { - pub async fn load(&mut self, config_dir: impl AsRef) { - let config_dir = config_dir.as_ref(); - let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| - error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) + pub async fn load(&mut self) { + let Ok(mut entries) = read_dir(&self.config_dir).await.map_err(|err| + error!(target: "MAPPING", "Failed to read filters from {}: {err}", self.config_dir.display()) ) else { return; }; @@ -53,26 +61,61 @@ impl GenMapperBuilder { continue; }; if let Ok(file_type) = entry.file_type().await { - if file_type.is_file() && path.extension() == Some("toml") { - info!(target: "MAPPING", "Loading pipeline: {path}"); - if let Err(err) = self.load_pipeline(path).await { - error!(target: "MAPPING", "Failed to load pipeline: {err}"); + if file_type.is_file() { + match path.extension() { + Some("toml") => { + info!(target: "MAPPING", "Loading pipeline: {path}"); + if let Err(err) = self.load_pipeline(path).await { + error!(target: "MAPPING", "Failed to load pipeline: {err}"); + } + } + Some("js") | Some("ts") => { + info!(target: "MAPPING", "Loading filter: {path}"); + if let Err(err) = self.load_filter(path).await { + error!(target: "MAPPING", "Failed to load filter: {err}"); + } + } + _ => { + info!(target: "MAPPING", "Skipping file which type is unknown: {path}"); + } } } } } + + // Done here to ease the computation of the topics to subscribe to + // as these topics have to be known when connect is called + self.compile() } async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { if let Some(name) = file.as_ref().file_name() { let specs = read_to_string(file.as_ref()).await?; - let pipeline: Pipeline = toml::from_str(&specs)?; - self.pipelines.insert(name.to_string(), pipeline); + let pipeline: PipelineConfig = toml::from_str(&specs)?; + self.pipeline_specs.insert(name.to_string(), pipeline); } Ok(()) } + async fn load_filter(&mut self, file: impl AsRef) -> Result<(), LoadError> { + self.js_runtime.load_file(file.as_ref()).await?; + Ok(()) + } + + fn compile(&mut self) { + for (name, specs) in self.pipeline_specs.drain() { + match specs.compile(&self.js_runtime, &self.config_dir) { + Ok(pipeline) => { + let _ = self.pipelines.insert(name, pipeline); + } + Err(err) => { + error!(target: "MAPPING", "Failed to compile pipeline {name}: {err}") + } + } + } + } + pub fn connect( &mut self, mqtt: &mut (impl MessageSource + MessageSink), @@ -107,15 +150,22 @@ impl Builder for GenMapperBuilder { GenMapper { mqtt: self.message_box.build(), pipelines: self.pipelines, + js_runtime: self.js_runtime, } } } #[derive(thiserror::Error, Debug)] pub enum LoadError { + #[error("Script not loaded: {path}")] + ScriptNotLoaded { path: PathBuf }, + #[error(transparent)] IoError(#[from] std::io::Error), #[error(transparent)] TomlError(#[from] toml::de::Error), + + #[error(transparent)] + JsError(#[from] rquickjs::Error), } diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 03adc975276..782fd577429 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -1,11 +1,10 @@ -use serde::Deserialize; +use crate::js_filter::JsFilter; +use crate::js_filter::JsRuntime; use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::TopicFilter; use time::OffsetDateTime; /// A chain of transformation of MQTT messages -#[derive(Deserialize)] -#[serde(try_from = "crate::config::PipelineConfig")] pub struct Pipeline { /// The source topics pub input_topics: TopicFilter, @@ -16,34 +15,10 @@ pub struct Pipeline { /// A message transformation stage pub struct Stage { - pub filter: Box, + pub filter: JsFilter, pub config_topics: TopicFilter, } -/// A filter process a stream of messages, producing a stream of transformed messages -/// -/// Filters are chained along pipelines, consuming MQTT messages as input -/// and producing MQTT messages as output. -/// -/// The behavior of a filter can be time related and -/// -/// Filters are dynamically configured. New partial configuration updates are sent overtime, -/// giving the opportunity for a filter to adapt its behavior. -pub trait Filter: 'static + Send + Sync { - /// Process a single message; producing zero, one or more transformed messages - fn process( - &mut self, - timestamp: OffsetDateTime, - message: &MqttMessage, - ) -> Result, FilterError>; - - /// Update the filter configuration - fn update_config(&mut self, config: &MqttMessage) -> Result<(), FilterError>; - - /// Close the current time-window; producing zero, one or more accumulated messages - fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError>; -} - #[derive(thiserror::Error, Debug)] pub enum FilterError { #[error("Input message cannot be processed: {0}")] @@ -62,21 +37,26 @@ impl Pipeline { topics } - pub fn update_config(&mut self, message: &MqttMessage) -> Result<(), FilterError> { + pub fn update_config( + &mut self, + js_runtime: &JsRuntime, + message: &MqttMessage, + ) -> Result<(), FilterError> { for stage in self.stages.iter_mut() { if stage.config_topics.accept(message) { - stage.filter.update_config(message)? + stage.filter.update_config(js_runtime, message)? } } Ok(()) } - pub fn process( + pub async fn process( &mut self, + js_runtime: &JsRuntime, timestamp: OffsetDateTime, message: &MqttMessage, ) -> Result, FilterError> { - self.update_config(message)?; + self.update_config(js_runtime, message)?; if !self.input_topics.accept(message) { return Ok(vec![]); } @@ -84,10 +64,8 @@ impl Pipeline { let mut messages = vec![message.clone()]; for stage in self.stages.iter_mut() { let mut transformed_messages = vec![]; - for filter_output in messages - .iter() - .map(|message| stage.filter.process(timestamp, message)) - { + for message in messages.iter() { + let filter_output = stage.filter.process(js_runtime, timestamp, message).await; transformed_messages.extend(filter_output?); } messages = transformed_messages; @@ -95,20 +73,22 @@ impl Pipeline { Ok(messages) } - pub fn tick(&mut self, timestamp: OffsetDateTime) -> Result, FilterError> { + pub async fn tick( + &mut self, + js_runtime: &JsRuntime, + timestamp: OffsetDateTime, + ) -> Result, FilterError> { let mut messages = vec![]; for stage in self.stages.iter_mut() { // Process first the messages triggered upstream by the tick let mut transformed_messages = vec![]; - for filter_output in messages - .iter() - .map(|message| stage.filter.process(timestamp, message)) - { + for message in messages.iter() { + let filter_output = stage.filter.process(js_runtime, timestamp, message).await; transformed_messages.extend(filter_output?); } // Only then process the tick - transformed_messages.extend(stage.filter.tick(timestamp)?); + transformed_messages.extend(stage.filter.tick(js_runtime, timestamp)?); // Iterate with all the messages collected at this stage messages = transformed_messages; From a63206ff64da5a4de9b98a6a7187486141075e01 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 16 May 2025 17:10:48 +0200 Subject: [PATCH 04/53] Update deprecated call tempfile into_path -> keep Signed-off-by: Didier Wenzek --- crates/core/plugin_sm/src/operation_logs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/plugin_sm/src/operation_logs.rs b/crates/core/plugin_sm/src/operation_logs.rs index 0f5ad92383d..424dfab61ba 100644 --- a/crates/core/plugin_sm/src/operation_logs.rs +++ b/crates/core/plugin_sm/src/operation_logs.rs @@ -152,7 +152,7 @@ mod tests { let unrelated_2 = create_file(log_dir.path(), "bar"); // Open the log dir - let _operation_logs = OperationLogs::try_new(log_dir.into_path().try_into().unwrap())?; + let _operation_logs = OperationLogs::try_new(log_dir.keep().try_into().unwrap())?; // Outdated logs are removed assert!(!update_log_1.exists()); From f0926352019d69336b1df25faafe6d812b0dcc19 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 19 May 2025 12:25:44 +0200 Subject: [PATCH 05/53] Avoid to deserialize MQTT at each pipeline stage. Signed-off-by: Didier Wenzek --- crates/common/mqtt_channel/src/topics.rs | 9 +- .../extensions/tedge_gen_mapper/src/actor.rs | 34 ++++-- .../tedge_gen_mapper/src/js_filter.rs | 102 +++--------------- .../tedge_gen_mapper/src/pipeline.rs | 102 +++++++++++++++--- 4 files changed, 138 insertions(+), 109 deletions(-) diff --git a/crates/common/mqtt_channel/src/topics.rs b/crates/common/mqtt_channel/src/topics.rs index 35885660eee..cad07cc2e6e 100644 --- a/crates/common/mqtt_channel/src/topics.rs +++ b/crates/common/mqtt_channel/src/topics.rs @@ -117,10 +117,15 @@ impl TopicFilter { } /// Check if the given topic matches this filter pattern. - pub fn accept_topic(&self, topic: &Topic) -> bool { + pub fn accept_topic_name(&self, topic: &str) -> bool { self.patterns .iter() - .any(|pattern| rumqttc::matches(&topic.name, pattern)) + .any(|pattern| rumqttc::matches(topic, pattern)) + } + + /// Check if the given topic matches this filter pattern. + pub fn accept_topic(&self, topic: &Topic) -> bool { + self.accept_topic_name(&topic.name) } /// Check if the given message matches this filter pattern. diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 69cc6c9b77f..084ef835573 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,4 +1,6 @@ use crate::js_filter::JsRuntime; +use crate::pipeline::DateTime; +use crate::pipeline::Message; use crate::pipeline::Pipeline; use async_trait::async_trait; use std::collections::HashMap; @@ -8,7 +10,6 @@ use tedge_actors::RuntimeError; use tedge_actors::Sender; use tedge_actors::SimpleMessageBox; use tedge_mqtt_ext::MqttMessage; -use time::OffsetDateTime; use tokio::time::interval; use tokio::time::Duration; use tracing::error; @@ -34,8 +35,11 @@ impl Actor for GenMapper { self.tick().await?; } message = self.mqtt.recv() => { - match message { - Some(message) => self.filter(message).await?, + match message.map(Message::try_from) { + Some(Ok(message)) => self.filter(message).await?, + Some(Err(err)) => { + error!(target: "gen-mapper", "Cannot process message: {err}"); + }, None => break, } } @@ -46,16 +50,21 @@ impl Actor for GenMapper { } impl GenMapper { - async fn filter(&mut self, message: MqttMessage) -> Result<(), RuntimeError> { - let timestamp = OffsetDateTime::now_utc(); + async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { + let timestamp = DateTime::now(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { match pipeline - .process(&self.js_runtime, timestamp, &message) + .process(&self.js_runtime, ×tamp, &message) .await { Ok(messages) => { for message in messages { - self.mqtt.send(message).await?; + match MqttMessage::try_from(message) { + Ok(message) => self.mqtt.send(message).await?, + Err(err) => { + error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") + } + } } } Err(err) => { @@ -68,12 +77,17 @@ impl GenMapper { } async fn tick(&mut self) -> Result<(), RuntimeError> { - let timestamp = OffsetDateTime::now_utc(); + let timestamp = DateTime::now(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - match pipeline.tick(&self.js_runtime, timestamp).await { + match pipeline.tick(&self.js_runtime, ×tamp).await { Ok(messages) => { for message in messages { - self.mqtt.send(message).await?; + match MqttMessage::try_from(message) { + Ok(message) => self.mqtt.send(message).await?, + Err(err) => { + error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") + } + } } } Err(err) => { diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 906f3287068..1691e2d352a 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -1,5 +1,7 @@ use crate::pipeline; +use crate::pipeline::DateTime; use crate::pipeline::FilterError; +use crate::pipeline::Message; use crate::LoadError; use rquickjs::Ctx; use rquickjs::FromJs; @@ -9,22 +11,8 @@ use rquickjs::Value; use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; -use tedge_mqtt_ext::MqttMessage; -use time::OffsetDateTime; use tracing::debug; -#[derive(serde::Deserialize, serde::Serialize)] -pub struct DateTime { - seconds: u64, - nanoseconds: u32, -} - -#[derive(serde::Deserialize, serde::Serialize)] -pub struct Message { - topic: String, - payload: String, -} - #[derive(Clone)] pub struct JsFilter { path: PathBuf, @@ -34,31 +22,23 @@ impl JsFilter { pub async fn process( &self, js: &JsRuntime, - timestamp: OffsetDateTime, - message: &MqttMessage, - ) -> Result, FilterError> { - debug!(target: "MAPPING", "{}: process({timestamp}, {message:?})", self.path.display()); - let timestamp = DateTime::try_from(timestamp)?; - let message = Message::try_from(message)?; - let input = (timestamp, message); - let output: Vec = js - .call_function(&self, "process", input) + timestamp: &DateTime, + message: &Message, + ) -> Result, FilterError> { + debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.path.display()); + let input = (timestamp.clone(), message.clone()); + js.call_function(&self, "process", input) .await - .map_err(error_from_js)?; - output.into_iter().map(MqttMessage::try_from).collect() + .map_err(pipeline::error_from_js) } - pub fn update_config(&self, _js: &JsRuntime, config: &MqttMessage) -> Result<(), FilterError> { + pub fn update_config(&self, _js: &JsRuntime, config: &Message) -> Result<(), FilterError> { debug!(target: "MAPPING", "{}: update_config({config:?})", self.path.display()); Ok(()) } - pub fn tick( - &self, - _js: &JsRuntime, - timestamp: OffsetDateTime, - ) -> Result, FilterError> { - debug!(target: "MAPPING", "{}: tick({timestamp})", self.path.display()); + pub fn tick(&self, _js: &JsRuntime, timestamp: &DateTime) -> Result, FilterError> { + debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.path.display()); Ok(vec![]) } } @@ -107,7 +87,7 @@ impl JsRuntime { ) -> Result where for<'a> Args: rquickjs::function::IntoArgs<'a> + Send + 'a, - for<'a> Ret: rquickjs::FromJs<'a> + Send + 'a, + for<'a> Ret: FromJs<'a> + Send + 'a, { let Some(source) = self.modules.get(&module.path) else { return Err(LoadError::ScriptNotLoaded { @@ -131,47 +111,6 @@ impl JsRuntime { } } -impl TryFrom for DateTime { - type Error = FilterError; - - fn try_from(value: OffsetDateTime) -> Result { - let seconds = u64::try_from(value.unix_timestamp()).map_err(|err| { - FilterError::UnsupportedMessage(format!("failed to convert timestamp: {}", err)) - })?; - - Ok(DateTime { - seconds, - nanoseconds: value.nanosecond(), - }) - } -} - -impl TryFrom<&MqttMessage> for Message { - type Error = FilterError; - - fn try_from(message: &MqttMessage) -> Result { - let topic = message.topic.to_string(); - let payload = message - .payload_str() - .map_err(|_| { - pipeline::FilterError::UnsupportedMessage("Not an UTF8 payload".to_string()) - })? - .to_string(); - Ok(Message { topic, payload }) - } -} - -impl TryFrom for MqttMessage { - type Error = FilterError; - - fn try_from(message: Message) -> Result { - let topic = message.topic.as_str().try_into().map_err(|_| { - FilterError::UnsupportedMessage(format!("invalid topic {}", message.topic)) - })?; - Ok(MqttMessage::new(&topic, message.payload)) - } -} - impl<'js> FromJs<'js> for Message { fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { match value.as_object() { @@ -205,14 +144,9 @@ impl<'js> IntoJs<'js> for DateTime { } } -fn error_from_js(err: LoadError) -> FilterError { - FilterError::IncorrectSetting(format!("{}", err)) -} - #[cfg(test)] mod tests { use super::*; - use tedge_mqtt_ext::Topic; #[tokio::test] async fn identity_filter() { @@ -220,12 +154,11 @@ mod tests { let mut runtime = JsRuntime::try_new().await.unwrap(); let filter = runtime.load_js("id.js", script).unwrap(); - let topic = Topic::new_unchecked("te/main/device///m/"); - let input = MqttMessage::new(&topic, "hello world"); + let input = Message::new("te/main/device///m/", "hello world"); let output = input.clone(); assert_eq!( filter - .process(&runtime, OffsetDateTime::now_utc(), &input) + .process(&runtime, &DateTime::now(), &input) .await .unwrap(), vec![output] @@ -238,10 +171,9 @@ mod tests { let mut runtime = JsRuntime::try_new().await.unwrap(); let filter = runtime.load_js("err.js", script).unwrap(); - let topic = Topic::new_unchecked("te/main/device///m/"); - let input = MqttMessage::new(&topic, "hello world"); + let input = Message::new("te/main/device///m/", "hello world"); let error = filter - .process(&runtime, OffsetDateTime::now_utc(), &input) + .process(&runtime, &DateTime::now(), &input) .await .unwrap_err(); eprintln!("{:?}", error); diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 782fd577429..90f756a2e09 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -1,5 +1,8 @@ use crate::js_filter::JsFilter; use crate::js_filter::JsRuntime; +use crate::LoadError; +use serde_json::json; +use serde_json::Value; use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::TopicFilter; use time::OffsetDateTime; @@ -19,6 +22,18 @@ pub struct Stage { pub config_topics: TopicFilter, } +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] +pub struct DateTime { + pub(crate) seconds: u64, + pub(crate) nanoseconds: u32, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] +pub struct Message { + pub(crate) topic: String, + pub(crate) payload: String, +} + #[derive(thiserror::Error, Debug)] pub enum FilterError { #[error("Input message cannot be processed: {0}")] @@ -38,12 +53,12 @@ impl Pipeline { } pub fn update_config( - &mut self, + &self, js_runtime: &JsRuntime, - message: &MqttMessage, + message: &Message, ) -> Result<(), FilterError> { - for stage in self.stages.iter_mut() { - if stage.config_topics.accept(message) { + for stage in self.stages.iter() { + if stage.config_topics.accept_topic_name(&message.topic) { stage.filter.update_config(js_runtime, message)? } } @@ -53,16 +68,16 @@ impl Pipeline { pub async fn process( &mut self, js_runtime: &JsRuntime, - timestamp: OffsetDateTime, - message: &MqttMessage, - ) -> Result, FilterError> { + timestamp: &DateTime, + message: &Message, + ) -> Result, FilterError> { self.update_config(js_runtime, message)?; - if !self.input_topics.accept(message) { + if !self.input_topics.accept_topic_name(&message.topic) { return Ok(vec![]); } let mut messages = vec![message.clone()]; - for stage in self.stages.iter_mut() { + for stage in self.stages.iter() { let mut transformed_messages = vec![]; for message in messages.iter() { let filter_output = stage.filter.process(js_runtime, timestamp, message).await; @@ -76,10 +91,10 @@ impl Pipeline { pub async fn tick( &mut self, js_runtime: &JsRuntime, - timestamp: OffsetDateTime, - ) -> Result, FilterError> { + timestamp: &DateTime, + ) -> Result, FilterError> { let mut messages = vec![]; - for stage in self.stages.iter_mut() { + for stage in self.stages.iter() { // Process first the messages triggered upstream by the tick let mut transformed_messages = vec![]; for message in messages.iter() { @@ -96,3 +111,66 @@ impl Pipeline { Ok(messages) } } + +impl DateTime { + pub fn now() -> Self { + DateTime::try_from(OffsetDateTime::now_utc()).unwrap() + } +} + +impl TryFrom for DateTime { + type Error = FilterError; + + fn try_from(value: OffsetDateTime) -> Result { + let seconds = u64::try_from(value.unix_timestamp()).map_err(|err| { + FilterError::UnsupportedMessage(format!("failed to convert timestamp: {}", err)) + })?; + + Ok(DateTime { + seconds, + nanoseconds: value.nanosecond(), + }) + } +} + +impl Message { + #[cfg(test)] + pub(crate) fn new(topic: &str, payload: &str) -> Self { + Message { + topic: topic.to_string(), + payload: payload.to_string(), + } + } + + pub fn json(&self) -> Value { + json!({"topic": self.topic, "payload": self.payload}) + } +} + +impl TryFrom for Message { + type Error = FilterError; + + fn try_from(message: MqttMessage) -> Result { + let topic = message.topic.to_string(); + let payload = message + .payload_str() + .map_err(|_| FilterError::UnsupportedMessage("Not an UTF8 payload".to_string()))? + .to_string(); + Ok(Message { topic, payload }) + } +} + +impl TryFrom for MqttMessage { + type Error = FilterError; + + fn try_from(message: Message) -> Result { + let topic = message.topic.as_str().try_into().map_err(|_| { + FilterError::UnsupportedMessage(format!("invalid topic {}", message.topic)) + })?; + Ok(MqttMessage::new(&topic, message.payload)) + } +} + +pub fn error_from_js(err: LoadError) -> FilterError { + FilterError::IncorrectSetting(format!("{}", err)) +} From 5871c215c8d7dc77711a36ad7f481601723b5412 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 19 May 2025 16:17:10 +0200 Subject: [PATCH 06/53] Add JS example: drop stragglers Signed-off-by: Didier Wenzek --- .../pipelines/add_timestamp.js | 4 ++- .../pipelines/drop_stragglers.js | 26 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js b/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js index 2c58927b104..f387141bfe3 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js @@ -1,6 +1,8 @@ export function process (timestamp, message) { let payload = JSON.parse(message.payload) - payload.time = Number(timestamp.seconds) + (timestamp.nanoseconds / 1e9) + if (!payload.time) { + payload.time = timestamp.seconds + (timestamp.nanoseconds / 1e9) + } return [{ topic: message.topic, diff --git a/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js b/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js new file mode 100644 index 00000000000..0b50afcfb46 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js @@ -0,0 +1,26 @@ +// Reject any message that is too old, too new or with no timestamp +export function process (timestamp, message, config) { + let payload = JSON.parse(message.payload) + let msg_time = payload.time + if (!msg_time) { + return [] + } + if (!config) { + config = {} + } + + let msg_timestamp = msg_time + if (typeof(msg_time) === "string") { + msg_timestamp = Date.parse(msg_time) / 1e3 + } + + let time = timestamp.seconds + (timestamp.nanoseconds / 1e9) + let max = time + (config?.max_advance || 1); + let min = time - (config?.max_delay || 10); + + if (min <= msg_timestamp && msg_timestamp <= max) { + return [message] + } else { + return [{"topic":" te/error", "payload":`straggler rejected on ${message.topic} with time=${msg_timestamp}`}] + } +} From 88ad9c4532a6d139573de17a1b3be1d27ab1dba1 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 19 May 2025 17:42:20 +0200 Subject: [PATCH 07/53] Add ability to configure each stages Signed-off-by: Didier Wenzek --- .../pipelines/drop_stragglers.js | 5 +- .../pipelines/measurements.toml | 1 + .../extensions/tedge_gen_mapper/src/config.rs | 6 +- .../tedge_gen_mapper/src/js_filter.rs | 67 ++++++++++++++++++- 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js b/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js index 0b50afcfb46..5ac019db494 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js @@ -5,9 +5,6 @@ export function process (timestamp, message, config) { if (!msg_time) { return [] } - if (!config) { - config = {} - } let msg_timestamp = msg_time if (typeof(msg_time) === "string") { @@ -21,6 +18,6 @@ export function process (timestamp, message, config) { if (min <= msg_timestamp && msg_timestamp <= max) { return [message] } else { - return [{"topic":" te/error", "payload":`straggler rejected on ${message.topic} with time=${msg_timestamp}`}] + return [{"topic":" te/error", "payload":`straggler rejected on ${message.topic} with time=${msg_timestamp} at ${time}`}] } } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml index 0d5e06d55a0..fea27e58da2 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml @@ -2,5 +2,6 @@ input_topics = ["te/+/+/+/+/m/+"] stages = [ { filter = "add_timestamp.js" }, + { filter = "drop_stragglers.js", config = { max_delay = 60 } }, { filter = "te_to_c8y.js" } ] diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index 694fb901451..cdba92cf6a3 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -4,6 +4,7 @@ use crate::pipeline::Stage; use crate::LoadError; use camino::Utf8PathBuf; use serde::Deserialize; +use serde_json::Value; use std::path::Path; use tedge_mqtt_ext::TopicFilter; @@ -17,6 +18,9 @@ pub struct PipelineConfig { pub struct StageConfig { filter: FilterSpec, + #[serde(default)] + config: Option, + #[serde(default)] config_topics: Vec, } @@ -61,7 +65,7 @@ impl StageConfig { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; - let filter = js_runtime.loaded_module(path)?; + let filter = js_runtime.loaded_module(path)?.with_config(self.config); let config_topics = topic_filters(&self.config_topics)?; Ok(Stage { filter, diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 1691e2d352a..de0edafcf0e 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -16,9 +16,31 @@ use tracing::debug; #[derive(Clone)] pub struct JsFilter { path: PathBuf, + config: JsonValue, } +#[derive(Clone, Default)] +pub struct JsonValue(serde_json::Value); + impl JsFilter { + pub fn new(path: PathBuf) -> Self { + JsFilter { + path, + config: JsonValue::default(), + } + } + + pub fn with_config(self, config: Option) -> Self { + if let Some(config) = config { + Self { + config: JsonValue(config), + ..self + } + } else { + self + } + } + pub async fn process( &self, js: &JsRuntime, @@ -26,7 +48,7 @@ impl JsFilter { message: &Message, ) -> Result, FilterError> { debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.path.display()); - let input = (timestamp.clone(), message.clone()); + let input = (timestamp.clone(), message.clone(), self.config.clone()); js.call_function(&self, "process", input) .await .map_err(pipeline::error_from_js) @@ -69,13 +91,13 @@ impl JsRuntime { ) -> Result { let path = path.as_ref().to_path_buf(); self.modules.insert(path.clone(), source.into()); - Ok(JsFilter { path }) + Ok(JsFilter::new(path)) } pub fn loaded_module(&self, path: PathBuf) -> Result { match self.modules.get(&path) { None => Err(LoadError::ScriptNotLoaded { path }), - Some(_) => Ok(JsFilter { path }), + Some(_) => Ok(JsFilter::new(path)), } } @@ -144,6 +166,45 @@ impl<'js> IntoJs<'js> for DateTime { } } +impl<'js> IntoJs<'js> for JsonValue { + fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { + match self.0 { + serde_json::Value::Null => Ok(Value::new_null(ctx.clone())), + serde_json::Value::Bool(value) => Ok(Value::new_bool(ctx.clone(), value)), + serde_json::Value::Number(value) => { + if let Some(n) = value.as_i64() { + if let Ok(n) = i32::try_from(n) { + return Ok(Value::new_int(ctx.clone(), n)); + } + } + if let Some(f) = value.as_f64() { + return Ok(Value::new_float(ctx.clone(), f)); + } + let nan = rquickjs::String::from_str(ctx.clone(), "NaN")?; + Ok(nan.into_value()) + } + serde_json::Value::String(value) => { + let string = rquickjs::String::from_str(ctx.clone(), &value)?; + Ok(string.into_value()) + } + serde_json::Value::Array(values) => { + let array = rquickjs::Array::new(ctx.clone())?; + for (i, value) in values.into_iter().enumerate() { + array.set(i, JsonValue(value))?; + } + Ok(array.into_value()) + } + serde_json::Value::Object(values) => { + let object = rquickjs::Object::new(ctx.clone())?; + for (key, value) in values.into_iter() { + object.set(key, JsonValue(value))?; + } + Ok(object.into_value()) + } + } + } +} + #[cfg(test)] mod tests { use super::*; From cb656fec3d00070e0b5c973c556272c26ae8f58b Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 20 May 2025 10:47:15 +0200 Subject: [PATCH 08/53] Add JS example: collectd pipeline Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/pipelines/collectd-to-te.js | 14 ++++++++++++++ .../pipelines/group_by_timestamp.js | 16 ++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js new file mode 100644 index 00000000000..e5aa71c0540 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js @@ -0,0 +1,14 @@ +export function process (timestamp, message, config) { + let groups = message.topic.split( '/') + let data = message.payload.split(':') + + let group = groups[2] + let measurement = groups[3] + let time = data[0] + let value = data[1] + + return [ { + topic: config.topic || "te/device/main///m/collectd", + payload: `{"time": ${time}, "${group}": {"${measurement}": ${value}}}` + }] +} \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js b/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js new file mode 100644 index 00000000000..ffa282ff475 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js @@ -0,0 +1,16 @@ +// Demonstrate that messages can be delayed +export function process (timestamp, message, config) { + if ( typeof process.batch == 'undefined' ) { + process.batch = []; + } + + let len = process.batch.push(message) + let batch_len = config.batch_len || 4 + if (len < batch_len) { + return [] + } + + let batch = process.batch + process.batch = [] + return batch +} \ No newline at end of file From a3a140feda499150e01a7a96e78125a4a04d0daa Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 20 May 2025 14:00:03 +0200 Subject: [PATCH 09/53] Add JS example: thin-edge JSON to c8y JSON Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/pipelines/te_to_c8y.js | 77 ++++++++++++++++++- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js index 9c7583a8bd3..fc2968db926 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -1,6 +1,75 @@ -export function process(t,msg) { - msg.topic = "te/error" - return [msg] -} +/// Transform: +/// +/// ``` +/// [te/device/main///m/example] { +/// "time": "2020-10-15T05:30:47+00:00", +/// "temperature": 25, +/// "location": { +/// "latitude": 32.54, +/// "longitude": -117.67, +/// "altitude": 98.6 +/// }, +/// "pressure": 98 +/// } +/// ``` +/// +/// into +/// +/// ``` +/// [c8y/measurement/measurements/create] { +/// "time": "2020-10-15T05:30:47Z", +/// "type": "example", +/// "temperature": { +/// "temperature": { +/// "value": 25 +/// } +/// }, +/// "location": { +/// "latitude": { +/// "value": 32.54 +/// }, +/// "longitude": { +/// "value": -117.67 +/// }, +/// "altitude": { +/// "value": 98.6 +/// } +/// }, +/// "pressure": { +/// "pressure": { +/// "value": 98 +/// } +/// } +/// } +/// ``` +export function process(t,message) { + let topic_parts = message.topic.split( '/') + let type = topic_parts[6] + let payload = JSON.parse(message.payload) + + let c8y_msg = { + type: type + } + for (let [k, v] of Object.entries(payload)) { + if (k === "time") { + let fragment = { time: v } + Object.assign(c8y_msg, fragment) + } + else if (typeof(v) === "number") { + let fragment = { [k]: { [k]: v } } + Object.assign(c8y_msg, fragment) + } else for (let [sub_k, sub_v] of Object.entries(v)) { + if (typeof(sub_v) === "number") { + let fragment = { [k]: { [sub_k]: sub_v } } + Object.assign(c8y_msg, fragment) + } + } + } + + return [{ + topic: "c8y/measurement/measurements/create", + payload: JSON.stringify(c8y_msg) + }] +} From 5d0ffd1424824cfcbe8da82784c53d6d81095aa5 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 20 May 2025 15:47:59 +0200 Subject: [PATCH 10/53] Fix tests broken by cargo update Signed-off-by: Didier Wenzek --- .../fixtures/invalid/reject_invalid_timestamp.expected_error | 2 +- .../fixtures/invalid/reject_partial_timestamp.expected_error | 2 +- .../reject_timestamp_missing_time_separator.expected_error | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/core/tedge_api/tests/fixtures/invalid/reject_invalid_timestamp.expected_error b/crates/core/tedge_api/tests/fixtures/invalid/reject_invalid_timestamp.expected_error index 081cbde607d..422d9e8e533 100644 --- a/crates/core/tedge_api/tests/fixtures/invalid/reject_invalid_timestamp.expected_error +++ b/crates/core/tedge_api/tests/fixtures/invalid/reject_invalid_timestamp.expected_error @@ -1,4 +1,4 @@ -Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-22 3am": a character literal was not valid at line 2 column 27: `", +Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-22 3am": the 'hour' component could not be parsed at line 2 column 27: `", "pressure": 220 } ` \ No newline at end of file diff --git a/crates/core/tedge_api/tests/fixtures/invalid/reject_partial_timestamp.expected_error b/crates/core/tedge_api/tests/fixtures/invalid/reject_partial_timestamp.expected_error index 54a5bc5a042..faac58c9fa6 100644 --- a/crates/core/tedge_api/tests/fixtures/invalid/reject_partial_timestamp.expected_error +++ b/crates/core/tedge_api/tests/fixtures/invalid/reject_partial_timestamp.expected_error @@ -1,4 +1,4 @@ -Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-22": a character literal was not valid at line 2 column 23: `", +Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-22": the 'separator' component could not be parsed at line 2 column 23: `", "pressure": 220 } ` \ No newline at end of file diff --git a/crates/core/tedge_api/tests/fixtures/invalid/reject_timestamp_missing_time_separator.expected_error b/crates/core/tedge_api/tests/fixtures/invalid/reject_timestamp_missing_time_separator.expected_error index 7a92d95256a..bf76a12e79a 100644 --- a/crates/core/tedge_api/tests/fixtures/invalid/reject_timestamp_missing_time_separator.expected_error +++ b/crates/core/tedge_api/tests/fixtures/invalid/reject_timestamp_missing_time_separator.expected_error @@ -1,3 +1,3 @@ -Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-2217:03:14.000658767+02:00": a character literal was not valid at line 2 column 47: `" +Invalid JSON: Invalid ISO8601 timestamp (expected YYYY-MM-DDThh:mm:ss.sss.±hh:mm): "2013-06-2217:03:14.000658767+02:00": the 'hour' component could not be parsed at line 2 column 47: `" } ` \ No newline at end of file From 9a84b39dad0a106d05b7e9e4f36a210a5485f10c Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 20 May 2025 17:56:25 +0200 Subject: [PATCH 11/53] Update JS filter config using MQTT Signed-off-by: Didier Wenzek --- .../pipelines/measurements.toml | 2 +- .../tedge_gen_mapper/pipelines/te_to_c8y.js | 53 ++++++++++++++- .../extensions/tedge_gen_mapper/src/config.rs | 4 +- .../tedge_gen_mapper/src/js_filter.rs | 65 ++++++++++++++++++- .../tedge_gen_mapper/src/pipeline.rs | 10 +-- 5 files changed, 122 insertions(+), 12 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml index fea27e58da2..1edde2696e6 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml @@ -3,5 +3,5 @@ input_topics = ["te/+/+/+/+/m/+"] stages = [ { filter = "add_timestamp.js" }, { filter = "drop_stragglers.js", config = { max_delay = 60 } }, - { filter = "te_to_c8y.js" } + { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } ] diff --git a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js index fc2968db926..ae2d8f157c1 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -42,7 +42,7 @@ /// } /// } /// ``` -export function process(t,message) { +export function process(t, message, config) { let topic_parts = message.topic.split( '/') let type = topic_parts[6] let payload = JSON.parse(message.payload) @@ -51,17 +51,26 @@ export function process(t,message) { type: type } + let meta = (config || {})[`${message.topic}/meta`] || {} + for (let [k, v] of Object.entries(payload)) { + let k_meta = (meta || {})[k] || {} if (k === "time") { let fragment = { time: v } Object.assign(c8y_msg, fragment) - } else if (typeof(v) === "number") { + if (Object.keys(k_meta).length>0) { + v = { value: v, ...k_meta } + } let fragment = { [k]: { [k]: v } } Object.assign(c8y_msg, fragment) } else for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_k_meta = k_meta[sub_k] if (typeof(sub_v) === "number") { + if (sub_k_meta) { + sub_v = { value: sub_v, ...sub_k_meta } + } let fragment = { [k]: { [sub_k]: sub_v } } Object.assign(c8y_msg, fragment) } @@ -73,3 +82,43 @@ export function process(t,message) { payload: JSON.stringify(c8y_msg) }] } + +/// Update the config with measurement metadata. +/// +/// These metadata are expected to have the same shape of the actual values. +/// +/// ``` +/// [te/device/main///m/example/meta] { "temperature": { "unit": "°C" }} +/// ``` +/// +/// and: +/// ``` +/// [te/device/main///m/example] { "temperature": { "unit": 23 }} +/// ``` +/// +/// will be merged by the process function into: +/// ``` +/// [c8y/measurement/measurements/create] { +/// "type": "example", +/// "temperature": { +/// "temperature": { +/// "value": 23, +/// "unit": "°C" +/// } +/// } +/// } +/// ``` +export function update_config(message, config) { + let type = message.topic + let metadata = JSON.parse(message.payload) + + let fragment = { + [type]: metadata + } + if (!config) { + config = {} + } + Object.assign(config, fragment) + + return config +} diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index cdba92cf6a3..fd6db236715 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -22,7 +22,7 @@ pub struct StageConfig { config: Option, #[serde(default)] - config_topics: Vec, + meta_topics: Vec, } #[derive(Deserialize)] @@ -66,7 +66,7 @@ impl StageConfig { FilterSpec::JavaScript(path) => config_dir.join(path), }; let filter = js_runtime.loaded_module(path)?.with_config(self.config); - let config_topics = topic_filters(&self.config_topics)?; + let config_topics = topic_filters(&self.meta_topics)?; Ok(Stage { filter, config_topics, diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index de0edafcf0e..c2fa1f71876 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -41,6 +41,14 @@ impl JsFilter { } } + /// Process a message returning zero, one or more messages + /// + /// The "process" function of the JS module is passed 3 arguments + /// - the current timestamp + /// - the message to be transformed + /// - the filter config (as configured for the pipeline stage, possibly updated by update_config messages) + /// + /// The returned value is expected to be an array of messages. pub async fn process( &self, js: &JsRuntime, @@ -54,8 +62,25 @@ impl JsFilter { .map_err(pipeline::error_from_js) } - pub fn update_config(&self, _js: &JsRuntime, config: &Message) -> Result<(), FilterError> { - debug!(target: "MAPPING", "{}: update_config({config:?})", self.path.display()); + /// Update the filter config using a metadata message + /// + /// The "update_config" function of the JS module is passed 2 arguments + /// - the message + /// - the current filter config + /// + /// The value returned by this function is used as the updated filter config + pub async fn update_config( + &mut self, + js: &JsRuntime, + message: &Message, + ) -> Result<(), FilterError> { + debug!(target: "MAPPING", "{}: update_config({message:?})", self.path.display()); + let input = (message.clone(), self.config.clone()); + let config = js + .call_function(&self, "update_config", input) + .await + .map_err(pipeline::error_from_js)?; + self.config = config; Ok(()) } @@ -205,6 +230,42 @@ impl<'js> IntoJs<'js> for JsonValue { } } +impl<'js> FromJs<'js> for JsonValue { + fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { + if let Some(b) = value.as_bool() { + return Ok(JsonValue(serde_json::Value::Bool(b))); + } + if let Some(n) = value.as_int() { + return Ok(JsonValue(serde_json::Value::Number(n.into()))); + } + if let Some(n) = value.as_float() { + let js_n = serde_json::Number::from_f64(n) + .map(serde_json::Value::Number) + .unwrap_or(serde_json::Value::Null); + return Ok(JsonValue(js_n)); + } + if let Some(string) = value.as_string() { + return Ok(JsonValue(serde_json::Value::String(string.to_string()?))); + } + if let Some(array) = value.as_array() { + let array: rquickjs::Result> = array.iter().collect(); + let array = array?.into_iter().map(|v| v.0).collect(); + return Ok(JsonValue(serde_json::Value::Array(array))); + } + if let Some(object) = value.as_object() { + let mut js_object = serde_json::Map::new(); + for key in object.keys::().flatten() { + if let Ok(JsonValue(v)) = object.get(&key) { + js_object.insert(key, v.clone()); + } + } + return Ok(JsonValue(serde_json::Value::Object(js_object))); + } + + Ok(JsonValue(serde_json::Value::Null)) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 90f756a2e09..787f87207ca 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -52,14 +52,14 @@ impl Pipeline { topics } - pub fn update_config( - &self, + pub async fn update_config( + &mut self, js_runtime: &JsRuntime, message: &Message, ) -> Result<(), FilterError> { - for stage in self.stages.iter() { + for stage in self.stages.iter_mut() { if stage.config_topics.accept_topic_name(&message.topic) { - stage.filter.update_config(js_runtime, message)? + stage.filter.update_config(js_runtime, message).await? } } Ok(()) @@ -71,7 +71,7 @@ impl Pipeline { timestamp: &DateTime, message: &Message, ) -> Result, FilterError> { - self.update_config(js_runtime, message)?; + self.update_config(js_runtime, message).await?; if !self.input_topics.accept_topic_name(&message.topic) { return Ok(vec![]); } From 7f8f9664bdddfd2d2b5043c090dff35778b2702a Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 20 May 2025 18:59:51 +0200 Subject: [PATCH 12/53] Filters can defer messages up to the end of a time window Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/pipelines/collectd.toml | 2 +- .../pipelines/group_by_timestamp.js | 22 +++++++------- .../extensions/tedge_gen_mapper/src/actor.rs | 2 +- .../extensions/tedge_gen_mapper/src/config.rs | 8 ++++- .../tedge_gen_mapper/src/js_filter.rs | 30 +++++++++++++++++-- .../tedge_gen_mapper/src/pipeline.rs | 6 +++- 6 files changed, 52 insertions(+), 18 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml index 2501140d5e0..44feb5b3323 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml @@ -2,5 +2,5 @@ input_topics = ["collectd/+/+/+"] stages = [ { filter = "collectd-to-te.js" }, - { filter = "group_by_timestamp.js" } + { filter = "group_by_timestamp.js", tick_every_seconds = 3 } ] diff --git a/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js b/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js index ffa282ff475..77b061ee1e5 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js @@ -1,16 +1,14 @@ -// Demonstrate that messages can be delayed -export function process (timestamp, message, config) { - if ( typeof process.batch == 'undefined' ) { - process.batch = []; - } +class State { + static batch = [] +} - let len = process.batch.push(message) - let batch_len = config.batch_len || 4 - if (len < batch_len) { - return [] - } +export function process (timestamp, message) { + State.batch.push(message) + return [] +} - let batch = process.batch - process.batch = [] +export function tick() { + let batch = State.batch + State.batch = [] return batch } \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 084ef835573..cb2584eb34f 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -27,7 +27,7 @@ impl Actor for GenMapper { } async fn run(mut self) -> Result<(), RuntimeError> { - let mut interval = interval(Duration::from_secs(5)); + let mut interval = interval(Duration::from_secs(1)); loop { tokio::select! { diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index fd6db236715..551e6707247 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -21,6 +21,9 @@ pub struct StageConfig { #[serde(default)] config: Option, + #[serde(default)] + tick_every_seconds: u64, + #[serde(default)] meta_topics: Vec, } @@ -65,7 +68,10 @@ impl StageConfig { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; - let filter = js_runtime.loaded_module(path)?.with_config(self.config); + let filter = js_runtime + .loaded_module(path)? + .with_config(self.config) + .with_tick_every_seconds(self.tick_every_seconds); let config_topics = topic_filters(&self.meta_topics)?; Ok(Stage { filter, diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index c2fa1f71876..f0a78cf6934 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -17,6 +17,7 @@ use tracing::debug; pub struct JsFilter { path: PathBuf, config: JsonValue, + tick_every_seconds: u64, } #[derive(Clone, Default)] @@ -27,6 +28,7 @@ impl JsFilter { JsFilter { path, config: JsonValue::default(), + tick_every_seconds: 0, } } @@ -41,6 +43,13 @@ impl JsFilter { } } + pub fn with_tick_every_seconds(self, tick_every_seconds: u64) -> Self { + Self { + tick_every_seconds, + ..self + } + } + /// Process a message returning zero, one or more messages /// /// The "process" function of the JS module is passed 3 arguments @@ -84,9 +93,26 @@ impl JsFilter { Ok(()) } - pub fn tick(&self, _js: &JsRuntime, timestamp: &DateTime) -> Result, FilterError> { + /// Trigger the tick function of the JS module + /// + /// The "tick" function is passed 2 arguments + /// - the current timestamp + /// - the current filter config + /// + /// Return zero, one or more messages + pub async fn tick( + &self, + js: &JsRuntime, + timestamp: &DateTime, + ) -> Result, FilterError> { + if !timestamp.tick_now(self.tick_every_seconds) { + return Ok(vec![]); + } debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.path.display()); - Ok(vec![]) + let input = (timestamp.clone(), self.config.clone()); + js.call_function(&self, "tick", input) + .await + .map_err(pipeline::error_from_js) } } diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 787f87207ca..4f5c8bb84fd 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -103,7 +103,7 @@ impl Pipeline { } // Only then process the tick - transformed_messages.extend(stage.filter.tick(js_runtime, timestamp)?); + transformed_messages.extend(stage.filter.tick(js_runtime, timestamp).await?); // Iterate with all the messages collected at this stage messages = transformed_messages; @@ -116,6 +116,10 @@ impl DateTime { pub fn now() -> Self { DateTime::try_from(OffsetDateTime::now_utc()).unwrap() } + + pub fn tick_now(&self, tick_every_seconds: u64) -> bool { + tick_every_seconds != 0 && (self.seconds % tick_every_seconds == 0) + } } impl TryFrom for DateTime { From efde5fee903276971cd963e2cbc9b2151e3f127c Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Wed, 21 May 2025 16:10:39 +0100 Subject: [PATCH 13/53] Tedge-gen-mapper dynamically reloads pipelines and filters Signed-off-by: James Rhodes --- Cargo.lock | 1 + crates/core/tedge_mapper/src/gen/mod.rs | 4 + crates/extensions/tedge_gen_mapper/Cargo.toml | 1 + .../extensions/tedge_gen_mapper/src/actor.rs | 93 +++++++++++++++++-- .../extensions/tedge_gen_mapper/src/config.rs | 2 + .../tedge_gen_mapper/src/js_filter.rs | 4 + crates/extensions/tedge_gen_mapper/src/lib.rs | 38 ++++++-- .../tedge_gen_mapper/src/pipeline.rs | 3 + 8 files changed, 129 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fda02b4ff11..b01164cf0aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4950,6 +4950,7 @@ dependencies = [ "serde", "serde_json", "tedge_actors", + "tedge_file_system_ext", "tedge_mqtt_ext", "thiserror 1.0.69", "time", diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs index 091fb47542b..acdaa3a6961 100644 --- a/crates/core/tedge_mapper/src/gen/mod.rs +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -1,6 +1,7 @@ use crate::core::mapper::start_basic_actors; use crate::TEdgeComponent; use tedge_config::TEdgeConfig; +use tedge_file_system_ext::FsWatchActorBuilder; use tedge_gen_mapper::GenMapperBuilder; pub struct GenMapper; @@ -15,12 +16,15 @@ impl TEdgeComponent for GenMapper { let (mut runtime, mut mqtt_actor) = start_basic_actors("tedge-gen-mapper", &tedge_config).await?; + let mut fs_actor = FsWatchActorBuilder::new(); let mut gen_mapper = GenMapperBuilder::try_new("/etc/tedge/gen-mapper").await?; gen_mapper.load().await; gen_mapper.connect(&mut mqtt_actor); + gen_mapper.connect_fs(&mut fs_actor); runtime.spawn(gen_mapper).await?; runtime.spawn(mqtt_actor).await?; + runtime.spawn(fs_actor).await?; runtime.run_to_completion().await?; Ok(()) } diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 32b302357e3..80c9b9cbb9b 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -15,6 +15,7 @@ rquickjs = { workspace = true, features = ["futures","parallel"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tedge_actors = { workspace = true } +tedge_file_system_ext = { workspace = true } tedge_mqtt_ext = { workspace = true } thiserror = { workspace = true } time = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index cb2584eb34f..8506cc2bc99 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,23 +1,31 @@ +use crate::config::PipelineConfig; use crate::js_filter::JsRuntime; use crate::pipeline::DateTime; use crate::pipeline::Message; use crate::pipeline::Pipeline; +use crate::InputMessage; +use crate::OutputMessage; use async_trait::async_trait; +use camino::Utf8PathBuf; use std::collections::HashMap; +use std::path::PathBuf; use tedge_actors::Actor; use tedge_actors::MessageReceiver; use tedge_actors::RuntimeError; use tedge_actors::Sender; use tedge_actors::SimpleMessageBox; +use tedge_file_system_ext::FsWatchEvent; use tedge_mqtt_ext::MqttMessage; use tokio::time::interval; use tokio::time::Duration; use tracing::error; +use tracing::info; pub struct GenMapper { - pub(super) mqtt: SimpleMessageBox, + pub(super) messages: SimpleMessageBox, pub(super) pipelines: HashMap, pub(super) js_runtime: JsRuntime, + pub(super) config_dir: PathBuf, } #[async_trait] @@ -34,11 +42,26 @@ impl Actor for GenMapper { _ = interval.tick() => { self.tick().await?; } - message = self.mqtt.recv() => { - match message.map(Message::try_from) { - Some(Ok(message)) => self.filter(message).await?, - Some(Err(err)) => { - error!(target: "gen-mapper", "Cannot process message: {err}"); + message = self.messages.recv() => { + match message { + Some(InputMessage::MqttMessage(message)) => match Message::try_from(message) { + Ok(message) => self.filter(message).await?, + Err(err) => { + error!(target: "gen-mapper", "Cannot process message: {err}"); + } + }, + Some(InputMessage::FsWatchEvent(FsWatchEvent::Modified(path))) => { + let Ok(path) = Utf8PathBuf::try_from(path) else { + continue; + }; + if matches!(path.extension(), Some("js" | "ts")) { + self.reload_filter(path).await; + } else if path.extension() == Some("toml") { + self.reload_pipeline(path).await; + } + }, + Some(InputMessage::FsWatchEvent(e)) => { + tracing::warn!("TODO do something with {e:?}") }, None => break, } @@ -50,6 +73,52 @@ impl Actor for GenMapper { } impl GenMapper { + async fn reload_filter(&mut self, path: Utf8PathBuf) { + for pipeline in self.pipelines.values_mut() { + for stage in &mut pipeline.stages { + if stage.filter.path() == path { + match self.js_runtime.load_file(&path).await { + Ok(filter) => { + info!("Reloaded filter {path}"); + stage.filter = filter + } + Err(e) => { + error!("Failed to reload filter {path}: {e}"); + return; + } + } + } + } + } + } + + async fn reload_pipeline(&mut self, path: Utf8PathBuf) { + for pipeline in self.pipelines.values_mut() { + if pipeline.source == path { + let Ok(source) = tokio::fs::read_to_string(&path).await else { + error!("Failed to read updated filter {path}"); + break; + }; + let config: PipelineConfig = match toml::from_str(&source) { + Ok(config) => config, + Err(e) => { + error!("Failed to parse toml for updated filter {path}: {e}"); + break; + } + }; + match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { + Ok(p) => { + *pipeline = p; + info!("Reloaded pipeline {path}"); + } + Err(e) => { + error!("Failed to load updated pipeline {path}: {e}") + } + }; + } + } + } + async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { @@ -60,7 +129,11 @@ impl GenMapper { Ok(messages) => { for message in messages { match MqttMessage::try_from(message) { - Ok(message) => self.mqtt.send(message).await?, + Ok(message) => { + self.messages + .send(OutputMessage::MqttMessage(message)) + .await? + } Err(err) => { error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") } @@ -83,7 +156,11 @@ impl GenMapper { Ok(messages) => { for message in messages { match MqttMessage::try_from(message) { - Ok(message) => self.mqtt.send(message).await?, + Ok(message) => { + self.messages + .send(OutputMessage::MqttMessage(message)) + .await? + } Err(err) => { error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index 551e6707247..b1af38c627e 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -48,6 +48,7 @@ impl PipelineConfig { self, js_runtime: &JsRuntime, config_dir: &Path, + source: Utf8PathBuf, ) -> Result { let input = topic_filters(&self.input_topics)?; let stages = self @@ -58,6 +59,7 @@ impl PipelineConfig { Ok(Pipeline { input_topics: input, stages, + source, }) } } diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index f0a78cf6934..5812b150f4b 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -50,6 +50,10 @@ impl JsFilter { } } + pub fn path(&self) -> &Path { + &self.path + } + /// Process a message returning zero, one or more messages /// /// The "process" function of the JS module is passed 3 arguments diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 8019c489ae2..95cc27a878d 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -8,10 +8,12 @@ use crate::config::PipelineConfig; use crate::js_filter::JsRuntime; use crate::pipeline::Pipeline; use camino::Utf8Path; +use camino::Utf8PathBuf; use std::collections::HashMap; use std::convert::Infallible; use std::path::Path; use std::path::PathBuf; +use tedge_actors::fan_in_message_type; use tedge_actors::Builder; use tedge_actors::DynSender; use tedge_actors::MessageSink; @@ -20,18 +22,23 @@ use tedge_actors::NoConfig; use tedge_actors::RuntimeRequest; use tedge_actors::RuntimeRequestSink; use tedge_actors::SimpleMessageBoxBuilder; +use tedge_file_system_ext::FsWatchEvent; use tedge_mqtt_ext::MqttMessage; +use tedge_mqtt_ext::PublishOrSubscribe; use tedge_mqtt_ext::TopicFilter; use tokio::fs::read_dir; use tokio::fs::read_to_string; use tracing::error; use tracing::info; +fan_in_message_type!(InputMessage[MqttMessage, FsWatchEvent]: Clone, Debug, Eq, PartialEq); +fan_in_message_type!(OutputMessage[MqttMessage]: Clone, Debug, Eq, PartialEq); + pub struct GenMapperBuilder { config_dir: PathBuf, - message_box: SimpleMessageBoxBuilder, + message_box: SimpleMessageBoxBuilder, pipelines: HashMap, - pipeline_specs: HashMap, + pipeline_specs: HashMap, js_runtime: JsRuntime, } @@ -92,7 +99,8 @@ impl GenMapperBuilder { if let Some(name) = file.as_ref().file_name() { let specs = read_to_string(file.as_ref()).await?; let pipeline: PipelineConfig = toml::from_str(&specs)?; - self.pipeline_specs.insert(name.to_string(), pipeline); + self.pipeline_specs + .insert(name.to_string(), (file.as_ref().to_owned(), pipeline)); } Ok(()) @@ -104,8 +112,8 @@ impl GenMapperBuilder { } fn compile(&mut self) { - for (name, specs) in self.pipeline_specs.drain() { - match specs.compile(&self.js_runtime, &self.config_dir) { + for (name, (source, specs)) in self.pipeline_specs.drain() { + match specs.compile(&self.js_runtime, &self.config_dir, source) { Ok(pipeline) => { let _ = self.pipelines.insert(name, pipeline); } @@ -118,10 +126,21 @@ impl GenMapperBuilder { pub fn connect( &mut self, - mqtt: &mut (impl MessageSource + MessageSink), + mqtt: &mut (impl MessageSource + MessageSink), ) { - mqtt.connect_sink(self.topics(), &self.message_box); - self.message_box.connect_sink(NoConfig, mqtt); + mqtt.connect_mapped_sink(self.topics(), &self.message_box, |msg| { + Some(InputMessage::MqttMessage(msg)) + }); + self.message_box + .connect_mapped_sink(NoConfig, mqtt, move |msg| match msg { + OutputMessage::MqttMessage(mqtt) => Some(PublishOrSubscribe::Publish(mqtt)), + }); + } + + pub fn connect_fs(&mut self, fs: &mut impl MessageSource) { + fs.connect_mapped_sink(self.config_dir.clone(), &self.message_box, |msg| { + Some(InputMessage::FsWatchEvent(msg)) + }); } fn topics(&self) -> TopicFilter { @@ -148,9 +167,10 @@ impl Builder for GenMapperBuilder { fn build(self) -> GenMapper { GenMapper { - mqtt: self.message_box.build(), + messages: self.message_box.build(), pipelines: self.pipelines, js_runtime: self.js_runtime, + config_dir: self.config_dir, } } } diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 4f5c8bb84fd..3cd9e92db81 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -1,6 +1,7 @@ use crate::js_filter::JsFilter; use crate::js_filter::JsRuntime; use crate::LoadError; +use camino::Utf8PathBuf; use serde_json::json; use serde_json::Value; use tedge_mqtt_ext::MqttMessage; @@ -14,6 +15,8 @@ pub struct Pipeline { /// Transformation stages to apply in order to the messages pub stages: Vec, + + pub source: Utf8PathBuf, } /// A message transformation stage From 185076d487d18c5f06557c82fb520d421fbc6701 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 26 May 2025 14:19:56 +0200 Subject: [PATCH 14/53] Tedge-gen-mapper dynamically subscribes to MQTT topics Signed-off-by: Didier Wenzek --- .../extensions/tedge_gen_mapper/src/actor.rs | 23 ++++++++ crates/extensions/tedge_gen_mapper/src/lib.rs | 18 +++++- crates/extensions/tedge_mqtt_ext/src/lib.rs | 58 ++++++++++++++++++- crates/extensions/tedge_mqtt_ext/src/trie.rs | 10 ++++ 4 files changed, 105 insertions(+), 4 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 8506cc2bc99..b2e832e99fb 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -9,6 +9,8 @@ use async_trait::async_trait; use camino::Utf8PathBuf; use std::collections::HashMap; use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; use tedge_actors::Actor; use tedge_actors::MessageReceiver; use tedge_actors::RuntimeError; @@ -16,6 +18,8 @@ use tedge_actors::Sender; use tedge_actors::SimpleMessageBox; use tedge_file_system_ext::FsWatchEvent; use tedge_mqtt_ext::MqttMessage; +use tedge_mqtt_ext::SubscriptionDiff; +use tedge_mqtt_ext::TopicFilter; use tokio::time::interval; use tokio::time::Duration; use tracing::error; @@ -24,6 +28,7 @@ use tracing::info; pub struct GenMapper { pub(super) messages: SimpleMessageBox, pub(super) pipelines: HashMap, + pub(super) subscriptions: Arc>, pub(super) js_runtime: JsRuntime, pub(super) config_dir: PathBuf, } @@ -58,6 +63,7 @@ impl Actor for GenMapper { self.reload_filter(path).await; } else if path.extension() == Some("toml") { self.reload_pipeline(path).await; + self.send_updated_subscriptions().await?; } }, Some(InputMessage::FsWatchEvent(e)) => { @@ -119,6 +125,23 @@ impl GenMapper { } } + async fn send_updated_subscriptions(&mut self) -> Result<(), RuntimeError> { + let topics = self.update_subscriptions(); + let diff = SubscriptionDiff::new(&topics, &TopicFilter::empty()); + self.messages + .send(OutputMessage::SubscriptionDiff(diff)) + .await?; + Ok(()) + } + + fn update_subscriptions(&self) -> TopicFilter { + let mut topics = self.subscriptions.lock().unwrap(); + for pipeline in self.pipelines.values() { + topics.add_all(pipeline.topics()) + } + topics.clone() + } + async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); for (pipeline_id, pipeline) in self.pipelines.iter_mut() { diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 95cc27a878d..7956be10d69 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -13,6 +13,8 @@ use std::collections::HashMap; use std::convert::Infallible; use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; use tedge_actors::fan_in_message_type; use tedge_actors::Builder; use tedge_actors::DynSender; @@ -23,8 +25,10 @@ use tedge_actors::RuntimeRequest; use tedge_actors::RuntimeRequestSink; use tedge_actors::SimpleMessageBoxBuilder; use tedge_file_system_ext::FsWatchEvent; +use tedge_mqtt_ext::DynSubscriptions; use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::PublishOrSubscribe; +use tedge_mqtt_ext::SubscriptionDiff; use tedge_mqtt_ext::TopicFilter; use tokio::fs::read_dir; use tokio::fs::read_to_string; @@ -32,13 +36,14 @@ use tracing::error; use tracing::info; fan_in_message_type!(InputMessage[MqttMessage, FsWatchEvent]: Clone, Debug, Eq, PartialEq); -fan_in_message_type!(OutputMessage[MqttMessage]: Clone, Debug, Eq, PartialEq); +fan_in_message_type!(OutputMessage[MqttMessage, SubscriptionDiff]: Clone, Debug, Eq, PartialEq); pub struct GenMapperBuilder { config_dir: PathBuf, message_box: SimpleMessageBoxBuilder, pipelines: HashMap, pipeline_specs: HashMap, + subscriptions: Arc>, js_runtime: JsRuntime, } @@ -51,6 +56,7 @@ impl GenMapperBuilder { message_box: SimpleMessageBoxBuilder::new("GenMapper", 16), pipelines: HashMap::default(), pipeline_specs: HashMap::default(), + subscriptions: Arc::new(Mutex::new(TopicFilter::empty())), js_runtime, }) } @@ -126,14 +132,19 @@ impl GenMapperBuilder { pub fn connect( &mut self, - mqtt: &mut (impl MessageSource + MessageSink), + mqtt: &mut (impl MessageSource + MessageSink), ) { - mqtt.connect_mapped_sink(self.topics(), &self.message_box, |msg| { + let dyn_subscriptions = DynSubscriptions::new(self.topics()); + mqtt.connect_mapped_sink(dyn_subscriptions.clone(), &self.message_box, |msg| { Some(InputMessage::MqttMessage(msg)) }); + let client_id = dyn_subscriptions.client_id(); self.message_box .connect_mapped_sink(NoConfig, mqtt, move |msg| match msg { OutputMessage::MqttMessage(mqtt) => Some(PublishOrSubscribe::Publish(mqtt)), + OutputMessage::SubscriptionDiff(diff) => { + Some(PublishOrSubscribe::subscribe(client_id, diff)) + } }); } @@ -169,6 +180,7 @@ impl Builder for GenMapperBuilder { GenMapper { messages: self.message_box.build(), pipelines: self.pipelines, + subscriptions: self.subscriptions, js_runtime: self.js_runtime, config_dir: self.config_dir, } diff --git a/crates/extensions/tedge_mqtt_ext/src/lib.rs b/crates/extensions/tedge_mqtt_ext/src/lib.rs index 2be5b2f3902..e0295a1d1cd 100644 --- a/crates/extensions/tedge_mqtt_ext/src/lib.rs +++ b/crates/extensions/tedge_mqtt_ext/src/lib.rs @@ -16,6 +16,8 @@ use mqtt_channel::SubscriberOps; pub use mqtt_channel::Topic; pub use mqtt_channel::TopicFilter; use std::convert::Infallible; +use std::sync::Arc; +use std::sync::Mutex; use tedge_actors::fan_in_message_type; use tedge_actors::futures::channel::mpsc; use tedge_actors::Actor; @@ -36,7 +38,7 @@ use tedge_actors::Server; use tedge_actors::ServerActorBuilder; use tedge_actors::ServerConfig; use trie::MqtTrie; -use trie::SubscriptionDiff; +pub use trie::SubscriptionDiff; pub type MqttConfig = mqtt_channel::Config; @@ -64,6 +66,12 @@ pub enum PublishOrSubscribe { Subscribe(SubscriptionRequest), } +impl PublishOrSubscribe { + pub fn subscribe(client_id: ClientId, diff: SubscriptionDiff) -> Self { + PublishOrSubscribe::Subscribe(SubscriptionRequest { diff, client_id }) + } +} + impl InputCombiner { pub fn close_input(&mut self) { self.publish_receiver.close(); @@ -161,6 +169,54 @@ impl MessageSource for MqttActorBuilder { } } +impl MessageSource for MqttActorBuilder { + fn connect_sink( + &mut self, + subscriptions: DynSubscriptions, + peer: &impl MessageSink, + ) { + let client_id = self.connect_id_sink(subscriptions.init_topics(), peer); + subscriptions.set_client_id(client_id); + } +} + +#[derive(Clone)] +pub struct DynSubscriptions { + inner: Arc>, +} +pub struct DynSubscriptionsInner { + init_topics: TopicFilter, + client_id: Option, +} + +impl DynSubscriptions { + pub fn new(init_topics: TopicFilter) -> Self { + let inner = DynSubscriptionsInner { + init_topics, + client_id: None, + }; + DynSubscriptions { + inner: Arc::new(Mutex::new(inner)), + } + } + + fn set_client_id(&self, client_id: ClientId) { + let mut inner = self.inner.lock().unwrap(); + inner.client_id = Some(client_id); + } + + fn init_topics(&self) -> TopicFilter { + self.inner.lock().unwrap().init_topics.clone() + } + + /// Return the client id + /// + /// Panic if not properly registered as a sink of the MqttActorBuilder + pub fn client_id(&self) -> ClientId { + self.inner.lock().unwrap().client_id.unwrap() + } +} + impl MqttActorBuilder { pub fn connect_id_sink( &mut self, diff --git a/crates/extensions/tedge_mqtt_ext/src/trie.rs b/crates/extensions/tedge_mqtt_ext/src/trie.rs index 4e1d1f47e6e..c284e4e4a44 100644 --- a/crates/extensions/tedge_mqtt_ext/src/trie.rs +++ b/crates/extensions/tedge_mqtt_ext/src/trie.rs @@ -156,6 +156,16 @@ impl SubscriptionDiff { } } + pub fn new( + subscribe: &mqtt_channel::TopicFilter, + unsubscribe: &mqtt_channel::TopicFilter, + ) -> Self { + Self { + subscribe: subscribe.patterns().iter().cloned().collect(), + unsubscribe: unsubscribe.patterns().iter().cloned().collect(), + } + } + fn with_topic_prefix(self, prefix: &str) -> Self { Self { subscribe: self From 50038c9442270f899095bb42555b877fe73ec06d Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 28 May 2025 16:19:19 +0200 Subject: [PATCH 15/53] Test JS example: collectd pipeline --- .../pipelines/collectd-to-te.js | 7 +- .../tedge_gen_mapper/src/js_filter.rs | 76 ++++++++++++++++--- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js index e5aa71c0540..53a56e13aff 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js @@ -7,8 +7,13 @@ export function process (timestamp, message, config) { let time = data[0] let value = data[1] + var topic = "te/device/main///m/collectd" + if (config && config.topic) { + topic = config.topic + } + return [ { - topic: config.topic || "te/device/main///m/collectd", + topic: topic, payload: `{"time": ${time}, "${group}": {"${measurement}": ${value}}}` }] } \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 5812b150f4b..3db0917c4dc 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -70,7 +70,7 @@ impl JsFilter { ) -> Result, FilterError> { debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.path.display()); let input = (timestamp.clone(), message.clone(), self.config.clone()); - js.call_function(&self, "process", input) + js.call_function(self, "process", input) .await .map_err(pipeline::error_from_js) } @@ -90,7 +90,7 @@ impl JsFilter { debug!(target: "MAPPING", "{}: update_config({message:?})", self.path.display()); let input = (message.clone(), self.config.clone()); let config = js - .call_function(&self, "update_config", input) + .call_function(self, "update_config", input) .await .map_err(pipeline::error_from_js)?; self.config = config; @@ -114,7 +114,7 @@ impl JsFilter { } debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.path.display()); let input = (timestamp.clone(), self.config.clone()); - js.call_function(&self, "tick", input) + js.call_function(self, "tick", input) .await .map_err(pipeline::error_from_js) } @@ -175,14 +175,24 @@ impl JsRuntime { let name = module.path.display().to_string(); rquickjs::async_with!(self.context => |ctx| { - let m = rquickjs::Module::declare(ctx, name, source.clone())?; + debug!(target: "MAPPING", "compile({name})"); + let m = rquickjs::Module::declare(ctx, name.clone(), source.clone())?; let (m,p) = m.eval()?; let () = p.finish()?; + debug!(target: "MAPPING", "link({name})"); let f: rquickjs::Value = m.get(function)?; let f = rquickjs::Function::from_value(f)?; - let r = f.call(args)?; - Ok(r) + + debug!(target: "MAPPING", "execute({name})"); + let r = f.call(args); + if r.is_err() { + let err = r.err().unwrap(); + debug!(target: "MAPPING", "execute({name}) => {err:?}"); + Err(err.into()) + } else { + Ok(r.unwrap()) + } }) .await } @@ -190,21 +200,28 @@ impl JsRuntime { impl<'js> FromJs<'js> for Message { fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { + debug!(target: "MAPPING", "from_js(...)"); match value.as_object() { None => Ok(Message { topic: "".to_string(), payload: "".to_string(), }), - Some(object) => Ok(Message { - topic: object.get("topic")?, - payload: object.get("payload")?, - }), + Some(object) => { + let topic = object.get("topic"); + let payload = object.get("payload"); + debug!(target: "MAPPING", "from_js(...) -> topic = {:?}, payload = {:?}", topic, payload); + Ok(Message { + topic: topic?, + payload: payload?, + }) + }, } } } impl<'js> IntoJs<'js> for Message { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { + debug!(target: "MAPPING", "into_js({self:?})"); let msg = Object::new(ctx.clone())?; msg.set("topic", self.topic)?; msg.set("payload", self.payload)?; @@ -214,6 +231,7 @@ impl<'js> IntoJs<'js> for Message { impl<'js> IntoJs<'js> for DateTime { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { + debug!(target: "MAPPING", "into_js({self:?})"); let msg = Object::new(ctx.clone())?; msg.set("topic", self.seconds)?; msg.set("payload", self.nanoseconds)?; @@ -331,4 +349,42 @@ mod tests { eprintln!("{:?}", error); assert!(error.to_string().contains("Exception generated by QuickJS")); } + + #[tokio::test] + async fn collectd_filter() { + let script = r#" +export function process (timestamp, message, config) { + let groups = message.topic.split( '/') + let data = message.payload.split(':') + + let group = groups[2] + let measurement = groups[3] + let time = data[0] + let value = data[1] + + var topic = "te/device/main///m/collectd" + if (config && config.topic) { + topic = config.topic + } + + return [ { + topic: topic, + payload: `{"time": ${time}, "${group}": {"${measurement}": ${value}}}` + }] +} + "#; + let mut runtime = JsRuntime::try_new().await.unwrap(); + let filter = runtime.load_js("collectd.js", script).unwrap(); + + let input = Message::new("collectd/h/memory/percent-used", "1748440192.104:19.9289468288182"); + let output = Message::new("te/device/main///m/collectd", r#"{"time": 1748440192.104, "memory": {"percent-used": 19.9289468288182}}"#); + assert_eq!( + filter + .process(&runtime, &DateTime::now(), &input) + .await + .unwrap(), + vec![output] + ); + } + } From 56480dd22d609055de375ef010cabdb6b39af3f1 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Fri, 30 May 2025 16:24:02 +0100 Subject: [PATCH 16/53] Tidy JS example Signed-off-by: James Rhodes --- .../pipelines/collectd-to-te.js | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js index 53a56e13aff..31edd135589 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js @@ -1,19 +1,14 @@ -export function process (timestamp, message, config) { - let groups = message.topic.split( '/') +export function process(_timestamp, message, config) { + let groups = message.topic.split('/') let data = message.payload.split(':') let group = groups[2] - let measurement = groups[3] - let time = data[0] - let value = data[1] + let measurement = groups[3] + let time = data[0] + let value = data[1] - var topic = "te/device/main///m/collectd" - if (config && config.topic) { - topic = config.topic - } - - return [ { - topic: topic, + return [{ + topic: config?.topic || "te/device/main///m/collectd", payload: `{"time": ${time}, "${group}": {"${measurement}": ${value}}}` }] } \ No newline at end of file From 779ff7f0920e42197991679e9cb9a727ff79c29a Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Fri, 30 May 2025 16:25:48 +0100 Subject: [PATCH 17/53] Relay error messages from Javascript to the generic mapper logs Signed-off-by: James Rhodes --- Cargo.lock | 1 + crates/extensions/tedge_gen_mapper/Cargo.toml | 1 + crates/extensions/tedge_gen_mapper/src/js_filter.rs | 11 ++++++++--- crates/extensions/tedge_gen_mapper/src/lib.rs | 3 +++ crates/extensions/tedge_gen_mapper/src/pipeline.rs | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b01164cf0aa..17c4cb4720f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4944,6 +4944,7 @@ dependencies = [ name = "tedge_gen_mapper" version = "1.5.1" dependencies = [ + "anyhow", "async-trait", "camino", "rquickjs", diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 80c9b9cbb9b..726dda5737f 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -9,6 +9,7 @@ homepage.workspace = true repository.workspace = true [dependencies] +anyhow = { workspace = true } async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } rquickjs = { workspace = true, features = ["futures","parallel"] } diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 3db0917c4dc..ce900cbfd3e 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -176,7 +176,7 @@ impl JsRuntime { rquickjs::async_with!(self.context => |ctx| { debug!(target: "MAPPING", "compile({name})"); - let m = rquickjs::Module::declare(ctx, name.clone(), source.clone())?; + let m = rquickjs::Module::declare(ctx.clone(), name.clone(), source.clone())?; let (m,p) = m.eval()?; let () = p.finish()?; @@ -187,9 +187,14 @@ impl JsRuntime { debug!(target: "MAPPING", "execute({name})"); let r = f.call(args); if r.is_err() { + if let Some(ex) = ctx.catch().as_exception() { + let err = anyhow::anyhow!("{ex}"); + Err(err.context("JS raised exception").into()) + } else { let err = r.err().unwrap(); debug!(target: "MAPPING", "execute({name}) => {err:?}"); Err(err.into()) + } } else { Ok(r.unwrap()) } @@ -214,7 +219,7 @@ impl<'js> FromJs<'js> for Message { topic: topic?, payload: payload?, }) - }, + } } } } @@ -347,7 +352,7 @@ mod tests { .await .unwrap_err(); eprintln!("{:?}", error); - assert!(error.to_string().contains("Exception generated by QuickJS")); + assert!(error.to_string().contains("Cannot process that message")); } #[tokio::test] diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 7956be10d69..ea8a7936e27 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -200,4 +200,7 @@ pub enum LoadError { #[error(transparent)] JsError(#[from] rquickjs::Error), + + #[error(transparent)] + Anyhow(#[from] anyhow::Error), } diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 3cd9e92db81..73da0e0a6f1 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -179,5 +179,5 @@ impl TryFrom for MqttMessage { } pub fn error_from_js(err: LoadError) -> FilterError { - FilterError::IncorrectSetting(format!("{}", err)) + FilterError::IncorrectSetting(format!("{err:#}")) } From 1c1e9e23b7312d8823eba1e63510bd0063020b06 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Fri, 30 May 2025 16:26:40 +0100 Subject: [PATCH 18/53] fixup! fixup! Add JS example: collectd pipline Signed-off-by: James Rhodes --- crates/extensions/tedge_gen_mapper/src/js_filter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index ce900cbfd3e..5d7c3374a05 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -238,8 +238,8 @@ impl<'js> IntoJs<'js> for DateTime { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { debug!(target: "MAPPING", "into_js({self:?})"); let msg = Object::new(ctx.clone())?; - msg.set("topic", self.seconds)?; - msg.set("payload", self.nanoseconds)?; + msg.set("seconds", self.seconds)?; + msg.set("nanoseconds", self.nanoseconds)?; Ok(Value::from_object(msg)) } } From f96d74bf4466f089078a90491e51855aecb6248e Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Fri, 30 May 2025 16:27:05 +0100 Subject: [PATCH 19/53] fixup! fixup! Add JS example: collectd pipline Signed-off-by: James Rhodes --- crates/extensions/tedge_gen_mapper/Cargo.toml | 2 +- crates/extensions/tedge_gen_mapper/src/js_filter.rs | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 726dda5737f..1c6349b02f4 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true anyhow = { workspace = true } async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } -rquickjs = { workspace = true, features = ["futures","parallel"] } +rquickjs = { workspace = true, features = ["futures", "parallel"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tedge_actors = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 5d7c3374a05..1809c6944de 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -381,8 +381,14 @@ export function process (timestamp, message, config) { let mut runtime = JsRuntime::try_new().await.unwrap(); let filter = runtime.load_js("collectd.js", script).unwrap(); - let input = Message::new("collectd/h/memory/percent-used", "1748440192.104:19.9289468288182"); - let output = Message::new("te/device/main///m/collectd", r#"{"time": 1748440192.104, "memory": {"percent-used": 19.9289468288182}}"#); + let input = Message::new( + "collectd/h/memory/percent-used", + "1748440192.104:19.9289468288182", + ); + let output = Message::new( + "te/device/main///m/collectd", + r#"{"time": 1748440192.104, "memory": {"percent-used": 19.9289468288182}}"#, + ); assert_eq!( filter .process(&runtime, &DateTime::now(), &input) @@ -391,5 +397,4 @@ export function process (timestamp, message, config) { vec![output] ); } - } From 0b2ad88db92de3ab71524abe7f4c3c91ceabffb7 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 4 Jun 2025 19:12:08 +0200 Subject: [PATCH 20/53] Fix memory leak The first design was naive (i.e. loading the javascript module for each and every function call) and leading to a memory leak (loading a module with the same name keeps the previous version of the module in memory). The new design fixes the issue. The memory is stable while translating to c8y 600 tedge measurements per second during 20 minutes (mode debug) ```shell $ watch ps -p 1382888 -o args,%cpu,etimes,times,%mem,rss,vsz COMMAND %CPU ELAPSED TIME %MEM RSS VSZ tedge-mapper gen 47.4 1322 627 0.0 41440 1359476 $ tedge mqtt sub 'test/output' --duration 1000 | pv | wc -l 89.3MiB 0:16:40 [91.4KiB/s] [ <=> ] 586311 ``` Signed-off-by: Didier Wenzek --- crates/extensions/tedge_gen_mapper/Cargo.toml | 4 +- .../tedge_gen_mapper/pipelines/set_topic.js | 6 + .../extensions/tedge_gen_mapper/src/actor.rs | 8 +- .../extensions/tedge_gen_mapper/src/config.rs | 8 +- .../tedge_gen_mapper/src/js_filter.rs | 194 ++++++----------- .../tedge_gen_mapper/src/js_runtime.rs | 204 ++++++++++++++++++ crates/extensions/tedge_gen_mapper/src/lib.rs | 13 +- .../tedge_gen_mapper/src/pipeline.rs | 9 +- 8 files changed, 309 insertions(+), 137 deletions(-) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/set_topic.js create mode 100644 crates/extensions/tedge_gen_mapper/src/js_runtime.rs diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 1c6349b02f4..91d491c5b61 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -12,7 +12,7 @@ repository.workspace = true anyhow = { workspace = true } async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } -rquickjs = { workspace = true, features = ["futures", "parallel"] } +rquickjs = { workspace = true, default-features = false, features = ["futures", "parallel"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tedge_actors = { workspace = true } @@ -20,7 +20,7 @@ tedge_file_system_ext = { workspace = true } tedge_mqtt_ext = { workspace = true } thiserror = { workspace = true } time = { workspace = true } -tokio = { workspace = true, features = ["fs", "macros", "time"] } +tokio = { workspace = true, features = ["fs", "macros", "time", "sync"] } toml = { workspace = true, features = ["parse"] } tracing = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js b/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js new file mode 100644 index 00000000000..16ce07f0589 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js @@ -0,0 +1,6 @@ +export function process (timestamp, message, config) { + return [{ + topic: config?.topic || "te/error", + payload: message.payload + }] +} diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index b2e832e99fb..121c6b3d38c 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,5 +1,5 @@ use crate::config::PipelineConfig; -use crate::js_filter::JsRuntime; +use crate::js_runtime::JsRuntime; use crate::pipeline::DateTime; use crate::pipeline::Message; use crate::pipeline::Pipeline; @@ -84,9 +84,8 @@ impl GenMapper { for stage in &mut pipeline.stages { if stage.filter.path() == path { match self.js_runtime.load_file(&path).await { - Ok(filter) => { + Ok(()) => { info!("Reloaded filter {path}"); - stage.filter = filter } Err(e) => { error!("Failed to reload filter {path}: {e}"); @@ -174,6 +173,9 @@ impl GenMapper { async fn tick(&mut self) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); + if timestamp.seconds % 300 == 0 { + self.js_runtime.dump_memory_stats().await; + } for (pipeline_id, pipeline) in self.pipelines.iter_mut() { match pipeline.tick(&self.js_runtime, ×tamp).await { Ok(messages) => { diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index b1af38c627e..dd389bd2bae 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -1,4 +1,5 @@ -use crate::js_filter::JsRuntime; +use crate::js_filter::JsFilter; +use crate::js_runtime::JsRuntime; use crate::pipeline::Pipeline; use crate::pipeline::Stage; use crate::LoadError; @@ -65,13 +66,12 @@ impl PipelineConfig { } impl StageConfig { - pub fn compile(self, js_runtime: &JsRuntime, config_dir: &Path) -> Result { + pub fn compile(self, _js_runtime: &JsRuntime, config_dir: &Path) -> Result { let path = match self.filter { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; - let filter = js_runtime - .loaded_module(path)? + let filter = JsFilter::new(path) .with_config(self.config) .with_tick_every_seconds(self.tick_every_seconds); let config_topics = topic_filters(&self.meta_topics)?; diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 1809c6944de..5e27b2b193d 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -1,14 +1,13 @@ +use crate::js_runtime::JsRuntime; use crate::pipeline; use crate::pipeline::DateTime; use crate::pipeline::FilterError; use crate::pipeline::Message; -use crate::LoadError; +use anyhow::Context; use rquickjs::Ctx; use rquickjs::FromJs; use rquickjs::IntoJs; -use rquickjs::Object; use rquickjs::Value; -use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; use tracing::debug; @@ -20,7 +19,7 @@ pub struct JsFilter { tick_every_seconds: u64, } -#[derive(Clone, Default)] +#[derive(Clone, Debug, Default)] pub struct JsonValue(serde_json::Value); impl JsFilter { @@ -32,6 +31,10 @@ impl JsFilter { } } + pub fn module_name(&self) -> String { + self.path.display().to_string() + } + pub fn with_config(self, config: Option) -> Self { if let Some(config) = config { Self { @@ -68,11 +71,16 @@ impl JsFilter { timestamp: &DateTime, message: &Message, ) -> Result, FilterError> { - debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.path.display()); - let input = (timestamp.clone(), message.clone(), self.config.clone()); - js.call_function(self, "process", input) + debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.module_name()); + let input = vec![ + timestamp.clone().into(), + message.clone().into(), + self.config.clone(), + ]; + js.call_function(&self.path, "process", input) .await - .map_err(pipeline::error_from_js) + .map_err(pipeline::error_from_js)? + .try_into() } /// Update the filter config using a metadata message @@ -87,10 +95,10 @@ impl JsFilter { js: &JsRuntime, message: &Message, ) -> Result<(), FilterError> { - debug!(target: "MAPPING", "{}: update_config({message:?})", self.path.display()); - let input = (message.clone(), self.config.clone()); + debug!(target: "MAPPING", "{}: update_config({message:?})", self.module_name()); + let input = vec![message.clone().into(), self.config.clone()]; let config = js - .call_function(self, "update_config", input) + .call_function(&self.path, "update_config", input) .await .map_err(pipeline::error_from_js)?; self.config = config; @@ -112,143 +120,78 @@ impl JsFilter { if !timestamp.tick_now(self.tick_every_seconds) { return Ok(vec![]); } - debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.path.display()); - let input = (timestamp.clone(), self.config.clone()); - js.call_function(self, "tick", input) + debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.module_name()); + let input = vec![timestamp.clone().into(), self.config.clone()]; + js.call_function(&self.path, "tick", input) .await - .map_err(pipeline::error_from_js) + .map_err(pipeline::error_from_js)? + .try_into() } } -pub struct JsRuntime { - context: rquickjs::AsyncContext, - modules: HashMap>, +impl From for JsonValue { + fn from(value: Message) -> Self { + JsonValue(value.json()) + } } -impl JsRuntime { - pub async fn try_new() -> Result { - let runtime = rquickjs::AsyncRuntime::new()?; - let context = rquickjs::AsyncContext::full(&runtime).await?; - let modules = HashMap::new(); - Ok(JsRuntime { context, modules }) +impl From for JsonValue { + fn from(value: DateTime) -> Self { + JsonValue(value.json()) } +} - pub async fn load_file(&mut self, path: impl AsRef) -> Result { - let path = path.as_ref(); - let source = tokio::fs::read_to_string(path).await?; - self.load_js(path, source) - } +impl TryFrom for Message { + type Error = FilterError; - pub fn load_js( - &mut self, - path: impl AsRef, - source: impl Into>, - ) -> Result { - let path = path.as_ref().to_path_buf(); - self.modules.insert(path.clone(), source.into()); - Ok(JsFilter::new(path)) + fn try_from(value: serde_json::Value) -> Result { + let message = serde_json::from_value(value) + .with_context(|| "Couldn't extract message payload and topic")?; + Ok(message) } +} - pub fn loaded_module(&self, path: PathBuf) -> Result { - match self.modules.get(&path) { - None => Err(LoadError::ScriptNotLoaded { path }), - Some(_) => Ok(JsFilter::new(path)), - } - } +impl TryFrom for Message { + type Error = FilterError; - pub async fn call_function( - &self, - module: &JsFilter, - function: &str, - args: Args, - ) -> Result - where - for<'a> Args: rquickjs::function::IntoArgs<'a> + Send + 'a, - for<'a> Ret: FromJs<'a> + Send + 'a, - { - let Some(source) = self.modules.get(&module.path) else { - return Err(LoadError::ScriptNotLoaded { - path: module.path.clone(), - }); - }; - - let name = module.path.display().to_string(); - - rquickjs::async_with!(self.context => |ctx| { - debug!(target: "MAPPING", "compile({name})"); - let m = rquickjs::Module::declare(ctx.clone(), name.clone(), source.clone())?; - let (m,p) = m.eval()?; - let () = p.finish()?; - - debug!(target: "MAPPING", "link({name})"); - let f: rquickjs::Value = m.get(function)?; - let f = rquickjs::Function::from_value(f)?; - - debug!(target: "MAPPING", "execute({name})"); - let r = f.call(args); - if r.is_err() { - if let Some(ex) = ctx.catch().as_exception() { - let err = anyhow::anyhow!("{ex}"); - Err(err.context("JS raised exception").into()) - } else { - let err = r.err().unwrap(); - debug!(target: "MAPPING", "execute({name}) => {err:?}"); - Err(err.into()) - } - } else { - Ok(r.unwrap()) - } - }) - .await + fn try_from(value: JsonValue) -> Result { + Message::try_from(value.0) } } -impl<'js> FromJs<'js> for Message { - fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { - debug!(target: "MAPPING", "from_js(...)"); - match value.as_object() { - None => Ok(Message { - topic: "".to_string(), - payload: "".to_string(), - }), - Some(object) => { - let topic = object.get("topic"); - let payload = object.get("payload"); - debug!(target: "MAPPING", "from_js(...) -> topic = {:?}, payload = {:?}", topic, payload); - Ok(Message { - topic: topic?, - payload: payload?, - }) +impl TryFrom for Vec { + type Error = FilterError; + + fn try_from(value: JsonValue) -> Result { + match value.0 { + serde_json::Value::Array(array) => array.into_iter().map(Message::try_from).collect(), + serde_json::Value::Object(map) => { + Message::try_from(serde_json::Value::Object(map)).map(|message| vec![message]) } + _ => Err(anyhow::anyhow!("Filters are expected to return an array of messages").into()), } } } -impl<'js> IntoJs<'js> for Message { +struct JsonValueRef<'a>(&'a serde_json::Value); + +impl<'js> IntoJs<'js> for JsonValue { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { - debug!(target: "MAPPING", "into_js({self:?})"); - let msg = Object::new(ctx.clone())?; - msg.set("topic", self.topic)?; - msg.set("payload", self.payload)?; - Ok(Value::from_object(msg)) + JsonValueRef(&self.0).into_js(ctx) } } -impl<'js> IntoJs<'js> for DateTime { +impl<'js> IntoJs<'js> for &JsonValue { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { - debug!(target: "MAPPING", "into_js({self:?})"); - let msg = Object::new(ctx.clone())?; - msg.set("seconds", self.seconds)?; - msg.set("nanoseconds", self.nanoseconds)?; - Ok(Value::from_object(msg)) + JsonValueRef(&self.0).into_js(ctx) } } -impl<'js> IntoJs<'js> for JsonValue { +impl<'a, 'js> IntoJs<'js> for JsonValueRef<'a> { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { match self.0 { serde_json::Value::Null => Ok(Value::new_null(ctx.clone())), - serde_json::Value::Bool(value) => Ok(Value::new_bool(ctx.clone(), value)), + serde_json::Value::Bool(value) => Ok(Value::new_bool(ctx.clone(), *value)), serde_json::Value::Number(value) => { if let Some(n) = value.as_i64() { if let Ok(n) = i32::try_from(n) { @@ -262,20 +205,20 @@ impl<'js> IntoJs<'js> for JsonValue { Ok(nan.into_value()) } serde_json::Value::String(value) => { - let string = rquickjs::String::from_str(ctx.clone(), &value)?; + let string = rquickjs::String::from_str(ctx.clone(), value)?; Ok(string.into_value()) } serde_json::Value::Array(values) => { let array = rquickjs::Array::new(ctx.clone())?; - for (i, value) in values.into_iter().enumerate() { - array.set(i, JsonValue(value))?; + for (i, value) in values.iter().enumerate() { + array.set(i, JsonValueRef(value))?; } Ok(array.into_value()) } serde_json::Value::Object(values) => { let object = rquickjs::Object::new(ctx.clone())?; for (key, value) in values.into_iter() { - object.set(key, JsonValue(value))?; + object.set(key, JsonValueRef(value))?; } Ok(object.into_value()) } @@ -327,7 +270,8 @@ mod tests { async fn identity_filter() { let script = "export function process(t,msg) { return [msg]; };"; let mut runtime = JsRuntime::try_new().await.unwrap(); - let filter = runtime.load_js("id.js", script).unwrap(); + runtime.load_js("id.js", script).await.unwrap(); + let filter = JsFilter::new("id.js".into()); let input = Message::new("te/main/device///m/", "hello world"); let output = input.clone(); @@ -344,7 +288,8 @@ mod tests { async fn error_filter() { let script = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; let mut runtime = JsRuntime::try_new().await.unwrap(); - let filter = runtime.load_js("err.js", script).unwrap(); + runtime.load_js("err.js", script).await.unwrap(); + let filter = JsFilter::new("err.js".into()); let input = Message::new("te/main/device///m/", "hello world"); let error = filter @@ -379,7 +324,8 @@ export function process (timestamp, message, config) { } "#; let mut runtime = JsRuntime::try_new().await.unwrap(); - let filter = runtime.load_js("collectd.js", script).unwrap(); + runtime.load_js("collectd.js", script).await.unwrap(); + let filter = JsFilter::new("collectd.js".into()); let input = Message::new( "collectd/h/memory/percent-used", diff --git a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs new file mode 100644 index 00000000000..892b9f3793d --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs @@ -0,0 +1,204 @@ +use crate::js_filter::JsonValue; +use crate::LoadError; +use anyhow::anyhow; +use rquickjs::module::Evaluated; +use rquickjs::Ctx; +use rquickjs::Module; +use std::collections::HashMap; +use std::path::Path; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tracing::debug; + +pub struct JsRuntime { + runtime: rquickjs::AsyncRuntime, + worker: mpsc::Sender, +} + +impl JsRuntime { + pub async fn try_new() -> Result { + let runtime = rquickjs::AsyncRuntime::new()?; + let context = rquickjs::AsyncContext::full(&runtime).await?; + let worker = JsWorker::spawn(context).await; + Ok(JsRuntime { runtime, worker }) + } + + pub async fn load_file(&mut self, path: impl AsRef) -> Result<(), LoadError> { + let path = path.as_ref(); + let source = tokio::fs::read_to_string(path).await?; + self.load_js(path, source).await + } + + pub async fn load_js( + &mut self, + path: impl AsRef, + source: impl Into>, + ) -> Result<(), LoadError> { + let (sender, receiver) = oneshot::channel(); + let path = path.as_ref().to_path_buf(); + let name = path.display().to_string(); + let source = source.into(); + self.worker + .send(JsRequest::LoadModule { + name, + source, + sender, + }) + .await + .map_err(|err| anyhow!(err))?; + receiver.await.map_err(|err| anyhow!(err))? + } + + pub async fn call_function( + &self, + module: &Path, + function: &str, + args: Vec, + ) -> Result { + let (sender, receiver) = oneshot::channel(); + self.worker + .send(JsRequest::CallFunction { + module: module.display().to_string(), + function: function.to_string(), + args, + sender, + }) + .await + .map_err(|err| anyhow!(err))?; + receiver.await.map_err(|err| anyhow!(err))? + } + + pub async fn dump_memory_stats(&self) { + let usage = self.runtime.memory_usage().await; + tracing::info!(target: "gen-mapper", "Memory usage:"); + tracing::info!(target: "gen-mapper", " - malloc size: {}", usage.malloc_size); + tracing::info!(target: "gen-mapper", " - used memory size: {}", usage.memory_used_size); + tracing::info!(target: "gen-mapper", " - function count: {}", usage.js_func_count); + tracing::info!(target: "gen-mapper", " - object count: {}", usage.obj_count); + tracing::info!(target: "gen-mapper", " - array count: {}", usage.array_count); + tracing::info!(target: "gen-mapper", " - string count: {}", usage.str_count); + tracing::info!(target: "gen-mapper", " - atom count: {}", usage.atom_count); + } +} + +enum JsRequest { + LoadModule { + name: String, + source: Vec, + sender: oneshot::Sender>, + }, + CallFunction { + module: String, + function: String, + args: Vec, + sender: oneshot::Sender>, + }, +} + +struct JsWorker { + context: rquickjs::AsyncContext, + requests: mpsc::Receiver, +} + +impl JsWorker { + pub async fn spawn(context: rquickjs::AsyncContext) -> mpsc::Sender { + let (sender, requests) = mpsc::channel(100); + tokio::spawn(async move { + let worker = JsWorker { context, requests }; + worker.run().await + }); + sender + } + + async fn run(mut self) { + rquickjs::async_with!(self.context => |ctx| { + let mut modules = JsModules::new(); + while let Some(request) = self.requests.recv().await { + match request { + JsRequest::LoadModule{name, source, sender} => { + let result = modules.load_module(ctx.clone(), name, source).await; + let _ = sender.send(result); + } + JsRequest::CallFunction{module, function, args, sender} => { + let result = modules.call_function(ctx.clone(), module, function, args).await; + let _ = sender.send(result); + } + } + } + }) + .await + } +} + +struct JsModules<'js> { + modules: HashMap>, +} + +impl<'js> JsModules<'js> { + fn new() -> Self { + JsModules { + modules: HashMap::new(), + } + } + + async fn load_module( + &mut self, + ctx: Ctx<'js>, + name: String, + source: Vec, + ) -> Result<(), LoadError> { + debug!(target: "MAPPING", "compile({name})"); + let module = Module::declare(ctx, name.clone(), source)?; + let (module, p) = module.eval()?; + let () = p.finish()?; + self.modules.insert(name, module); + Ok(()) + } + + async fn call_function( + &mut self, + ctx: Ctx<'js>, + module_name: String, + function: String, + args: Vec, + ) -> Result { + debug!(target: "MAPPING", "link({module_name}.{function})"); + let module = self + .modules + .get(&module_name) + .ok_or_else(|| LoadError::UnknownModule { + module_name: module_name.clone(), + })?; + let f: rquickjs::Value = module + .get(&function) + .map_err(|_| LoadError::UnknownFunction { + module_name: module_name.clone(), + function: function.clone(), + })?; + let f = rquickjs::Function::from_value(f)?; + + debug!(target: "MAPPING", "execute({module_name}.{function})"); + let r = match &args[..] { + [] => f.call(()), + [v0] => f.call((v0,)), + [v0, v1] => f.call((v0, v1)), + [v0, v1, v2] => f.call((v0, v1, v2)), + [v0, v1, v2, v3] => f.call((v0, v1, v2, v3)), + [v0, v1, v2, v3, v4] => f.call((v0, v1, v2, v3, v4)), + [v0, v1, v2, v3, v4, v5] => f.call((v0, v1, v2, v3, v4, v5)), + [v0, v1, v2, v3, v4, v5, v6] => f.call((v0, v1, v2, v3, v4, v5, v6)), + _ => return Err(anyhow::anyhow!("Too many args").into()), + }; + + debug!(target: "MAPPING", "execute({module_name}.{function}) => {r:?}"); + r.map_err(|err| { + if let Some(ex) = ctx.catch().as_exception() { + let err = anyhow::anyhow!("{ex}"); + err.context("JS raised exception").into() + } else { + debug!(target: "MAPPING", "execute({module_name}.{function}) => {err:?}"); + err.into() + } + }) + } +} diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index ea8a7936e27..130094195a5 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -1,11 +1,12 @@ mod actor; mod config; mod js_filter; +mod js_runtime; mod pipeline; use crate::actor::GenMapper; use crate::config::PipelineConfig; -use crate::js_filter::JsRuntime; +use crate::js_runtime::JsRuntime; use crate::pipeline::Pipeline; use camino::Utf8Path; use camino::Utf8PathBuf; @@ -189,8 +190,14 @@ impl Builder for GenMapperBuilder { #[derive(thiserror::Error, Debug)] pub enum LoadError { - #[error("Script not loaded: {path}")] - ScriptNotLoaded { path: PathBuf }, + #[error("JavaScript module not found: {module_name}")] + UnknownModule { module_name: String }, + + #[error("JavaScript function not found: {function} in {module_name}")] + UnknownFunction { + module_name: String, + function: String, + }, #[error(transparent)] IoError(#[from] std::io::Error), diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 73da0e0a6f1..e9e993a03f2 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -1,5 +1,5 @@ use crate::js_filter::JsFilter; -use crate::js_filter::JsRuntime; +use crate::js_runtime::JsRuntime; use crate::LoadError; use camino::Utf8PathBuf; use serde_json::json; @@ -44,6 +44,9 @@ pub enum FilterError { #[error("No messages can be processed due to an incorrect setting: {0}")] IncorrectSetting(String), + + #[error(transparent)] + Anyhow(#[from] anyhow::Error), } impl Pipeline { @@ -123,6 +126,10 @@ impl DateTime { pub fn tick_now(&self, tick_every_seconds: u64) -> bool { tick_every_seconds != 0 && (self.seconds % tick_every_seconds == 0) } + + pub fn json(&self) -> Value { + json!({"seconds": self.seconds, "nanoseconds": self.nanoseconds}) + } } impl TryFrom for DateTime { From b5faa994235602f1cf769b2c470a0910ce2ebbfb Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 4 Jun 2025 19:20:23 +0200 Subject: [PATCH 21/53] Add benchmark options to tedge mqtt pub Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mqtt/cli.rs | 10 ++++++++++ crates/core/tedge/src/cli/mqtt/publish.rs | 12 +++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/core/tedge/src/cli/mqtt/cli.rs b/crates/core/tedge/src/cli/mqtt/cli.rs index a99931e6415..5739d6ac541 100644 --- a/crates/core/tedge/src/cli/mqtt/cli.rs +++ b/crates/core/tedge/src/cli/mqtt/cli.rs @@ -30,6 +30,12 @@ pub enum TEdgeMqttCli { /// Retain flag #[clap(short, long = "retain")] retain: bool, + /// Repeat the message + #[clap(long)] + repeat: Option, + /// Pause between repeated messages (e.g., 60s, 1h) + #[clap(long, default_value = "1s")] + sleep: SecondsOrHumanTime, }, /// Subscribe a MQTT topic. @@ -69,6 +75,8 @@ impl BuildCommand for TEdgeMqttCli { message, qos, retain, + repeat, + sleep, } => MqttPublishCommand { host: config.mqtt.client.host.clone(), port: config.mqtt.client.port.into(), @@ -80,6 +88,8 @@ impl BuildCommand for TEdgeMqttCli { ca_file: auth_config.ca_file.clone(), ca_dir: auth_config.ca_dir, client_auth_config: auth_config.client, + count: repeat.unwrap_or(1), + sleep: sleep.duration(), } .into_boxed(), TEdgeMqttCli::Sub { diff --git a/crates/core/tedge/src/cli/mqtt/publish.rs b/crates/core/tedge/src/cli/mqtt/publish.rs index 48873ba61b1..9f19507ee94 100644 --- a/crates/core/tedge/src/cli/mqtt/publish.rs +++ b/crates/core/tedge/src/cli/mqtt/publish.rs @@ -22,6 +22,8 @@ pub struct MqttPublishCommand { pub ca_file: Option, pub ca_dir: Option, pub client_auth_config: Option, + pub count: u32, + pub sleep: std::time::Duration, } #[async_trait::async_trait] @@ -36,7 +38,15 @@ impl Command for MqttPublishCommand { } async fn execute(&self, _: TEdgeConfig) -> Result<(), MaybeFancy> { - Ok(publish(self).await?) + let mut i = 0; + loop { + publish(self).await?; + i += 1; + if i == self.count { + return Ok(()); + } + tokio::time::sleep(self.sleep).await; + } } } From b97ab44e537ca83c14c515ff55b62cb9705bd15e Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 6 Jun 2025 10:50:08 +0200 Subject: [PATCH 22/53] Add gen-mapper system tests Signed-off-by: Didier Wenzek --- ci/build_scripts/build.sh | 2 +- .../pipelines/add_timestamp.js | 11 ++ .../pipelines/measurements.toml | 7 + .../tedge_gen_mapper/pipelines/set_topic.js | 6 + .../tedge_gen_mapper/pipelines/te_to_c8y.js | 124 ++++++++++++++++++ .../tedge_gen_mapper/tedge_gen_mapper.robot | 27 ++++ 6 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/add_timestamp.js create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot diff --git a/ci/build_scripts/build.sh b/ci/build_scripts/build.sh index a9b66114c56..0e232293a37 100755 --- a/ci/build_scripts/build.sh +++ b/ci/build_scripts/build.sh @@ -101,7 +101,7 @@ BUILD_WITH="${BUILD_WITH:-zig}" COMMON_BUILD_OPTIONS=( "--release" ) -TOOLCHAIN="${TOOLCHAIN:-+1.78}" +TOOLCHAIN="${TOOLCHAIN:-+1.82}" # Note: Minimum version that is supported with riscv64gc-unknown-linux-gnu is 2.27 GLIBC_VERSION="${GLIBC_VERSION:-2.17}" RISCV_GLIBC_VERSION="${RISCV_GLIBC_VERSION:-2.27}" diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/add_timestamp.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/add_timestamp.js new file mode 100644 index 00000000000..f387141bfe3 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/add_timestamp.js @@ -0,0 +1,11 @@ +export function process (timestamp, message) { + let payload = JSON.parse(message.payload) + if (!payload.time) { + payload.time = timestamp.seconds + (timestamp.nanoseconds / 1e9) + } + + return [{ + topic: message.topic, + payload: JSON.stringify(payload) + }] +} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml new file mode 100644 index 00000000000..a777177165b --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml @@ -0,0 +1,7 @@ +input_topics = ["te/+/+/+/+/m/+"] + +stages = [ + { filter = "add_timestamp.js" }, + { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] }, + { filter = "set_topic.js", config = { topic = "gen-mapper/c8y" } } +] diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js new file mode 100644 index 00000000000..16ce07f0589 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js @@ -0,0 +1,6 @@ +export function process (timestamp, message, config) { + return [{ + topic: config?.topic || "te/error", + payload: message.payload + }] +} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js new file mode 100644 index 00000000000..ae2d8f157c1 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -0,0 +1,124 @@ +/// Transform: +/// +/// ``` +/// [te/device/main///m/example] { +/// "time": "2020-10-15T05:30:47+00:00", +/// "temperature": 25, +/// "location": { +/// "latitude": 32.54, +/// "longitude": -117.67, +/// "altitude": 98.6 +/// }, +/// "pressure": 98 +/// } +/// ``` +/// +/// into +/// +/// ``` +/// [c8y/measurement/measurements/create] { +/// "time": "2020-10-15T05:30:47Z", +/// "type": "example", +/// "temperature": { +/// "temperature": { +/// "value": 25 +/// } +/// }, +/// "location": { +/// "latitude": { +/// "value": 32.54 +/// }, +/// "longitude": { +/// "value": -117.67 +/// }, +/// "altitude": { +/// "value": 98.6 +/// } +/// }, +/// "pressure": { +/// "pressure": { +/// "value": 98 +/// } +/// } +/// } +/// ``` +export function process(t, message, config) { + let topic_parts = message.topic.split( '/') + let type = topic_parts[6] + let payload = JSON.parse(message.payload) + + let c8y_msg = { + type: type + } + + let meta = (config || {})[`${message.topic}/meta`] || {} + + for (let [k, v] of Object.entries(payload)) { + let k_meta = (meta || {})[k] || {} + if (k === "time") { + let fragment = { time: v } + Object.assign(c8y_msg, fragment) + } + else if (typeof(v) === "number") { + if (Object.keys(k_meta).length>0) { + v = { value: v, ...k_meta } + } + let fragment = { [k]: { [k]: v } } + Object.assign(c8y_msg, fragment) + } else for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_k_meta = k_meta[sub_k] + if (typeof(sub_v) === "number") { + if (sub_k_meta) { + sub_v = { value: sub_v, ...sub_k_meta } + } + let fragment = { [k]: { [sub_k]: sub_v } } + Object.assign(c8y_msg, fragment) + } + } + } + + return [{ + topic: "c8y/measurement/measurements/create", + payload: JSON.stringify(c8y_msg) + }] +} + +/// Update the config with measurement metadata. +/// +/// These metadata are expected to have the same shape of the actual values. +/// +/// ``` +/// [te/device/main///m/example/meta] { "temperature": { "unit": "°C" }} +/// ``` +/// +/// and: +/// ``` +/// [te/device/main///m/example] { "temperature": { "unit": 23 }} +/// ``` +/// +/// will be merged by the process function into: +/// ``` +/// [c8y/measurement/measurements/create] { +/// "type": "example", +/// "temperature": { +/// "temperature": { +/// "value": 23, +/// "unit": "°C" +/// } +/// } +/// } +/// ``` +export function update_config(message, config) { + let type = message.topic + let metadata = JSON.parse(message.payload) + + let fragment = { + [type]: metadata + } + if (!config) { + config = {} + } + Object.assign(config, fragment) + + return config +} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot new file mode 100644 index 00000000000..67ed106835e --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -0,0 +1,27 @@ +*** Settings *** +Library ThinEdgeIO + +Test Setup Custom Setup +Test Teardown Get Logs + +Test Tags theme:tedge_mapper + +*** Test Cases *** +Add missing timestamps + Execute Command tedge mqtt pub te/device/main// '{}' + ${transformed_msg} Should Have MQTT Messages gen-mapper/c8y + Should Contain ${transformed_msg} item=time + +*** Keywords *** +Custom Setup + ${DEVICE_SN}= Setup + Set Suite Variable $DEVICE_SN + Copy Configuration Files + Start Generic Mapper + +Copy Configuration Files + Execute Command mkdir /etc/tedge/gen-mapper/ + ThinEdgeIO.Transfer To Device ${CURDIR}/pipelines/* /etc/tedge/gen-mapper/ + +Start Generic Mapper + Execute Command nohup tedge run tedge-mapper gen & From 7673515c247dfd0a77a28052013bc20db58638f3 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 6 Jun 2025 19:42:00 +0200 Subject: [PATCH 23/53] Add a reference guide Signed-off-by: Didier Wenzek --- docs/src/references/mappers/gen-mapper.md | 168 ++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 docs/src/references/mappers/gen-mapper.md diff --git a/docs/src/references/mappers/gen-mapper.md b/docs/src/references/mappers/gen-mapper.md new file mode 100644 index 00000000000..2696ddf73e5 --- /dev/null +++ b/docs/src/references/mappers/gen-mapper.md @@ -0,0 +1,168 @@ +--- +title: Generic Mapper +tags: [Reference, Mappers, Cloud] +sidebar_position: 2 +draft: true +--- + +import ProposalBanner from '@site/src/components/ProposalBanner' + + + +:::note +This section is actually a design document. +It includes a reference guide for the POC, but also proposes a plan toward a generic mapper. +::: + +## Motivation + +In theory, %%te%% users can implement customized mappers to transform data published on the MQTT bus +or to interact with the cloud. In practice, they don't. +Implementing a mapper is costly while what is provided out-the-box by %%te%% already meets most requirements. +The need is not to write new mappers but to adapt existing ones. + +The aim of the generic mapper it to let users extend and adapt the mappers with their own filtering and mapping rules, +leveraging the core mapping rules and mapper mechanisms (bridge connections, HTTP proxies, operations). + +## Vision + +The %%te%% mappers for Cumulocity, Azure, AWS and Collectd are implemented on top of a so-called generic mapper +which is used to drive all MQTT message transformations. +- Transformations are implemented as pipelines consuming MQTT messages, feeding a chain of filters and producing MQTT messages. + - `MQTT sub| filter-1 | filter-2 | ... | filter-n | MQTT pub` +- A pipeline can combine builtin and user-provided filters. +- The user can configure all the transformations used by a mapper, + editing MQTT sources, pipelines, filters and MQTT sinks. +- By contrast with the current implementation, where the translation of measurements from %%te%% JSON to Cumulocity JSON + is fully hard-coded, with the generic mapper a user can re-use the core of this transformation while adding customized steps: + - consuming measurement from a non-standard topic + - filtering out part of the measurements + - normalizing units + - adding units read from some config + - producing transformed measurements on a non-standard topic. + +## POC reference + +- The generic mapper loads pipeline and filters stored in `/etc/tedge/gen-mapper/`. +- A pipeline is defined by a TOML file with `.toml` extension. +- A filter is defined by a Javascript file with `.js` extension. +- The definition of pipeline must provide a list of MQTT topics to subscribe to. + - The pipeline will be feed with all the messages received on these topics. +- A pipeline definition also provides a list of stages. + - Each stage is built from a javascript and is possibly given a config (arbitrary json that will be passed to the script) + - Each stage can also subscribe to a list of MQTT topics (which messages will be passed to the script to update its config) + +```toml +input_topics = ["te/+/+/+/+/m/+"] + +stages = [ + { filter = "add_timestamp.js" }, + { filter = "drop_stragglers.js", config = { max_delay = 60 } }, + { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } +] +``` + +- A filter has to export at least one `process` function. + - `process(t: Timestamp, msg: Message, config: Json) -> Vec` + - This function is called for each message to be transformed + - The arguments passed to the function are: + - The current time as `{ seconds: u64, nanoseconds: u32 }` + - The message `{ topic: string, payload: string }` + - The config as read from the pipeline config or updated by the script + - The function is expected to return zero, one or many transformed messages `[{ topic: string, payload: string }]` + - An exception can be thrown if the input message cannot be transformed. +- A filter can also export an `update_config` function + - This function is called on each message received on the `meta_topics` as defined in the config. + - The arguments are: + - The message to be interpreted as a config update `{ topic: string, payload: string }` + - The current config + - The returned value (an arbitrary JSON value) is then used as the new config for the filter. +- A filter can also export a `tick` function + - This function is called at a regular pace with the current time and config. + - The filter can then return zero, one or many transformed messages + - By sharing an internal state between the `process` and `tick` functions, + the filter can implement aggregations over a time window. + When messages are received they are pushed by the `process` function into that state + and the final outcome is extracted by the `tick` function at the end of the time window. + +## First release + +While the POC provides a generic mapper that is fully independent of the legacy mappers, +the plan is not to abandon the latter in favor of the former +but to revisit the legacy mappers to include the ability for users to add their own mapping rules. + +To be lovable, the first release of an extensible mapper should at least: + +- be a drop-in replacement of the current mapper (for c8y, aws, az or collect) +- feature the ability to customize MEA processing by combining builtin filters with user-provided functions written in JavaScript +- provide tools to create, test, monitor and debug filters and pipelines +- be stable enough that user-defined filters will still work without changes with future releases. + +To keep things simple for the first release, the following questions are deferred: + +- Could a generic mapper let users define bridge rules as well as message transformation pipelines? +- Does it make sense to run such a mapper on child-devices? +- Could a pipeline send HTTP messages? Or could a filter tell the runtime to send messages over HTTP? +- How to handle binary payloads on the MQTT bus? +- Could operations be managed is a similar way with user-provided functions to transform commands? +- To handle operations, would the plugins be expanded to do more complex things like HTTP calls, file-system interactions, etc.? +- What are the pros and cons to persist filter states? +- Split a pipeline, forwarding transformed messages to different pipelines for further processing + +### API + +The POC expects the filter to implement a bunch of functions. This gives a quite expressive interface +(filtering, mapping, splitting, dynamic configuration, aggregation over time windows), but at the cost of some complexity. + +- `process(t: Timestamp, msg: Message, config: Json) -> Vec` +- `tick(t: Timestamp) -> Vec` +- `update_config(msg: Message, config: Json) -> Json` + +An alternative is to let the user implement more specific functions with simpler type signatures: + +- `filter(msg: Message, config: Json) -> bool` +- `map(msg: Message, config: Json) -> Message` +- `filter_map(msg: Message, config: Json) -> Option` +- `flat_map(msg: Message, config: Json) -> Vec` + +One can also rearrange the argument order for these functions, +making life easier when a transformation does need a config or the current time +leveraging that one can pass more arguments than declared to a javascript function: + +- `process(msg: Message, config: Json, t: Timestamp) -> Vec` +- `process(msg: Message, config: Json) -> Vec` +- `process(msg: Message) -> Vec` + +One can even use a bit further the flexibility of javascript, to let the process function freely return: +- An array of message objects +- A single message object +- A null value interpreted as no messages +- A boolean + +Other ideas to explore to make the API more flexible: + +- Interaction with the entity store and tedge config. +- Allow a pipeline to subscribe to topics related to the device/entity it is running on +- Feed filters with message excerpts as done for the workflows + +### Devops tools + +The flexibility to customize MQTT message processing with user-provided functions comes with risks: +- a filter might not behave as expected, +- pipelines might be overlapping or conflicting, possibly sending duplicate messages or creating infinite loops +- builtin pipelines might be accidentally disconnected or broken +- a filter might introduce a performance bottleneck. + +To help mitigating these risks, the `tedge mapping` sub-commands provide the tools to test, monitor and debug filters and pipelines. + +- `tedge mapping flow [topic]` displays pipelines and filters messages received on this topic will flow through + - can be used with a set of pipelines not configured yet for a mapper +- `tedge mapping test [filter]` feeds a filter or pipeline with input messages and produces the transformed output messages + - allow users to run an assertion based on the input/output of a filter + - ability to pipe `tedge mqtt sub` and `tedge mapping test` + - control of the timestamps + - test aggregation over ticks + - can be used with a set of pipelines not configured yet for a mapper +- `tedge mapping stats [pipeline]` returns statistics on the messages processed by a pipeline + - count message in, message out + - processing time min, median, max per filter From 2a40620e4f6c160a55fe582459ea3683479c5f0a Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 20 Jun 2025 10:41:04 +0200 Subject: [PATCH 24/53] Implement a circuit breaker filter Signed-off-by: Didier Wenzek --- .../pipelines/circuit-breaker.js | 65 +++++++++++++++++++ .../tedge_gen_mapper/pipelines/loop.toml | 7 ++ 2 files changed, 72 insertions(+) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/loop.toml diff --git a/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js b/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js new file mode 100644 index 00000000000..3ecb09e4bf4 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js @@ -0,0 +1,65 @@ +// A filter that let messages go through, unless too many messages are received within a given period +// +// This filter is configured by the following settings: +// - tick_every_seconds: the frequency at which the sliding window is moved +// - tick_count: size of the time windows +// - too_many: how many messages is too many (received during the last tick_count*tick_every_seconds seconds) +// - back_to_normal: how many messages is okay to reactivate the filter if bellow +// - message_on_too_many: message sent when the upper threshold is crossed +// - message_on_back_to_normal: message sent when the lower threshold is crossed +// - stats_topic: topic for statistic messages +class State { + static open = false + static total = 0 + static batch = [0] +} + + +export function process (timestamp, message, config) { + State.total += 1 + State.batch[0] += 1 + if (State.open) { + let back_to_normal = config?.back_to_normal || 100 + if (State.total < back_to_normal) { + State.open = false + if (config?.message_on_back_to_normal) { + return [config?.message_on_back_to_normal, message] + } else { + return [message] + } + } else { + return [] + } + } else { + let too_many = config?.too_many || 1000 + if (State.total < too_many) { + return [message] + } else { + State.open = true + if (config?.message_on_too_many) { + return [config?.message_on_too_many] + } else { + return [] + } + } + } +} + + +export function tick(timestamp, config) { + let max_batch_count = config?.tick_count || 10 + let new_batch_count = State.batch.unshift(0) + if (new_batch_count > max_batch_count) { + State.total -= State.batch.pop() + } + + if (config?.stats_topic) { + return [{ + topic: config?.stats_topic, + payload: `{"circuit-breaker-open": ${State.open}, "total": ${State.total}, "batch": ${State.batch}}` + }] + } else { + return [] + } + +} \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/pipelines/loop.toml b/crates/extensions/tedge_gen_mapper/pipelines/loop.toml new file mode 100644 index 00000000000..64304daf4a3 --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/loop.toml @@ -0,0 +1,7 @@ +# This pipeline is on purpose looping: the messages are published to the same topic +input_topics = ["loopback/#"] + +stages = [ + { filter = "add_timestamp.js" }, + { filter = "circuit-breaker.js", tick_every_seconds = 1, config = { stats_topic = "te/error", too_many = 10000, message_on_too_many = { topic = "te/device/main///a/too-many-messages", payload = "too many messages" }, message_on_back_to_normal = { topic = "te/device/main///a/too-many-messages", payload = "back to normal" } } } +] From f797861e3bd71c6501cdd8c04d59fe78dbe55326 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 20 Jun 2025 16:37:30 +0200 Subject: [PATCH 25/53] Add a filter to compute average over a time window Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/pipelines/average.js | 99 +++++++++++++++++++ .../tedge_gen_mapper/pipelines/collectd.toml | 2 +- 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 crates/extensions/tedge_gen_mapper/pipelines/average.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/average.js b/crates/extensions/tedge_gen_mapper/pipelines/average.js new file mode 100644 index 00000000000..1a3f1c7e8dd --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/pipelines/average.js @@ -0,0 +1,99 @@ +// Compute the average value of a series of measurements received during a time windows +// - Take care of the topic: messages received over different topics are not mixed +// - Ignore messages which are not formated as thin-edge JSON +// - Ignore values which are not numbers +// - Use the first timestamp as the timestamp for the aggregate +class State { + static agg_for_topic = {} +} + +export function process (timestamp, message) { + let topic = message.topic + let payload = JSON.parse(message.payload) + let agg_payload = State.agg_for_topic[topic] + + if (agg_payload) { + for (let [k, v] of Object.entries(payload)) { + let agg = agg_payload[k] + if (k === "time") { + if (!agg) { + let fragment = {time: v} + Object.assign(agg_payload, fragment) + } + } else if (typeof (v) === "number") { + if (!agg) { + let fragment = {k: {sum: v, count: 1}} + Object.assign(agg_payload, fragment) + } else { + agg.sum += v + agg.count += 1 + } + } else { + if (!agg) { + for (let [sub_k, sub_v] of Object.entries(v)) { + let fragment = { [k]: { [sub_k]: { sum: sub_v, count: 1 } } } + Object.assign(agg_payload, fragment) + } + } else { + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_agg = agg_payload[sub_k] + if (!sub_agg) { + let fragment = {k: { [sub_k]: { sum: sub_v, count: 1 } } } + Object.assign(agg_payload, fragment) + } else { + sub_agg.sum += sub_v + sub_agg.count += 1 + } + } + } + } + } + } else { + let agg_payload = {} + for (let [k, v] of Object.entries(payload)) { + if (k === "time") { + let fragment = { time: v } + Object.assign(agg_payload, fragment) + } + else if (typeof(v) === "number") { + let fragment = { k: { sum: v, count: 1 } } + Object.assign(agg_payload, fragment) + } else for (let [sub_k, sub_v] of Object.entries(v)) { + let fragment = { [k]: { [sub_k]: { sum: sub_v, count: 1 } } } + Object.assign(agg_payload, fragment) + } + } + State.agg_for_topic[topic] = agg_payload + } + + return [] +} + +export function tick() { + let messages = [] + + for (let [topic, agg] of Object.entries(State.agg_for_topic)) { + let payload = {} + for (let [k, v] of Object.entries(agg)) { + if (k === "time") { + let fragment = { time: v } + Object.assign(payload, fragment) + } + else if (v.sum && v.count) { + let fragment = { k: v.sum / v.count } + Object.assign(payload, fragment) + } else for (let [sub_k, sub_v] of Object.entries(v)) { + let fragment = { [k]: { [sub_k]: sub_v.sum / sub_v.count } } + Object.assign(payload, fragment) + } + } + + messages.push ({ + topic: topic, + payload: JSON.stringify(payload) + }) + } + + State.agg_for_topic = {} + return messages +} \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml index 44feb5b3323..05330b870f8 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml @@ -2,5 +2,5 @@ input_topics = ["collectd/+/+/+"] stages = [ { filter = "collectd-to-te.js" }, - { filter = "group_by_timestamp.js", tick_every_seconds = 3 } + { filter = "average.js", tick_every_seconds = 10 } ] From ed33143ab191f7ad1f67661684d8f0f551aa2ca3 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 25 Jun 2025 19:35:04 +0200 Subject: [PATCH 26/53] Extract message processing from gen-mapper actor The goal is to be able to use the message processor from within the tedge cli command, notably for testing pipelines and filters. Signed-off-by: Didier Wenzek --- crates/core/tedge_mapper/src/gen/mod.rs | 1 - .../extensions/tedge_gen_mapper/src/actor.rs | 77 +------ crates/extensions/tedge_gen_mapper/src/lib.rs | 111 +--------- .../tedge_gen_mapper/src/runtime.rs | 205 ++++++++++++++++++ 4 files changed, 227 insertions(+), 167 deletions(-) create mode 100644 crates/extensions/tedge_gen_mapper/src/runtime.rs diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs index acdaa3a6961..e89644506de 100644 --- a/crates/core/tedge_mapper/src/gen/mod.rs +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -18,7 +18,6 @@ impl TEdgeComponent for GenMapper { let mut fs_actor = FsWatchActorBuilder::new(); let mut gen_mapper = GenMapperBuilder::try_new("/etc/tedge/gen-mapper").await?; - gen_mapper.load().await; gen_mapper.connect(&mut mqtt_actor); gen_mapper.connect_fs(&mut fs_actor); diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 121c6b3d38c..3497e3f6c9e 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,14 +1,10 @@ -use crate::config::PipelineConfig; -use crate::js_runtime::JsRuntime; use crate::pipeline::DateTime; use crate::pipeline::Message; -use crate::pipeline::Pipeline; +use crate::runtime::MessageProcessor; use crate::InputMessage; use crate::OutputMessage; use async_trait::async_trait; use camino::Utf8PathBuf; -use std::collections::HashMap; -use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; use tedge_actors::Actor; @@ -23,14 +19,11 @@ use tedge_mqtt_ext::TopicFilter; use tokio::time::interval; use tokio::time::Duration; use tracing::error; -use tracing::info; pub struct GenMapper { pub(super) messages: SimpleMessageBox, - pub(super) pipelines: HashMap, pub(super) subscriptions: Arc>, - pub(super) js_runtime: JsRuntime, - pub(super) config_dir: PathBuf, + pub(super) processor: MessageProcessor, } #[async_trait] @@ -60,9 +53,9 @@ impl Actor for GenMapper { continue; }; if matches!(path.extension(), Some("js" | "ts")) { - self.reload_filter(path).await; + self.processor.reload_filter(path).await; } else if path.extension() == Some("toml") { - self.reload_pipeline(path).await; + self.processor.reload_pipeline(path).await; self.send_updated_subscriptions().await?; } }, @@ -79,51 +72,6 @@ impl Actor for GenMapper { } impl GenMapper { - async fn reload_filter(&mut self, path: Utf8PathBuf) { - for pipeline in self.pipelines.values_mut() { - for stage in &mut pipeline.stages { - if stage.filter.path() == path { - match self.js_runtime.load_file(&path).await { - Ok(()) => { - info!("Reloaded filter {path}"); - } - Err(e) => { - error!("Failed to reload filter {path}: {e}"); - return; - } - } - } - } - } - } - - async fn reload_pipeline(&mut self, path: Utf8PathBuf) { - for pipeline in self.pipelines.values_mut() { - if pipeline.source == path { - let Ok(source) = tokio::fs::read_to_string(&path).await else { - error!("Failed to read updated filter {path}"); - break; - }; - let config: PipelineConfig = match toml::from_str(&source) { - Ok(config) => config, - Err(e) => { - error!("Failed to parse toml for updated filter {path}: {e}"); - break; - } - }; - match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { - Ok(p) => { - *pipeline = p; - info!("Reloaded pipeline {path}"); - } - Err(e) => { - error!("Failed to load updated pipeline {path}: {e}") - } - }; - } - } - } - async fn send_updated_subscriptions(&mut self) -> Result<(), RuntimeError> { let topics = self.update_subscriptions(); let diff = SubscriptionDiff::new(&topics, &TopicFilter::empty()); @@ -135,19 +83,14 @@ impl GenMapper { fn update_subscriptions(&self) -> TopicFilter { let mut topics = self.subscriptions.lock().unwrap(); - for pipeline in self.pipelines.values() { - topics.add_all(pipeline.topics()) - } + topics.add_all(self.processor.subscriptions()); topics.clone() } async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); - for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - match pipeline - .process(&self.js_runtime, ×tamp, &message) - .await - { + for (pipeline_id, pipeline_messages) in self.processor.process(×tamp, &message).await { + match pipeline_messages { Ok(messages) => { for message in messages { match MqttMessage::try_from(message) { @@ -174,10 +117,10 @@ impl GenMapper { async fn tick(&mut self) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); if timestamp.seconds % 300 == 0 { - self.js_runtime.dump_memory_stats().await; + self.processor.dump_memory_stats().await; } - for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - match pipeline.tick(&self.js_runtime, ×tamp).await { + for (pipeline_id, pipeline_messages) in self.processor.tick(×tamp).await { + match pipeline_messages { Ok(messages) => { for message in messages { match MqttMessage::try_from(message) { diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 130094195a5..cad842c304d 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -3,14 +3,10 @@ mod config; mod js_filter; mod js_runtime; mod pipeline; +mod runtime; use crate::actor::GenMapper; -use crate::config::PipelineConfig; -use crate::js_runtime::JsRuntime; -use crate::pipeline::Pipeline; -use camino::Utf8Path; -use camino::Utf8PathBuf; -use std::collections::HashMap; +use crate::runtime::MessageProcessor; use std::convert::Infallible; use std::path::Path; use std::path::PathBuf; @@ -31,106 +27,27 @@ use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::PublishOrSubscribe; use tedge_mqtt_ext::SubscriptionDiff; use tedge_mqtt_ext::TopicFilter; -use tokio::fs::read_dir; -use tokio::fs::read_to_string; use tracing::error; -use tracing::info; fan_in_message_type!(InputMessage[MqttMessage, FsWatchEvent]: Clone, Debug, Eq, PartialEq); fan_in_message_type!(OutputMessage[MqttMessage, SubscriptionDiff]: Clone, Debug, Eq, PartialEq); pub struct GenMapperBuilder { - config_dir: PathBuf, message_box: SimpleMessageBoxBuilder, - pipelines: HashMap, - pipeline_specs: HashMap, subscriptions: Arc>, - js_runtime: JsRuntime, + processor: MessageProcessor, } impl GenMapperBuilder { pub async fn try_new(config_dir: impl AsRef) -> Result { - let config_dir = config_dir.as_ref().to_owned(); - let js_runtime = JsRuntime::try_new().await?; + let processor = MessageProcessor::try_new(config_dir).await?; Ok(GenMapperBuilder { - config_dir, message_box: SimpleMessageBoxBuilder::new("GenMapper", 16), - pipelines: HashMap::default(), - pipeline_specs: HashMap::default(), subscriptions: Arc::new(Mutex::new(TopicFilter::empty())), - js_runtime, + processor, }) } - pub async fn load(&mut self) { - let Ok(mut entries) = read_dir(&self.config_dir).await.map_err(|err| - error!(target: "MAPPING", "Failed to read filters from {}: {err}", self.config_dir.display()) - ) else { - return; - }; - - while let Ok(Some(entry)) = entries.next_entry().await { - let Some(path) = Utf8Path::from_path(&entry.path()).map(|p| p.to_path_buf()) else { - error!(target: "MAPPING", "Skipping non UTF8 path: {}", entry.path().display()); - continue; - }; - if let Ok(file_type) = entry.file_type().await { - if file_type.is_file() { - match path.extension() { - Some("toml") => { - info!(target: "MAPPING", "Loading pipeline: {path}"); - if let Err(err) = self.load_pipeline(path).await { - error!(target: "MAPPING", "Failed to load pipeline: {err}"); - } - } - Some("js") | Some("ts") => { - info!(target: "MAPPING", "Loading filter: {path}"); - if let Err(err) = self.load_filter(path).await { - error!(target: "MAPPING", "Failed to load filter: {err}"); - } - } - _ => { - info!(target: "MAPPING", "Skipping file which type is unknown: {path}"); - } - } - } - } - } - - // Done here to ease the computation of the topics to subscribe to - // as these topics have to be known when connect is called - self.compile() - } - - async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { - if let Some(name) = file.as_ref().file_name() { - let specs = read_to_string(file.as_ref()).await?; - let pipeline: PipelineConfig = toml::from_str(&specs)?; - self.pipeline_specs - .insert(name.to_string(), (file.as_ref().to_owned(), pipeline)); - } - - Ok(()) - } - - async fn load_filter(&mut self, file: impl AsRef) -> Result<(), LoadError> { - self.js_runtime.load_file(file.as_ref()).await?; - Ok(()) - } - - fn compile(&mut self) { - for (name, (source, specs)) in self.pipeline_specs.drain() { - match specs.compile(&self.js_runtime, &self.config_dir, source) { - Ok(pipeline) => { - let _ = self.pipelines.insert(name, pipeline); - } - Err(err) => { - error!(target: "MAPPING", "Failed to compile pipeline {name}: {err}") - } - } - } - } - pub fn connect( &mut self, mqtt: &mut (impl MessageSource + MessageSink), @@ -150,17 +67,15 @@ impl GenMapperBuilder { } pub fn connect_fs(&mut self, fs: &mut impl MessageSource) { - fs.connect_mapped_sink(self.config_dir.clone(), &self.message_box, |msg| { - Some(InputMessage::FsWatchEvent(msg)) - }); + fs.connect_mapped_sink( + self.processor.config_dir.clone(), + &self.message_box, + |msg| Some(InputMessage::FsWatchEvent(msg)), + ); } fn topics(&self) -> TopicFilter { - let mut topics = TopicFilter::empty(); - for pipeline in self.pipelines.values() { - topics.add_all(pipeline.topics()) - } - topics + self.processor.subscriptions() } } @@ -180,10 +95,8 @@ impl Builder for GenMapperBuilder { fn build(self) -> GenMapper { GenMapper { messages: self.message_box.build(), - pipelines: self.pipelines, subscriptions: self.subscriptions, - js_runtime: self.js_runtime, - config_dir: self.config_dir, + processor: self.processor, } } } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs new file mode 100644 index 00000000000..31417716f9d --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -0,0 +1,205 @@ +use crate::config::PipelineConfig; +use crate::js_runtime::JsRuntime; +use crate::pipeline::DateTime; +use crate::pipeline::FilterError; +use crate::pipeline::Message; +use crate::pipeline::Pipeline; +use crate::LoadError; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use tedge_mqtt_ext::TopicFilter; +use tokio::fs::read_dir; +use tokio::fs::read_to_string; +use tracing::error; +use tracing::info; + +pub struct MessageProcessor { + pub(super) config_dir: PathBuf, + pub(super) pipelines: HashMap, + pub(super) js_runtime: JsRuntime, +} + +impl MessageProcessor { + pub async fn try_new(config_dir: impl AsRef) -> Result { + let config_dir = config_dir.as_ref().to_owned(); + let mut js_runtime = JsRuntime::try_new().await?; + let mut pipeline_specs = PipelineSpecs::default(); + pipeline_specs.load(&mut js_runtime, &config_dir).await; + let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir); + + Ok(MessageProcessor { + config_dir, + pipelines, + js_runtime, + }) + } + + pub fn subscriptions(&self) -> TopicFilter { + let mut topics = TopicFilter::empty(); + for pipeline in self.pipelines.values() { + topics.add_all(pipeline.topics()) + } + topics + } + + pub async fn process( + &mut self, + timestamp: &DateTime, + message: &Message, + ) -> Vec<(String, Result, FilterError>)> { + let mut out_messages = vec![]; + for (pipeline_id, pipeline) in self.pipelines.iter_mut() { + let pipeline_output = pipeline + .process(&self.js_runtime, ×tamp, &message) + .await; + out_messages.push((pipeline_id.clone(), pipeline_output)); + } + out_messages + } + + pub async fn tick( + &mut self, + timestamp: &DateTime, + ) -> Vec<(String, Result, FilterError>)> { + let mut out_messages = vec![]; + for (pipeline_id, pipeline) in self.pipelines.iter_mut() { + let pipeline_output = pipeline.tick(&self.js_runtime, ×tamp).await; + out_messages.push((pipeline_id.clone(), pipeline_output)); + } + out_messages + } + + pub async fn dump_memory_stats(&self) { + self.js_runtime.dump_memory_stats().await; + } + + pub async fn reload_filter(&mut self, path: Utf8PathBuf) { + for pipeline in self.pipelines.values_mut() { + for stage in &mut pipeline.stages { + if stage.filter.path() == path { + match self.js_runtime.load_file(&path).await { + Ok(()) => { + info!("Reloaded filter {path}"); + } + Err(e) => { + error!("Failed to reload filter {path}: {e}"); + return; + } + } + } + } + } + } + + pub async fn reload_pipeline(&mut self, path: Utf8PathBuf) { + for pipeline in self.pipelines.values_mut() { + if pipeline.source == path { + let Ok(source) = tokio::fs::read_to_string(&path).await else { + error!("Failed to read updated filter {path}"); + break; + }; + let config: PipelineConfig = match toml::from_str(&source) { + Ok(config) => config, + Err(e) => { + error!("Failed to parse toml for updated filter {path}: {e}"); + break; + } + }; + match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { + Ok(p) => { + *pipeline = p; + info!("Reloaded pipeline {path}"); + } + Err(e) => { + error!("Failed to load updated pipeline {path}: {e}") + } + }; + } + } + } +} + +#[derive(Default)] +struct PipelineSpecs { + pipeline_specs: HashMap, +} + +impl PipelineSpecs { + pub async fn load(&mut self, js_runtime: &mut JsRuntime, config_dir: &PathBuf) { + let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| + error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) + ) else { + return; + }; + + while let Ok(Some(entry)) = entries.next_entry().await { + let Some(path) = Utf8Path::from_path(&entry.path()).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", entry.path().display()); + continue; + }; + if let Ok(file_type) = entry.file_type().await { + if file_type.is_file() { + match path.extension() { + Some("toml") => { + info!(target: "MAPPING", "Loading pipeline: {path}"); + if let Err(err) = self.load_pipeline(path).await { + error!(target: "MAPPING", "Failed to load pipeline: {err}"); + } + } + Some("js") | Some("ts") => { + info!(target: "MAPPING", "Loading filter: {path}"); + if let Err(err) = self.load_filter(js_runtime, path).await { + error!(target: "MAPPING", "Failed to load filter: {err}"); + } + } + _ => { + info!(target: "MAPPING", "Skipping file which type is unknown: {path}"); + } + } + } + } + } + } + + async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { + if let Some(name) = file.as_ref().file_name() { + let specs = read_to_string(file.as_ref()).await?; + let pipeline: PipelineConfig = toml::from_str(&specs)?; + self.pipeline_specs + .insert(name.to_string(), (file.as_ref().to_owned(), pipeline)); + } + + Ok(()) + } + + async fn load_filter( + &mut self, + js_runtime: &mut JsRuntime, + file: impl AsRef, + ) -> Result<(), LoadError> { + js_runtime.load_file(file.as_ref()).await?; + Ok(()) + } + + fn compile( + mut self, + js_runtime: &JsRuntime, + config_dir: &PathBuf, + ) -> HashMap { + let mut pipelines = HashMap::new(); + for (name, (source, specs)) in self.pipeline_specs.drain() { + match specs.compile(js_runtime, config_dir, source) { + Ok(pipeline) => { + let _ = pipelines.insert(name, pipeline); + } + Err(err) => { + error!(target: "MAPPING", "Failed to compile pipeline {name}: {err}") + } + } + } + pipelines + } +} From b2233a49d2d2e924032c3af2100b27ac48e9712e Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 26 Jun 2025 13:18:17 +0200 Subject: [PATCH 27/53] Add cli commands to test user-defined mappings Signed-off-by: Didier Wenzek --- Cargo.lock | 1 + crates/core/tedge/Cargo.toml | 1 + crates/core/tedge/src/cli/mapping/cli.rs | 104 ++++++++++++++++++ crates/core/tedge/src/cli/mapping/list.rs | 47 ++++++++ crates/core/tedge/src/cli/mapping/mod.rs | 5 + crates/core/tedge/src/cli/mapping/test.rs | 61 ++++++++++ crates/core/tedge/src/cli/mod.rs | 6 + .../tedge_gen_mapper/src/js_filter.rs | 6 +- crates/extensions/tedge_gen_mapper/src/lib.rs | 4 +- .../tedge_gen_mapper/src/pipeline.rs | 8 +- .../tedge_gen_mapper/src/runtime.rs | 31 ++++-- 11 files changed, 253 insertions(+), 21 deletions(-) create mode 100644 crates/core/tedge/src/cli/mapping/cli.rs create mode 100644 crates/core/tedge/src/cli/mapping/list.rs create mode 100644 crates/core/tedge/src/cli/mapping/mod.rs create mode 100644 crates/core/tedge/src/cli/mapping/test.rs diff --git a/Cargo.lock b/Cargo.lock index 17c4cb4720f..db467c7863f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4582,6 +4582,7 @@ dependencies = [ "tedge-write", "tedge_api", "tedge_config", + "tedge_gen_mapper", "tedge_test_utils", "tedge_utils", "tempfile", diff --git a/crates/core/tedge/Cargo.toml b/crates/core/tedge/Cargo.toml index fd930cae168..83315dc2197 100644 --- a/crates/core/tedge/Cargo.toml +++ b/crates/core/tedge/Cargo.toml @@ -49,6 +49,7 @@ strum_macros = { workspace = true } tar = { workspace = true } tedge-agent = { workspace = true } tedge-apt-plugin = { workspace = true } +tedge_gen_mapper = { workspace = true } tedge-mapper = { workspace = true, default-features = false } tedge-watchdog = { workspace = true } tedge-write = { workspace = true } diff --git a/crates/core/tedge/src/cli/mapping/cli.rs b/crates/core/tedge/src/cli/mapping/cli.rs new file mode 100644 index 00000000000..729bf0ededb --- /dev/null +++ b/crates/core/tedge/src/cli/mapping/cli.rs @@ -0,0 +1,104 @@ +use crate::cli::mapping::list::ListCommand; +use crate::cli::mapping::test::TestCommand; +use crate::command::BuildCommand; +use crate::command::Command; +use crate::ConfigError; +use anyhow::anyhow; +use anyhow::Context; +use anyhow::Error; +use std::path::PathBuf; +use tedge_config::TEdgeConfig; +use tedge_gen_mapper::pipeline::Message; +use tedge_gen_mapper::MessageProcessor; + +#[derive(clap::Subcommand, Debug)] +pub enum TEdgeMappingCli { + /// List pipelines and filters + List { + /// Path to pipeline and filter specs + /// + /// Default to /etc/tedge/gen-mapper + #[clap(long)] + mapping_dir: Option, + + /// List pipelines processing messages published on this topic + /// + /// If none is provided, lists all the pipelines + #[clap(long)] + topic: Option, + }, + + /// Process message samples + Test { + /// Path to pipeline and filter specs + /// + /// Default to /etc/tedge/gen-mapper + #[clap(long)] + mapping_dir: Option, + + /// Path to the javascript filter or TOML pipeline definition + /// + /// If none is provided, applies all the matching pipelines + #[clap(long)] + filter: Option, + + /// Topic of the message sample + /// + /// If none is provided, messages are read from stdout + topic: Option, + + /// Payload of the message sample + /// + /// If none is provided, payloads are read from stdout + payload: Option, + }, +} + +impl BuildCommand for TEdgeMappingCli { + fn build_command(self, config: &TEdgeConfig) -> Result, ConfigError> { + match self { + TEdgeMappingCli::List { mapping_dir, topic } => { + let mapping_dir = mapping_dir.unwrap_or_else(|| Self::default_mapping_dir(config)); + Ok(ListCommand { mapping_dir, topic }.into_boxed()) + } + + TEdgeMappingCli::Test { + mapping_dir, + filter, + topic, + payload, + } => { + let mapping_dir = mapping_dir.unwrap_or_else(|| Self::default_mapping_dir(config)); + let message = match (topic, payload) { + (Some(topic), Some(payload)) => Some(Message { topic, payload }), + (Some(_), None) => Err(anyhow!("Missing sample payload"))?, + (None, Some(_)) => Err(anyhow!("Missing sample topic"))?, + (None, None) => None, + }; + Ok(TestCommand { + mapping_dir, + filter, + message, + } + .into_boxed()) + } + } + } +} + +impl TEdgeMappingCli { + fn default_mapping_dir(config: &TEdgeConfig) -> PathBuf { + config.root_dir().join("gen-mapper").into() + } + + pub async fn load_pipelines(mapping_dir: &PathBuf) -> Result { + MessageProcessor::try_new(mapping_dir) + .await + .with_context(|| { + format!( + "loading pipelines and filters from {}", + mapping_dir.display() + ) + }) + } +} diff --git a/crates/core/tedge/src/cli/mapping/list.rs b/crates/core/tedge/src/cli/mapping/list.rs new file mode 100644 index 00000000000..8c7577350df --- /dev/null +++ b/crates/core/tedge/src/cli/mapping/list.rs @@ -0,0 +1,47 @@ +use crate::cli::mapping::TEdgeMappingCli; +use crate::command::Command; +use crate::log::MaybeFancy; +use anyhow::Error; +use std::path::PathBuf; +use tedge_config::TEdgeConfig; +use tedge_gen_mapper::pipeline::Pipeline; + +pub struct ListCommand { + pub mapping_dir: PathBuf, + pub topic: Option, +} + +#[async_trait::async_trait] +impl Command for ListCommand { + fn description(&self) -> String { + format!( + "list pipelines and filters in {:}", + self.mapping_dir.display() + ) + } + + async fn execute(&self, _config: TEdgeConfig) -> Result<(), MaybeFancy> { + let processor = TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?; + + match &self.topic { + Some(topic) => processor + .pipelines + .iter() + .filter(|(_, pipeline)| pipeline.topics().accept_topic_name(topic)) + .for_each(Self::display), + + None => processor.pipelines.iter().for_each(Self::display), + } + + Ok(()) + } +} + +impl ListCommand { + fn display((pipeline_id, pipeline): (&String, &Pipeline)) { + println!("{pipeline_id}"); + for stage in pipeline.stages.iter() { + println!("\t{}", stage.filter.path.display()); + } + } +} diff --git a/crates/core/tedge/src/cli/mapping/mod.rs b/crates/core/tedge/src/cli/mapping/mod.rs new file mode 100644 index 00000000000..753d9370f59 --- /dev/null +++ b/crates/core/tedge/src/cli/mapping/mod.rs @@ -0,0 +1,5 @@ +mod cli; +mod list; +mod test; + +pub use cli::TEdgeMappingCli; diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs new file mode 100644 index 00000000000..2737720bc4d --- /dev/null +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -0,0 +1,61 @@ +use crate::cli::mapping::TEdgeMappingCli; +use crate::command::Command; +use crate::log::MaybeFancy; +use anyhow::Error; +use std::path::PathBuf; +use tedge_config::TEdgeConfig; +use tedge_gen_mapper::pipeline::*; + +pub struct TestCommand { + pub mapping_dir: PathBuf, + pub filter: Option, + pub message: Option, +} + +#[async_trait::async_trait] +impl Command for TestCommand { + fn description(&self) -> String { + format!( + "process message samples using pipelines and filters in {:}", + self.mapping_dir.display() + ) + } + + async fn execute(&self, _config: TEdgeConfig) -> Result<(), MaybeFancy> { + let mut processor = TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?; + if let Some(message) = &self.message { + let timestamp = DateTime::now(); + match &self.filter { + Some(filter) => { + let filter_name = filter.display().to_string(); + print( + processor + .process_with_pipeline(&filter_name, ×tamp, message) + .await, + ) + } + None => processor + .process(×tamp, message) + .await + .into_iter() + .map(|(_, v)| v) + .for_each(print), + } + } + + Ok(()) + } +} + +fn print(messages: Result, FilterError>) { + match messages { + Ok(messages) => { + for message in messages { + println!("[{}] {}", message.topic, message.payload); + } + } + Err(err) => { + eprintln!("Error: {}", err) + } + } +} diff --git a/crates/core/tedge/src/cli/mod.rs b/crates/core/tedge/src/cli/mod.rs index 7d155ccbdd4..4c42a0efa7b 100644 --- a/crates/core/tedge/src/cli/mod.rs +++ b/crates/core/tedge/src/cli/mod.rs @@ -25,6 +25,7 @@ mod disconnect; mod http; mod init; pub mod log; +mod mapping; mod mqtt; mod reconnect; mod refresh_bridges; @@ -138,6 +139,10 @@ pub enum TEdgeOpt { #[clap(subcommand)] Http(http::TEdgeHttpCli), + /// Monitor and test mapping rules + #[clap(subcommand)] + Mapping(mapping::TEdgeMappingCli), + /// Run thin-edge services and plugins Run(ComponentOpt), @@ -208,6 +213,7 @@ impl BuildCommand for TEdgeOpt { TEdgeOpt::Mqtt(opt) => opt.build_command(config), TEdgeOpt::Http(opt) => opt.build_command(config), TEdgeOpt::Reconnect(opt) => opt.build_command(config), + TEdgeOpt::Mapping(opt) => opt.build_command(config), TEdgeOpt::Run(_) => { // This method has to be kept in sync with tedge::redirect_if_multicall() panic!("tedge mapper|agent|write commands are launched as multicall") diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 5e27b2b193d..2f4fc094a4f 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -14,9 +14,9 @@ use tracing::debug; #[derive(Clone)] pub struct JsFilter { - path: PathBuf, - config: JsonValue, - tick_every_seconds: u64, + pub path: PathBuf, + pub config: JsonValue, + pub tick_every_seconds: u64, } #[derive(Clone, Debug, Default)] diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index cad842c304d..84ac172a3f9 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -2,11 +2,11 @@ mod actor; mod config; mod js_filter; mod js_runtime; -mod pipeline; +pub mod pipeline; mod runtime; use crate::actor::GenMapper; -use crate::runtime::MessageProcessor; +pub use crate::runtime::MessageProcessor; use std::convert::Infallible; use std::path::Path; use std::path::PathBuf; diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index e9e993a03f2..a87daef8359 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -27,14 +27,14 @@ pub struct Stage { #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] pub struct DateTime { - pub(crate) seconds: u64, - pub(crate) nanoseconds: u32, + pub seconds: u64, + pub nanoseconds: u32, } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, Eq, PartialEq)] pub struct Message { - pub(crate) topic: String, - pub(crate) payload: String, + pub topic: String, + pub payload: String, } #[derive(thiserror::Error, Debug)] diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 31417716f9d..af0c0d9f053 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -17,8 +17,8 @@ use tracing::error; use tracing::info; pub struct MessageProcessor { - pub(super) config_dir: PathBuf, - pub(super) pipelines: HashMap, + pub config_dir: PathBuf, + pub pipelines: HashMap, pub(super) js_runtime: JsRuntime, } @@ -28,7 +28,7 @@ impl MessageProcessor { let mut js_runtime = JsRuntime::try_new().await?; let mut pipeline_specs = PipelineSpecs::default(); pipeline_specs.load(&mut js_runtime, &config_dir).await; - let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir); + let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); Ok(MessageProcessor { config_dir, @@ -52,21 +52,32 @@ impl MessageProcessor { ) -> Vec<(String, Result, FilterError>)> { let mut out_messages = vec![]; for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - let pipeline_output = pipeline - .process(&self.js_runtime, ×tamp, &message) - .await; + let pipeline_output = pipeline.process(&self.js_runtime, timestamp, message).await; out_messages.push((pipeline_id.clone(), pipeline_output)); } out_messages } + pub async fn process_with_pipeline( + &mut self, + pipeline_id: &String, + timestamp: &DateTime, + message: &Message, + ) -> Result, FilterError> { + let pipeline = self + .pipelines + .get_mut(pipeline_id) + .ok_or_else(|| anyhow::anyhow!("No such pipeline: {pipeline_id}"))?; + pipeline.process(&self.js_runtime, timestamp, message).await + } + pub async fn tick( &mut self, timestamp: &DateTime, ) -> Vec<(String, Result, FilterError>)> { let mut out_messages = vec![]; for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - let pipeline_output = pipeline.tick(&self.js_runtime, ×tamp).await; + let pipeline_output = pipeline.tick(&self.js_runtime, timestamp).await; out_messages.push((pipeline_id.clone(), pipeline_output)); } out_messages @@ -184,11 +195,7 @@ impl PipelineSpecs { Ok(()) } - fn compile( - mut self, - js_runtime: &JsRuntime, - config_dir: &PathBuf, - ) -> HashMap { + fn compile(mut self, js_runtime: &JsRuntime, config_dir: &Path) -> HashMap { let mut pipelines = HashMap::new(); for (name, (source, specs)) in self.pipeline_specs.drain() { match specs.compile(js_runtime, config_dir, source) { From 8ccf5163c5824b6243ed77e3d5f65d40acb2f533 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 26 Jun 2025 14:57:36 +0200 Subject: [PATCH 28/53] Simplify gen-mapper subscription updates Signed-off-by: Didier Wenzek --- .../extensions/tedge_gen_mapper/src/actor.rs | 16 +++++----- crates/extensions/tedge_gen_mapper/src/lib.rs | 7 ++--- .../tedge_gen_mapper/src/runtime.rs | 12 ++++---- crates/extensions/tedge_mqtt_ext/src/trie.rs | 29 +++++++++++-------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 3497e3f6c9e..e6b4eea94a8 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -5,8 +5,6 @@ use crate::InputMessage; use crate::OutputMessage; use async_trait::async_trait; use camino::Utf8PathBuf; -use std::sync::Arc; -use std::sync::Mutex; use tedge_actors::Actor; use tedge_actors::MessageReceiver; use tedge_actors::RuntimeError; @@ -22,7 +20,7 @@ use tracing::error; pub struct GenMapper { pub(super) messages: SimpleMessageBox, - pub(super) subscriptions: Arc>, + pub(super) subscriptions: TopicFilter, pub(super) processor: MessageProcessor, } @@ -73,18 +71,18 @@ impl Actor for GenMapper { impl GenMapper { async fn send_updated_subscriptions(&mut self) -> Result<(), RuntimeError> { - let topics = self.update_subscriptions(); - let diff = SubscriptionDiff::new(&topics, &TopicFilter::empty()); + let diff = self.update_subscriptions(); self.messages .send(OutputMessage::SubscriptionDiff(diff)) .await?; Ok(()) } - fn update_subscriptions(&self) -> TopicFilter { - let mut topics = self.subscriptions.lock().unwrap(); - topics.add_all(self.processor.subscriptions()); - topics.clone() + fn update_subscriptions(&mut self) -> SubscriptionDiff { + let new_subscriptions = self.processor.subscriptions(); + let diff = SubscriptionDiff::new(&new_subscriptions, &self.subscriptions); + self.subscriptions = new_subscriptions; + diff } async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 84ac172a3f9..f2e22e8a283 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -10,8 +10,6 @@ pub use crate::runtime::MessageProcessor; use std::convert::Infallible; use std::path::Path; use std::path::PathBuf; -use std::sync::Arc; -use std::sync::Mutex; use tedge_actors::fan_in_message_type; use tedge_actors::Builder; use tedge_actors::DynSender; @@ -34,7 +32,6 @@ fan_in_message_type!(OutputMessage[MqttMessage, SubscriptionDiff]: Clone, Debug, pub struct GenMapperBuilder { message_box: SimpleMessageBoxBuilder, - subscriptions: Arc>, processor: MessageProcessor, } @@ -43,7 +40,6 @@ impl GenMapperBuilder { let processor = MessageProcessor::try_new(config_dir).await?; Ok(GenMapperBuilder { message_box: SimpleMessageBoxBuilder::new("GenMapper", 16), - subscriptions: Arc::new(Mutex::new(TopicFilter::empty())), processor, }) } @@ -93,9 +89,10 @@ impl Builder for GenMapperBuilder { } fn build(self) -> GenMapper { + let subscriptions = self.topics().clone(); GenMapper { messages: self.message_box.build(), - subscriptions: self.subscriptions, + subscriptions, processor: self.processor, } } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index af0c0d9f053..63688c7c923 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -93,10 +93,10 @@ impl MessageProcessor { if stage.filter.path() == path { match self.js_runtime.load_file(&path).await { Ok(()) => { - info!("Reloaded filter {path}"); + info!(target: "gen-mapper", "Reloaded filter {path}"); } Err(e) => { - error!("Failed to reload filter {path}: {e}"); + error!(target: "gen-mapper", "Failed to reload filter {path}: {e}"); return; } } @@ -109,23 +109,23 @@ impl MessageProcessor { for pipeline in self.pipelines.values_mut() { if pipeline.source == path { let Ok(source) = tokio::fs::read_to_string(&path).await else { - error!("Failed to read updated filter {path}"); + error!(target: "gen-mapper", "Failed to read updated pipeline {path}"); break; }; let config: PipelineConfig = match toml::from_str(&source) { Ok(config) => config, Err(e) => { - error!("Failed to parse toml for updated filter {path}: {e}"); + error!(target: "gen-mapper", "Failed to parse toml for updated pipeline {path}: {e}"); break; } }; match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { Ok(p) => { *pipeline = p; - info!("Reloaded pipeline {path}"); + info!(target: "gen-mapper", "Reloaded pipeline {path}"); } Err(e) => { - error!("Failed to load updated pipeline {path}: {e}") + error!(target: "gen-mapper", "Failed to load updated pipeline {path}: {e}") } }; } diff --git a/crates/extensions/tedge_mqtt_ext/src/trie.rs b/crates/extensions/tedge_mqtt_ext/src/trie.rs index c284e4e4a44..6f60bbe76d5 100644 --- a/crates/extensions/tedge_mqtt_ext/src/trie.rs +++ b/crates/extensions/tedge_mqtt_ext/src/trie.rs @@ -135,16 +135,7 @@ impl AddAssign for SubscriptionDiff { fn add_assign(&mut self, rhs: Self) { self.subscribe.extend(rhs.subscribe); self.unsubscribe.extend(rhs.unsubscribe); - - let overlap = self - .subscribe - .intersection(&self.unsubscribe) - .cloned() - .collect::>(); - for topic in overlap { - self.subscribe.remove(&topic); - self.unsubscribe.remove(&topic); - } + self.simplify() } } @@ -160,10 +151,12 @@ impl SubscriptionDiff { subscribe: &mqtt_channel::TopicFilter, unsubscribe: &mqtt_channel::TopicFilter, ) -> Self { - Self { + let mut diff = Self { subscribe: subscribe.patterns().iter().cloned().collect(), unsubscribe: unsubscribe.patterns().iter().cloned().collect(), - } + }; + diff.simplify(); + diff } fn with_topic_prefix(self, prefix: &str) -> Self { @@ -180,6 +173,18 @@ impl SubscriptionDiff { .collect(), } } + + fn simplify(&mut self) { + let overlap = self + .subscribe + .intersection(&self.unsubscribe) + .cloned() + .collect::>(); + for topic in overlap { + self.subscribe.remove(&topic); + self.unsubscribe.remove(&topic); + } + } } #[derive(PartialEq)] From 883efe8a35c14db2e8547e5c07074db1593a7e49 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 26 Jun 2025 16:20:10 +0200 Subject: [PATCH 29/53] Gen mapper dynamically load/reload/remove pipelines and filters Still to be improved: some event are processed twice - `mv pipeline.toml /tmp` leads to update then remove - `rm pipeline.toml` correctly triggers only remove - `cp /tmp/pipeline.toml .` triggers creation and update Signed-off-by: Didier Wenzek --- .../extensions/tedge_gen_mapper/src/actor.rs | 25 ++++- .../tedge_gen_mapper/src/runtime.rs | 94 ++++++++++++++----- 2 files changed, 93 insertions(+), 26 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index e6b4eea94a8..4e1daa381f7 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -57,10 +57,29 @@ impl Actor for GenMapper { self.send_updated_subscriptions().await?; } }, - Some(InputMessage::FsWatchEvent(e)) => { - tracing::warn!("TODO do something with {e:?}") + Some(InputMessage::FsWatchEvent(FsWatchEvent::FileCreated(path))) => { + let Ok(path) = Utf8PathBuf::try_from(path) else { + continue; + }; + if matches!(path.extension(), Some("js" | "ts")) { + self.processor.add_filter(path).await; + } else if path.extension() == Some("toml") { + self.processor.add_pipeline(path).await; + self.send_updated_subscriptions().await?; + } + }, + Some(InputMessage::FsWatchEvent(FsWatchEvent::FileDeleted(path))) => { + let Ok(path) = Utf8PathBuf::try_from(path) else { + continue; + }; + if matches!(path.extension(), Some("js" | "ts")) { + self.processor.remove_filter(path).await; + } else if path.extension() == Some("toml") { + self.processor.remove_pipeline(path).await; + self.send_updated_subscriptions().await?; + } }, - None => break, + _ => break, } } } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 63688c7c923..0e81b2a485e 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -15,6 +15,7 @@ use tokio::fs::read_dir; use tokio::fs::read_to_string; use tracing::error; use tracing::info; +use tracing::warn; pub struct MessageProcessor { pub config_dir: PathBuf, @@ -87,6 +88,17 @@ impl MessageProcessor { self.js_runtime.dump_memory_stats().await; } + pub async fn add_filter(&mut self, path: Utf8PathBuf) { + match self.js_runtime.load_file(&path).await { + Ok(()) => { + info!(target: "gen-mapper", "Loaded filter {path}"); + } + Err(e) => { + error!(target: "gen-mapper", "Failed to load filter {path}: {e}"); + } + } + } + pub async fn reload_filter(&mut self, path: Utf8PathBuf) { for pipeline in self.pipelines.values_mut() { for stage in &mut pipeline.stages { @@ -105,32 +117,68 @@ impl MessageProcessor { } } - pub async fn reload_pipeline(&mut self, path: Utf8PathBuf) { - for pipeline in self.pipelines.values_mut() { - if pipeline.source == path { - let Ok(source) = tokio::fs::read_to_string(&path).await else { - error!(target: "gen-mapper", "Failed to read updated pipeline {path}"); - break; - }; - let config: PipelineConfig = match toml::from_str(&source) { - Ok(config) => config, - Err(e) => { - error!(target: "gen-mapper", "Failed to parse toml for updated pipeline {path}: {e}"); - break; - } - }; - match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { - Ok(p) => { - *pipeline = p; - info!(target: "gen-mapper", "Reloaded pipeline {path}"); - } - Err(e) => { - error!(target: "gen-mapper", "Failed to load updated pipeline {path}: {e}") - } - }; + pub async fn remove_filter(&mut self, path: Utf8PathBuf) { + for (pipeline_id, pipeline) in self.pipelines.iter() { + for stage in pipeline.stages.iter() { + if stage.filter.path() == path { + warn!(target: "gen-mapper", "Removing a filter used by {pipeline_id}: {path}"); + return; + } } } } + + pub async fn load_pipeline(&mut self, pipeline_id: String, path: Utf8PathBuf) -> bool { + let Ok(source) = tokio::fs::read_to_string(&path).await else { + self.remove_pipeline(path).await; + return false; + }; + let config: PipelineConfig = match toml::from_str(&source) { + Ok(config) => config, + Err(e) => { + error!(target: "gen-mapper", "Failed to parse toml for pipeline {path}: {e}"); + return false; + } + }; + match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { + Ok(pipeline) => { + self.pipelines.insert(pipeline_id, pipeline); + true + } + Err(e) => { + error!(target: "gen-mapper", "Failed to compile pipeline {path}: {e}"); + false + } + } + } + + pub async fn add_pipeline(&mut self, path: Utf8PathBuf) { + let pipeline_id = path.file_name().unwrap(); + if !self.pipelines.contains_key(pipeline_id) + && self + .load_pipeline(pipeline_id.to_string(), path.clone()) + .await + { + info!(target: "gen-mapper", "Loaded new pipeline {path}"); + } + } + + pub async fn reload_pipeline(&mut self, path: Utf8PathBuf) { + let pipeline_id = path.file_name().unwrap(); + if self.pipelines.contains_key(pipeline_id) + && self + .load_pipeline(pipeline_id.to_string(), path.clone()) + .await + { + info!(target: "gen-mapper", "Reloaded updated pipeline {path}"); + } + } + + pub async fn remove_pipeline(&mut self, path: Utf8PathBuf) { + let pipeline_id = path.file_name().unwrap(); + self.pipelines.remove(pipeline_id); + info!(target: "gen-mapper", "Removed deleted pipeline {path}"); + } } #[derive(Default)] From 61ef1e3f5f96c044aee50a1048c31e8d23b8a2ff Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 26 Jun 2025 19:01:04 +0200 Subject: [PATCH 30/53] Index pipelines using full paths not just filenames Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/src/runtime.rs | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 0e81b2a485e..3abfab4f1a8 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -24,6 +24,10 @@ pub struct MessageProcessor { } impl MessageProcessor { + pub fn pipeline_id(path: impl AsRef) -> String { + format!("{}", path.as_ref().display()) + } + pub async fn try_new(config_dir: impl AsRef) -> Result { let config_dir = config_dir.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; @@ -153,30 +157,26 @@ impl MessageProcessor { } pub async fn add_pipeline(&mut self, path: Utf8PathBuf) { - let pipeline_id = path.file_name().unwrap(); - if !self.pipelines.contains_key(pipeline_id) - && self - .load_pipeline(pipeline_id.to_string(), path.clone()) - .await + let pipeline_id = Self::pipeline_id(&path); + if !self.pipelines.contains_key(&pipeline_id) + && self.load_pipeline(pipeline_id, path.clone()).await { info!(target: "gen-mapper", "Loaded new pipeline {path}"); } } pub async fn reload_pipeline(&mut self, path: Utf8PathBuf) { - let pipeline_id = path.file_name().unwrap(); - if self.pipelines.contains_key(pipeline_id) - && self - .load_pipeline(pipeline_id.to_string(), path.clone()) - .await + let pipeline_id = Self::pipeline_id(&path); + if self.pipelines.contains_key(&pipeline_id) + && self.load_pipeline(pipeline_id, path.clone()).await { info!(target: "gen-mapper", "Reloaded updated pipeline {path}"); } } pub async fn remove_pipeline(&mut self, path: Utf8PathBuf) { - let pipeline_id = path.file_name().unwrap(); - self.pipelines.remove(pipeline_id); + let pipeline_id = Self::pipeline_id(&path); + self.pipelines.remove(&pipeline_id); info!(target: "gen-mapper", "Removed deleted pipeline {path}"); } } @@ -224,12 +224,12 @@ impl PipelineSpecs { } async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { - if let Some(name) = file.as_ref().file_name() { - let specs = read_to_string(file.as_ref()).await?; - let pipeline: PipelineConfig = toml::from_str(&specs)?; - self.pipeline_specs - .insert(name.to_string(), (file.as_ref().to_owned(), pipeline)); - } + let path = file.as_ref(); + let pipeline_id = MessageProcessor::pipeline_id(path); + let specs = read_to_string(file.as_ref()).await?; + let pipeline: PipelineConfig = toml::from_str(&specs)?; + self.pipeline_specs + .insert(pipeline_id, (path.to_owned(), pipeline)); Ok(()) } From a7108e362aa0f11fc03d7cef7d97ca67f1c8b3ed Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 27 Jun 2025 09:16:36 +0200 Subject: [PATCH 31/53] tedge mapping test consumes messages from stdin Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/cli.rs | 5 +- crates/core/tedge/src/cli/mapping/test.rs | 88 ++++++++++++++++++----- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/crates/core/tedge/src/cli/mapping/cli.rs b/crates/core/tedge/src/cli/mapping/cli.rs index 729bf0ededb..8ff07b4c135 100644 --- a/crates/core/tedge/src/cli/mapping/cli.rs +++ b/crates/core/tedge/src/cli/mapping/cli.rs @@ -44,12 +44,13 @@ pub enum TEdgeMappingCli { /// Topic of the message sample /// - /// If none is provided, messages are read from stdout + /// If none is provided, messages are read from stdin expecting a line per message: + /// [topic] payload topic: Option, /// Payload of the message sample /// - /// If none is provided, payloads are read from stdout + /// If none is provided, payloads are read from stdin payload: Option, }, } diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index 2737720bc4d..d597567c0bc 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -5,6 +5,10 @@ use anyhow::Error; use std::path::PathBuf; use tedge_config::TEdgeConfig; use tedge_gen_mapper::pipeline::*; +use tedge_gen_mapper::MessageProcessor; +use tokio::io::AsyncBufReadExt; +use tokio::io::BufReader; +use tokio::io::Stdin; pub struct TestCommand { pub mapping_dir: PathBuf, @@ -25,28 +29,44 @@ impl Command for TestCommand { let mut processor = TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?; if let Some(message) = &self.message { let timestamp = DateTime::now(); - match &self.filter { - Some(filter) => { - let filter_name = filter.display().to_string(); - print( - processor - .process_with_pipeline(&filter_name, ×tamp, message) - .await, - ) - } - None => processor - .process(×tamp, message) - .await - .into_iter() - .map(|(_, v)| v) - .for_each(print), + self.process(&mut processor, message, ×tamp).await; + } else { + let mut stdin = BufReader::new(tokio::io::stdin()); + while let Some(message) = next_message(&mut stdin).await { + let timestamp = DateTime::now(); + self.process(&mut processor, &message, ×tamp).await; } } - Ok(()) } } +impl TestCommand { + async fn process( + &self, + processor: &mut MessageProcessor, + message: &Message, + timestamp: &DateTime, + ) { + match &self.filter { + Some(filter) => { + let filter_name = filter.display().to_string(); + print( + processor + .process_with_pipeline(&filter_name, timestamp, message) + .await, + ) + } + None => processor + .process(timestamp, message) + .await + .into_iter() + .map(|(_, v)| v) + .for_each(print), + } + } +} + fn print(messages: Result, FilterError>) { match messages { Ok(messages) => { @@ -59,3 +79,39 @@ fn print(messages: Result, FilterError>) { } } } + +fn parse(line: String) -> Result, Error> { + let line = line.trim(); + if line.is_empty() { + return Ok(None); + } + if !line.starts_with("[") { + return Err(anyhow::anyhow!("Missing opening bracket: {}", line)); + } + let Some(closing_bracket) = line.find(']') else { + return Err(anyhow::anyhow!("Missing closing bracket: {}", line)); + }; + + let topic = line[1..closing_bracket].to_string(); + let payload = line[closing_bracket + 1..].to_string(); + + Ok(Some(Message { topic, payload })) +} + +async fn next_message(input: &mut BufReader) -> Option { + let mut line = String::new(); + match input.read_line(&mut line).await { + Ok(0) => None, + Ok(_) => match parse(line) { + Ok(message) => message, + Err(err) => { + eprintln!("Fail to parse input message {}", err); + None + } + }, + Err(err) => { + eprintln!("Fail to read input stream {}", err); + None + } + } +} From db0f9a71d23fd15e3aca1a9abf8b2897a07b7c27 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 27 Jun 2025 13:42:06 +0200 Subject: [PATCH 32/53] Test and fix te_2_c8y.js Signed-off-by: Didier Wenzek --- crates/core/tedge/Cargo.toml | 2 +- crates/extensions/tedge_gen_mapper/Cargo.toml | 5 ++- .../tedge_gen_mapper/pipelines/te_to_c8y.js | 26 +++++++++----- .../pipelines/measurements.toml | 1 - .../tedge_gen_mapper/pipelines/te_to_c8y.js | 26 +++++++++----- .../tedge_gen_mapper/tedge_gen_mapper.robot | 36 +++++++++++++++---- 6 files changed, 69 insertions(+), 27 deletions(-) diff --git a/crates/core/tedge/Cargo.toml b/crates/core/tedge/Cargo.toml index 83315dc2197..cf18af21e6f 100644 --- a/crates/core/tedge/Cargo.toml +++ b/crates/core/tedge/Cargo.toml @@ -49,12 +49,12 @@ strum_macros = { workspace = true } tar = { workspace = true } tedge-agent = { workspace = true } tedge-apt-plugin = { workspace = true } -tedge_gen_mapper = { workspace = true } tedge-mapper = { workspace = true, default-features = false } tedge-watchdog = { workspace = true } tedge-write = { workspace = true } tedge_api = { workspace = true } tedge_config = { workspace = true } +tedge_gen_mapper = { workspace = true } tedge_utils = { workspace = true } thiserror = { workspace = true } time = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index 91d491c5b61..d77da72de61 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -12,7 +12,10 @@ repository.workspace = true anyhow = { workspace = true } async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } -rquickjs = { workspace = true, default-features = false, features = ["futures", "parallel"] } +rquickjs = { workspace = true, default-features = false, features = [ + "futures", + "parallel", +] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tedge_actors = { workspace = true } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js index ae2d8f157c1..d054c53742e 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -44,7 +44,7 @@ /// ``` export function process(t, message, config) { let topic_parts = message.topic.split( '/') - let type = topic_parts[6] + let type = topic_parts[6] || "ThinEdgeMeasurement" let payload = JSON.parse(message.payload) let c8y_msg = { @@ -56,7 +56,11 @@ export function process(t, message, config) { for (let [k, v] of Object.entries(payload)) { let k_meta = (meta || {})[k] || {} if (k === "time") { - let fragment = { time: v } + let t = v + if (typeof(v) === "number") { + t = (new Date(v * 1000)).toISOString() + } + let fragment = { time: t } Object.assign(c8y_msg, fragment) } else if (typeof(v) === "number") { @@ -65,15 +69,19 @@ export function process(t, message, config) { } let fragment = { [k]: { [k]: v } } Object.assign(c8y_msg, fragment) - } else for (let [sub_k, sub_v] of Object.entries(v)) { - let sub_k_meta = k_meta[sub_k] - if (typeof(sub_v) === "number") { - if (sub_k_meta) { - sub_v = { value: sub_v, ...sub_k_meta } + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_k_meta = k_meta[sub_k] + if (typeof(sub_v) === "number") { + if (sub_k_meta) { + sub_v = { value: sub_v, ...sub_k_meta } + } + let sub_fragment = { [sub_k]: sub_v } + Object.assign(fragment, sub_fragment) } - let fragment = { [k]: { [sub_k]: sub_v } } - Object.assign(c8y_msg, fragment) } + Object.assign(c8y_msg, { [k]: fragment}) } } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml index a777177165b..3262580a673 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml @@ -3,5 +3,4 @@ input_topics = ["te/+/+/+/+/m/+"] stages = [ { filter = "add_timestamp.js" }, { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] }, - { filter = "set_topic.js", config = { topic = "gen-mapper/c8y" } } ] diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js index ae2d8f157c1..d054c53742e 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -44,7 +44,7 @@ /// ``` export function process(t, message, config) { let topic_parts = message.topic.split( '/') - let type = topic_parts[6] + let type = topic_parts[6] || "ThinEdgeMeasurement" let payload = JSON.parse(message.payload) let c8y_msg = { @@ -56,7 +56,11 @@ export function process(t, message, config) { for (let [k, v] of Object.entries(payload)) { let k_meta = (meta || {})[k] || {} if (k === "time") { - let fragment = { time: v } + let t = v + if (typeof(v) === "number") { + t = (new Date(v * 1000)).toISOString() + } + let fragment = { time: t } Object.assign(c8y_msg, fragment) } else if (typeof(v) === "number") { @@ -65,15 +69,19 @@ export function process(t, message, config) { } let fragment = { [k]: { [k]: v } } Object.assign(c8y_msg, fragment) - } else for (let [sub_k, sub_v] of Object.entries(v)) { - let sub_k_meta = k_meta[sub_k] - if (typeof(sub_v) === "number") { - if (sub_k_meta) { - sub_v = { value: sub_v, ...sub_k_meta } + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_k_meta = k_meta[sub_k] + if (typeof(sub_v) === "number") { + if (sub_k_meta) { + sub_v = { value: sub_v, ...sub_k_meta } + } + let sub_fragment = { [sub_k]: sub_v } + Object.assign(fragment, sub_fragment) } - let fragment = { [k]: { [sub_k]: sub_v } } - Object.assign(c8y_msg, fragment) } + Object.assign(c8y_msg, { [k]: fragment}) } } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index 67ed106835e..eaea57904eb 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -8,20 +8,44 @@ Test Tags theme:tedge_mapper *** Test Cases *** Add missing timestamps - Execute Command tedge mqtt pub te/device/main// '{}' - ${transformed_msg} Should Have MQTT Messages gen-mapper/c8y + ${transformed_msg} Execute Command tedge mapping test te/device/main///m/ '{}' Should Contain ${transformed_msg} item=time +Convert timestamps to ISO + ${transformed_msg} Execute Command tedge mapping test te/device/main///m/ '{"time": 1751023862.000}' + Should Contain ${transformed_msg} item="time":"2025-06-27T11:31:02.000Z" + +Extract measurement type from topic + ${transformed_msg} Execute Command + ... tedge mapping test te/device/main///m/environment '{"temperature": 258}' + Should Contain + ... ${transformed_msg} + ... item="type":"environment" + +Use default measurement type + ${transformed_msg} Execute Command + ... tedge mapping test te/device/main///m/ '{"temperature": 258}' + Should Contain + ... ${transformed_msg} + ... item="type":"ThinEdgeMeasurement" + +Translate complex tedge json to c8y json + ${transformed_msg} Execute Command + ... tedge mapping test te/device/main///m/environment '{"time":"2025-06-27T08:11:05.301804125Z", "temperature": 258, "location": {"latitude": 32.54, "longitude": -117.67, "altitude": 98.6 }, "pressure": 98}' + ... strip=True + Should Be Equal + ... ${transformed_msg} + ... [c8y/measurement/measurements/create] {"type":"environment","time":"2025-06-27T08:11:05.301804125Z","temperature":{"temperature":258},"location":{"latitude":32.54,"longitude":-117.67,"altitude":98.6},"pressure":{"pressure":98}} + + *** Keywords *** Custom Setup - ${DEVICE_SN}= Setup + ${DEVICE_SN}= Setup skip_bootstrap=${True} + Execute Command ./bootstrap.sh --no-bootstrap --no-connect Set Suite Variable $DEVICE_SN Copy Configuration Files - Start Generic Mapper Copy Configuration Files Execute Command mkdir /etc/tedge/gen-mapper/ ThinEdgeIO.Transfer To Device ${CURDIR}/pipelines/* /etc/tedge/gen-mapper/ -Start Generic Mapper - Execute Command nohup tedge run tedge-mapper gen & From f36568cb886d53688ebece4d9714079fded49910 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 27 Jun 2025 16:05:17 +0200 Subject: [PATCH 33/53] Test dynamic update of filter config Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/test.rs | 31 +++++++++++++------ .../pipelines/measurements.samples | 24 ++++++++++++++ .../tedge_gen_mapper/tedge_gen_mapper.robot | 11 +++++++ 3 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.samples diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index d597567c0bc..bbf4d9992cc 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -98,19 +98,30 @@ fn parse(line: String) -> Result, Error> { Ok(Some(Message { topic, payload })) } -async fn next_message(input: &mut BufReader) -> Option { - let mut line = String::new(); - match input.read_line(&mut line).await { - Ok(0) => None, - Ok(_) => match parse(line) { - Ok(message) => message, +async fn next_line(input: &mut BufReader) -> Option { + loop { + let mut line = String::new(); + match input.read_line(&mut line).await { + Ok(0) => return None, + Ok(_) => { + let line = line.trim(); + if !line.is_empty() { + return Some(line.to_string()); + } + } Err(err) => { - eprintln!("Fail to parse input message {}", err); - None + eprintln!("Fail to read input stream {}", err); + return None } - }, + } + } +} +async fn next_message(input: &mut BufReader) -> Option { + let line = next_line(input).await?; + match parse(line) { + Ok(message) => message, Err(err) => { - eprintln!("Fail to read input stream {}", err); + eprintln!("Fail to parse input message {}", err); None } } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.samples b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.samples new file mode 100644 index 00000000000..2528b355503 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.samples @@ -0,0 +1,24 @@ +# The default is to have no units +INPUT: [te/device/main///m/] {"temperature": 25, "time":"2025-06-27T13:33:53.493Z"} +OUTPUT: [c8y/measurement/measurements/create] {"type":"ThinEdgeMeasurement","temperature":{"temperature":25},"time":"2025-06-27T13:33:53.493Z"} + +# Units can be set using the meta topic +INPUT: [te/device/main///m//meta] {"temperature": { "unit": "°C" }} + +# All the subsequent messages use then the configured units +INPUT: [te/device/main///m/] {"temperature": 25, "time":"2025-06-27T12:40:24.122Z"} +OUTPUT: [c8y/measurement/measurements/create] {"type":"ThinEdgeMeasurement","temperature":{"temperature":{"value":25,"unit":"°C"}},"time":"2025-06-27T12:40:24.122Z"} + +# Units can be dynamically updated +INPUT: [te/device/main///m//meta] {"temperature": { "unit": "°F" }} +INPUT: [te/device/main///m/] {"temperature": 77, "time":"2025-06-27T12:40:24.122Z"} +OUTPUT: [c8y/measurement/measurements/create] {"type":"ThinEdgeMeasurement","temperature":{"temperature":{"value":77,"unit":"°F"}},"time":"2025-06-27T12:40:24.122Z"} + +# Units can be set to across measurement types +INPUT: [te/device/main///m/environment/meta] { "temperature": { "unit": "°C" }, "location": { "altitude": { "unit": "m" } }, "pressure": { "unit": "pascal" } } +INPUT: [te/device/main///m/environment] {"time":"2025-06-27T08:11:05.301804125Z", "temperature": 25, "location": {"latitude": 32.54, "longitude": -117.67, "altitude": 98.6 }, "pressure": 98} +OUTPUT: [c8y/measurement/measurements/create] {"type":"environment","time":"2025-06-27T08:11:05.301804125Z","temperature":{"temperature":{"value":25,"unit":"°C"}},"location":{"latitude":32.54,"longitude":-117.67,"altitude":{"value":98.6,"unit":"m"}},"pressure":{"pressure":{"value":98,"unit":"pascal"}}} + +# For the default type °F are still used, while changed to °C for environment measurements +INPUT: [te/device/main///m/] {"temperature": 77, "time":"2025-06-27T12:40:24.122Z"} +OUTPUT: [c8y/measurement/measurements/create] {"type":"ThinEdgeMeasurement","temperature":{"temperature":{"value":77,"unit":"°F"}},"time":"2025-06-27T12:40:24.122Z"} \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index eaea57904eb..cc0060dccca 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -37,6 +37,17 @@ Translate complex tedge json to c8y json ... ${transformed_msg} ... [c8y/measurement/measurements/create] {"type":"environment","time":"2025-06-27T08:11:05.301804125Z","temperature":{"temperature":258},"location":{"latitude":32.54,"longitude":-117.67,"altitude":98.6},"pressure":{"pressure":98}} +Units are configured using topic metadata + ${transformed_msg} Execute Command + ... cat /etc/tedge/gen-mapper/measurements.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test + ... strip=True + ${expected_msg} Execute Command + ... cat /etc/tedge/gen-mapper/measurements.samples | awk '{ if ($2) print $2 }' FS\='OUTPUT: ' + ... strip=True + Should Be Equal + ... ${transformed_msg} + ... ${expected_msg} + *** Keywords *** Custom Setup From 203f21e1becc5b8787ca5c4d333f7d14cb5b3006 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 27 Jun 2025 16:27:03 +0200 Subject: [PATCH 34/53] Update system tests Setup Signed-off-by: Didier Wenzek --- .../tests/tedge_gen_mapper/tedge_gen_mapper.robot | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index cc0060dccca..0b877cb9682 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -51,8 +51,7 @@ Units are configured using topic metadata *** Keywords *** Custom Setup - ${DEVICE_SN}= Setup skip_bootstrap=${True} - Execute Command ./bootstrap.sh --no-bootstrap --no-connect + ${DEVICE_SN}= Setup connect=${False} Set Suite Variable $DEVICE_SN Copy Configuration Files From 024fe6f2891aa81a51570194fe1df84010040894 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 30 Jun 2025 10:54:04 +0200 Subject: [PATCH 35/53] Support for testing the tick function of a filter $ tedge mapping test --final-tick Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/cli.rs | 6 + crates/core/tedge/src/cli/mapping/test.rs | 22 +++- .../tedge_gen_mapper/pipelines/average.js | 28 +++-- .../tedge_gen_mapper/src/runtime.rs | 12 ++ .../tedge_gen_mapper/pipelines/average.js | 103 ++++++++++++++++++ .../pipelines/average.samples | 16 +++ .../tedge_gen_mapper/pipelines/average.toml | 5 + .../tedge_gen_mapper/tedge_gen_mapper.robot | 11 ++ 8 files changed, 190 insertions(+), 13 deletions(-) create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml diff --git a/crates/core/tedge/src/cli/mapping/cli.rs b/crates/core/tedge/src/cli/mapping/cli.rs index 8ff07b4c135..4adaff621d4 100644 --- a/crates/core/tedge/src/cli/mapping/cli.rs +++ b/crates/core/tedge/src/cli/mapping/cli.rs @@ -42,6 +42,10 @@ pub enum TEdgeMappingCli { #[clap(long)] filter: Option, + /// Send a tick after all the message samples + #[clap(long = "final-tick")] + final_tick: bool, + /// Topic of the message sample /// /// If none is provided, messages are read from stdin expecting a line per message: @@ -66,6 +70,7 @@ impl BuildCommand for TEdgeMappingCli { TEdgeMappingCli::Test { mapping_dir, filter, + final_tick, topic, payload, } => { @@ -80,6 +85,7 @@ impl BuildCommand for TEdgeMappingCli { mapping_dir, filter, message, + final_tick, } .into_boxed()) } diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index bbf4d9992cc..79c208b0aab 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -14,6 +14,7 @@ pub struct TestCommand { pub mapping_dir: PathBuf, pub filter: Option, pub message: Option, + pub final_tick: bool, } #[async_trait::async_trait] @@ -37,6 +38,10 @@ impl Command for TestCommand { self.process(&mut processor, &message, ×tamp).await; } } + if self.final_tick { + let timestamp = DateTime::now(); + self.tick(&mut processor, ×tamp).await; + } Ok(()) } } @@ -65,6 +70,21 @@ impl TestCommand { .for_each(print), } } + + async fn tick(&self, processor: &mut MessageProcessor, timestamp: &DateTime) { + match &self.filter { + Some(filter) => { + let filter_name = filter.display().to_string(); + print(processor.tick_with_pipeline(&filter_name, timestamp).await) + } + None => processor + .tick(timestamp) + .await + .into_iter() + .map(|(_, v)| v) + .for_each(print), + } + } } fn print(messages: Result, FilterError>) { @@ -111,7 +131,7 @@ async fn next_line(input: &mut BufReader) -> Option { } Err(err) => { eprintln!("Fail to read input stream {}", err); - return None + return None; } } } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/average.js b/crates/extensions/tedge_gen_mapper/pipelines/average.js index 1a3f1c7e8dd..128f8774194 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/average.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/average.js @@ -11,7 +11,6 @@ export function process (timestamp, message) { let topic = message.topic let payload = JSON.parse(message.payload) let agg_payload = State.agg_for_topic[topic] - if (agg_payload) { for (let [k, v] of Object.entries(payload)) { let agg = agg_payload[k] @@ -22,7 +21,7 @@ export function process (timestamp, message) { } } else if (typeof (v) === "number") { if (!agg) { - let fragment = {k: {sum: v, count: 1}} + let fragment = {[k]: {sum: v, count: 1}} Object.assign(agg_payload, fragment) } else { agg.sum += v @@ -30,16 +29,17 @@ export function process (timestamp, message) { } } else { if (!agg) { + let fragment = {} for (let [sub_k, sub_v] of Object.entries(v)) { - let fragment = { [k]: { [sub_k]: { sum: sub_v, count: 1 } } } - Object.assign(agg_payload, fragment) + let sub_fragment = { [sub_k]: { sum: sub_v, count: 1 } } + Object.assign(fragment, sub_fragment) } + Object.assign(agg_payload, { [k]: fragment }) } else { for (let [sub_k, sub_v] of Object.entries(v)) { - let sub_agg = agg_payload[sub_k] + let sub_agg = agg[sub_k] if (!sub_agg) { - let fragment = {k: { [sub_k]: { sum: sub_v, count: 1 } } } - Object.assign(agg_payload, fragment) + agg[sub_k] = { sum: sub_v, count: 1 } } else { sub_agg.sum += sub_v sub_agg.count += 1 @@ -56,11 +56,15 @@ export function process (timestamp, message) { Object.assign(agg_payload, fragment) } else if (typeof(v) === "number") { - let fragment = { k: { sum: v, count: 1 } } + let fragment = { [k]: { sum: v, count: 1 } } Object.assign(agg_payload, fragment) - } else for (let [sub_k, sub_v] of Object.entries(v)) { - let fragment = { [k]: { [sub_k]: { sum: sub_v, count: 1 } } } - Object.assign(agg_payload, fragment) + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_fragment = { [sub_k]: { sum: sub_v, count: 1 } } + Object.assign(fragment, sub_fragment) + } + Object.assign(agg_payload, { [k]: fragment }) } } State.agg_for_topic[topic] = agg_payload @@ -80,7 +84,7 @@ export function tick() { Object.assign(payload, fragment) } else if (v.sum && v.count) { - let fragment = { k: v.sum / v.count } + let fragment = { [k]: v.sum / v.count } Object.assign(payload, fragment) } else for (let [sub_k, sub_v] of Object.entries(v)) { let fragment = { [k]: { [sub_k]: sub_v.sum / sub_v.count } } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 3abfab4f1a8..830ec3bc5c7 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -88,6 +88,18 @@ impl MessageProcessor { out_messages } + pub async fn tick_with_pipeline( + &mut self, + pipeline_id: &String, + timestamp: &DateTime, + ) -> Result, FilterError> { + let pipeline = self + .pipelines + .get_mut(pipeline_id) + .ok_or_else(|| anyhow::anyhow!("No such pipeline: {pipeline_id}"))?; + pipeline.tick(&self.js_runtime, timestamp).await + } + pub async fn dump_memory_stats(&self) { self.js_runtime.dump_memory_stats().await; } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js new file mode 100644 index 00000000000..128f8774194 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js @@ -0,0 +1,103 @@ +// Compute the average value of a series of measurements received during a time windows +// - Take care of the topic: messages received over different topics are not mixed +// - Ignore messages which are not formated as thin-edge JSON +// - Ignore values which are not numbers +// - Use the first timestamp as the timestamp for the aggregate +class State { + static agg_for_topic = {} +} + +export function process (timestamp, message) { + let topic = message.topic + let payload = JSON.parse(message.payload) + let agg_payload = State.agg_for_topic[topic] + if (agg_payload) { + for (let [k, v] of Object.entries(payload)) { + let agg = agg_payload[k] + if (k === "time") { + if (!agg) { + let fragment = {time: v} + Object.assign(agg_payload, fragment) + } + } else if (typeof (v) === "number") { + if (!agg) { + let fragment = {[k]: {sum: v, count: 1}} + Object.assign(agg_payload, fragment) + } else { + agg.sum += v + agg.count += 1 + } + } else { + if (!agg) { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_fragment = { [sub_k]: { sum: sub_v, count: 1 } } + Object.assign(fragment, sub_fragment) + } + Object.assign(agg_payload, { [k]: fragment }) + } else { + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_agg = agg[sub_k] + if (!sub_agg) { + agg[sub_k] = { sum: sub_v, count: 1 } + } else { + sub_agg.sum += sub_v + sub_agg.count += 1 + } + } + } + } + } + } else { + let agg_payload = {} + for (let [k, v] of Object.entries(payload)) { + if (k === "time") { + let fragment = { time: v } + Object.assign(agg_payload, fragment) + } + else if (typeof(v) === "number") { + let fragment = { [k]: { sum: v, count: 1 } } + Object.assign(agg_payload, fragment) + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_fragment = { [sub_k]: { sum: sub_v, count: 1 } } + Object.assign(fragment, sub_fragment) + } + Object.assign(agg_payload, { [k]: fragment }) + } + } + State.agg_for_topic[topic] = agg_payload + } + + return [] +} + +export function tick() { + let messages = [] + + for (let [topic, agg] of Object.entries(State.agg_for_topic)) { + let payload = {} + for (let [k, v] of Object.entries(agg)) { + if (k === "time") { + let fragment = { time: v } + Object.assign(payload, fragment) + } + else if (v.sum && v.count) { + let fragment = { [k]: v.sum / v.count } + Object.assign(payload, fragment) + } else for (let [sub_k, sub_v] of Object.entries(v)) { + let fragment = { [k]: { [sub_k]: sub_v.sum / sub_v.count } } + Object.assign(payload, fragment) + } + } + + messages.push ({ + topic: topic, + payload: JSON.stringify(payload) + }) + } + + State.agg_for_topic = {} + return messages +} \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples new file mode 100644 index 00000000000..cee47197527 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples @@ -0,0 +1,16 @@ +INPUT: [test/average] {"temperature": 20} +INPUT: [test/average] {"humidity": 50} +INPUT: [test/average] {"temperature": 25, "location": {"altitude": 100.0 }} +INPUT: [test/average] {"temperature": 30} +INPUT: [test/average] {"temperature": 25} +INPUT: [test/average] {"humidity": 70, "location": {"altitude": 120.0 }} +INPUT: [test/average] {"temperature": 20} +INPUT: [test/average/another_topic] {"temperature": 2} +INPUT: [test/average/another_topic] {"temperature": 3} +INPUT: [test/average/another_topic] {"temperature": 5} +INPUT: [test/average/another_topic] {"temperature": 6} +INPUT: [test/average/another_topic] {"temperature": 4} + + +OUTPUT: [test/average] {"temperature":24,"humidity":60,"location":{"altitude":110}} +OUTPUT: [test/average/another_topic] {"temperature":4} \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml new file mode 100644 index 00000000000..52f2bdf7c4a --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml @@ -0,0 +1,5 @@ +input_topics = ["test/average/#"] + +stages = [ + { filter = "average.js", tick_every_seconds = 1 }, +] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index 0b877cb9682..0092d7d400a 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -48,6 +48,17 @@ Units are configured using topic metadata ... ${transformed_msg} ... ${expected_msg} +Computing average over a time window + ${transformed_msg} Execute Command + ... cat /etc/tedge/gen-mapper/average.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test --final-tick + ... strip=True + ${expected_msg} Execute Command + ... cat /etc/tedge/gen-mapper/average.samples | awk '{ if ($2) print $2 }' FS\='OUTPUT: ' + ... strip=True + Should Be Equal + ... ${transformed_msg} + ... ${expected_msg} + *** Keywords *** Custom Setup From 59a3279e5616403f05b3b93d444da1b1b20f6b0c Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Mon, 30 Jun 2025 18:30:59 +0200 Subject: [PATCH 36/53] Add support for console.log() Signed-off-by: Didier Wenzek --- Cargo.lock | 49 +++++++++++++++++ crates/extensions/tedge_gen_mapper/Cargo.toml | 1 + .../tedge_gen_mapper/pipelines/average.js | 11 +++- .../tedge_gen_mapper/src/js_filter.rs | 11 ++++ .../tedge_gen_mapper/src/js_runtime.rs | 55 ++++++++++++++++++- .../tedge_gen_mapper/pipelines/average.js | 11 +++- .../pipelines/average.samples | 4 +- 7 files changed, 133 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db467c7863f..e29d364310a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1178,6 +1178,15 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2459fc9262a1aa204eb4b5764ad4f189caec88aea9634389c0a25f8be7f6265e" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.10.0" @@ -3354,6 +3363,15 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -3715,6 +3733,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.15" @@ -3815,6 +3839,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c5227859c4dfc83f428e58f9569bf439e628c8d139020e7faff437e6f5abaa0" dependencies = [ "rquickjs-core", + "rquickjs-macro", ] [[package]] @@ -3824,9 +3849,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e82e0ca83028ad5b533b53b96c395bbaab905a5774de4aaf1004eeacafa3d85d" dependencies = [ "async-lock", + "relative-path", "rquickjs-sys", ] +[[package]] +name = "rquickjs-macro" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4d2eccd988a924a470a76fbd81a191b22d1f5f4f4619cf5662a8c1ab4ca1db7" +dependencies = [ + "convert_case", + "fnv", + "ident_case", + "indexmap 2.9.0", + "proc-macro-crate", + "proc-macro2", + "quote", + "rquickjs-core", + "syn 2.0.101", +] + [[package]] name = "rquickjs-sys" version = "0.9.0" @@ -5734,6 +5777,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unicode-width" version = "0.1.14" diff --git a/crates/extensions/tedge_gen_mapper/Cargo.toml b/crates/extensions/tedge_gen_mapper/Cargo.toml index d77da72de61..9b5a4b47a23 100644 --- a/crates/extensions/tedge_gen_mapper/Cargo.toml +++ b/crates/extensions/tedge_gen_mapper/Cargo.toml @@ -14,6 +14,7 @@ async-trait = { workspace = true } camino = { workspace = true, features = ["serde1"] } rquickjs = { workspace = true, default-features = false, features = [ "futures", + "macro", "parallel", ] } serde = { workspace = true, features = ["derive"] } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/average.js b/crates/extensions/tedge_gen_mapper/pipelines/average.js index 128f8774194..c9fcd1f4cf8 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/average.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/average.js @@ -70,6 +70,7 @@ export function process (timestamp, message) { State.agg_for_topic[topic] = agg_payload } + console.log("average.state", State.agg_for_topic) return [] } @@ -86,9 +87,13 @@ export function tick() { else if (v.sum && v.count) { let fragment = { [k]: v.sum / v.count } Object.assign(payload, fragment) - } else for (let [sub_k, sub_v] of Object.entries(v)) { - let fragment = { [k]: { [sub_k]: sub_v.sum / sub_v.count } } - Object.assign(payload, fragment) + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_fragment = { [sub_k]: sub_v.sum / sub_v.count } + Object.assign(fragment, sub_fragment) + } + Object.assign(payload, { [k]: fragment }) } } diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 2f4fc094a4f..316ad827133 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -228,6 +228,12 @@ impl<'a, 'js> IntoJs<'js> for JsonValueRef<'a> { impl<'js> FromJs<'js> for JsonValue { fn from_js(_ctx: &Ctx<'js>, value: Value<'js>) -> rquickjs::Result { + JsonValue::from_js_value(value) + } +} + +impl JsonValue { + fn from_js_value(value: Value<'_>) -> rquickjs::Result { if let Some(b) = value.as_bool() { return Ok(JsonValue(serde_json::Value::Bool(b))); } @@ -260,6 +266,11 @@ impl<'js> FromJs<'js> for JsonValue { Ok(JsonValue(serde_json::Value::Null)) } + + pub(crate) fn display(value: Value<'_>) -> String { + let json = JsonValue::from_js_value(value).unwrap_or_default(); + serde_json::to_string_pretty(&json.0).unwrap() + } } #[cfg(test)] diff --git a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs index 892b9f3793d..b88118b0758 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs @@ -112,6 +112,7 @@ impl JsWorker { async fn run(mut self) { rquickjs::async_with!(self.context => |ctx| { + console::init(&ctx); let mut modules = JsModules::new(); while let Some(request) = self.requests.recv().await { match request { @@ -177,7 +178,6 @@ impl<'js> JsModules<'js> { })?; let f = rquickjs::Function::from_value(f)?; - debug!(target: "MAPPING", "execute({module_name}.{function})"); let r = match &args[..] { [] => f.call(()), [v0] => f.call((v0,)), @@ -202,3 +202,56 @@ impl<'js> JsModules<'js> { }) } } + +mod console { + use crate::js_filter::JsonValue; + use rquickjs::class::Trace; + use rquickjs::function::Rest; + use rquickjs::Ctx; + use rquickjs::JsLifetime; + use rquickjs::Result; + use rquickjs::Value; + use std::fmt::Write; + + #[derive(Clone, Trace, JsLifetime)] + #[rquickjs::class(frozen)] + struct Console {} + + pub fn init(ctx: &Ctx<'_>) { + let console = Console {}; + let _ = ctx.globals().set("console", console); + } + + impl Console { + fn print(&self, _level: tracing::Level, values: Rest>) -> Result<()> { + let mut message = String::new(); + for (i, value) in values.0.into_iter().enumerate() { + if i > 0 { + let _ = write!(&mut message, ", "); + } + let _ = write!(&mut message, "{}", JsonValue::display(value)); + } + eprintln!("JavaScript.Console: {message}"); + Ok(()) + } + } + + #[rquickjs::methods] + impl Console { + fn debug(&self, values: Rest>) -> Result<()> { + self.print(tracing::Level::DEBUG, values) + } + + fn log(&self, values: Rest>) -> Result<()> { + self.print(tracing::Level::INFO, values) + } + + fn warn(&self, values: Rest>) -> Result<()> { + self.print(tracing::Level::WARN, values) + } + + fn error(&self, values: Rest>) -> Result<()> { + self.print(tracing::Level::ERROR, values) + } + } +} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js index 128f8774194..c9fcd1f4cf8 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js @@ -70,6 +70,7 @@ export function process (timestamp, message) { State.agg_for_topic[topic] = agg_payload } + console.log("average.state", State.agg_for_topic) return [] } @@ -86,9 +87,13 @@ export function tick() { else if (v.sum && v.count) { let fragment = { [k]: v.sum / v.count } Object.assign(payload, fragment) - } else for (let [sub_k, sub_v] of Object.entries(v)) { - let fragment = { [k]: { [sub_k]: sub_v.sum / sub_v.count } } - Object.assign(payload, fragment) + } else { + let fragment = {} + for (let [sub_k, sub_v] of Object.entries(v)) { + let sub_fragment = { [sub_k]: sub_v.sum / sub_v.count } + Object.assign(fragment, sub_fragment) + } + Object.assign(payload, { [k]: fragment }) } } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples index cee47197527..64f7fbf336e 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples @@ -1,7 +1,7 @@ INPUT: [test/average] {"temperature": 20} INPUT: [test/average] {"humidity": 50} INPUT: [test/average] {"temperature": 25, "location": {"altitude": 100.0 }} -INPUT: [test/average] {"temperature": 30} +INPUT: [test/average] {"temperature": 30, "location": {"latitude": 45.0 }} INPUT: [test/average] {"temperature": 25} INPUT: [test/average] {"humidity": 70, "location": {"altitude": 120.0 }} INPUT: [test/average] {"temperature": 20} @@ -12,5 +12,5 @@ INPUT: [test/average/another_topic] {"temperature": 6} INPUT: [test/average/another_topic] {"temperature": 4} -OUTPUT: [test/average] {"temperature":24,"humidity":60,"location":{"altitude":110}} +OUTPUT: [test/average] {"temperature":24,"humidity":60,"location":{"altitude":110,"latitude":45}} OUTPUT: [test/average/another_topic] {"temperature":4} \ No newline at end of file From 5850d6e459367007db5e8682ff05ec7c5e0b1308 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 1 Jul 2025 10:33:20 +0200 Subject: [PATCH 37/53] Tedge mapping test a single single filter Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/cli.rs | 15 ++ crates/core/tedge/src/cli/mapping/test.rs | 45 +++--- .../extensions/tedge_gen_mapper/src/config.rs | 15 ++ .../tedge_gen_mapper/src/runtime.rs | 129 ++++++++++++++---- .../tedge_gen_mapper/pipelines/average.toml | 5 - .../tedge_gen_mapper/tedge_gen_mapper.robot | 2 +- 6 files changed, 148 insertions(+), 63 deletions(-) delete mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml diff --git a/crates/core/tedge/src/cli/mapping/cli.rs b/crates/core/tedge/src/cli/mapping/cli.rs index 4adaff621d4..04345a1c978 100644 --- a/crates/core/tedge/src/cli/mapping/cli.rs +++ b/crates/core/tedge/src/cli/mapping/cli.rs @@ -108,4 +108,19 @@ impl TEdgeMappingCli { ) }) } + + pub async fn load_filter( + mapping_dir: &PathBuf, + path: &PathBuf, + ) -> Result { + if let Some("toml") = path.extension().and_then(|s| s.to_str()) { + MessageProcessor::try_new_single_pipeline(mapping_dir, path) + .await + .with_context(|| format!("loading pipeline {pipeline}", pipeline = path.display())) + } else { + MessageProcessor::try_new_single_filter(mapping_dir, path) + .await + .with_context(|| format!("loading filter {filter}", filter = path.display())) + } + } } diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index 79c208b0aab..0b6009faef6 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -27,7 +27,10 @@ impl Command for TestCommand { } async fn execute(&self, _config: TEdgeConfig) -> Result<(), MaybeFancy> { - let mut processor = TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?; + let mut processor = match &self.filter { + None => TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?, + Some(filter) => TEdgeMappingCli::load_filter(&self.mapping_dir, filter).await?, + }; if let Some(message) = &self.message { let timestamp = DateTime::now(); self.process(&mut processor, message, ×tamp).await; @@ -53,37 +56,21 @@ impl TestCommand { message: &Message, timestamp: &DateTime, ) { - match &self.filter { - Some(filter) => { - let filter_name = filter.display().to_string(); - print( - processor - .process_with_pipeline(&filter_name, timestamp, message) - .await, - ) - } - None => processor - .process(timestamp, message) - .await - .into_iter() - .map(|(_, v)| v) - .for_each(print), - } + processor + .process(timestamp, message) + .await + .into_iter() + .map(|(_, v)| v) + .for_each(print) } async fn tick(&self, processor: &mut MessageProcessor, timestamp: &DateTime) { - match &self.filter { - Some(filter) => { - let filter_name = filter.display().to_string(); - print(processor.tick_with_pipeline(&filter_name, timestamp).await) - } - None => processor - .tick(timestamp) - .await - .into_iter() - .map(|(_, v)| v) - .for_each(print), - } + processor + .tick(timestamp) + .await + .into_iter() + .map(|(_, v)| v) + .for_each(print) } } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index dd389bd2bae..c2822adef79 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -45,6 +45,20 @@ pub enum ConfigError { } impl PipelineConfig { + pub fn from_filter(filter: Utf8PathBuf) -> Self { + let input_topic = "#".to_string(); + let stage = StageConfig { + filter: FilterSpec::JavaScript(filter), + config: None, + tick_every_seconds: 1, + meta_topics: vec![], + }; + Self { + input_topics: vec![input_topic], + stages: vec![stage], + } + } + pub fn compile( self, js_runtime: &JsRuntime, @@ -69,6 +83,7 @@ impl StageConfig { pub fn compile(self, _js_runtime: &JsRuntime, config_dir: &Path) -> Result { let path = match self.filter { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), + FilterSpec::JavaScript(path) if path.starts_with(config_dir) => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; let filter = JsFilter::new(path) diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 830ec3bc5c7..f2b2fcbe4c2 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -42,6 +42,43 @@ impl MessageProcessor { }) } + pub async fn try_new_single_pipeline( + config_dir: impl AsRef, + pipeline: impl AsRef, + ) -> Result { + let config_dir = config_dir.as_ref().to_owned(); + let pipeline = pipeline.as_ref().to_owned(); + let mut js_runtime = JsRuntime::try_new().await?; + let mut pipeline_specs = PipelineSpecs::default(); + pipeline_specs + .load_single_pipeline(&mut js_runtime, &config_dir, &pipeline) + .await; + let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); + Ok(MessageProcessor { + config_dir, + pipelines, + js_runtime, + }) + } + + pub async fn try_new_single_filter( + config_dir: impl AsRef, + filter: impl AsRef, + ) -> Result { + let config_dir = config_dir.as_ref().to_owned(); + let mut js_runtime = JsRuntime::try_new().await?; + let mut pipeline_specs = PipelineSpecs::default(); + pipeline_specs + .load_single_filter(&mut js_runtime, &filter) + .await; + let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); + Ok(MessageProcessor { + config_dir, + pipelines, + js_runtime, + }) + } + pub fn subscriptions(&self) -> TopicFilter { let mut topics = TopicFilter::empty(); for pipeline in self.pipelines.values() { @@ -63,19 +100,6 @@ impl MessageProcessor { out_messages } - pub async fn process_with_pipeline( - &mut self, - pipeline_id: &String, - timestamp: &DateTime, - message: &Message, - ) -> Result, FilterError> { - let pipeline = self - .pipelines - .get_mut(pipeline_id) - .ok_or_else(|| anyhow::anyhow!("No such pipeline: {pipeline_id}"))?; - pipeline.process(&self.js_runtime, timestamp, message).await - } - pub async fn tick( &mut self, timestamp: &DateTime, @@ -88,18 +112,6 @@ impl MessageProcessor { out_messages } - pub async fn tick_with_pipeline( - &mut self, - pipeline_id: &String, - timestamp: &DateTime, - ) -> Result, FilterError> { - let pipeline = self - .pipelines - .get_mut(pipeline_id) - .ok_or_else(|| anyhow::anyhow!("No such pipeline: {pipeline_id}"))?; - pipeline.tick(&self.js_runtime, timestamp).await - } - pub async fn dump_memory_stats(&self) { self.js_runtime.dump_memory_stats().await; } @@ -235,10 +247,71 @@ impl PipelineSpecs { } } + pub async fn load_single_pipeline( + &mut self, + js_runtime: &mut JsRuntime, + config_dir: &PathBuf, + pipeline: &Path, + ) { + let Some(path) = Utf8Path::from_path(pipeline).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", pipeline.display()); + return; + }; + if let Err(err) = self.load_pipeline(&path).await { + error!(target: "MAPPING", "Failed to load pipeline {path}: {err}"); + return; + } + + let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| + error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) + ) else { + return; + }; + + while let Ok(Some(entry)) = entries.next_entry().await { + let Some(path) = Utf8Path::from_path(&entry.path()).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", entry.path().display()); + continue; + }; + if let Ok(file_type) = entry.file_type().await { + if file_type.is_file() { + match path.extension() { + Some("js") | Some("ts") => { + info!(target: "MAPPING", "Loading filter: {path}"); + if let Err(err) = self.load_filter(js_runtime, path).await { + error!(target: "MAPPING", "Failed to load filter: {err}"); + } + } + _ => {} + } + } + } + } + } + + pub async fn load_single_filter( + &mut self, + js_runtime: &mut JsRuntime, + filter: impl AsRef, + ) { + let filter = filter.as_ref(); + let Some(path) = Utf8Path::from_path(filter).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", filter.display()); + return; + }; + if let Err(err) = js_runtime.load_file(&path).await { + error!(target: "MAPPING", "Failed to load filter {path}: {err}"); + } + let pipeline_id = MessageProcessor::pipeline_id(&path); + let pipeline = PipelineConfig::from_filter(path.to_owned()); + self.pipeline_specs + .insert(pipeline_id, (path.to_owned(), pipeline)); + } + async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { let path = file.as_ref(); let pipeline_id = MessageProcessor::pipeline_id(path); - let specs = read_to_string(file.as_ref()).await?; + let specs = read_to_string(path).await?; let pipeline: PipelineConfig = toml::from_str(&specs)?; self.pipeline_specs .insert(pipeline_id, (path.to_owned(), pipeline)); @@ -249,9 +322,9 @@ impl PipelineSpecs { async fn load_filter( &mut self, js_runtime: &mut JsRuntime, - file: impl AsRef, + file: impl AsRef, ) -> Result<(), LoadError> { - js_runtime.load_file(file.as_ref()).await?; + js_runtime.load_file(file).await?; Ok(()) } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml deleted file mode 100644 index 52f2bdf7c4a..00000000000 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.toml +++ /dev/null @@ -1,5 +0,0 @@ -input_topics = ["test/average/#"] - -stages = [ - { filter = "average.js", tick_every_seconds = 1 }, -] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index 0092d7d400a..fdd9518bfdd 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -50,7 +50,7 @@ Units are configured using topic metadata Computing average over a time window ${transformed_msg} Execute Command - ... cat /etc/tedge/gen-mapper/average.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test --final-tick + ... cat /etc/tedge/gen-mapper/average.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test --final-tick --filter /etc/tedge/gen-mapper/average.js ... strip=True ${expected_msg} Execute Command ... cat /etc/tedge/gen-mapper/average.samples | awk '{ if ($2) print $2 }' FS\='OUTPUT: ' From 5b137e847834ba5d0c304796e103a64077400317 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 1 Jul 2025 20:06:50 +0200 Subject: [PATCH 38/53] Fix JS engine with one module per filter instance Now, each instance of a script is given its own static state Signed-off-by: Didier Wenzek --- .../extensions/tedge_gen_mapper/src/actor.rs | 4 +- .../extensions/tedge_gen_mapper/src/config.rs | 33 +++-- .../tedge_gen_mapper/src/js_filter.rs | 25 ++-- .../tedge_gen_mapper/src/js_runtime.rs | 16 +-- .../tedge_gen_mapper/src/runtime.rs | 117 ++++-------------- .../pipelines/count-events.toml | 5 + .../pipelines/count-measurements.toml | 5 + .../pipelines/count-messages.js | 22 ++++ .../pipelines/count-messages.samples | 19 +++ .../tedge_gen_mapper/tedge_gen_mapper.robot | 11 ++ 10 files changed, 135 insertions(+), 122 deletions(-) create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-events.toml create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-measurements.toml create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js create mode 100644 tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 4e1daa381f7..2c2e12a1d7f 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -61,9 +61,7 @@ impl Actor for GenMapper { let Ok(path) = Utf8PathBuf::try_from(path) else { continue; }; - if matches!(path.extension(), Some("js" | "ts")) { - self.processor.add_filter(path).await; - } else if path.extension() == Some("toml") { + if matches!(path.extension(), Some("toml")) { self.processor.add_pipeline(path).await; self.send_updated_subscriptions().await?; } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index c2822adef79..f3e3c80ab3c 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -3,9 +3,11 @@ use crate::js_runtime::JsRuntime; use crate::pipeline::Pipeline; use crate::pipeline::Stage; use crate::LoadError; +use camino::Utf8Path; use camino::Utf8PathBuf; use serde::Deserialize; use serde_json::Value; +use std::fmt::Debug; use std::path::Path; use tedge_mqtt_ext::TopicFilter; @@ -59,20 +61,24 @@ impl PipelineConfig { } } - pub fn compile( + pub async fn compile( self, - js_runtime: &JsRuntime, + js_runtime: &mut JsRuntime, config_dir: &Path, source: Utf8PathBuf, ) -> Result { - let input = topic_filters(&self.input_topics)?; - let stages = self - .stages - .into_iter() - .map(|stage| stage.compile(js_runtime, config_dir)) - .collect::, _>>()?; + let input_topics = topic_filters(&self.input_topics)?; + let mut stages = vec![]; + for (i, stage) in self.stages.into_iter().enumerate() { + let stage = stage.compile(config_dir, i, &source).await?; + let filter = &stage.filter; + js_runtime + .load_file(filter.module_name(), filter.path()) + .await?; + stages.push(stage); + } Ok(Pipeline { - input_topics: input, + input_topics, stages, source, }) @@ -80,13 +86,18 @@ impl PipelineConfig { } impl StageConfig { - pub fn compile(self, _js_runtime: &JsRuntime, config_dir: &Path) -> Result { + pub async fn compile( + self, + config_dir: &Path, + index: usize, + pipeline: &Utf8Path, + ) -> Result { let path = match self.filter { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), FilterSpec::JavaScript(path) if path.starts_with(config_dir) => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; - let filter = JsFilter::new(path) + let filter = JsFilter::new(pipeline.to_owned().into(), index, path) .with_config(self.config) .with_tick_every_seconds(self.tick_every_seconds); let config_topics = topic_filters(&self.meta_topics)?; diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 316ad827133..9c1c006b756 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -14,6 +14,7 @@ use tracing::debug; #[derive(Clone)] pub struct JsFilter { + pub module_name: String, pub path: PathBuf, pub config: JsonValue, pub tick_every_seconds: u64, @@ -23,8 +24,10 @@ pub struct JsFilter { pub struct JsonValue(serde_json::Value); impl JsFilter { - pub fn new(path: PathBuf) -> Self { + pub fn new(pipeline: PathBuf, index: usize, path: PathBuf) -> Self { + let module_name = format!("{}|{}|{}", pipeline.display(), index, path.display()); JsFilter { + module_name, path, config: JsonValue::default(), tick_every_seconds: 0, @@ -32,7 +35,7 @@ impl JsFilter { } pub fn module_name(&self) -> String { - self.path.display().to_string() + self.module_name.to_owned() } pub fn with_config(self, config: Option) -> Self { @@ -77,7 +80,7 @@ impl JsFilter { message.clone().into(), self.config.clone(), ]; - js.call_function(&self.path, "process", input) + js.call_function(&self.module_name(), "process", input) .await .map_err(pipeline::error_from_js)? .try_into() @@ -98,7 +101,7 @@ impl JsFilter { debug!(target: "MAPPING", "{}: update_config({message:?})", self.module_name()); let input = vec![message.clone().into(), self.config.clone()]; let config = js - .call_function(&self.path, "update_config", input) + .call_function(&self.module_name(), "update_config", input) .await .map_err(pipeline::error_from_js)?; self.config = config; @@ -122,7 +125,7 @@ impl JsFilter { } debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.module_name()); let input = vec![timestamp.clone().into(), self.config.clone()]; - js.call_function(&self.path, "tick", input) + js.call_function(&self.module_name(), "tick", input) .await .map_err(pipeline::error_from_js)? .try_into() @@ -281,8 +284,8 @@ mod tests { async fn identity_filter() { let script = "export function process(t,msg) { return [msg]; };"; let mut runtime = JsRuntime::try_new().await.unwrap(); - runtime.load_js("id.js", script).await.unwrap(); - let filter = JsFilter::new("id.js".into()); + let filter = JsFilter::new("id.toml".into(), 1, "id.js".into()); + runtime.load_js(filter.module_name(), script).await.unwrap(); let input = Message::new("te/main/device///m/", "hello world"); let output = input.clone(); @@ -299,8 +302,8 @@ mod tests { async fn error_filter() { let script = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; let mut runtime = JsRuntime::try_new().await.unwrap(); - runtime.load_js("err.js", script).await.unwrap(); - let filter = JsFilter::new("err.js".into()); + let filter = JsFilter::new("err.toml".into(), 1, "err.js".into()); + runtime.load_js(filter.module_name(), script).await.unwrap(); let input = Message::new("te/main/device///m/", "hello world"); let error = filter @@ -335,8 +338,8 @@ export function process (timestamp, message, config) { } "#; let mut runtime = JsRuntime::try_new().await.unwrap(); - runtime.load_js("collectd.js", script).await.unwrap(); - let filter = JsFilter::new("collectd.js".into()); + let filter = JsFilter::new("collectd.toml".into(), 1, "collectd.js".into()); + runtime.load_js(filter.module_name(), script).await.unwrap(); let input = Message::new( "collectd/h/memory/percent-used", diff --git a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs index b88118b0758..bd58bdb6a80 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs @@ -23,20 +23,22 @@ impl JsRuntime { Ok(JsRuntime { runtime, worker }) } - pub async fn load_file(&mut self, path: impl AsRef) -> Result<(), LoadError> { + pub async fn load_file( + &mut self, + module_name: String, + path: impl AsRef, + ) -> Result<(), LoadError> { let path = path.as_ref(); let source = tokio::fs::read_to_string(path).await?; - self.load_js(path, source).await + self.load_js(module_name, source).await } pub async fn load_js( &mut self, - path: impl AsRef, + name: String, source: impl Into>, ) -> Result<(), LoadError> { let (sender, receiver) = oneshot::channel(); - let path = path.as_ref().to_path_buf(); - let name = path.display().to_string(); let source = source.into(); self.worker .send(JsRequest::LoadModule { @@ -51,14 +53,14 @@ impl JsRuntime { pub async fn call_function( &self, - module: &Path, + module: &str, function: &str, args: Vec, ) -> Result { let (sender, receiver) = oneshot::channel(); self.worker .send(JsRequest::CallFunction { - module: module.display().to_string(), + module: module.to_string(), function: function.to_string(), args, sender, diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index f2b2fcbe4c2..8991f199b63 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -32,8 +32,8 @@ impl MessageProcessor { let config_dir = config_dir.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; let mut pipeline_specs = PipelineSpecs::default(); - pipeline_specs.load(&mut js_runtime, &config_dir).await; - let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); + pipeline_specs.load(&config_dir).await; + let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; Ok(MessageProcessor { config_dir, @@ -50,10 +50,8 @@ impl MessageProcessor { let pipeline = pipeline.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; let mut pipeline_specs = PipelineSpecs::default(); - pipeline_specs - .load_single_pipeline(&mut js_runtime, &config_dir, &pipeline) - .await; - let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); + pipeline_specs.load_single_pipeline(&pipeline).await; + let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; Ok(MessageProcessor { config_dir, pipelines, @@ -68,10 +66,8 @@ impl MessageProcessor { let config_dir = config_dir.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; let mut pipeline_specs = PipelineSpecs::default(); - pipeline_specs - .load_single_filter(&mut js_runtime, &filter) - .await; - let pipelines = pipeline_specs.compile(&js_runtime, &config_dir); + pipeline_specs.load_single_filter(&filter).await; + let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; Ok(MessageProcessor { config_dir, pipelines, @@ -116,22 +112,15 @@ impl MessageProcessor { self.js_runtime.dump_memory_stats().await; } - pub async fn add_filter(&mut self, path: Utf8PathBuf) { - match self.js_runtime.load_file(&path).await { - Ok(()) => { - info!(target: "gen-mapper", "Loaded filter {path}"); - } - Err(e) => { - error!(target: "gen-mapper", "Failed to load filter {path}: {e}"); - } - } - } - pub async fn reload_filter(&mut self, path: Utf8PathBuf) { for pipeline in self.pipelines.values_mut() { for stage in &mut pipeline.stages { if stage.filter.path() == path { - match self.js_runtime.load_file(&path).await { + match self + .js_runtime + .load_file(stage.filter.module_name(), &path) + .await + { Ok(()) => { info!(target: "gen-mapper", "Reloaded filter {path}"); } @@ -168,7 +157,10 @@ impl MessageProcessor { return false; } }; - match config.compile(&self.js_runtime, &self.config_dir, path.clone()) { + match config + .compile(&mut self.js_runtime, &self.config_dir, path.clone()) + .await + { Ok(pipeline) => { self.pipelines.insert(pipeline_id, pipeline); true @@ -211,7 +203,7 @@ struct PipelineSpecs { } impl PipelineSpecs { - pub async fn load(&mut self, js_runtime: &mut JsRuntime, config_dir: &PathBuf) { + pub async fn load(&mut self, config_dir: &PathBuf) { let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) ) else { @@ -225,21 +217,10 @@ impl PipelineSpecs { }; if let Ok(file_type) = entry.file_type().await { if file_type.is_file() { - match path.extension() { - Some("toml") => { - info!(target: "MAPPING", "Loading pipeline: {path}"); - if let Err(err) = self.load_pipeline(path).await { - error!(target: "MAPPING", "Failed to load pipeline: {err}"); - } - } - Some("js") | Some("ts") => { - info!(target: "MAPPING", "Loading filter: {path}"); - if let Err(err) = self.load_filter(js_runtime, path).await { - error!(target: "MAPPING", "Failed to load filter: {err}"); - } - } - _ => { - info!(target: "MAPPING", "Skipping file which type is unknown: {path}"); + if let Some("toml") = path.extension() { + info!(target: "MAPPING", "Loading pipeline: {path}"); + if let Err(err) = self.load_pipeline(path).await { + error!(target: "MAPPING", "Failed to load pipeline: {err}"); } } } @@ -247,61 +228,22 @@ impl PipelineSpecs { } } - pub async fn load_single_pipeline( - &mut self, - js_runtime: &mut JsRuntime, - config_dir: &PathBuf, - pipeline: &Path, - ) { + pub async fn load_single_pipeline(&mut self, pipeline: &Path) { let Some(path) = Utf8Path::from_path(pipeline).map(|p| p.to_path_buf()) else { error!(target: "MAPPING", "Skipping non UTF8 path: {}", pipeline.display()); return; }; if let Err(err) = self.load_pipeline(&path).await { error!(target: "MAPPING", "Failed to load pipeline {path}: {err}"); - return; - } - - let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| - error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) - ) else { - return; - }; - - while let Ok(Some(entry)) = entries.next_entry().await { - let Some(path) = Utf8Path::from_path(&entry.path()).map(|p| p.to_path_buf()) else { - error!(target: "MAPPING", "Skipping non UTF8 path: {}", entry.path().display()); - continue; - }; - if let Ok(file_type) = entry.file_type().await { - if file_type.is_file() { - match path.extension() { - Some("js") | Some("ts") => { - info!(target: "MAPPING", "Loading filter: {path}"); - if let Err(err) = self.load_filter(js_runtime, path).await { - error!(target: "MAPPING", "Failed to load filter: {err}"); - } - } - _ => {} - } - } - } } } - pub async fn load_single_filter( - &mut self, - js_runtime: &mut JsRuntime, - filter: impl AsRef, - ) { + pub async fn load_single_filter(&mut self, filter: impl AsRef) { let filter = filter.as_ref(); let Some(path) = Utf8Path::from_path(filter).map(|p| p.to_path_buf()) else { error!(target: "MAPPING", "Skipping non UTF8 path: {}", filter.display()); return; }; - if let Err(err) = js_runtime.load_file(&path).await { - error!(target: "MAPPING", "Failed to load filter {path}: {err}"); - } let pipeline_id = MessageProcessor::pipeline_id(&path); let pipeline = PipelineConfig::from_filter(path.to_owned()); self.pipeline_specs @@ -319,19 +261,14 @@ impl PipelineSpecs { Ok(()) } - async fn load_filter( - &mut self, + async fn compile( + mut self, js_runtime: &mut JsRuntime, - file: impl AsRef, - ) -> Result<(), LoadError> { - js_runtime.load_file(file).await?; - Ok(()) - } - - fn compile(mut self, js_runtime: &JsRuntime, config_dir: &Path) -> HashMap { + config_dir: &Path, + ) -> HashMap { let mut pipelines = HashMap::new(); for (name, (source, specs)) in self.pipeline_specs.drain() { - match specs.compile(js_runtime, config_dir, source) { + match specs.compile(js_runtime, config_dir, source).await { Ok(pipeline) => { let _ = pipelines.insert(name, pipeline); } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-events.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-events.toml new file mode 100644 index 00000000000..9dcc8e32f2f --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-events.toml @@ -0,0 +1,5 @@ +input_topics = ["test/+/+/+/+/e/+"] + +stages = [ + { filter = "count-messages.js", config = { topic = "test/count/e" }, tick_every_seconds = 1 }, +] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-measurements.toml new file mode 100644 index 00000000000..81895dde972 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-measurements.toml @@ -0,0 +1,5 @@ +input_topics = ["test/+/+/+/+/m/+"] + +stages = [ + { filter = "count-messages.js", config = { topic = "test/count/m" }, tick_every_seconds = 1 }, +] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js new file mode 100644 index 00000000000..5be6c29aa22 --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js @@ -0,0 +1,22 @@ +class State { + static count_per_topic = {} +} + +export function process (timestamp, message) { + let topic = message.topic + let count = State.count_per_topic[topic] || 0 + State.count_per_topic[topic] = count + 1 + + console.log("current count", State.count_per_topic) + return [] +} + +export function tick(timestamp, config) { + let message = { + topic: config?.topic || "te/error", + payload: JSON.stringify(State.count_per_topic) + } + + State.count_per_topic = {} + return [message] +} \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples new file mode 100644 index 00000000000..180e028e3ae --- /dev/null +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples @@ -0,0 +1,19 @@ +INPUT: [test/device/main///m/] "some measurement" +INPUT: [test/device/child1///m/] "some measurement" +INPUT: [test/device/child2///m/] "some measurement" +INPUT: [test/device/main///m/] "some measurement" +INPUT: [test/device/child1///m/] "some measurement" +INPUT: [test/device/child2///m/] "some measurement" +INPUT: [test/device/main///m/] "some measurement" +INPUT: [test/device/child1///m/] "some measurement" +INPUT: [test/device/main///m/] "some measurement" +INPUT: [test/device/main///e/] "some event" +INPUT: [test/device/child2///e/] "some event" +INPUT: [test/device/main///e/] "some event" +INPUT: [test/device/child1///e/] "some event" +INPUT: [test/device/child2///e/] "some event" + +# Since we have two pipelines using the same javascript filter, one expect two output messages +# A first one for all the measurements, another one for all the events +OUTPUT: [test/count/m] {"test/device/main///m/":4,"test/device/child1///m/":3,"test/device/child2///m/":2} +OUTPUT: [test/count/e] {"test/device/main///e/":2,"test/device/child2///e/":2,"test/device/child1///e/":1} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index fdd9518bfdd..5ec963cb7ee 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -59,6 +59,17 @@ Computing average over a time window ... ${transformed_msg} ... ${expected_msg} +Each instance of a script must have its own static state + ${transformed_msg} Execute Command + ... cat /etc/tedge/gen-mapper/count-messages.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test --final-tick | sort + ... strip=True + ${expected_msg} Execute Command + ... cat /etc/tedge/gen-mapper/count-messages.samples | awk '{ if ($2) print $2 }' FS\='OUTPUT: ' | sort + ... strip=True + Should Be Equal + ... ${transformed_msg} + ... ${expected_msg} + *** Keywords *** Custom Setup From c26ed7b7ce7ddec0e5a9e436fb9955126e352a8b Mon Sep 17 00:00:00 2001 From: reubenmiller Date: Mon, 7 Jul 2025 08:54:46 +0200 Subject: [PATCH 39/53] use config_dir variable to get configuration file location Signed-off-by: reubenmiller --- crates/core/tedge_mapper/src/gen/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/core/tedge_mapper/src/gen/mod.rs b/crates/core/tedge_mapper/src/gen/mod.rs index e89644506de..ee6995e4748 100644 --- a/crates/core/tedge_mapper/src/gen/mod.rs +++ b/crates/core/tedge_mapper/src/gen/mod.rs @@ -11,13 +11,13 @@ impl TEdgeComponent for GenMapper { async fn start( &self, tedge_config: TEdgeConfig, - _config_dir: &tedge_config::Path, + config_dir: &tedge_config::Path, ) -> Result<(), anyhow::Error> { let (mut runtime, mut mqtt_actor) = start_basic_actors("tedge-gen-mapper", &tedge_config).await?; let mut fs_actor = FsWatchActorBuilder::new(); - let mut gen_mapper = GenMapperBuilder::try_new("/etc/tedge/gen-mapper").await?; + let mut gen_mapper = GenMapperBuilder::try_new(config_dir.join("gen-mapper")).await?; gen_mapper.connect(&mut mqtt_actor); gen_mapper.connect_fs(&mut fs_actor); From ebd8af270e447631b83071f17aee1e8866742cd5 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 10 Jul 2025 15:22:36 +0200 Subject: [PATCH 40/53] The default config of flow transformation is {} The default config was previously null, forcing explicit checks in the scripts. Signed-off-by: Didier Wenzek --- .../pipelines/circuit-breaker.js | 16 ++++++++-------- .../tedge_gen_mapper/pipelines/collectd-to-te.js | 2 +- .../pipelines/drop_stragglers.js | 4 ++-- .../tedge_gen_mapper/pipelines/measurements.toml | 1 - .../tedge_gen_mapper/pipelines/set_topic.js | 2 +- .../tedge_gen_mapper/pipelines/te_to_c8y.js | 7 ++----- .../extensions/tedge_gen_mapper/src/js_filter.rs | 8 +++++++- .../tedge_gen_mapper/pipelines/count-messages.js | 2 +- .../tedge_gen_mapper/pipelines/set_topic.js | 2 +- .../tedge_gen_mapper/pipelines/te_to_c8y.js | 7 ++----- 10 files changed, 25 insertions(+), 26 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js b/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js index 3ecb09e4bf4..f65742a20c7 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js @@ -19,10 +19,10 @@ export function process (timestamp, message, config) { State.total += 1 State.batch[0] += 1 if (State.open) { - let back_to_normal = config?.back_to_normal || 100 + let back_to_normal = config.back_to_normal || 100 if (State.total < back_to_normal) { State.open = false - if (config?.message_on_back_to_normal) { + if (config.message_on_back_to_normal) { return [config?.message_on_back_to_normal, message] } else { return [message] @@ -31,13 +31,13 @@ export function process (timestamp, message, config) { return [] } } else { - let too_many = config?.too_many || 1000 + let too_many = config.too_many || 1000 if (State.total < too_many) { return [message] } else { State.open = true - if (config?.message_on_too_many) { - return [config?.message_on_too_many] + if (config.message_on_too_many) { + return [config.message_on_too_many] } else { return [] } @@ -47,15 +47,15 @@ export function process (timestamp, message, config) { export function tick(timestamp, config) { - let max_batch_count = config?.tick_count || 10 + let max_batch_count = config.tick_count || 10 let new_batch_count = State.batch.unshift(0) if (new_batch_count > max_batch_count) { State.total -= State.batch.pop() } - if (config?.stats_topic) { + if (config.stats_topic) { return [{ - topic: config?.stats_topic, + topic: config.stats_topic, payload: `{"circuit-breaker-open": ${State.open}, "total": ${State.total}, "batch": ${State.batch}}` }] } else { diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js index 31edd135589..5600155a0c5 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js @@ -8,7 +8,7 @@ export function process(_timestamp, message, config) { let value = data[1] return [{ - topic: config?.topic || "te/device/main///m/collectd", + topic: config.topic || "te/device/main///m/collectd", payload: `{"time": ${time}, "${group}": {"${measurement}": ${value}}}` }] } \ No newline at end of file diff --git a/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js b/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js index 5ac019db494..b663ac5959c 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js @@ -12,8 +12,8 @@ export function process (timestamp, message, config) { } let time = timestamp.seconds + (timestamp.nanoseconds / 1e9) - let max = time + (config?.max_advance || 1); - let min = time - (config?.max_delay || 10); + let max = time + (config.max_advance || 1); + let min = time - (config.max_delay || 10); if (min <= msg_timestamp && msg_timestamp <= max) { return [message] diff --git a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml index 1edde2696e6..cd339bb7556 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml +++ b/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml @@ -2,6 +2,5 @@ input_topics = ["te/+/+/+/+/m/+"] stages = [ { filter = "add_timestamp.js" }, - { filter = "drop_stragglers.js", config = { max_delay = 60 } }, { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } ] diff --git a/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js b/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js index 16ce07f0589..1e28b490f17 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js @@ -1,6 +1,6 @@ export function process (timestamp, message, config) { return [{ - topic: config?.topic || "te/error", + topic: config.topic || "te/error", payload: message.payload }] } diff --git a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js index d054c53742e..5c1e6c47fa9 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js +++ b/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -51,10 +51,10 @@ export function process(t, message, config) { type: type } - let meta = (config || {})[`${message.topic}/meta`] || {} + let meta = config[`${message.topic}/meta`] || {} for (let [k, v] of Object.entries(payload)) { - let k_meta = (meta || {})[k] || {} + let k_meta = meta[k] || {} if (k === "time") { let t = v if (typeof(v) === "number") { @@ -123,9 +123,6 @@ export function update_config(message, config) { let fragment = { [type]: metadata } - if (!config) { - config = {} - } Object.assign(config, fragment) return config diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 9c1c006b756..bc7d238c4c0 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -20,9 +20,15 @@ pub struct JsFilter { pub tick_every_seconds: u64, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug)] pub struct JsonValue(serde_json::Value); +impl Default for JsonValue { + fn default() -> Self { + JsonValue(serde_json::Value::Object(Default::default())) + } +} + impl JsFilter { pub fn new(pipeline: PathBuf, index: usize, path: PathBuf) -> Self { let module_name = format!("{}|{}|{}", pipeline.display(), index, path.display()); diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js index 5be6c29aa22..0c4e1bba81c 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js @@ -13,7 +13,7 @@ export function process (timestamp, message) { export function tick(timestamp, config) { let message = { - topic: config?.topic || "te/error", + topic: config.topic || "te/error", payload: JSON.stringify(State.count_per_topic) } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js index 16ce07f0589..1e28b490f17 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js @@ -1,6 +1,6 @@ export function process (timestamp, message, config) { return [{ - topic: config?.topic || "te/error", + topic: config.topic || "te/error", payload: message.payload }] } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js index d054c53742e..5c1e6c47fa9 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js @@ -51,10 +51,10 @@ export function process(t, message, config) { type: type } - let meta = (config || {})[`${message.topic}/meta`] || {} + let meta = config[`${message.topic}/meta`] || {} for (let [k, v] of Object.entries(payload)) { - let k_meta = (meta || {})[k] || {} + let k_meta = meta[k] || {} if (k === "time") { let t = v if (typeof(v) === "number") { @@ -123,9 +123,6 @@ export function update_config(message, config) { let fragment = { [type]: metadata } - if (!config) { - config = {} - } Object.assign(config, fragment) return config From 7c2049f2ebf53623f758cb8b21d5b494b158e8d7 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 10 Jul 2025 17:23:42 +0200 Subject: [PATCH 41/53] Warn when javascript module and config are not consistent Signed-off-by: Didier Wenzek --- crates/common/mqtt_channel/src/topics.rs | 4 ++ .../extensions/tedge_gen_mapper/src/config.rs | 8 ++-- .../tedge_gen_mapper/src/js_filter.rs | 17 +++++++ .../tedge_gen_mapper/src/js_runtime.rs | 47 +++++++++++++++---- .../tedge_gen_mapper/src/pipeline.rs | 17 +++++++ .../tedge_gen_mapper/src/runtime.rs | 6 +-- 6 files changed, 81 insertions(+), 18 deletions(-) diff --git a/crates/common/mqtt_channel/src/topics.rs b/crates/common/mqtt_channel/src/topics.rs index cad07cc2e6e..0e76cbf5c64 100644 --- a/crates/common/mqtt_channel/src/topics.rs +++ b/crates/common/mqtt_channel/src/topics.rs @@ -116,6 +116,10 @@ impl TopicFilter { } } + pub fn is_empty(&self) -> bool { + self.patterns.is_empty() + } + /// Check if the given topic matches this filter pattern. pub fn accept_topic_name(&self, topic: &str) -> bool { self.patterns diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index f3e3c80ab3c..55add53d09c 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -70,11 +70,9 @@ impl PipelineConfig { let input_topics = topic_filters(&self.input_topics)?; let mut stages = vec![]; for (i, stage) in self.stages.into_iter().enumerate() { - let stage = stage.compile(config_dir, i, &source).await?; - let filter = &stage.filter; - js_runtime - .load_file(filter.module_name(), filter.path()) - .await?; + let mut stage = stage.compile(config_dir, i, &source).await?; + js_runtime.load_filter(&mut stage.filter).await?; + stage.check(&source); stages.push(stage); } Ok(Pipeline { diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index bc7d238c4c0..4ec69135e1d 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -18,6 +18,9 @@ pub struct JsFilter { pub path: PathBuf, pub config: JsonValue, pub tick_every_seconds: u64, + pub no_js_process: bool, + pub no_js_update_config: bool, + pub no_js_tick: bool, } #[derive(Clone, Debug)] @@ -37,6 +40,9 @@ impl JsFilter { path, config: JsonValue::default(), tick_every_seconds: 0, + no_js_process: true, + no_js_update_config: true, + no_js_tick: true, } } @@ -81,6 +87,10 @@ impl JsFilter { message: &Message, ) -> Result, FilterError> { debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.module_name()); + if self.no_js_process { + return Ok(vec![message.clone()]); + } + let input = vec![ timestamp.clone().into(), message.clone().into(), @@ -105,6 +115,10 @@ impl JsFilter { message: &Message, ) -> Result<(), FilterError> { debug!(target: "MAPPING", "{}: update_config({message:?})", self.module_name()); + if self.no_js_update_config { + return Ok(()); + } + let input = vec![message.clone().into(), self.config.clone()]; let config = js .call_function(&self.module_name(), "update_config", input) @@ -126,6 +140,9 @@ impl JsFilter { js: &JsRuntime, timestamp: &DateTime, ) -> Result, FilterError> { + if self.no_js_tick { + return Ok(vec![]); + } if !timestamp.tick_now(self.tick_every_seconds) { return Ok(vec![]); } diff --git a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs index bd58bdb6a80..f6a41f9ae40 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs @@ -1,3 +1,4 @@ +use crate::js_filter::JsFilter; use crate::js_filter::JsonValue; use crate::LoadError; use anyhow::anyhow; @@ -23,11 +24,24 @@ impl JsRuntime { Ok(JsRuntime { runtime, worker }) } + pub async fn load_filter(&mut self, filter: &mut JsFilter) -> Result<(), LoadError> { + let exports = self.load_file(filter.module_name(), filter.path()).await?; + for export in exports { + match export { + "process" => filter.no_js_process = false, + "update_config" => filter.no_js_update_config = false, + "tick" => filter.no_js_tick = false, + _ => (), + } + } + Ok(()) + } + pub async fn load_file( &mut self, module_name: String, path: impl AsRef, - ) -> Result<(), LoadError> { + ) -> Result, LoadError> { let path = path.as_ref(); let source = tokio::fs::read_to_string(path).await?; self.load_js(module_name, source).await @@ -37,13 +51,15 @@ impl JsRuntime { &mut self, name: String, source: impl Into>, - ) -> Result<(), LoadError> { + ) -> Result, LoadError> { let (sender, receiver) = oneshot::channel(); let source = source.into(); + let imports = vec!["process", "update_config", "tick"]; self.worker .send(JsRequest::LoadModule { name, source, + imports, sender, }) .await @@ -87,7 +103,8 @@ enum JsRequest { LoadModule { name: String, source: Vec, - sender: oneshot::Sender>, + imports: Vec<&'static str>, + sender: oneshot::Sender, LoadError>>, }, CallFunction { module: String, @@ -118,8 +135,8 @@ impl JsWorker { let mut modules = JsModules::new(); while let Some(request) = self.requests.recv().await { match request { - JsRequest::LoadModule{name, source, sender} => { - let result = modules.load_module(ctx.clone(), name, source).await; + JsRequest::LoadModule{name, source, sender, imports} => { + let result = modules.load_module(ctx.clone(), name, source, imports).await; let _ = sender.send(result); } JsRequest::CallFunction{module, function, args, sender} => { @@ -149,13 +166,24 @@ impl<'js> JsModules<'js> { ctx: Ctx<'js>, name: String, source: Vec, - ) -> Result<(), LoadError> { + imports: Vec<&'static str>, + ) -> Result, LoadError> { debug!(target: "MAPPING", "compile({name})"); let module = Module::declare(ctx, name.clone(), source)?; let (module, p) = module.eval()?; let () = p.finish()?; + + let mut exports = vec![]; + for import in imports { + if let Ok(Some(v)) = module.get(import) { + if rquickjs::Function::from_value(v).is_ok() { + exports.push(import); + } + } + } + self.modules.insert(name, module); - Ok(()) + Ok(exports) } async fn call_function( @@ -178,7 +206,10 @@ impl<'js> JsModules<'js> { module_name: module_name.clone(), function: function.clone(), })?; - let f = rquickjs::Function::from_value(f)?; + let f = rquickjs::Function::from_value(f).map_err(|_| LoadError::UnknownFunction { + module_name: module_name.clone(), + function: function.clone(), + })?; let r = match &args[..] { [] => f.call(()), diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index a87daef8359..4d51164022a 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -1,12 +1,14 @@ use crate::js_filter::JsFilter; use crate::js_runtime::JsRuntime; use crate::LoadError; +use camino::Utf8Path; use camino::Utf8PathBuf; use serde_json::json; use serde_json::Value; use tedge_mqtt_ext::MqttMessage; use tedge_mqtt_ext::TopicFilter; use time::OffsetDateTime; +use tracing::warn; /// A chain of transformation of MQTT messages pub struct Pipeline { @@ -118,6 +120,21 @@ impl Pipeline { } } +impl Stage { + pub(crate) fn check(&self, pipeline: &Utf8Path) { + let filter = &self.filter; + if filter.no_js_process { + warn!(target: "MAPPING", "Filter with no 'process' function: {}", filter.path.display()); + } + if filter.no_js_update_config && !self.config_topics.is_empty() { + warn!(target: "MAPPING", "Filter with no 'config_update' function: {}; but configured with 'config_topics' in {pipeline}", filter.path.display()); + } + if filter.no_js_tick && filter.tick_every_seconds != 0 { + warn!(target: "MAPPING", "Filter with no 'tick' function: {}; but configured with 'tick_every_seconds' in {pipeline}", filter.path.display()); + } + } +} + impl DateTime { pub fn now() -> Self { DateTime::try_from(OffsetDateTime::now_utc()).unwrap() diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 8991f199b63..fc720fa5bf7 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -116,11 +116,7 @@ impl MessageProcessor { for pipeline in self.pipelines.values_mut() { for stage in &mut pipeline.stages { if stage.filter.path() == path { - match self - .js_runtime - .load_file(stage.filter.module_name(), &path) - .await - { + match self.js_runtime.load_filter(&mut stage.filter).await { Ok(()) => { info!(target: "gen-mapper", "Reloaded filter {path}"); } From d3df51556a2a3be617f8b1d53ea21b506020bb6b Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 10 Jul 2025 18:10:05 +0200 Subject: [PATCH 42/53] Trigger ticks every seconds per default when a tick handler is provided Signed-off-by: Didier Wenzek --- crates/extensions/tedge_gen_mapper/src/config.rs | 3 ++- crates/extensions/tedge_gen_mapper/src/pipeline.rs | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index 55add53d09c..86d49057fc3 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -52,7 +52,7 @@ impl PipelineConfig { let stage = StageConfig { filter: FilterSpec::JavaScript(filter), config: None, - tick_every_seconds: 1, + tick_every_seconds: 0, meta_topics: vec![], }; Self { @@ -73,6 +73,7 @@ impl PipelineConfig { let mut stage = stage.compile(config_dir, i, &source).await?; js_runtime.load_filter(&mut stage.filter).await?; stage.check(&source); + stage.fix(); stages.push(stage); } Ok(Pipeline { diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index 4d51164022a..c668e29fdf9 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -133,6 +133,14 @@ impl Stage { warn!(target: "MAPPING", "Filter with no 'tick' function: {}; but configured with 'tick_every_seconds' in {pipeline}", filter.path.display()); } } + + pub(crate) fn fix(&mut self) { + let filter = &mut self.filter; + if !filter.no_js_tick && filter.tick_every_seconds == 0 { + // 0 as a default is not appropriate for a filter with a tick handler + filter.tick_every_seconds = 1; + } + } } impl DateTime { From d23100e5a45bee6fa99f6cfbf8d731d0522fd2b1 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 11 Jul 2025 19:56:29 +0200 Subject: [PATCH 43/53] fixup! Warn when javascript module and config are not consistent --- crates/extensions/tedge_gen_mapper/src/js_filter.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 4ec69135e1d..07c841e9d68 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -325,7 +325,8 @@ mod tests { async fn error_filter() { let script = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; let mut runtime = JsRuntime::try_new().await.unwrap(); - let filter = JsFilter::new("err.toml".into(), 1, "err.js".into()); + let mut filter = JsFilter::new("err.toml".into(), 1, "err.js".into()); + filter.no_js_process = false; runtime.load_js(filter.module_name(), script).await.unwrap(); let input = Message::new("te/main/device///m/", "hello world"); @@ -361,7 +362,8 @@ export function process (timestamp, message, config) { } "#; let mut runtime = JsRuntime::try_new().await.unwrap(); - let filter = JsFilter::new("collectd.toml".into(), 1, "collectd.js".into()); + let mut filter = JsFilter::new("collectd.toml".into(), 1, "collectd.js".into()); + filter.no_js_process = false; runtime.load_js(filter.module_name(), script).await.unwrap(); let input = Message::new( From c111bcc82f49fda681b7ae138365c6488285ef93 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 11 Jul 2025 20:00:33 +0200 Subject: [PATCH 44/53] collect message transformation stats Signed-off-by: Didier Wenzek --- .../extensions/tedge_gen_mapper/src/actor.rs | 1 + .../tedge_gen_mapper/src/js_filter.rs | 4 + crates/extensions/tedge_gen_mapper/src/lib.rs | 1 + .../tedge_gen_mapper/src/pipeline.rs | 32 ++- .../tedge_gen_mapper/src/runtime.rs | 32 ++- .../extensions/tedge_gen_mapper/src/stats.rs | 203 ++++++++++++++++++ 6 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 crates/extensions/tedge_gen_mapper/src/stats.rs diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 2c2e12a1d7f..a8003ce16e5 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -133,6 +133,7 @@ impl GenMapper { let timestamp = DateTime::now(); if timestamp.seconds % 300 == 0 { self.processor.dump_memory_stats().await; + self.processor.dump_processing_stats().await; } for (pipeline_id, pipeline_messages) in self.processor.tick(×tamp).await { match pipeline_messages { diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index 07c841e9d68..bde96f4e5b2 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -72,6 +72,10 @@ impl JsFilter { &self.path } + pub fn source(&self) -> String { + format!("{}", self.path.display()) + } + /// Process a message returning zero, one or more messages /// /// The "process" function of the JS module is passed 3 arguments diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index f2e22e8a283..a2b9eaed248 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -4,6 +4,7 @@ mod js_filter; mod js_runtime; pub mod pipeline; mod runtime; +mod stats; use crate::actor::GenMapper; pub use crate::runtime::MessageProcessor; diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/pipeline.rs index c668e29fdf9..d9f49354e74 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/pipeline.rs @@ -1,5 +1,6 @@ use crate::js_filter::JsFilter; use crate::js_runtime::JsRuntime; +use crate::stats::Counter; use crate::LoadError; use camino::Utf8Path; use camino::Utf8PathBuf; @@ -76,6 +77,7 @@ impl Pipeline { pub async fn process( &mut self, js_runtime: &JsRuntime, + stats: &mut Counter, timestamp: &DateTime, message: &Message, ) -> Result, FilterError> { @@ -84,38 +86,66 @@ impl Pipeline { return Ok(vec![]); } + let stated_at = stats.pipeline_process_start(self.source.as_str()); let mut messages = vec![message.clone()]; for stage in self.stages.iter() { + let js = stage.filter.source(); let mut transformed_messages = vec![]; for message in messages.iter() { + let filter_started_at = stats.filter_start(&js, "process"); let filter_output = stage.filter.process(js_runtime, timestamp, message).await; + match &filter_output { + Ok(messages) => { + stats.filter_done(&js, "process", filter_started_at, messages.len()) + } + Err(_) => stats.filter_failed(&js, "process"), + } transformed_messages.extend(filter_output?); } messages = transformed_messages; } + + stats.pipeline_process_done(self.source.as_str(), stated_at, messages.len()); Ok(messages) } pub async fn tick( &mut self, js_runtime: &JsRuntime, + stats: &mut Counter, timestamp: &DateTime, ) -> Result, FilterError> { + let stated_at = stats.pipeline_tick_start(self.source.as_str()); let mut messages = vec![]; for stage in self.stages.iter() { + let js = stage.filter.source(); // Process first the messages triggered upstream by the tick let mut transformed_messages = vec![]; for message in messages.iter() { + let filter_started_at = stats.filter_start(&js, "process"); let filter_output = stage.filter.process(js_runtime, timestamp, message).await; + match &filter_output { + Ok(messages) => { + stats.filter_done(&js, "process", filter_started_at, messages.len()) + } + Err(_) => stats.filter_failed(&js, "process"), + } transformed_messages.extend(filter_output?); } // Only then process the tick - transformed_messages.extend(stage.filter.tick(js_runtime, timestamp).await?); + let filter_started_at = stats.filter_start(&js, "tick"); + let tick_output = stage.filter.tick(js_runtime, timestamp).await; + match &tick_output { + Ok(messages) => stats.filter_done(&js, "tick", filter_started_at, messages.len()), + Err(_) => stats.filter_failed(&js, "tick"), + } + transformed_messages.extend(tick_output?); // Iterate with all the messages collected at this stage messages = transformed_messages; } + stats.pipeline_tick_done(self.source.as_str(), stated_at, messages.len()); Ok(messages) } } diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index fc720fa5bf7..8a42c0a6ba6 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -4,6 +4,7 @@ use crate::pipeline::DateTime; use crate::pipeline::FilterError; use crate::pipeline::Message; use crate::pipeline::Pipeline; +use crate::stats::Counter; use crate::LoadError; use camino::Utf8Path; use camino::Utf8PathBuf; @@ -21,6 +22,7 @@ pub struct MessageProcessor { pub config_dir: PathBuf, pub pipelines: HashMap, pub(super) js_runtime: JsRuntime, + pub stats: Counter, } impl MessageProcessor { @@ -34,11 +36,13 @@ impl MessageProcessor { let mut pipeline_specs = PipelineSpecs::default(); pipeline_specs.load(&config_dir).await; let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; + let stats = Counter::default(); Ok(MessageProcessor { config_dir, pipelines, js_runtime, + stats, }) } @@ -52,10 +56,13 @@ impl MessageProcessor { let mut pipeline_specs = PipelineSpecs::default(); pipeline_specs.load_single_pipeline(&pipeline).await; let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; + let stats = Counter::default(); + Ok(MessageProcessor { config_dir, pipelines, js_runtime, + stats, }) } @@ -68,10 +75,13 @@ impl MessageProcessor { let mut pipeline_specs = PipelineSpecs::default(); pipeline_specs.load_single_filter(&filter).await; let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; + let stats = Counter::default(); + Ok(MessageProcessor { config_dir, pipelines, js_runtime, + stats, }) } @@ -88,11 +98,20 @@ impl MessageProcessor { timestamp: &DateTime, message: &Message, ) -> Vec<(String, Result, FilterError>)> { + let started_at = self.stats.runtime_process_start(); + let mut out_messages = vec![]; for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - let pipeline_output = pipeline.process(&self.js_runtime, timestamp, message).await; + let pipeline_output = pipeline + .process(&self.js_runtime, &mut self.stats, timestamp, message) + .await; + if pipeline_output.is_err() { + self.stats.pipeline_process_failed(pipeline_id); + } out_messages.push((pipeline_id.clone(), pipeline_output)); } + + self.stats.runtime_process_done(started_at); out_messages } @@ -102,12 +121,21 @@ impl MessageProcessor { ) -> Vec<(String, Result, FilterError>)> { let mut out_messages = vec![]; for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - let pipeline_output = pipeline.tick(&self.js_runtime, timestamp).await; + let pipeline_output = pipeline + .tick(&self.js_runtime, &mut self.stats, timestamp) + .await; + if pipeline_output.is_err() { + self.stats.pipeline_tick_failed(pipeline_id); + } out_messages.push((pipeline_id.clone(), pipeline_output)); } out_messages } + pub async fn dump_processing_stats(&self) { + self.stats.dump_processing_stats(); + } + pub async fn dump_memory_stats(&self) { self.js_runtime.dump_memory_stats().await; } diff --git a/crates/extensions/tedge_gen_mapper/src/stats.rs b/crates/extensions/tedge_gen_mapper/src/stats.rs new file mode 100644 index 00000000000..1e964e19ceb --- /dev/null +++ b/crates/extensions/tedge_gen_mapper/src/stats.rs @@ -0,0 +1,203 @@ +use std::collections::HashMap; +use std::fmt::Display; +use std::time::Duration; +use std::time::Instant; + +#[derive(Default)] +pub struct Counter { + from_start: HashMap, +} + +#[derive(Clone, Eq, Hash, PartialEq)] +pub enum Dimension { + Runtime, + Pipeline(String), + Process(String), + Tick(String), + Update(String), +} + +pub enum Sample { + MessageIn, + MessageOut(usize), + ErrorRaised, + ProcessingTime(Duration), +} + +#[derive(Default)] +pub struct Stats { + messages_in: usize, + messages_out: usize, + error_raised: usize, + processing_time: Option, +} + +pub struct DurationStats { + min: Duration, + max: Duration, +} + +impl Counter { + pub fn runtime_process_start(&mut self) -> Instant { + self.add(Dimension::Runtime, Sample::MessageIn); + Instant::now() + } + + pub fn runtime_process_done(&mut self, started_at: Instant) { + self.add( + Dimension::Runtime, + Sample::ProcessingTime(started_at.elapsed()), + ); + } + + pub fn pipeline_process_start(&mut self, pipeline_id: &str) -> Instant { + self.add( + Dimension::Pipeline(pipeline_id.to_owned()), + Sample::MessageIn, + ); + Instant::now() + } + + pub fn pipeline_process_done(&mut self, pipeline_id: &str, started_at: Instant, count: usize) { + self.add(Dimension::Runtime, Sample::MessageOut(count)); + self.add( + Dimension::Pipeline(pipeline_id.to_owned()), + Sample::MessageOut(count), + ); + self.add( + Dimension::Pipeline(pipeline_id.to_owned()), + Sample::ProcessingTime(started_at.elapsed()), + ); + } + + pub fn pipeline_process_failed(&mut self, pipeline_id: &str) { + self.add(Dimension::Runtime, Sample::ErrorRaised); + self.add( + Dimension::Pipeline(pipeline_id.to_owned()), + Sample::ErrorRaised, + ); + } + + pub fn pipeline_tick_start(&mut self, _pipeline_id: &str) -> Instant { + Instant::now() + } + + pub fn pipeline_tick_done(&mut self, pipeline_id: &str, _started_at: Instant, count: usize) { + self.add(Dimension::Runtime, Sample::MessageOut(count)); + self.add( + Dimension::Pipeline(pipeline_id.to_owned()), + Sample::MessageOut(count), + ); + } + + pub fn pipeline_tick_failed(&mut self, pipeline_id: &str) { + self.add(Dimension::Runtime, Sample::ErrorRaised); + self.add( + Dimension::Pipeline(pipeline_id.to_owned()), + Sample::ErrorRaised, + ); + } + + pub fn filter_start(&mut self, js: &str, f: &str) -> Instant { + if let Some(dim) = Dimension::function_call(js, f) { + self.add(dim, Sample::MessageIn); + } + Instant::now() + } + + pub fn filter_done(&mut self, js: &str, f: &str, started_at: Instant, count: usize) { + if let Some(dim) = Dimension::function_call(js, f) { + self.add(dim.clone(), Sample::MessageOut(count)); + self.add(dim, Sample::ProcessingTime(started_at.elapsed())); + } + } + + pub fn filter_failed(&mut self, js: &str, f: &str) { + if let Some(dim) = Dimension::function_call(js, f) { + self.add(dim.clone(), Sample::ErrorRaised); + } + } + + fn add(&mut self, dim: Dimension, sample: Sample) { + self.from_start.entry(dim).or_default().add(sample); + } + + pub fn dump_processing_stats(&self) { + tracing::info!(target: "gen-mapper", "Processing statistics:"); + for (dim, stats) in &self.from_start { + stats.dump_statistics(dim) + } + } +} + +impl Stats { + pub fn add(&mut self, sample: Sample) { + match sample { + Sample::MessageIn => { + self.messages_in += 1; + } + Sample::MessageOut(count) => { + self.messages_out += count; + } + Sample::ErrorRaised => { + self.error_raised += 1; + } + Sample::ProcessingTime(t) => match self.processing_time.as_mut() { + None => self.processing_time = Some(DurationStats::new(t)), + Some(stats) => stats.add(t), + }, + } + } + + pub fn dump_statistics(&self, dim: &Dimension) { + tracing::info!(target: "gen-mapper", " - {dim}"); + tracing::info!(target: "gen-mapper", " - input count: {}", self.messages_in); + tracing::info!(target: "gen-mapper", " - output count: {}", self.messages_out); + tracing::info!(target: "gen-mapper", " - error count: {}", self.error_raised); + if let Some(duration_stats) = &self.processing_time { + tracing::info!(target: "gen-mapper", " - min processing time: {:?}", duration_stats.min); + tracing::info!(target: "gen-mapper", " - max processing time: {:?}", duration_stats.max); + } + } +} + +impl DurationStats { + pub fn new(duration: Duration) -> Self { + DurationStats { + min: duration, + max: duration, + } + } + + pub fn add(&mut self, duration: Duration) { + if duration < self.min { + self.min = duration; + } + if self.max < duration { + self.max = duration; + } + } +} + +impl Display for Dimension { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Dimension::Runtime => write!(f, "runtime"), + Dimension::Pipeline(toml) => write!(f, "pipeline {toml}"), + Dimension::Process(js) => write!(f, "process filter {js}"), + Dimension::Tick(js) => write!(f, "tick filter {js}"), + Dimension::Update(js) => write!(f, "update_config filter {js}"), + } + } +} + +impl Dimension { + pub fn function_call(js: &str, f: &str) -> Option { + match f { + "process" => Some(Dimension::Process(js.to_owned())), + "tick" => Some(Dimension::Tick(js.to_owned())), + "update_config" => Some(Dimension::Update(js.to_owned())), + _ => None, + } + } +} From 34a5fe2e8453f1af898c76c39137057890e07008 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 15 Jul 2025 14:07:07 +0200 Subject: [PATCH 45/53] Naming fix: Pipeline -> Flow Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/cli.rs | 29 ++- crates/core/tedge/src/cli/mapping/list.rs | 21 +-- crates/core/tedge/src/cli/mapping/test.rs | 6 +- .../{pipelines => flows}/add_timestamp.js | 0 .../{pipelines => flows}/average.js | 0 .../{pipelines => flows}/circuit-breaker.js | 0 .../{pipelines => flows}/collectd-to-te.js | 0 .../{pipelines => flows}/collectd.toml | 0 .../{pipelines => flows}/drop_stragglers.js | 0 .../group_by_timestamp.js | 0 .../{pipelines => flows}/loop.toml | 2 +- .../{pipelines => flows}/measurements.toml | 0 .../{pipelines => flows}/set_topic.js | 0 .../{pipelines => flows}/te_to_c8y.js | 0 .../extensions/tedge_gen_mapper/src/actor.rs | 26 +-- .../extensions/tedge_gen_mapper/src/config.rs | 16 +- .../src/{pipeline.rs => flow.rs} | 18 +- .../tedge_gen_mapper/src/js_filter.rs | 20 +- crates/extensions/tedge_gen_mapper/src/lib.rs | 2 +- .../tedge_gen_mapper/src/runtime.rs | 172 +++++++++--------- .../extensions/tedge_gen_mapper/src/stats.rs | 37 ++-- docs/src/references/mappers/gen-mapper.md | 44 ++--- .../{pipelines => flows}/add_timestamp.js | 0 .../{pipelines => flows}/average.js | 0 .../{pipelines => flows}/average.samples | 0 .../{pipelines => flows}/count-events.toml | 0 .../count-measurements.toml | 0 .../{pipelines => flows}/count-messages.js | 0 .../count-messages.samples | 2 +- .../{pipelines => flows}/measurements.samples | 0 .../{pipelines => flows}/measurements.toml | 0 .../{pipelines => flows}/set_topic.js | 0 .../{pipelines => flows}/te_to_c8y.js | 0 .../tedge_gen_mapper/tedge_gen_mapper.robot | 2 +- 34 files changed, 187 insertions(+), 210 deletions(-) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/add_timestamp.js (100%) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/average.js (100%) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/circuit-breaker.js (100%) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/collectd-to-te.js (100%) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/collectd.toml (100%) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/drop_stragglers.js (100%) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/group_by_timestamp.js (100%) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/loop.toml (83%) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/measurements.toml (100%) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/set_topic.js (100%) rename crates/extensions/tedge_gen_mapper/{pipelines => flows}/te_to_c8y.js (100%) rename crates/extensions/tedge_gen_mapper/src/{pipeline.rs => flow.rs} (92%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/add_timestamp.js (100%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/average.js (100%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/average.samples (100%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/count-events.toml (100%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/count-measurements.toml (100%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/count-messages.js (100%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/count-messages.samples (90%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/measurements.samples (100%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/measurements.toml (100%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/set_topic.js (100%) rename tests/RobotFramework/tests/tedge_gen_mapper/{pipelines => flows}/te_to_c8y.js (100%) diff --git a/crates/core/tedge/src/cli/mapping/cli.rs b/crates/core/tedge/src/cli/mapping/cli.rs index 04345a1c978..8ff98ef9771 100644 --- a/crates/core/tedge/src/cli/mapping/cli.rs +++ b/crates/core/tedge/src/cli/mapping/cli.rs @@ -8,37 +8,37 @@ use anyhow::Context; use anyhow::Error; use std::path::PathBuf; use tedge_config::TEdgeConfig; -use tedge_gen_mapper::pipeline::Message; +use tedge_gen_mapper::flow::Message; use tedge_gen_mapper::MessageProcessor; #[derive(clap::Subcommand, Debug)] pub enum TEdgeMappingCli { - /// List pipelines and filters + /// List flows and filters List { - /// Path to pipeline and filter specs + /// Path to flow and filter specs /// /// Default to /etc/tedge/gen-mapper #[clap(long)] mapping_dir: Option, - /// List pipelines processing messages published on this topic + /// List flows processing messages published on this topic /// - /// If none is provided, lists all the pipelines + /// If none is provided, lists all the flows #[clap(long)] topic: Option, }, /// Process message samples Test { - /// Path to pipeline and filter specs + /// Path to flow and filter specs /// /// Default to /etc/tedge/gen-mapper #[clap(long)] mapping_dir: Option, - /// Path to the javascript filter or TOML pipeline definition + /// Path to the javascript filter or TOML flow definition /// - /// If none is provided, applies all the matching pipelines + /// If none is provided, applies all the matching flows #[clap(long)] filter: Option, @@ -98,15 +98,10 @@ impl TEdgeMappingCli { config.root_dir().join("gen-mapper").into() } - pub async fn load_pipelines(mapping_dir: &PathBuf) -> Result { + pub async fn load_flows(mapping_dir: &PathBuf) -> Result { MessageProcessor::try_new(mapping_dir) .await - .with_context(|| { - format!( - "loading pipelines and filters from {}", - mapping_dir.display() - ) - }) + .with_context(|| format!("loading flows and filters from {}", mapping_dir.display())) } pub async fn load_filter( @@ -114,9 +109,9 @@ impl TEdgeMappingCli { path: &PathBuf, ) -> Result { if let Some("toml") = path.extension().and_then(|s| s.to_str()) { - MessageProcessor::try_new_single_pipeline(mapping_dir, path) + MessageProcessor::try_new_single_flow(mapping_dir, path) .await - .with_context(|| format!("loading pipeline {pipeline}", pipeline = path.display())) + .with_context(|| format!("loading flow {flow}", flow = path.display())) } else { MessageProcessor::try_new_single_filter(mapping_dir, path) .await diff --git a/crates/core/tedge/src/cli/mapping/list.rs b/crates/core/tedge/src/cli/mapping/list.rs index 8c7577350df..add11c92507 100644 --- a/crates/core/tedge/src/cli/mapping/list.rs +++ b/crates/core/tedge/src/cli/mapping/list.rs @@ -4,7 +4,7 @@ use crate::log::MaybeFancy; use anyhow::Error; use std::path::PathBuf; use tedge_config::TEdgeConfig; -use tedge_gen_mapper::pipeline::Pipeline; +use tedge_gen_mapper::flow::Flow; pub struct ListCommand { pub mapping_dir: PathBuf, @@ -14,23 +14,20 @@ pub struct ListCommand { #[async_trait::async_trait] impl Command for ListCommand { fn description(&self) -> String { - format!( - "list pipelines and filters in {:}", - self.mapping_dir.display() - ) + format!("list flows and filters in {:}", self.mapping_dir.display()) } async fn execute(&self, _config: TEdgeConfig) -> Result<(), MaybeFancy> { - let processor = TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?; + let processor = TEdgeMappingCli::load_flows(&self.mapping_dir).await?; match &self.topic { Some(topic) => processor - .pipelines + .flows .iter() - .filter(|(_, pipeline)| pipeline.topics().accept_topic_name(topic)) + .filter(|(_, flow)| flow.topics().accept_topic_name(topic)) .for_each(Self::display), - None => processor.pipelines.iter().for_each(Self::display), + None => processor.flows.iter().for_each(Self::display), } Ok(()) @@ -38,9 +35,9 @@ impl Command for ListCommand { } impl ListCommand { - fn display((pipeline_id, pipeline): (&String, &Pipeline)) { - println!("{pipeline_id}"); - for stage in pipeline.stages.iter() { + fn display((flow_id, flow): (&String, &Flow)) { + println!("{flow_id}"); + for stage in flow.stages.iter() { println!("\t{}", stage.filter.path.display()); } } diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index 0b6009faef6..6d813164174 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -4,7 +4,7 @@ use crate::log::MaybeFancy; use anyhow::Error; use std::path::PathBuf; use tedge_config::TEdgeConfig; -use tedge_gen_mapper::pipeline::*; +use tedge_gen_mapper::flow::*; use tedge_gen_mapper::MessageProcessor; use tokio::io::AsyncBufReadExt; use tokio::io::BufReader; @@ -21,14 +21,14 @@ pub struct TestCommand { impl Command for TestCommand { fn description(&self) -> String { format!( - "process message samples using pipelines and filters in {:}", + "process message samples using flows and filters in {:}", self.mapping_dir.display() ) } async fn execute(&self, _config: TEdgeConfig) -> Result<(), MaybeFancy> { let mut processor = match &self.filter { - None => TEdgeMappingCli::load_pipelines(&self.mapping_dir).await?, + None => TEdgeMappingCli::load_flows(&self.mapping_dir).await?, Some(filter) => TEdgeMappingCli::load_filter(&self.mapping_dir, filter).await?, }; if let Some(message) = &self.message { diff --git a/crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js b/crates/extensions/tedge_gen_mapper/flows/add_timestamp.js similarity index 100% rename from crates/extensions/tedge_gen_mapper/pipelines/add_timestamp.js rename to crates/extensions/tedge_gen_mapper/flows/add_timestamp.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/average.js b/crates/extensions/tedge_gen_mapper/flows/average.js similarity index 100% rename from crates/extensions/tedge_gen_mapper/pipelines/average.js rename to crates/extensions/tedge_gen_mapper/flows/average.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js b/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js similarity index 100% rename from crates/extensions/tedge_gen_mapper/pipelines/circuit-breaker.js rename to crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js b/crates/extensions/tedge_gen_mapper/flows/collectd-to-te.js similarity index 100% rename from crates/extensions/tedge_gen_mapper/pipelines/collectd-to-te.js rename to crates/extensions/tedge_gen_mapper/flows/collectd-to-te.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/collectd.toml b/crates/extensions/tedge_gen_mapper/flows/collectd.toml similarity index 100% rename from crates/extensions/tedge_gen_mapper/pipelines/collectd.toml rename to crates/extensions/tedge_gen_mapper/flows/collectd.toml diff --git a/crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js b/crates/extensions/tedge_gen_mapper/flows/drop_stragglers.js similarity index 100% rename from crates/extensions/tedge_gen_mapper/pipelines/drop_stragglers.js rename to crates/extensions/tedge_gen_mapper/flows/drop_stragglers.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js b/crates/extensions/tedge_gen_mapper/flows/group_by_timestamp.js similarity index 100% rename from crates/extensions/tedge_gen_mapper/pipelines/group_by_timestamp.js rename to crates/extensions/tedge_gen_mapper/flows/group_by_timestamp.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/loop.toml b/crates/extensions/tedge_gen_mapper/flows/loop.toml similarity index 83% rename from crates/extensions/tedge_gen_mapper/pipelines/loop.toml rename to crates/extensions/tedge_gen_mapper/flows/loop.toml index 64304daf4a3..6cb4e290d71 100644 --- a/crates/extensions/tedge_gen_mapper/pipelines/loop.toml +++ b/crates/extensions/tedge_gen_mapper/flows/loop.toml @@ -1,4 +1,4 @@ -# This pipeline is on purpose looping: the messages are published to the same topic +# This flow is on purpose looping: the messages are published to the same topic input_topics = ["loopback/#"] stages = [ diff --git a/crates/extensions/tedge_gen_mapper/pipelines/measurements.toml b/crates/extensions/tedge_gen_mapper/flows/measurements.toml similarity index 100% rename from crates/extensions/tedge_gen_mapper/pipelines/measurements.toml rename to crates/extensions/tedge_gen_mapper/flows/measurements.toml diff --git a/crates/extensions/tedge_gen_mapper/pipelines/set_topic.js b/crates/extensions/tedge_gen_mapper/flows/set_topic.js similarity index 100% rename from crates/extensions/tedge_gen_mapper/pipelines/set_topic.js rename to crates/extensions/tedge_gen_mapper/flows/set_topic.js diff --git a/crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/flows/te_to_c8y.js similarity index 100% rename from crates/extensions/tedge_gen_mapper/pipelines/te_to_c8y.js rename to crates/extensions/tedge_gen_mapper/flows/te_to_c8y.js diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index a8003ce16e5..605f56d7c97 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -1,5 +1,5 @@ -use crate::pipeline::DateTime; -use crate::pipeline::Message; +use crate::flow::DateTime; +use crate::flow::Message; use crate::runtime::MessageProcessor; use crate::InputMessage; use crate::OutputMessage; @@ -53,7 +53,7 @@ impl Actor for GenMapper { if matches!(path.extension(), Some("js" | "ts")) { self.processor.reload_filter(path).await; } else if path.extension() == Some("toml") { - self.processor.reload_pipeline(path).await; + self.processor.reload_flow(path).await; self.send_updated_subscriptions().await?; } }, @@ -62,7 +62,7 @@ impl Actor for GenMapper { continue; }; if matches!(path.extension(), Some("toml")) { - self.processor.add_pipeline(path).await; + self.processor.add_flow(path).await; self.send_updated_subscriptions().await?; } }, @@ -73,7 +73,7 @@ impl Actor for GenMapper { if matches!(path.extension(), Some("js" | "ts")) { self.processor.remove_filter(path).await; } else if path.extension() == Some("toml") { - self.processor.remove_pipeline(path).await; + self.processor.remove_flow(path).await; self.send_updated_subscriptions().await?; } }, @@ -104,8 +104,8 @@ impl GenMapper { async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); - for (pipeline_id, pipeline_messages) in self.processor.process(×tamp, &message).await { - match pipeline_messages { + for (flow_id, flow_messages) in self.processor.process(×tamp, &message).await { + match flow_messages { Ok(messages) => { for message in messages { match MqttMessage::try_from(message) { @@ -115,13 +115,13 @@ impl GenMapper { .await? } Err(err) => { - error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") + error!(target: "gen-mapper", "{flow_id}: cannot send transformed message: {err}") } } } } Err(err) => { - error!(target: "gen-mapper", "{pipeline_id}: {err}"); + error!(target: "gen-mapper", "{flow_id}: {err}"); } } } @@ -135,8 +135,8 @@ impl GenMapper { self.processor.dump_memory_stats().await; self.processor.dump_processing_stats().await; } - for (pipeline_id, pipeline_messages) in self.processor.tick(×tamp).await { - match pipeline_messages { + for (flow_id, flow_messages) in self.processor.tick(×tamp).await { + match flow_messages { Ok(messages) => { for message in messages { match MqttMessage::try_from(message) { @@ -146,13 +146,13 @@ impl GenMapper { .await? } Err(err) => { - error!(target: "gen-mapper", "{pipeline_id}: cannot send transformed message: {err}") + error!(target: "gen-mapper", "{flow_id}: cannot send transformed message: {err}") } } } } Err(err) => { - error!(target: "gen-mapper", "{pipeline_id}: {err}"); + error!(target: "gen-mapper", "{flow_id}: {err}"); } } } diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index 86d49057fc3..fcc5c865690 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -1,7 +1,7 @@ +use crate::flow::Flow; +use crate::flow::Stage; use crate::js_filter::JsFilter; use crate::js_runtime::JsRuntime; -use crate::pipeline::Pipeline; -use crate::pipeline::Stage; use crate::LoadError; use camino::Utf8Path; use camino::Utf8PathBuf; @@ -12,7 +12,7 @@ use std::path::Path; use tedge_mqtt_ext::TopicFilter; #[derive(Deserialize)] -pub struct PipelineConfig { +pub struct FlowConfig { input_topics: Vec, stages: Vec, } @@ -46,7 +46,7 @@ pub enum ConfigError { LoadError(#[from] LoadError), } -impl PipelineConfig { +impl FlowConfig { pub fn from_filter(filter: Utf8PathBuf) -> Self { let input_topic = "#".to_string(); let stage = StageConfig { @@ -66,7 +66,7 @@ impl PipelineConfig { js_runtime: &mut JsRuntime, config_dir: &Path, source: Utf8PathBuf, - ) -> Result { + ) -> Result { let input_topics = topic_filters(&self.input_topics)?; let mut stages = vec![]; for (i, stage) in self.stages.into_iter().enumerate() { @@ -76,7 +76,7 @@ impl PipelineConfig { stage.fix(); stages.push(stage); } - Ok(Pipeline { + Ok(Flow { input_topics, stages, source, @@ -89,14 +89,14 @@ impl StageConfig { self, config_dir: &Path, index: usize, - pipeline: &Utf8Path, + flow: &Utf8Path, ) -> Result { let path = match self.filter { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), FilterSpec::JavaScript(path) if path.starts_with(config_dir) => path.into(), FilterSpec::JavaScript(path) => config_dir.join(path), }; - let filter = JsFilter::new(pipeline.to_owned().into(), index, path) + let filter = JsFilter::new(flow.to_owned().into(), index, path) .with_config(self.config) .with_tick_every_seconds(self.tick_every_seconds); let config_topics = topic_filters(&self.meta_topics)?; diff --git a/crates/extensions/tedge_gen_mapper/src/pipeline.rs b/crates/extensions/tedge_gen_mapper/src/flow.rs similarity index 92% rename from crates/extensions/tedge_gen_mapper/src/pipeline.rs rename to crates/extensions/tedge_gen_mapper/src/flow.rs index d9f49354e74..feedabcb61a 100644 --- a/crates/extensions/tedge_gen_mapper/src/pipeline.rs +++ b/crates/extensions/tedge_gen_mapper/src/flow.rs @@ -12,7 +12,7 @@ use time::OffsetDateTime; use tracing::warn; /// A chain of transformation of MQTT messages -pub struct Pipeline { +pub struct Flow { /// The source topics pub input_topics: TopicFilter, @@ -52,7 +52,7 @@ pub enum FilterError { Anyhow(#[from] anyhow::Error), } -impl Pipeline { +impl Flow { pub fn topics(&self) -> TopicFilter { let mut topics = self.input_topics.clone(); for stage in self.stages.iter() { @@ -86,7 +86,7 @@ impl Pipeline { return Ok(vec![]); } - let stated_at = stats.pipeline_process_start(self.source.as_str()); + let stated_at = stats.flow_process_start(self.source.as_str()); let mut messages = vec![message.clone()]; for stage in self.stages.iter() { let js = stage.filter.source(); @@ -105,7 +105,7 @@ impl Pipeline { messages = transformed_messages; } - stats.pipeline_process_done(self.source.as_str(), stated_at, messages.len()); + stats.flow_process_done(self.source.as_str(), stated_at, messages.len()); Ok(messages) } @@ -115,7 +115,7 @@ impl Pipeline { stats: &mut Counter, timestamp: &DateTime, ) -> Result, FilterError> { - let stated_at = stats.pipeline_tick_start(self.source.as_str()); + let stated_at = stats.flow_tick_start(self.source.as_str()); let mut messages = vec![]; for stage in self.stages.iter() { let js = stage.filter.source(); @@ -145,22 +145,22 @@ impl Pipeline { // Iterate with all the messages collected at this stage messages = transformed_messages; } - stats.pipeline_tick_done(self.source.as_str(), stated_at, messages.len()); + stats.flow_tick_done(self.source.as_str(), stated_at, messages.len()); Ok(messages) } } impl Stage { - pub(crate) fn check(&self, pipeline: &Utf8Path) { + pub(crate) fn check(&self, flow: &Utf8Path) { let filter = &self.filter; if filter.no_js_process { warn!(target: "MAPPING", "Filter with no 'process' function: {}", filter.path.display()); } if filter.no_js_update_config && !self.config_topics.is_empty() { - warn!(target: "MAPPING", "Filter with no 'config_update' function: {}; but configured with 'config_topics' in {pipeline}", filter.path.display()); + warn!(target: "MAPPING", "Filter with no 'config_update' function: {}; but configured with 'config_topics' in {flow}", filter.path.display()); } if filter.no_js_tick && filter.tick_every_seconds != 0 { - warn!(target: "MAPPING", "Filter with no 'tick' function: {}; but configured with 'tick_every_seconds' in {pipeline}", filter.path.display()); + warn!(target: "MAPPING", "Filter with no 'tick' function: {}; but configured with 'tick_every_seconds' in {flow}", filter.path.display()); } } diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_filter.rs index bde96f4e5b2..6e29aeedf8d 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_filter.rs @@ -1,8 +1,8 @@ +use crate::flow; +use crate::flow::DateTime; +use crate::flow::FilterError; +use crate::flow::Message; use crate::js_runtime::JsRuntime; -use crate::pipeline; -use crate::pipeline::DateTime; -use crate::pipeline::FilterError; -use crate::pipeline::Message; use anyhow::Context; use rquickjs::Ctx; use rquickjs::FromJs; @@ -33,8 +33,8 @@ impl Default for JsonValue { } impl JsFilter { - pub fn new(pipeline: PathBuf, index: usize, path: PathBuf) -> Self { - let module_name = format!("{}|{}|{}", pipeline.display(), index, path.display()); + pub fn new(flow: PathBuf, index: usize, path: PathBuf) -> Self { + let module_name = format!("{}|{}|{}", flow.display(), index, path.display()); JsFilter { module_name, path, @@ -81,7 +81,7 @@ impl JsFilter { /// The "process" function of the JS module is passed 3 arguments /// - the current timestamp /// - the message to be transformed - /// - the filter config (as configured for the pipeline stage, possibly updated by update_config messages) + /// - the filter config (as configured for the flow step, possibly updated by update_config messages) /// /// The returned value is expected to be an array of messages. pub async fn process( @@ -102,7 +102,7 @@ impl JsFilter { ]; js.call_function(&self.module_name(), "process", input) .await - .map_err(pipeline::error_from_js)? + .map_err(flow::error_from_js)? .try_into() } @@ -127,7 +127,7 @@ impl JsFilter { let config = js .call_function(&self.module_name(), "update_config", input) .await - .map_err(pipeline::error_from_js)?; + .map_err(flow::error_from_js)?; self.config = config; Ok(()) } @@ -154,7 +154,7 @@ impl JsFilter { let input = vec![timestamp.clone().into(), self.config.clone()]; js.call_function(&self.module_name(), "tick", input) .await - .map_err(pipeline::error_from_js)? + .map_err(flow::error_from_js)? .try_into() } } diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index a2b9eaed248..7bc891fbb2b 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -1,8 +1,8 @@ mod actor; mod config; +pub mod flow; mod js_filter; mod js_runtime; -pub mod pipeline; mod runtime; mod stats; diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 8a42c0a6ba6..2265376362d 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -1,9 +1,9 @@ -use crate::config::PipelineConfig; +use crate::config::FlowConfig; +use crate::flow::DateTime; +use crate::flow::FilterError; +use crate::flow::Flow; +use crate::flow::Message; use crate::js_runtime::JsRuntime; -use crate::pipeline::DateTime; -use crate::pipeline::FilterError; -use crate::pipeline::Message; -use crate::pipeline::Pipeline; use crate::stats::Counter; use crate::LoadError; use camino::Utf8Path; @@ -20,47 +20,47 @@ use tracing::warn; pub struct MessageProcessor { pub config_dir: PathBuf, - pub pipelines: HashMap, + pub flows: HashMap, pub(super) js_runtime: JsRuntime, pub stats: Counter, } impl MessageProcessor { - pub fn pipeline_id(path: impl AsRef) -> String { + pub fn flow_id(path: impl AsRef) -> String { format!("{}", path.as_ref().display()) } pub async fn try_new(config_dir: impl AsRef) -> Result { let config_dir = config_dir.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; - let mut pipeline_specs = PipelineSpecs::default(); - pipeline_specs.load(&config_dir).await; - let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; + let mut flow_specs = FlowSpecs::default(); + flow_specs.load(&config_dir).await; + let flows = flow_specs.compile(&mut js_runtime, &config_dir).await; let stats = Counter::default(); Ok(MessageProcessor { config_dir, - pipelines, + flows, js_runtime, stats, }) } - pub async fn try_new_single_pipeline( + pub async fn try_new_single_flow( config_dir: impl AsRef, - pipeline: impl AsRef, + flow: impl AsRef, ) -> Result { let config_dir = config_dir.as_ref().to_owned(); - let pipeline = pipeline.as_ref().to_owned(); + let flow = flow.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; - let mut pipeline_specs = PipelineSpecs::default(); - pipeline_specs.load_single_pipeline(&pipeline).await; - let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; + let mut flow_specs = FlowSpecs::default(); + flow_specs.load_single_flow(&flow).await; + let flows = flow_specs.compile(&mut js_runtime, &config_dir).await; let stats = Counter::default(); Ok(MessageProcessor { config_dir, - pipelines, + flows, js_runtime, stats, }) @@ -72,14 +72,14 @@ impl MessageProcessor { ) -> Result { let config_dir = config_dir.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; - let mut pipeline_specs = PipelineSpecs::default(); - pipeline_specs.load_single_filter(&filter).await; - let pipelines = pipeline_specs.compile(&mut js_runtime, &config_dir).await; + let mut flow_specs = FlowSpecs::default(); + flow_specs.load_single_filter(&filter).await; + let flows = flow_specs.compile(&mut js_runtime, &config_dir).await; let stats = Counter::default(); Ok(MessageProcessor { config_dir, - pipelines, + flows, js_runtime, stats, }) @@ -87,8 +87,8 @@ impl MessageProcessor { pub fn subscriptions(&self) -> TopicFilter { let mut topics = TopicFilter::empty(); - for pipeline in self.pipelines.values() { - topics.add_all(pipeline.topics()) + for flow in self.flows.values() { + topics.add_all(flow.topics()) } topics } @@ -101,14 +101,14 @@ impl MessageProcessor { let started_at = self.stats.runtime_process_start(); let mut out_messages = vec![]; - for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - let pipeline_output = pipeline + for (flow_id, flow) in self.flows.iter_mut() { + let flow_output = flow .process(&self.js_runtime, &mut self.stats, timestamp, message) .await; - if pipeline_output.is_err() { - self.stats.pipeline_process_failed(pipeline_id); + if flow_output.is_err() { + self.stats.flow_process_failed(flow_id); } - out_messages.push((pipeline_id.clone(), pipeline_output)); + out_messages.push((flow_id.clone(), flow_output)); } self.stats.runtime_process_done(started_at); @@ -120,14 +120,14 @@ impl MessageProcessor { timestamp: &DateTime, ) -> Vec<(String, Result, FilterError>)> { let mut out_messages = vec![]; - for (pipeline_id, pipeline) in self.pipelines.iter_mut() { - let pipeline_output = pipeline + for (flow_id, flow) in self.flows.iter_mut() { + let flow_output = flow .tick(&self.js_runtime, &mut self.stats, timestamp) .await; - if pipeline_output.is_err() { - self.stats.pipeline_tick_failed(pipeline_id); + if flow_output.is_err() { + self.stats.flow_tick_failed(flow_id); } - out_messages.push((pipeline_id.clone(), pipeline_output)); + out_messages.push((flow_id.clone(), flow_output)); } out_messages } @@ -141,8 +141,8 @@ impl MessageProcessor { } pub async fn reload_filter(&mut self, path: Utf8PathBuf) { - for pipeline in self.pipelines.values_mut() { - for stage in &mut pipeline.stages { + for flow in self.flows.values_mut() { + for stage in &mut flow.stages { if stage.filter.path() == path { match self.js_runtime.load_filter(&mut stage.filter).await { Ok(()) => { @@ -159,25 +159,25 @@ impl MessageProcessor { } pub async fn remove_filter(&mut self, path: Utf8PathBuf) { - for (pipeline_id, pipeline) in self.pipelines.iter() { - for stage in pipeline.stages.iter() { + for (flow_id, flow) in self.flows.iter() { + for stage in flow.stages.iter() { if stage.filter.path() == path { - warn!(target: "gen-mapper", "Removing a filter used by {pipeline_id}: {path}"); + warn!(target: "gen-mapper", "Removing a filter used by {flow_id}: {path}"); return; } } } } - pub async fn load_pipeline(&mut self, pipeline_id: String, path: Utf8PathBuf) -> bool { + pub async fn load_flow(&mut self, flow_id: String, path: Utf8PathBuf) -> bool { let Ok(source) = tokio::fs::read_to_string(&path).await else { - self.remove_pipeline(path).await; + self.remove_flow(path).await; return false; }; - let config: PipelineConfig = match toml::from_str(&source) { + let config: FlowConfig = match toml::from_str(&source) { Ok(config) => config, Err(e) => { - error!(target: "gen-mapper", "Failed to parse toml for pipeline {path}: {e}"); + error!(target: "gen-mapper", "Failed to parse toml for flow {path}: {e}"); return false; } }; @@ -185,48 +185,44 @@ impl MessageProcessor { .compile(&mut self.js_runtime, &self.config_dir, path.clone()) .await { - Ok(pipeline) => { - self.pipelines.insert(pipeline_id, pipeline); + Ok(flow) => { + self.flows.insert(flow_id, flow); true } Err(e) => { - error!(target: "gen-mapper", "Failed to compile pipeline {path}: {e}"); + error!(target: "gen-mapper", "Failed to compile flow {path}: {e}"); false } } } - pub async fn add_pipeline(&mut self, path: Utf8PathBuf) { - let pipeline_id = Self::pipeline_id(&path); - if !self.pipelines.contains_key(&pipeline_id) - && self.load_pipeline(pipeline_id, path.clone()).await - { - info!(target: "gen-mapper", "Loaded new pipeline {path}"); + pub async fn add_flow(&mut self, path: Utf8PathBuf) { + let flow_id = Self::flow_id(&path); + if !self.flows.contains_key(&flow_id) && self.load_flow(flow_id, path.clone()).await { + info!(target: "gen-mapper", "Loaded new flow {path}"); } } - pub async fn reload_pipeline(&mut self, path: Utf8PathBuf) { - let pipeline_id = Self::pipeline_id(&path); - if self.pipelines.contains_key(&pipeline_id) - && self.load_pipeline(pipeline_id, path.clone()).await - { - info!(target: "gen-mapper", "Reloaded updated pipeline {path}"); + pub async fn reload_flow(&mut self, path: Utf8PathBuf) { + let flow_id = Self::flow_id(&path); + if self.flows.contains_key(&flow_id) && self.load_flow(flow_id, path.clone()).await { + info!(target: "gen-mapper", "Reloaded updated flow {path}"); } } - pub async fn remove_pipeline(&mut self, path: Utf8PathBuf) { - let pipeline_id = Self::pipeline_id(&path); - self.pipelines.remove(&pipeline_id); - info!(target: "gen-mapper", "Removed deleted pipeline {path}"); + pub async fn remove_flow(&mut self, path: Utf8PathBuf) { + let flow_id = Self::flow_id(&path); + self.flows.remove(&flow_id); + info!(target: "gen-mapper", "Removed deleted flow {path}"); } } #[derive(Default)] -struct PipelineSpecs { - pipeline_specs: HashMap, +struct FlowSpecs { + flow_specs: HashMap, } -impl PipelineSpecs { +impl FlowSpecs { pub async fn load(&mut self, config_dir: &PathBuf) { let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) @@ -242,9 +238,9 @@ impl PipelineSpecs { if let Ok(file_type) = entry.file_type().await { if file_type.is_file() { if let Some("toml") = path.extension() { - info!(target: "MAPPING", "Loading pipeline: {path}"); - if let Err(err) = self.load_pipeline(path).await { - error!(target: "MAPPING", "Failed to load pipeline: {err}"); + info!(target: "MAPPING", "Loading flow: {path}"); + if let Err(err) = self.load_flow(path).await { + error!(target: "MAPPING", "Failed to load flow: {err}"); } } } @@ -252,13 +248,13 @@ impl PipelineSpecs { } } - pub async fn load_single_pipeline(&mut self, pipeline: &Path) { - let Some(path) = Utf8Path::from_path(pipeline).map(|p| p.to_path_buf()) else { - error!(target: "MAPPING", "Skipping non UTF8 path: {}", pipeline.display()); + pub async fn load_single_flow(&mut self, flow: &Path) { + let Some(path) = Utf8Path::from_path(flow).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", flow.display()); return; }; - if let Err(err) = self.load_pipeline(&path).await { - error!(target: "MAPPING", "Failed to load pipeline {path}: {err}"); + if let Err(err) = self.load_flow(&path).await { + error!(target: "MAPPING", "Failed to load flow {path}: {err}"); } } @@ -268,19 +264,17 @@ impl PipelineSpecs { error!(target: "MAPPING", "Skipping non UTF8 path: {}", filter.display()); return; }; - let pipeline_id = MessageProcessor::pipeline_id(&path); - let pipeline = PipelineConfig::from_filter(path.to_owned()); - self.pipeline_specs - .insert(pipeline_id, (path.to_owned(), pipeline)); + let flow_id = MessageProcessor::flow_id(&path); + let flow = FlowConfig::from_filter(path.to_owned()); + self.flow_specs.insert(flow_id, (path.to_owned(), flow)); } - async fn load_pipeline(&mut self, file: impl AsRef) -> Result<(), LoadError> { + async fn load_flow(&mut self, file: impl AsRef) -> Result<(), LoadError> { let path = file.as_ref(); - let pipeline_id = MessageProcessor::pipeline_id(path); + let flow_id = MessageProcessor::flow_id(path); let specs = read_to_string(path).await?; - let pipeline: PipelineConfig = toml::from_str(&specs)?; - self.pipeline_specs - .insert(pipeline_id, (path.to_owned(), pipeline)); + let flow: FlowConfig = toml::from_str(&specs)?; + self.flow_specs.insert(flow_id, (path.to_owned(), flow)); Ok(()) } @@ -289,18 +283,18 @@ impl PipelineSpecs { mut self, js_runtime: &mut JsRuntime, config_dir: &Path, - ) -> HashMap { - let mut pipelines = HashMap::new(); - for (name, (source, specs)) in self.pipeline_specs.drain() { + ) -> HashMap { + let mut flows = HashMap::new(); + for (name, (source, specs)) in self.flow_specs.drain() { match specs.compile(js_runtime, config_dir, source).await { - Ok(pipeline) => { - let _ = pipelines.insert(name, pipeline); + Ok(flow) => { + let _ = flows.insert(name, flow); } Err(err) => { - error!(target: "MAPPING", "Failed to compile pipeline {name}: {err}") + error!(target: "MAPPING", "Failed to compile flow {name}: {err}") } } } - pipelines + flows } } diff --git a/crates/extensions/tedge_gen_mapper/src/stats.rs b/crates/extensions/tedge_gen_mapper/src/stats.rs index 1e964e19ceb..bcd0182f2a1 100644 --- a/crates/extensions/tedge_gen_mapper/src/stats.rs +++ b/crates/extensions/tedge_gen_mapper/src/stats.rs @@ -11,7 +11,7 @@ pub struct Counter { #[derive(Clone, Eq, Hash, PartialEq)] pub enum Dimension { Runtime, - Pipeline(String), + Flow(String), Process(String), Tick(String), Update(String), @@ -50,52 +50,43 @@ impl Counter { ); } - pub fn pipeline_process_start(&mut self, pipeline_id: &str) -> Instant { - self.add( - Dimension::Pipeline(pipeline_id.to_owned()), - Sample::MessageIn, - ); + pub fn flow_process_start(&mut self, flow_id: &str) -> Instant { + self.add(Dimension::Flow(flow_id.to_owned()), Sample::MessageIn); Instant::now() } - pub fn pipeline_process_done(&mut self, pipeline_id: &str, started_at: Instant, count: usize) { + pub fn flow_process_done(&mut self, flow_id: &str, started_at: Instant, count: usize) { self.add(Dimension::Runtime, Sample::MessageOut(count)); self.add( - Dimension::Pipeline(pipeline_id.to_owned()), + Dimension::Flow(flow_id.to_owned()), Sample::MessageOut(count), ); self.add( - Dimension::Pipeline(pipeline_id.to_owned()), + Dimension::Flow(flow_id.to_owned()), Sample::ProcessingTime(started_at.elapsed()), ); } - pub fn pipeline_process_failed(&mut self, pipeline_id: &str) { + pub fn flow_process_failed(&mut self, flow_id: &str) { self.add(Dimension::Runtime, Sample::ErrorRaised); - self.add( - Dimension::Pipeline(pipeline_id.to_owned()), - Sample::ErrorRaised, - ); + self.add(Dimension::Flow(flow_id.to_owned()), Sample::ErrorRaised); } - pub fn pipeline_tick_start(&mut self, _pipeline_id: &str) -> Instant { + pub fn flow_tick_start(&mut self, _flow_id: &str) -> Instant { Instant::now() } - pub fn pipeline_tick_done(&mut self, pipeline_id: &str, _started_at: Instant, count: usize) { + pub fn flow_tick_done(&mut self, flow_id: &str, _started_at: Instant, count: usize) { self.add(Dimension::Runtime, Sample::MessageOut(count)); self.add( - Dimension::Pipeline(pipeline_id.to_owned()), + Dimension::Flow(flow_id.to_owned()), Sample::MessageOut(count), ); } - pub fn pipeline_tick_failed(&mut self, pipeline_id: &str) { + pub fn flow_tick_failed(&mut self, flow_id: &str) { self.add(Dimension::Runtime, Sample::ErrorRaised); - self.add( - Dimension::Pipeline(pipeline_id.to_owned()), - Sample::ErrorRaised, - ); + self.add(Dimension::Flow(flow_id.to_owned()), Sample::ErrorRaised); } pub fn filter_start(&mut self, js: &str, f: &str) -> Instant { @@ -183,7 +174,7 @@ impl Display for Dimension { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Dimension::Runtime => write!(f, "runtime"), - Dimension::Pipeline(toml) => write!(f, "pipeline {toml}"), + Dimension::Flow(toml) => write!(f, "flow {toml}"), Dimension::Process(js) => write!(f, "process filter {js}"), Dimension::Tick(js) => write!(f, "tick filter {js}"), Dimension::Update(js) => write!(f, "update_config filter {js}"), diff --git a/docs/src/references/mappers/gen-mapper.md b/docs/src/references/mappers/gen-mapper.md index 2696ddf73e5..99d2239e32c 100644 --- a/docs/src/references/mappers/gen-mapper.md +++ b/docs/src/references/mappers/gen-mapper.md @@ -28,11 +28,11 @@ leveraging the core mapping rules and mapper mechanisms (bridge connections, HTT The %%te%% mappers for Cumulocity, Azure, AWS and Collectd are implemented on top of a so-called generic mapper which is used to drive all MQTT message transformations. -- Transformations are implemented as pipelines consuming MQTT messages, feeding a chain of filters and producing MQTT messages. +- Transformations are implemented by flows which consume MQTT messages, apply a sequence of transformation steps and produce MQTT messages. - `MQTT sub| filter-1 | filter-2 | ... | filter-n | MQTT pub` -- A pipeline can combine builtin and user-provided filters. +- A flow can combine builtin and user-provided filters. - The user can configure all the transformations used by a mapper, - editing MQTT sources, pipelines, filters and MQTT sinks. + editing MQTT sources, flows, filters and MQTT sinks. - By contrast with the current implementation, where the translation of measurements from %%te%% JSON to Cumulocity JSON is fully hard-coded, with the generic mapper a user can re-use the core of this transformation while adding customized steps: - consuming measurement from a non-standard topic @@ -43,12 +43,12 @@ which is used to drive all MQTT message transformations. ## POC reference -- The generic mapper loads pipeline and filters stored in `/etc/tedge/gen-mapper/`. -- A pipeline is defined by a TOML file with `.toml` extension. +- The generic mapper loads flows and filters stored in `/etc/tedge/gen-mapper/`. +- A flow is defined by a TOML file with `.toml` extension. - A filter is defined by a Javascript file with `.js` extension. -- The definition of pipeline must provide a list of MQTT topics to subscribe to. - - The pipeline will be feed with all the messages received on these topics. -- A pipeline definition also provides a list of stages. +- The definition of flows must provide a list of MQTT topics to subscribe to. + - The flow will be feed with all the messages received on these topics. +- A flow definition also provides a list of stages. - Each stage is built from a javascript and is possibly given a config (arbitrary json that will be passed to the script) - Each stage can also subscribe to a list of MQTT topics (which messages will be passed to the script to update its config) @@ -68,7 +68,7 @@ stages = [ - The arguments passed to the function are: - The current time as `{ seconds: u64, nanoseconds: u32 }` - The message `{ topic: string, payload: string }` - - The config as read from the pipeline config or updated by the script + - The config as read from the flow config or updated by the script - The function is expected to return zero, one or many transformed messages `[{ topic: string, payload: string }]` - An exception can be thrown if the input message cannot be transformed. - A filter can also export an `update_config` function @@ -95,19 +95,19 @@ To be lovable, the first release of an extensible mapper should at least: - be a drop-in replacement of the current mapper (for c8y, aws, az or collect) - feature the ability to customize MEA processing by combining builtin filters with user-provided functions written in JavaScript -- provide tools to create, test, monitor and debug filters and pipelines +- provide tools to create, test, monitor and debug filters and flows - be stable enough that user-defined filters will still work without changes with future releases. To keep things simple for the first release, the following questions are deferred: -- Could a generic mapper let users define bridge rules as well as message transformation pipelines? +- Could a generic mapper let users define bridge rules as well as message transformation flows? - Does it make sense to run such a mapper on child-devices? -- Could a pipeline send HTTP messages? Or could a filter tell the runtime to send messages over HTTP? +- Could a flow send HTTP messages? Or could a filter tell the runtime to send messages over HTTP? - How to handle binary payloads on the MQTT bus? - Could operations be managed is a similar way with user-provided functions to transform commands? - To handle operations, would the plugins be expanded to do more complex things like HTTP calls, file-system interactions, etc.? - What are the pros and cons to persist filter states? -- Split a pipeline, forwarding transformed messages to different pipelines for further processing +- Split a flow, forwarding transformed messages to different flows for further processing ### API @@ -142,27 +142,27 @@ One can even use a bit further the flexibility of javascript, to let the process Other ideas to explore to make the API more flexible: - Interaction with the entity store and tedge config. -- Allow a pipeline to subscribe to topics related to the device/entity it is running on +- Allow a flow to subscribe to topics related to the device/entity it is running on - Feed filters with message excerpts as done for the workflows ### Devops tools The flexibility to customize MQTT message processing with user-provided functions comes with risks: - a filter might not behave as expected, -- pipelines might be overlapping or conflicting, possibly sending duplicate messages or creating infinite loops -- builtin pipelines might be accidentally disconnected or broken +- flows might be overlapping or conflicting, possibly sending duplicate messages or creating infinite loops +- builtin flows might be accidentally disconnected or broken - a filter might introduce a performance bottleneck. -To help mitigating these risks, the `tedge mapping` sub-commands provide the tools to test, monitor and debug filters and pipelines. +To help mitigating these risks, the `tedge mapping` sub-commands provide the tools to test, monitor and debug filters and flows. -- `tedge mapping flow [topic]` displays pipelines and filters messages received on this topic will flow through - - can be used with a set of pipelines not configured yet for a mapper -- `tedge mapping test [filter]` feeds a filter or pipeline with input messages and produces the transformed output messages +- `tedge mapping flow [topic]` displays flows and filters messages received on this topic will flow through + - can be used with a set of flows not configured yet for a mapper +- `tedge mapping test [filter]` feeds a filter or flow with input messages and produces the transformed output messages - allow users to run an assertion based on the input/output of a filter - ability to pipe `tedge mqtt sub` and `tedge mapping test` - control of the timestamps - test aggregation over ticks - - can be used with a set of pipelines not configured yet for a mapper -- `tedge mapping stats [pipeline]` returns statistics on the messages processed by a pipeline + - can be used with a set of flows not configured yet for a mapper +- `tedge mapping stats [flow]` returns statistics on the messages processed by a flow - count message in, message out - processing time min, median, max per filter diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/add_timestamp.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/add_timestamp.js similarity index 100% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/add_timestamp.js rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/add_timestamp.js diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/average.js similarity index 100% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.js rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/average.js diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples b/tests/RobotFramework/tests/tedge_gen_mapper/flows/average.samples similarity index 100% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/average.samples rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/average.samples diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-events.toml b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-events.toml similarity index 100% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-events.toml rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/count-events.toml diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-measurements.toml similarity index 100% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-measurements.toml rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/count-measurements.toml diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.js similarity index 100% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.js rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.js diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.samples similarity index 90% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.samples index 180e028e3ae..70b27cc48a0 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/count-messages.samples +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.samples @@ -13,7 +13,7 @@ INPUT: [test/device/main///e/] "some event" INPUT: [test/device/child1///e/] "some event" INPUT: [test/device/child2///e/] "some event" -# Since we have two pipelines using the same javascript filter, one expect two output messages +# Since we have two flows using the same javascript transformation, one expect two output messages # A first one for all the measurements, another one for all the events OUTPUT: [test/count/m] {"test/device/main///m/":4,"test/device/child1///m/":3,"test/device/child2///m/":2} OUTPUT: [test/count/e] {"test/device/main///e/":2,"test/device/child2///e/":2,"test/device/child1///e/":1} diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.samples b/tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.samples similarity index 100% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.samples rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.samples diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.toml similarity index 100% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/measurements.toml rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.toml diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/set_topic.js similarity index 100% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/set_topic.js rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/set_topic.js diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/te_to_c8y.js similarity index 100% rename from tests/RobotFramework/tests/tedge_gen_mapper/pipelines/te_to_c8y.js rename to tests/RobotFramework/tests/tedge_gen_mapper/flows/te_to_c8y.js diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index 5ec963cb7ee..cc5ca8a8806 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -79,5 +79,5 @@ Custom Setup Copy Configuration Files Execute Command mkdir /etc/tedge/gen-mapper/ - ThinEdgeIO.Transfer To Device ${CURDIR}/pipelines/* /etc/tedge/gen-mapper/ + ThinEdgeIO.Transfer To Device ${CURDIR}/flows/* /etc/tedge/gen-mapper/ From ec21ead0d27ff3335fdb5a110c49e68c84e1d374 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 15 Jul 2025 14:30:13 +0200 Subject: [PATCH 46/53] Naming fix: Stage -> Step Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/list.rs | 9 +++-- .../tedge_gen_mapper/flows/collectd.toml | 2 +- .../tedge_gen_mapper/flows/loop.toml | 2 +- .../tedge_gen_mapper/flows/measurements.toml | 2 +- .../extensions/tedge_gen_mapper/src/config.rs | 32 ++++++++--------- .../extensions/tedge_gen_mapper/src/flow.rs | 36 +++++++++---------- .../tedge_gen_mapper/src/runtime.rs | 10 +++--- docs/src/references/mappers/gen-mapper.md | 8 ++--- .../tedge_gen_mapper/flows/count-events.toml | 2 +- .../flows/count-measurements.toml | 2 +- .../tedge_gen_mapper/flows/measurements.toml | 2 +- 11 files changed, 55 insertions(+), 52 deletions(-) diff --git a/crates/core/tedge/src/cli/mapping/list.rs b/crates/core/tedge/src/cli/mapping/list.rs index add11c92507..e68454c6216 100644 --- a/crates/core/tedge/src/cli/mapping/list.rs +++ b/crates/core/tedge/src/cli/mapping/list.rs @@ -14,7 +14,10 @@ pub struct ListCommand { #[async_trait::async_trait] impl Command for ListCommand { fn description(&self) -> String { - format!("list flows and filters in {:}", self.mapping_dir.display()) + format!( + "list flows and flow steps in {:}", + self.mapping_dir.display() + ) } async fn execute(&self, _config: TEdgeConfig) -> Result<(), MaybeFancy> { @@ -37,8 +40,8 @@ impl Command for ListCommand { impl ListCommand { fn display((flow_id, flow): (&String, &Flow)) { println!("{flow_id}"); - for stage in flow.stages.iter() { - println!("\t{}", stage.filter.path.display()); + for step in flow.steps.iter() { + println!("\t{}", step.filter.path.display()); } } } diff --git a/crates/extensions/tedge_gen_mapper/flows/collectd.toml b/crates/extensions/tedge_gen_mapper/flows/collectd.toml index 05330b870f8..a744d2d7e29 100644 --- a/crates/extensions/tedge_gen_mapper/flows/collectd.toml +++ b/crates/extensions/tedge_gen_mapper/flows/collectd.toml @@ -1,6 +1,6 @@ input_topics = ["collectd/+/+/+"] -stages = [ +steps = [ { filter = "collectd-to-te.js" }, { filter = "average.js", tick_every_seconds = 10 } ] diff --git a/crates/extensions/tedge_gen_mapper/flows/loop.toml b/crates/extensions/tedge_gen_mapper/flows/loop.toml index 6cb4e290d71..87b1d9ff85d 100644 --- a/crates/extensions/tedge_gen_mapper/flows/loop.toml +++ b/crates/extensions/tedge_gen_mapper/flows/loop.toml @@ -1,7 +1,7 @@ # This flow is on purpose looping: the messages are published to the same topic input_topics = ["loopback/#"] -stages = [ +steps = [ { filter = "add_timestamp.js" }, { filter = "circuit-breaker.js", tick_every_seconds = 1, config = { stats_topic = "te/error", too_many = 10000, message_on_too_many = { topic = "te/device/main///a/too-many-messages", payload = "too many messages" }, message_on_back_to_normal = { topic = "te/device/main///a/too-many-messages", payload = "back to normal" } } } ] diff --git a/crates/extensions/tedge_gen_mapper/flows/measurements.toml b/crates/extensions/tedge_gen_mapper/flows/measurements.toml index cd339bb7556..2980650efc1 100644 --- a/crates/extensions/tedge_gen_mapper/flows/measurements.toml +++ b/crates/extensions/tedge_gen_mapper/flows/measurements.toml @@ -1,6 +1,6 @@ input_topics = ["te/+/+/+/+/m/+"] -stages = [ +steps = [ { filter = "add_timestamp.js" }, { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } ] diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index fcc5c865690..01ea61c57c7 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -1,5 +1,5 @@ use crate::flow::Flow; -use crate::flow::Stage; +use crate::flow::FlowStep; use crate::js_filter::JsFilter; use crate::js_runtime::JsRuntime; use crate::LoadError; @@ -14,11 +14,11 @@ use tedge_mqtt_ext::TopicFilter; #[derive(Deserialize)] pub struct FlowConfig { input_topics: Vec, - stages: Vec, + steps: Vec, } #[derive(Deserialize)] -pub struct StageConfig { +pub struct StepConfig { filter: FilterSpec, #[serde(default)] @@ -49,7 +49,7 @@ pub enum ConfigError { impl FlowConfig { pub fn from_filter(filter: Utf8PathBuf) -> Self { let input_topic = "#".to_string(); - let stage = StageConfig { + let step = StepConfig { filter: FilterSpec::JavaScript(filter), config: None, tick_every_seconds: 0, @@ -57,7 +57,7 @@ impl FlowConfig { }; Self { input_topics: vec![input_topic], - stages: vec![stage], + steps: vec![step], } } @@ -68,29 +68,29 @@ impl FlowConfig { source: Utf8PathBuf, ) -> Result { let input_topics = topic_filters(&self.input_topics)?; - let mut stages = vec![]; - for (i, stage) in self.stages.into_iter().enumerate() { - let mut stage = stage.compile(config_dir, i, &source).await?; - js_runtime.load_filter(&mut stage.filter).await?; - stage.check(&source); - stage.fix(); - stages.push(stage); + let mut steps = vec![]; + for (i, step) in self.steps.into_iter().enumerate() { + let mut step = step.compile(config_dir, i, &source).await?; + js_runtime.load_filter(&mut step.filter).await?; + step.check(&source); + step.fix(); + steps.push(step); } Ok(Flow { input_topics, - stages, + steps, source, }) } } -impl StageConfig { +impl StepConfig { pub async fn compile( self, config_dir: &Path, index: usize, flow: &Utf8Path, - ) -> Result { + ) -> Result { let path = match self.filter { FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), FilterSpec::JavaScript(path) if path.starts_with(config_dir) => path.into(), @@ -100,7 +100,7 @@ impl StageConfig { .with_config(self.config) .with_tick_every_seconds(self.tick_every_seconds); let config_topics = topic_filters(&self.meta_topics)?; - Ok(Stage { + Ok(FlowStep { filter, config_topics, }) diff --git a/crates/extensions/tedge_gen_mapper/src/flow.rs b/crates/extensions/tedge_gen_mapper/src/flow.rs index feedabcb61a..8e993173183 100644 --- a/crates/extensions/tedge_gen_mapper/src/flow.rs +++ b/crates/extensions/tedge_gen_mapper/src/flow.rs @@ -16,14 +16,14 @@ pub struct Flow { /// The source topics pub input_topics: TopicFilter, - /// Transformation stages to apply in order to the messages - pub stages: Vec, + /// Transformation steps to apply in order to the messages + pub steps: Vec, pub source: Utf8PathBuf, } -/// A message transformation stage -pub struct Stage { +/// A message transformation step +pub struct FlowStep { pub filter: JsFilter, pub config_topics: TopicFilter, } @@ -55,8 +55,8 @@ pub enum FilterError { impl Flow { pub fn topics(&self) -> TopicFilter { let mut topics = self.input_topics.clone(); - for stage in self.stages.iter() { - topics.add_all(stage.config_topics.clone()) + for step in self.steps.iter() { + topics.add_all(step.config_topics.clone()) } topics } @@ -66,9 +66,9 @@ impl Flow { js_runtime: &JsRuntime, message: &Message, ) -> Result<(), FilterError> { - for stage in self.stages.iter_mut() { - if stage.config_topics.accept_topic_name(&message.topic) { - stage.filter.update_config(js_runtime, message).await? + for step in self.steps.iter_mut() { + if step.config_topics.accept_topic_name(&message.topic) { + step.filter.update_config(js_runtime, message).await? } } Ok(()) @@ -88,12 +88,12 @@ impl Flow { let stated_at = stats.flow_process_start(self.source.as_str()); let mut messages = vec![message.clone()]; - for stage in self.stages.iter() { - let js = stage.filter.source(); + for step in self.steps.iter() { + let js = step.filter.source(); let mut transformed_messages = vec![]; for message in messages.iter() { let filter_started_at = stats.filter_start(&js, "process"); - let filter_output = stage.filter.process(js_runtime, timestamp, message).await; + let filter_output = step.filter.process(js_runtime, timestamp, message).await; match &filter_output { Ok(messages) => { stats.filter_done(&js, "process", filter_started_at, messages.len()) @@ -117,13 +117,13 @@ impl Flow { ) -> Result, FilterError> { let stated_at = stats.flow_tick_start(self.source.as_str()); let mut messages = vec![]; - for stage in self.stages.iter() { - let js = stage.filter.source(); + for step in self.steps.iter() { + let js = step.filter.source(); // Process first the messages triggered upstream by the tick let mut transformed_messages = vec![]; for message in messages.iter() { let filter_started_at = stats.filter_start(&js, "process"); - let filter_output = stage.filter.process(js_runtime, timestamp, message).await; + let filter_output = step.filter.process(js_runtime, timestamp, message).await; match &filter_output { Ok(messages) => { stats.filter_done(&js, "process", filter_started_at, messages.len()) @@ -135,14 +135,14 @@ impl Flow { // Only then process the tick let filter_started_at = stats.filter_start(&js, "tick"); - let tick_output = stage.filter.tick(js_runtime, timestamp).await; + let tick_output = step.filter.tick(js_runtime, timestamp).await; match &tick_output { Ok(messages) => stats.filter_done(&js, "tick", filter_started_at, messages.len()), Err(_) => stats.filter_failed(&js, "tick"), } transformed_messages.extend(tick_output?); - // Iterate with all the messages collected at this stage + // Iterate with all the messages collected at this step messages = transformed_messages; } stats.flow_tick_done(self.source.as_str(), stated_at, messages.len()); @@ -150,7 +150,7 @@ impl Flow { } } -impl Stage { +impl FlowStep { pub(crate) fn check(&self, flow: &Utf8Path) { let filter = &self.filter; if filter.no_js_process { diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 2265376362d..9a632c8a130 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -142,9 +142,9 @@ impl MessageProcessor { pub async fn reload_filter(&mut self, path: Utf8PathBuf) { for flow in self.flows.values_mut() { - for stage in &mut flow.stages { - if stage.filter.path() == path { - match self.js_runtime.load_filter(&mut stage.filter).await { + for step in &mut flow.steps { + if step.filter.path() == path { + match self.js_runtime.load_filter(&mut step.filter).await { Ok(()) => { info!(target: "gen-mapper", "Reloaded filter {path}"); } @@ -160,8 +160,8 @@ impl MessageProcessor { pub async fn remove_filter(&mut self, path: Utf8PathBuf) { for (flow_id, flow) in self.flows.iter() { - for stage in flow.stages.iter() { - if stage.filter.path() == path { + for step in flow.steps.iter() { + if step.filter.path() == path { warn!(target: "gen-mapper", "Removing a filter used by {flow_id}: {path}"); return; } diff --git a/docs/src/references/mappers/gen-mapper.md b/docs/src/references/mappers/gen-mapper.md index 99d2239e32c..347021997c5 100644 --- a/docs/src/references/mappers/gen-mapper.md +++ b/docs/src/references/mappers/gen-mapper.md @@ -48,14 +48,14 @@ which is used to drive all MQTT message transformations. - A filter is defined by a Javascript file with `.js` extension. - The definition of flows must provide a list of MQTT topics to subscribe to. - The flow will be feed with all the messages received on these topics. -- A flow definition also provides a list of stages. - - Each stage is built from a javascript and is possibly given a config (arbitrary json that will be passed to the script) - - Each stage can also subscribe to a list of MQTT topics (which messages will be passed to the script to update its config) +- A flow definition provides a list of steps. + - Each step is built from a javascript and is possibly given a config (arbitrary json that will be passed to the script) + - Each step can also subscribe to a list of MQTT topics (which messages will be passed to the script to update its config) ```toml input_topics = ["te/+/+/+/+/m/+"] -stages = [ +steps = [ { filter = "add_timestamp.js" }, { filter = "drop_stragglers.js", config = { max_delay = 60 } }, { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-events.toml b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-events.toml index 9dcc8e32f2f..27d36587559 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-events.toml +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-events.toml @@ -1,5 +1,5 @@ input_topics = ["test/+/+/+/+/e/+"] -stages = [ +steps = [ { filter = "count-messages.js", config = { topic = "test/count/e" }, tick_every_seconds = 1 }, ] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-measurements.toml index 81895dde972..358e8a9e9e1 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-measurements.toml +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-measurements.toml @@ -1,5 +1,5 @@ input_topics = ["test/+/+/+/+/m/+"] -stages = [ +steps = [ { filter = "count-messages.js", config = { topic = "test/count/m" }, tick_every_seconds = 1 }, ] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.toml index 3262580a673..dec7044ddc5 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.toml +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.toml @@ -1,6 +1,6 @@ input_topics = ["te/+/+/+/+/m/+"] -stages = [ +steps = [ { filter = "add_timestamp.js" }, { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] }, ] From 208986a5708825497ac657919af4048ee1a4c8c3 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Tue, 15 Jul 2025 15:58:06 +0200 Subject: [PATCH 47/53] Naming fix: Filter -> Flow Step In some context `Flow script` is more appropriate. For instance, when two steps use the same script. Signed-off-by: Didier Wenzek --- crates/core/tedge/src/cli/mapping/cli.rs | 22 ++--- crates/core/tedge/src/cli/mapping/list.rs | 2 +- crates/core/tedge/src/cli/mapping/test.rs | 10 +-- .../tedge_gen_mapper/flows/circuit-breaker.js | 6 +- .../tedge_gen_mapper/flows/collectd.toml | 4 +- .../tedge_gen_mapper/flows/loop.toml | 4 +- .../tedge_gen_mapper/flows/measurements.toml | 4 +- .../extensions/tedge_gen_mapper/src/actor.rs | 8 +- .../extensions/tedge_gen_mapper/src/config.rs | 24 ++--- .../extensions/tedge_gen_mapper/src/flow.rs | 88 +++++++++---------- .../tedge_gen_mapper/src/js_runtime.rs | 16 ++-- .../src/{js_filter.rs => js_script.rs} | 68 +++++++------- crates/extensions/tedge_gen_mapper/src/lib.rs | 2 +- .../tedge_gen_mapper/src/runtime.rs | 40 ++++----- .../extensions/tedge_gen_mapper/src/stats.rs | 12 +-- docs/src/references/mappers/gen-mapper.md | 56 ++++++------ .../tedge_gen_mapper/flows/count-events.toml | 2 +- .../flows/count-measurements.toml | 2 +- .../tedge_gen_mapper/flows/measurements.toml | 4 +- .../tedge_gen_mapper/tedge_gen_mapper.robot | 2 +- 20 files changed, 189 insertions(+), 187 deletions(-) rename crates/extensions/tedge_gen_mapper/src/{js_filter.rs => js_script.rs} (87%) diff --git a/crates/core/tedge/src/cli/mapping/cli.rs b/crates/core/tedge/src/cli/mapping/cli.rs index 8ff98ef9771..9ea56651ad6 100644 --- a/crates/core/tedge/src/cli/mapping/cli.rs +++ b/crates/core/tedge/src/cli/mapping/cli.rs @@ -13,9 +13,9 @@ use tedge_gen_mapper::MessageProcessor; #[derive(clap::Subcommand, Debug)] pub enum TEdgeMappingCli { - /// List flows and filters + /// List flows and steps List { - /// Path to flow and filter specs + /// Path to the directory of flows and steps /// /// Default to /etc/tedge/gen-mapper #[clap(long)] @@ -30,17 +30,17 @@ pub enum TEdgeMappingCli { /// Process message samples Test { - /// Path to flow and filter specs + /// Path to the directory of flows and steps /// /// Default to /etc/tedge/gen-mapper #[clap(long)] mapping_dir: Option, - /// Path to the javascript filter or TOML flow definition + /// Path to the flow step script or TOML flow definition /// /// If none is provided, applies all the matching flows #[clap(long)] - filter: Option, + flow: Option, /// Send a tick after all the message samples #[clap(long = "final-tick")] @@ -69,7 +69,7 @@ impl BuildCommand for TEdgeMappingCli { TEdgeMappingCli::Test { mapping_dir, - filter, + flow, final_tick, topic, payload, @@ -83,7 +83,7 @@ impl BuildCommand for TEdgeMappingCli { }; Ok(TestCommand { mapping_dir, - filter, + flow, message, final_tick, } @@ -101,10 +101,10 @@ impl TEdgeMappingCli { pub async fn load_flows(mapping_dir: &PathBuf) -> Result { MessageProcessor::try_new(mapping_dir) .await - .with_context(|| format!("loading flows and filters from {}", mapping_dir.display())) + .with_context(|| format!("loading flows and steps from {}", mapping_dir.display())) } - pub async fn load_filter( + pub async fn load_file( mapping_dir: &PathBuf, path: &PathBuf, ) -> Result { @@ -113,9 +113,9 @@ impl TEdgeMappingCli { .await .with_context(|| format!("loading flow {flow}", flow = path.display())) } else { - MessageProcessor::try_new_single_filter(mapping_dir, path) + MessageProcessor::try_new_single_step_flow(mapping_dir, path) .await - .with_context(|| format!("loading filter {filter}", filter = path.display())) + .with_context(|| format!("loading flow script {script}", script = path.display())) } } } diff --git a/crates/core/tedge/src/cli/mapping/list.rs b/crates/core/tedge/src/cli/mapping/list.rs index e68454c6216..95903ee6362 100644 --- a/crates/core/tedge/src/cli/mapping/list.rs +++ b/crates/core/tedge/src/cli/mapping/list.rs @@ -41,7 +41,7 @@ impl ListCommand { fn display((flow_id, flow): (&String, &Flow)) { println!("{flow_id}"); for step in flow.steps.iter() { - println!("\t{}", step.filter.path.display()); + println!("\t{}", step.script.path.display()); } } } diff --git a/crates/core/tedge/src/cli/mapping/test.rs b/crates/core/tedge/src/cli/mapping/test.rs index 6d813164174..36a95ecb846 100644 --- a/crates/core/tedge/src/cli/mapping/test.rs +++ b/crates/core/tedge/src/cli/mapping/test.rs @@ -12,7 +12,7 @@ use tokio::io::Stdin; pub struct TestCommand { pub mapping_dir: PathBuf, - pub filter: Option, + pub flow: Option, pub message: Option, pub final_tick: bool, } @@ -21,15 +21,15 @@ pub struct TestCommand { impl Command for TestCommand { fn description(&self) -> String { format!( - "process message samples using flows and filters in {:}", + "process message samples using flows and steps in {:}", self.mapping_dir.display() ) } async fn execute(&self, _config: TEdgeConfig) -> Result<(), MaybeFancy> { - let mut processor = match &self.filter { + let mut processor = match &self.flow { None => TEdgeMappingCli::load_flows(&self.mapping_dir).await?, - Some(filter) => TEdgeMappingCli::load_filter(&self.mapping_dir, filter).await?, + Some(flow) => TEdgeMappingCli::load_file(&self.mapping_dir, flow).await?, }; if let Some(message) = &self.message { let timestamp = DateTime::now(); @@ -74,7 +74,7 @@ impl TestCommand { } } -fn print(messages: Result, FilterError>) { +fn print(messages: Result, FlowError>) { match messages { Ok(messages) => { for message in messages { diff --git a/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js b/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js index f65742a20c7..36dc8a7932a 100644 --- a/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js +++ b/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js @@ -1,10 +1,10 @@ -// A filter that let messages go through, unless too many messages are received within a given period +// A flow step that let messages go through, unless too many messages are received within a given period // -// This filter is configured by the following settings: +// This flow step is configured by the following settings: // - tick_every_seconds: the frequency at which the sliding window is moved // - tick_count: size of the time windows // - too_many: how many messages is too many (received during the last tick_count*tick_every_seconds seconds) -// - back_to_normal: how many messages is okay to reactivate the filter if bellow +// - back_to_normal: how many messages is okay to reactivate the flow step if bellow // - message_on_too_many: message sent when the upper threshold is crossed // - message_on_back_to_normal: message sent when the lower threshold is crossed // - stats_topic: topic for statistic messages diff --git a/crates/extensions/tedge_gen_mapper/flows/collectd.toml b/crates/extensions/tedge_gen_mapper/flows/collectd.toml index a744d2d7e29..5df8740e541 100644 --- a/crates/extensions/tedge_gen_mapper/flows/collectd.toml +++ b/crates/extensions/tedge_gen_mapper/flows/collectd.toml @@ -1,6 +1,6 @@ input_topics = ["collectd/+/+/+"] steps = [ - { filter = "collectd-to-te.js" }, - { filter = "average.js", tick_every_seconds = 10 } + { script = "collectd-to-te.js" }, + { script = "average.js", tick_every_seconds = 10 } ] diff --git a/crates/extensions/tedge_gen_mapper/flows/loop.toml b/crates/extensions/tedge_gen_mapper/flows/loop.toml index 87b1d9ff85d..2126a32fc01 100644 --- a/crates/extensions/tedge_gen_mapper/flows/loop.toml +++ b/crates/extensions/tedge_gen_mapper/flows/loop.toml @@ -2,6 +2,6 @@ input_topics = ["loopback/#"] steps = [ - { filter = "add_timestamp.js" }, - { filter = "circuit-breaker.js", tick_every_seconds = 1, config = { stats_topic = "te/error", too_many = 10000, message_on_too_many = { topic = "te/device/main///a/too-many-messages", payload = "too many messages" }, message_on_back_to_normal = { topic = "te/device/main///a/too-many-messages", payload = "back to normal" } } } + { script = "add_timestamp.js" }, + { script = "circuit-breaker.js", tick_every_seconds = 1, config = { stats_topic = "te/error", too_many = 10000, message_on_too_many = { topic = "te/device/main///a/too-many-messages", payload = "too many messages" }, message_on_back_to_normal = { topic = "te/device/main///a/too-many-messages", payload = "back to normal" } } } ] diff --git a/crates/extensions/tedge_gen_mapper/flows/measurements.toml b/crates/extensions/tedge_gen_mapper/flows/measurements.toml index 2980650efc1..4d87b48ef3b 100644 --- a/crates/extensions/tedge_gen_mapper/flows/measurements.toml +++ b/crates/extensions/tedge_gen_mapper/flows/measurements.toml @@ -1,6 +1,6 @@ input_topics = ["te/+/+/+/+/m/+"] steps = [ - { filter = "add_timestamp.js" }, - { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } + { script = "add_timestamp.js" }, + { script = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } ] diff --git a/crates/extensions/tedge_gen_mapper/src/actor.rs b/crates/extensions/tedge_gen_mapper/src/actor.rs index 605f56d7c97..ce873c4c2b5 100644 --- a/crates/extensions/tedge_gen_mapper/src/actor.rs +++ b/crates/extensions/tedge_gen_mapper/src/actor.rs @@ -41,7 +41,7 @@ impl Actor for GenMapper { message = self.messages.recv() => { match message { Some(InputMessage::MqttMessage(message)) => match Message::try_from(message) { - Ok(message) => self.filter(message).await?, + Ok(message) => self.process(message).await?, Err(err) => { error!(target: "gen-mapper", "Cannot process message: {err}"); } @@ -51,7 +51,7 @@ impl Actor for GenMapper { continue; }; if matches!(path.extension(), Some("js" | "ts")) { - self.processor.reload_filter(path).await; + self.processor.reload_script(path).await; } else if path.extension() == Some("toml") { self.processor.reload_flow(path).await; self.send_updated_subscriptions().await?; @@ -71,7 +71,7 @@ impl Actor for GenMapper { continue; }; if matches!(path.extension(), Some("js" | "ts")) { - self.processor.remove_filter(path).await; + self.processor.remove_script(path).await; } else if path.extension() == Some("toml") { self.processor.remove_flow(path).await; self.send_updated_subscriptions().await?; @@ -102,7 +102,7 @@ impl GenMapper { diff } - async fn filter(&mut self, message: Message) -> Result<(), RuntimeError> { + async fn process(&mut self, message: Message) -> Result<(), RuntimeError> { let timestamp = DateTime::now(); for (flow_id, flow_messages) in self.processor.process(×tamp, &message).await { match flow_messages { diff --git a/crates/extensions/tedge_gen_mapper/src/config.rs b/crates/extensions/tedge_gen_mapper/src/config.rs index 01ea61c57c7..8344fcc3cd4 100644 --- a/crates/extensions/tedge_gen_mapper/src/config.rs +++ b/crates/extensions/tedge_gen_mapper/src/config.rs @@ -1,7 +1,7 @@ use crate::flow::Flow; use crate::flow::FlowStep; -use crate::js_filter::JsFilter; use crate::js_runtime::JsRuntime; +use crate::js_script::JsScript; use crate::LoadError; use camino::Utf8Path; use camino::Utf8PathBuf; @@ -19,7 +19,7 @@ pub struct FlowConfig { #[derive(Deserialize)] pub struct StepConfig { - filter: FilterSpec, + script: ScriptSpec, #[serde(default)] config: Option, @@ -33,7 +33,7 @@ pub struct StepConfig { #[derive(Deserialize)] #[serde(untagged)] -pub enum FilterSpec { +pub enum ScriptSpec { JavaScript(Utf8PathBuf), } @@ -47,10 +47,10 @@ pub enum ConfigError { } impl FlowConfig { - pub fn from_filter(filter: Utf8PathBuf) -> Self { + pub fn from_step(script: Utf8PathBuf) -> Self { let input_topic = "#".to_string(); let step = StepConfig { - filter: FilterSpec::JavaScript(filter), + script: ScriptSpec::JavaScript(script), config: None, tick_every_seconds: 0, meta_topics: vec![], @@ -71,7 +71,7 @@ impl FlowConfig { let mut steps = vec![]; for (i, step) in self.steps.into_iter().enumerate() { let mut step = step.compile(config_dir, i, &source).await?; - js_runtime.load_filter(&mut step.filter).await?; + js_runtime.load_script(&mut step.script).await?; step.check(&source); step.fix(); steps.push(step); @@ -91,17 +91,17 @@ impl StepConfig { index: usize, flow: &Utf8Path, ) -> Result { - let path = match self.filter { - FilterSpec::JavaScript(path) if path.is_absolute() => path.into(), - FilterSpec::JavaScript(path) if path.starts_with(config_dir) => path.into(), - FilterSpec::JavaScript(path) => config_dir.join(path), + let path = match self.script { + ScriptSpec::JavaScript(path) if path.is_absolute() => path.into(), + ScriptSpec::JavaScript(path) if path.starts_with(config_dir) => path.into(), + ScriptSpec::JavaScript(path) => config_dir.join(path), }; - let filter = JsFilter::new(flow.to_owned().into(), index, path) + let script = JsScript::new(flow.to_owned().into(), index, path) .with_config(self.config) .with_tick_every_seconds(self.tick_every_seconds); let config_topics = topic_filters(&self.meta_topics)?; Ok(FlowStep { - filter, + script, config_topics, }) } diff --git a/crates/extensions/tedge_gen_mapper/src/flow.rs b/crates/extensions/tedge_gen_mapper/src/flow.rs index 8e993173183..ae4560f38cd 100644 --- a/crates/extensions/tedge_gen_mapper/src/flow.rs +++ b/crates/extensions/tedge_gen_mapper/src/flow.rs @@ -1,5 +1,5 @@ -use crate::js_filter::JsFilter; use crate::js_runtime::JsRuntime; +use crate::js_script::JsScript; use crate::stats::Counter; use crate::LoadError; use camino::Utf8Path; @@ -24,7 +24,7 @@ pub struct Flow { /// A message transformation step pub struct FlowStep { - pub filter: JsFilter, + pub script: JsScript, pub config_topics: TopicFilter, } @@ -41,7 +41,7 @@ pub struct Message { } #[derive(thiserror::Error, Debug)] -pub enum FilterError { +pub enum FlowError { #[error("Input message cannot be processed: {0}")] UnsupportedMessage(String), @@ -65,10 +65,10 @@ impl Flow { &mut self, js_runtime: &JsRuntime, message: &Message, - ) -> Result<(), FilterError> { + ) -> Result<(), FlowError> { for step in self.steps.iter_mut() { if step.config_topics.accept_topic_name(&message.topic) { - step.filter.update_config(js_runtime, message).await? + step.script.update_config(js_runtime, message).await? } } Ok(()) @@ -80,7 +80,7 @@ impl Flow { stats: &mut Counter, timestamp: &DateTime, message: &Message, - ) -> Result, FilterError> { + ) -> Result, FlowError> { self.update_config(js_runtime, message).await?; if !self.input_topics.accept_topic_name(&message.topic) { return Ok(vec![]); @@ -89,18 +89,18 @@ impl Flow { let stated_at = stats.flow_process_start(self.source.as_str()); let mut messages = vec![message.clone()]; for step in self.steps.iter() { - let js = step.filter.source(); + let js = step.script.source(); let mut transformed_messages = vec![]; for message in messages.iter() { - let filter_started_at = stats.filter_start(&js, "process"); - let filter_output = step.filter.process(js_runtime, timestamp, message).await; - match &filter_output { + let step_started_at = stats.flow_step_start(&js, "process"); + let step_output = step.script.process(js_runtime, timestamp, message).await; + match &step_output { Ok(messages) => { - stats.filter_done(&js, "process", filter_started_at, messages.len()) + stats.flow_step_done(&js, "process", step_started_at, messages.len()) } - Err(_) => stats.filter_failed(&js, "process"), + Err(_) => stats.flow_step_failed(&js, "process"), } - transformed_messages.extend(filter_output?); + transformed_messages.extend(step_output?); } messages = transformed_messages; } @@ -114,31 +114,31 @@ impl Flow { js_runtime: &JsRuntime, stats: &mut Counter, timestamp: &DateTime, - ) -> Result, FilterError> { + ) -> Result, FlowError> { let stated_at = stats.flow_tick_start(self.source.as_str()); let mut messages = vec![]; for step in self.steps.iter() { - let js = step.filter.source(); + let js = step.script.source(); // Process first the messages triggered upstream by the tick let mut transformed_messages = vec![]; for message in messages.iter() { - let filter_started_at = stats.filter_start(&js, "process"); - let filter_output = step.filter.process(js_runtime, timestamp, message).await; - match &filter_output { + let step_started_at = stats.flow_step_start(&js, "process"); + let step_output = step.script.process(js_runtime, timestamp, message).await; + match &step_output { Ok(messages) => { - stats.filter_done(&js, "process", filter_started_at, messages.len()) + stats.flow_step_done(&js, "process", step_started_at, messages.len()) } - Err(_) => stats.filter_failed(&js, "process"), + Err(_) => stats.flow_step_failed(&js, "process"), } - transformed_messages.extend(filter_output?); + transformed_messages.extend(step_output?); } // Only then process the tick - let filter_started_at = stats.filter_start(&js, "tick"); - let tick_output = step.filter.tick(js_runtime, timestamp).await; + let step_started_at = stats.flow_step_start(&js, "tick"); + let tick_output = step.script.tick(js_runtime, timestamp).await; match &tick_output { - Ok(messages) => stats.filter_done(&js, "tick", filter_started_at, messages.len()), - Err(_) => stats.filter_failed(&js, "tick"), + Ok(messages) => stats.flow_step_done(&js, "tick", step_started_at, messages.len()), + Err(_) => stats.flow_step_failed(&js, "tick"), } transformed_messages.extend(tick_output?); @@ -152,23 +152,23 @@ impl Flow { impl FlowStep { pub(crate) fn check(&self, flow: &Utf8Path) { - let filter = &self.filter; - if filter.no_js_process { - warn!(target: "MAPPING", "Filter with no 'process' function: {}", filter.path.display()); + let script = &self.script; + if script.no_js_process { + warn!(target: "MAPPING", "Flow script with no 'process' function: {}", script.path.display()); } - if filter.no_js_update_config && !self.config_topics.is_empty() { - warn!(target: "MAPPING", "Filter with no 'config_update' function: {}; but configured with 'config_topics' in {flow}", filter.path.display()); + if script.no_js_update_config && !self.config_topics.is_empty() { + warn!(target: "MAPPING", "Flow script with no 'config_update' function: {}; but configured with 'config_topics' in {flow}", script.path.display()); } - if filter.no_js_tick && filter.tick_every_seconds != 0 { - warn!(target: "MAPPING", "Filter with no 'tick' function: {}; but configured with 'tick_every_seconds' in {flow}", filter.path.display()); + if script.no_js_tick && script.tick_every_seconds != 0 { + warn!(target: "MAPPING", "Flow script with no 'tick' function: {}; but configured with 'tick_every_seconds' in {flow}", script.path.display()); } } pub(crate) fn fix(&mut self) { - let filter = &mut self.filter; - if !filter.no_js_tick && filter.tick_every_seconds == 0 { - // 0 as a default is not appropriate for a filter with a tick handler - filter.tick_every_seconds = 1; + let script = &mut self.script; + if !script.no_js_tick && script.tick_every_seconds == 0 { + // 0 as a default is not appropriate for a script with a tick handler + script.tick_every_seconds = 1; } } } @@ -188,11 +188,11 @@ impl DateTime { } impl TryFrom for DateTime { - type Error = FilterError; + type Error = FlowError; fn try_from(value: OffsetDateTime) -> Result { let seconds = u64::try_from(value.unix_timestamp()).map_err(|err| { - FilterError::UnsupportedMessage(format!("failed to convert timestamp: {}", err)) + FlowError::UnsupportedMessage(format!("failed to convert timestamp: {}", err)) })?; Ok(DateTime { @@ -217,29 +217,29 @@ impl Message { } impl TryFrom for Message { - type Error = FilterError; + type Error = FlowError; fn try_from(message: MqttMessage) -> Result { let topic = message.topic.to_string(); let payload = message .payload_str() - .map_err(|_| FilterError::UnsupportedMessage("Not an UTF8 payload".to_string()))? + .map_err(|_| FlowError::UnsupportedMessage("Not an UTF8 payload".to_string()))? .to_string(); Ok(Message { topic, payload }) } } impl TryFrom for MqttMessage { - type Error = FilterError; + type Error = FlowError; fn try_from(message: Message) -> Result { let topic = message.topic.as_str().try_into().map_err(|_| { - FilterError::UnsupportedMessage(format!("invalid topic {}", message.topic)) + FlowError::UnsupportedMessage(format!("invalid topic {}", message.topic)) })?; Ok(MqttMessage::new(&topic, message.payload)) } } -pub fn error_from_js(err: LoadError) -> FilterError { - FilterError::IncorrectSetting(format!("{err:#}")) +pub fn error_from_js(err: LoadError) -> FlowError { + FlowError::IncorrectSetting(format!("{err:#}")) } diff --git a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs index f6a41f9ae40..8718263f051 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs @@ -1,5 +1,5 @@ -use crate::js_filter::JsFilter; -use crate::js_filter::JsonValue; +use crate::js_script::JsScript; +use crate::js_script::JsonValue; use crate::LoadError; use anyhow::anyhow; use rquickjs::module::Evaluated; @@ -24,13 +24,13 @@ impl JsRuntime { Ok(JsRuntime { runtime, worker }) } - pub async fn load_filter(&mut self, filter: &mut JsFilter) -> Result<(), LoadError> { - let exports = self.load_file(filter.module_name(), filter.path()).await?; + pub async fn load_script(&mut self, script: &mut JsScript) -> Result<(), LoadError> { + let exports = self.load_file(script.module_name(), script.path()).await?; for export in exports { match export { - "process" => filter.no_js_process = false, - "update_config" => filter.no_js_update_config = false, - "tick" => filter.no_js_tick = false, + "process" => script.no_js_process = false, + "update_config" => script.no_js_update_config = false, + "tick" => script.no_js_tick = false, _ => (), } } @@ -237,7 +237,7 @@ impl<'js> JsModules<'js> { } mod console { - use crate::js_filter::JsonValue; + use crate::js_script::JsonValue; use rquickjs::class::Trace; use rquickjs::function::Rest; use rquickjs::Ctx; diff --git a/crates/extensions/tedge_gen_mapper/src/js_filter.rs b/crates/extensions/tedge_gen_mapper/src/js_script.rs similarity index 87% rename from crates/extensions/tedge_gen_mapper/src/js_filter.rs rename to crates/extensions/tedge_gen_mapper/src/js_script.rs index 6e29aeedf8d..9f901261274 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_filter.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_script.rs @@ -1,6 +1,6 @@ use crate::flow; use crate::flow::DateTime; -use crate::flow::FilterError; +use crate::flow::FlowError; use crate::flow::Message; use crate::js_runtime::JsRuntime; use anyhow::Context; @@ -13,7 +13,7 @@ use std::path::PathBuf; use tracing::debug; #[derive(Clone)] -pub struct JsFilter { +pub struct JsScript { pub module_name: String, pub path: PathBuf, pub config: JsonValue, @@ -32,10 +32,10 @@ impl Default for JsonValue { } } -impl JsFilter { +impl JsScript { pub fn new(flow: PathBuf, index: usize, path: PathBuf) -> Self { let module_name = format!("{}|{}|{}", flow.display(), index, path.display()); - JsFilter { + JsScript { module_name, path, config: JsonValue::default(), @@ -81,7 +81,7 @@ impl JsFilter { /// The "process" function of the JS module is passed 3 arguments /// - the current timestamp /// - the message to be transformed - /// - the filter config (as configured for the flow step, possibly updated by update_config messages) + /// - the flow step config (as configured for the flow step, possibly updated by update_config messages) /// /// The returned value is expected to be an array of messages. pub async fn process( @@ -89,7 +89,7 @@ impl JsFilter { js: &JsRuntime, timestamp: &DateTime, message: &Message, - ) -> Result, FilterError> { + ) -> Result, FlowError> { debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.module_name()); if self.no_js_process { return Ok(vec![message.clone()]); @@ -106,18 +106,18 @@ impl JsFilter { .try_into() } - /// Update the filter config using a metadata message + /// Update the flow step config using a metadata message /// /// The "update_config" function of the JS module is passed 2 arguments /// - the message - /// - the current filter config + /// - the current flow step config /// - /// The value returned by this function is used as the updated filter config + /// The value returned by this function is used as the updated flow step config pub async fn update_config( &mut self, js: &JsRuntime, message: &Message, - ) -> Result<(), FilterError> { + ) -> Result<(), FlowError> { debug!(target: "MAPPING", "{}: update_config({message:?})", self.module_name()); if self.no_js_update_config { return Ok(()); @@ -136,14 +136,14 @@ impl JsFilter { /// /// The "tick" function is passed 2 arguments /// - the current timestamp - /// - the current filter config + /// - the current flow step config /// /// Return zero, one or more messages pub async fn tick( &self, js: &JsRuntime, timestamp: &DateTime, - ) -> Result, FilterError> { + ) -> Result, FlowError> { if self.no_js_tick { return Ok(vec![]); } @@ -172,7 +172,7 @@ impl From for JsonValue { } impl TryFrom for Message { - type Error = FilterError; + type Error = FlowError; fn try_from(value: serde_json::Value) -> Result { let message = serde_json::from_value(value) @@ -182,7 +182,7 @@ impl TryFrom for Message { } impl TryFrom for Message { - type Error = FilterError; + type Error = FlowError; fn try_from(value: JsonValue) -> Result { Message::try_from(value.0) @@ -190,7 +190,7 @@ impl TryFrom for Message { } impl TryFrom for Vec { - type Error = FilterError; + type Error = FlowError; fn try_from(value: JsonValue) -> Result { match value.0 { @@ -198,7 +198,9 @@ impl TryFrom for Vec { serde_json::Value::Object(map) => { Message::try_from(serde_json::Value::Object(map)).map(|message| vec![message]) } - _ => Err(anyhow::anyhow!("Filters are expected to return an array of messages").into()), + _ => Err( + anyhow::anyhow!("Flow scripts are expected to return an array of messages").into(), + ), } } } @@ -308,16 +310,16 @@ mod tests { use super::*; #[tokio::test] - async fn identity_filter() { - let script = "export function process(t,msg) { return [msg]; };"; + async fn identity_script() { + let js = "export function process(t,msg) { return [msg]; };"; let mut runtime = JsRuntime::try_new().await.unwrap(); - let filter = JsFilter::new("id.toml".into(), 1, "id.js".into()); - runtime.load_js(filter.module_name(), script).await.unwrap(); + let script = JsScript::new("id.toml".into(), 1, "id.js".into()); + runtime.load_js(script.module_name(), js).await.unwrap(); let input = Message::new("te/main/device///m/", "hello world"); let output = input.clone(); assert_eq!( - filter + script .process(&runtime, &DateTime::now(), &input) .await .unwrap(), @@ -326,15 +328,15 @@ mod tests { } #[tokio::test] - async fn error_filter() { - let script = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; + async fn error_script() { + let js = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; let mut runtime = JsRuntime::try_new().await.unwrap(); - let mut filter = JsFilter::new("err.toml".into(), 1, "err.js".into()); - filter.no_js_process = false; - runtime.load_js(filter.module_name(), script).await.unwrap(); + let mut script = JsScript::new("err.toml".into(), 1, "err.js".into()); + script.no_js_process = false; + runtime.load_js(script.module_name(), js).await.unwrap(); let input = Message::new("te/main/device///m/", "hello world"); - let error = filter + let error = script .process(&runtime, &DateTime::now(), &input) .await .unwrap_err(); @@ -343,8 +345,8 @@ mod tests { } #[tokio::test] - async fn collectd_filter() { - let script = r#" + async fn collectd_script() { + let js = r#" export function process (timestamp, message, config) { let groups = message.topic.split( '/') let data = message.payload.split(':') @@ -366,9 +368,9 @@ export function process (timestamp, message, config) { } "#; let mut runtime = JsRuntime::try_new().await.unwrap(); - let mut filter = JsFilter::new("collectd.toml".into(), 1, "collectd.js".into()); - filter.no_js_process = false; - runtime.load_js(filter.module_name(), script).await.unwrap(); + let mut script = JsScript::new("collectd.toml".into(), 1, "collectd.js".into()); + script.no_js_process = false; + runtime.load_js(script.module_name(), js).await.unwrap(); let input = Message::new( "collectd/h/memory/percent-used", @@ -379,7 +381,7 @@ export function process (timestamp, message, config) { r#"{"time": 1748440192.104, "memory": {"percent-used": 19.9289468288182}}"#, ); assert_eq!( - filter + script .process(&runtime, &DateTime::now(), &input) .await .unwrap(), diff --git a/crates/extensions/tedge_gen_mapper/src/lib.rs b/crates/extensions/tedge_gen_mapper/src/lib.rs index 7bc891fbb2b..c21f5b2a82c 100644 --- a/crates/extensions/tedge_gen_mapper/src/lib.rs +++ b/crates/extensions/tedge_gen_mapper/src/lib.rs @@ -1,8 +1,8 @@ mod actor; mod config; pub mod flow; -mod js_filter; mod js_runtime; +mod js_script; mod runtime; mod stats; diff --git a/crates/extensions/tedge_gen_mapper/src/runtime.rs b/crates/extensions/tedge_gen_mapper/src/runtime.rs index 9a632c8a130..d20b41d0335 100644 --- a/crates/extensions/tedge_gen_mapper/src/runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/runtime.rs @@ -1,7 +1,7 @@ use crate::config::FlowConfig; use crate::flow::DateTime; -use crate::flow::FilterError; use crate::flow::Flow; +use crate::flow::FlowError; use crate::flow::Message; use crate::js_runtime::JsRuntime; use crate::stats::Counter; @@ -66,14 +66,14 @@ impl MessageProcessor { }) } - pub async fn try_new_single_filter( + pub async fn try_new_single_step_flow( config_dir: impl AsRef, - filter: impl AsRef, + script: impl AsRef, ) -> Result { let config_dir = config_dir.as_ref().to_owned(); let mut js_runtime = JsRuntime::try_new().await?; let mut flow_specs = FlowSpecs::default(); - flow_specs.load_single_filter(&filter).await; + flow_specs.load_single_script(&script).await; let flows = flow_specs.compile(&mut js_runtime, &config_dir).await; let stats = Counter::default(); @@ -97,7 +97,7 @@ impl MessageProcessor { &mut self, timestamp: &DateTime, message: &Message, - ) -> Vec<(String, Result, FilterError>)> { + ) -> Vec<(String, Result, FlowError>)> { let started_at = self.stats.runtime_process_start(); let mut out_messages = vec![]; @@ -118,7 +118,7 @@ impl MessageProcessor { pub async fn tick( &mut self, timestamp: &DateTime, - ) -> Vec<(String, Result, FilterError>)> { + ) -> Vec<(String, Result, FlowError>)> { let mut out_messages = vec![]; for (flow_id, flow) in self.flows.iter_mut() { let flow_output = flow @@ -140,16 +140,16 @@ impl MessageProcessor { self.js_runtime.dump_memory_stats().await; } - pub async fn reload_filter(&mut self, path: Utf8PathBuf) { + pub async fn reload_script(&mut self, path: Utf8PathBuf) { for flow in self.flows.values_mut() { for step in &mut flow.steps { - if step.filter.path() == path { - match self.js_runtime.load_filter(&mut step.filter).await { + if step.script.path() == path { + match self.js_runtime.load_script(&mut step.script).await { Ok(()) => { - info!(target: "gen-mapper", "Reloaded filter {path}"); + info!(target: "gen-mapper", "Reloaded flow script {path}"); } Err(e) => { - error!(target: "gen-mapper", "Failed to reload filter {path}: {e}"); + error!(target: "gen-mapper", "Failed to reload flow script {path}: {e}"); return; } } @@ -158,11 +158,11 @@ impl MessageProcessor { } } - pub async fn remove_filter(&mut self, path: Utf8PathBuf) { + pub async fn remove_script(&mut self, path: Utf8PathBuf) { for (flow_id, flow) in self.flows.iter() { for step in flow.steps.iter() { - if step.filter.path() == path { - warn!(target: "gen-mapper", "Removing a filter used by {flow_id}: {path}"); + if step.script.path() == path { + warn!(target: "gen-mapper", "Removing a script used by a flow {flow_id}: {path}"); return; } } @@ -225,7 +225,7 @@ struct FlowSpecs { impl FlowSpecs { pub async fn load(&mut self, config_dir: &PathBuf) { let Ok(mut entries) = read_dir(config_dir).await.map_err(|err| - error!(target: "MAPPING", "Failed to read filters from {}: {err}", config_dir.display()) + error!(target: "MAPPING", "Failed to read flows from {}: {err}", config_dir.display()) ) else { return; }; @@ -258,14 +258,14 @@ impl FlowSpecs { } } - pub async fn load_single_filter(&mut self, filter: impl AsRef) { - let filter = filter.as_ref(); - let Some(path) = Utf8Path::from_path(filter).map(|p| p.to_path_buf()) else { - error!(target: "MAPPING", "Skipping non UTF8 path: {}", filter.display()); + pub async fn load_single_script(&mut self, script: impl AsRef) { + let script = script.as_ref(); + let Some(path) = Utf8Path::from_path(script).map(|p| p.to_path_buf()) else { + error!(target: "MAPPING", "Skipping non UTF8 path: {}", script.display()); return; }; let flow_id = MessageProcessor::flow_id(&path); - let flow = FlowConfig::from_filter(path.to_owned()); + let flow = FlowConfig::from_step(path.to_owned()); self.flow_specs.insert(flow_id, (path.to_owned(), flow)); } diff --git a/crates/extensions/tedge_gen_mapper/src/stats.rs b/crates/extensions/tedge_gen_mapper/src/stats.rs index bcd0182f2a1..c834192a8ef 100644 --- a/crates/extensions/tedge_gen_mapper/src/stats.rs +++ b/crates/extensions/tedge_gen_mapper/src/stats.rs @@ -89,21 +89,21 @@ impl Counter { self.add(Dimension::Flow(flow_id.to_owned()), Sample::ErrorRaised); } - pub fn filter_start(&mut self, js: &str, f: &str) -> Instant { + pub fn flow_step_start(&mut self, js: &str, f: &str) -> Instant { if let Some(dim) = Dimension::function_call(js, f) { self.add(dim, Sample::MessageIn); } Instant::now() } - pub fn filter_done(&mut self, js: &str, f: &str, started_at: Instant, count: usize) { + pub fn flow_step_done(&mut self, js: &str, f: &str, started_at: Instant, count: usize) { if let Some(dim) = Dimension::function_call(js, f) { self.add(dim.clone(), Sample::MessageOut(count)); self.add(dim, Sample::ProcessingTime(started_at.elapsed())); } } - pub fn filter_failed(&mut self, js: &str, f: &str) { + pub fn flow_step_failed(&mut self, js: &str, f: &str) { if let Some(dim) = Dimension::function_call(js, f) { self.add(dim.clone(), Sample::ErrorRaised); } @@ -175,9 +175,9 @@ impl Display for Dimension { match self { Dimension::Runtime => write!(f, "runtime"), Dimension::Flow(toml) => write!(f, "flow {toml}"), - Dimension::Process(js) => write!(f, "process filter {js}"), - Dimension::Tick(js) => write!(f, "tick filter {js}"), - Dimension::Update(js) => write!(f, "update_config filter {js}"), + Dimension::Process(js) => write!(f, "process step {js}"), + Dimension::Tick(js) => write!(f, "tick step {js}"), + Dimension::Update(js) => write!(f, "update_config step {js}"), } } } diff --git a/docs/src/references/mappers/gen-mapper.md b/docs/src/references/mappers/gen-mapper.md index 347021997c5..15011a388bf 100644 --- a/docs/src/references/mappers/gen-mapper.md +++ b/docs/src/references/mappers/gen-mapper.md @@ -29,10 +29,10 @@ leveraging the core mapping rules and mapper mechanisms (bridge connections, HTT The %%te%% mappers for Cumulocity, Azure, AWS and Collectd are implemented on top of a so-called generic mapper which is used to drive all MQTT message transformations. - Transformations are implemented by flows which consume MQTT messages, apply a sequence of transformation steps and produce MQTT messages. - - `MQTT sub| filter-1 | filter-2 | ... | filter-n | MQTT pub` -- A flow can combine builtin and user-provided filters. + - `MQTT sub| step-1 | step-2 | ... | step-n | MQTT pub` +- A flow can combine builtin and user-provided steps. - The user can configure all the transformations used by a mapper, - editing MQTT sources, flows, filters and MQTT sinks. + editing MQTT sources, flows, steps and MQTT sinks. - By contrast with the current implementation, where the translation of measurements from %%te%% JSON to Cumulocity JSON is fully hard-coded, with the generic mapper a user can re-use the core of this transformation while adding customized steps: - consuming measurement from a non-standard topic @@ -43,9 +43,9 @@ which is used to drive all MQTT message transformations. ## POC reference -- The generic mapper loads flows and filters stored in `/etc/tedge/gen-mapper/`. +- The generic mapper loads flows and steps stored in `/etc/tedge/gen-mapper/`. - A flow is defined by a TOML file with `.toml` extension. -- A filter is defined by a Javascript file with `.js` extension. +- A step is defined by a Javascript file with `.js` extension. - The definition of flows must provide a list of MQTT topics to subscribe to. - The flow will be feed with all the messages received on these topics. - A flow definition provides a list of steps. @@ -56,13 +56,13 @@ which is used to drive all MQTT message transformations. input_topics = ["te/+/+/+/+/m/+"] steps = [ - { filter = "add_timestamp.js" }, - { filter = "drop_stragglers.js", config = { max_delay = 60 } }, - { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } + { script = "add_timestamp.js" }, + { script = "drop_stragglers.js", config = { max_delay = 60 } }, + { script = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } ] ``` -- A filter has to export at least one `process` function. +- A flow script has to export at least one `process` function. - `process(t: Timestamp, msg: Message, config: Json) -> Vec` - This function is called for each message to be transformed - The arguments passed to the function are: @@ -71,17 +71,17 @@ steps = [ - The config as read from the flow config or updated by the script - The function is expected to return zero, one or many transformed messages `[{ topic: string, payload: string }]` - An exception can be thrown if the input message cannot be transformed. -- A filter can also export an `update_config` function +- A flow script can also export an `update_config` function - This function is called on each message received on the `meta_topics` as defined in the config. - The arguments are: - The message to be interpreted as a config update `{ topic: string, payload: string }` - The current config - - The returned value (an arbitrary JSON value) is then used as the new config for the filter. -- A filter can also export a `tick` function + - The returned value (an arbitrary JSON value) is then used as the new config for the flow script. +- A flow script can also export a `tick` function - This function is called at a regular pace with the current time and config. - - The filter can then return zero, one or many transformed messages + - The flow script can then return zero, one or many transformed messages - By sharing an internal state between the `process` and `tick` functions, - the filter can implement aggregations over a time window. + the flow script can implement aggregations over a time window. When messages are received they are pushed by the `process` function into that state and the final outcome is extracted by the `tick` function at the end of the time window. @@ -94,24 +94,24 @@ but to revisit the legacy mappers to include the ability for users to add their To be lovable, the first release of an extensible mapper should at least: - be a drop-in replacement of the current mapper (for c8y, aws, az or collect) -- feature the ability to customize MEA processing by combining builtin filters with user-provided functions written in JavaScript -- provide tools to create, test, monitor and debug filters and flows -- be stable enough that user-defined filters will still work without changes with future releases. +- feature the ability to customize MEA processing by combining builtin flow steps with user-provided functions written in JavaScript +- provide tools to create, test, monitor and debug steps and flows +- be stable enough that user-defined flow scripts will still work without changes with future releases. To keep things simple for the first release, the following questions are deferred: - Could a generic mapper let users define bridge rules as well as message transformation flows? - Does it make sense to run such a mapper on child-devices? -- Could a flow send HTTP messages? Or could a filter tell the runtime to send messages over HTTP? +- Could a flow send HTTP messages? Or could a flow step tell the runtime to send messages over HTTP? - How to handle binary payloads on the MQTT bus? - Could operations be managed is a similar way with user-provided functions to transform commands? - To handle operations, would the plugins be expanded to do more complex things like HTTP calls, file-system interactions, etc.? -- What are the pros and cons to persist filter states? +- What are the pros and cons to persist flow step states? - Split a flow, forwarding transformed messages to different flows for further processing ### API -The POC expects the filter to implement a bunch of functions. This gives a quite expressive interface +The POC expects the flow scripts to implement a bunch of functions. This gives a quite expressive interface (filtering, mapping, splitting, dynamic configuration, aggregation over time windows), but at the cost of some complexity. - `process(t: Timestamp, msg: Message, config: Json) -> Vec` @@ -143,26 +143,26 @@ Other ideas to explore to make the API more flexible: - Interaction with the entity store and tedge config. - Allow a flow to subscribe to topics related to the device/entity it is running on -- Feed filters with message excerpts as done for the workflows +- Feed flow scripts with message excerpts as done for the workflows ### Devops tools The flexibility to customize MQTT message processing with user-provided functions comes with risks: -- a filter might not behave as expected, +- a step might not behave as expected, - flows might be overlapping or conflicting, possibly sending duplicate messages or creating infinite loops - builtin flows might be accidentally disconnected or broken -- a filter might introduce a performance bottleneck. +- a step might introduce a performance bottleneck. -To help mitigating these risks, the `tedge mapping` sub-commands provide the tools to test, monitor and debug filters and flows. +To help mitigating these risks, the `tedge mapping` sub-commands provide the tools to test, monitor and debug steps and flows. -- `tedge mapping flow [topic]` displays flows and filters messages received on this topic will flow through +- `tedge mapping list [topic]` displays flows and steps messages received on this topic will flow through - can be used with a set of flows not configured yet for a mapper -- `tedge mapping test [filter]` feeds a filter or flow with input messages and produces the transformed output messages - - allow users to run an assertion based on the input/output of a filter +- `tedge mapping test [flow]` feeds a step or flow with input messages and produces the transformed output messages + - allow users to run an assertion based on the input/output of a flow - ability to pipe `tedge mqtt sub` and `tedge mapping test` - control of the timestamps - test aggregation over ticks - can be used with a set of flows not configured yet for a mapper - `tedge mapping stats [flow]` returns statistics on the messages processed by a flow - count message in, message out - - processing time min, median, max per filter + - processing time min, median, max for each flow and step diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-events.toml b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-events.toml index 27d36587559..9304ced0c8c 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-events.toml +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-events.toml @@ -1,5 +1,5 @@ input_topics = ["test/+/+/+/+/e/+"] steps = [ - { filter = "count-messages.js", config = { topic = "test/count/e" }, tick_every_seconds = 1 }, + { script = "count-messages.js", config = { topic = "test/count/e" }, tick_every_seconds = 1 }, ] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-measurements.toml index 358e8a9e9e1..17e4de59942 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-measurements.toml +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-measurements.toml @@ -1,5 +1,5 @@ input_topics = ["test/+/+/+/+/m/+"] steps = [ - { filter = "count-messages.js", config = { topic = "test/count/m" }, tick_every_seconds = 1 }, + { script = "count-messages.js", config = { topic = "test/count/m" }, tick_every_seconds = 1 }, ] \ No newline at end of file diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.toml b/tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.toml index dec7044ddc5..c520f48f16f 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.toml +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/measurements.toml @@ -1,6 +1,6 @@ input_topics = ["te/+/+/+/+/m/+"] steps = [ - { filter = "add_timestamp.js" }, - { filter = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] }, + { script = "add_timestamp.js" }, + { script = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] }, ] diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot index cc5ca8a8806..e5adf4de671 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot +++ b/tests/RobotFramework/tests/tedge_gen_mapper/tedge_gen_mapper.robot @@ -50,7 +50,7 @@ Units are configured using topic metadata Computing average over a time window ${transformed_msg} Execute Command - ... cat /etc/tedge/gen-mapper/average.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test --final-tick --filter /etc/tedge/gen-mapper/average.js + ... cat /etc/tedge/gen-mapper/average.samples | awk '{ print $2 }' FS\='INPUT:' | tedge mapping test --final-tick --flow /etc/tedge/gen-mapper/average.js ... strip=True ${expected_msg} Execute Command ... cat /etc/tedge/gen-mapper/average.samples | awk '{ if ($2) print $2 }' FS\='OUTPUT: ' From 3e83ba47fac8d94973e86b66a69b048a0af67edd Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 16 Jul 2025 10:13:34 +0200 Subject: [PATCH 48/53] fixup! Add a new mapper: tedge-gen-mapper --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index e29d364310a..601e31c211e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4986,7 +4986,7 @@ dependencies = [ [[package]] name = "tedge_gen_mapper" -version = "1.5.1" +version = "1.6.0" dependencies = [ "anyhow", "async-trait", From abe12fe1dd55c1ad6ffc913d2650a143596afac6 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Wed, 16 Jul 2025 22:51:57 +0200 Subject: [PATCH 49/53] Concept documentation Signed-off-by: Didier Wenzek --- design/decisions/0006-tedge-flows.md | 157 +++++++++++++++++++++ docs/src/references/mappers/gen-mapper.md | 161 +++++++--------------- 2 files changed, 208 insertions(+), 110 deletions(-) create mode 100644 design/decisions/0006-tedge-flows.md diff --git a/design/decisions/0006-tedge-flows.md b/design/decisions/0006-tedge-flows.md new file mode 100644 index 00000000000..7775e2fcc61 --- /dev/null +++ b/design/decisions/0006-tedge-flows.md @@ -0,0 +1,157 @@ +# Extensible mapper and user-provided flows + +* Date: __2025-07-1-__ +* Status: __New__ + +## Motivation + +In theory, %%te%% users can implement customized mappers to transform data published on the MQTT bus +or to interact with the cloud. In practice, they don't. +Implementing a mapper is costly while what is provided out-the-box by %%te%% already meets most requirements. +The need is not to write new mappers but to adapt existing ones. + +The aim of the extensible mapper it to let users extend and adapt the mappers with their own filtering and mapping rules, +leveraging the core mapping rules and mapper mechanisms (bridge connections, HTTP proxies, operations). + +## Vision + +The %%te%% mappers for Cumulocity, Azure, AWS and Collectd are implemented on top of an extensible mapper +which is used to drive all MQTT message transformations. +- Transformations are implemented by flows which consume MQTT messages, apply a sequence of transformation steps and produce MQTT messages. + - `MQTT sub| step-1 | step-2 | ... | step-n | MQTT pub` +- A flow can combine builtin and user-provided steps. +- The user can configure all the transformations used by a mapper, + editing MQTT sources, flows, steps and MQTT sinks. +- By contrast with the current implementation, where the translation of measurements from %%te%% JSON to Cumulocity JSON + is fully hard-coded, with the generic mapper a user can re-use the core of this transformation while adding customized steps: + - consuming measurement from a non-standard topic + - filtering out part of the measurements + - normalizing units + - adding units read from some config + - producing transformed measurements on a non-standard topic. + +## POC reference + +- The generic mapper loads flows and steps stored in `/etc/tedge/gen-mapper/`. +- A flow is defined by a TOML file with `.toml` extension. +- A step is defined by a Javascript file with `.js` extension. +- The definition of flows must provide a list of MQTT topics to subscribe to. + - The flow will be feed with all the messages received on these topics. +- A flow definition provides a list of steps. + - Each step is built from a javascript and is possibly given a config (arbitrary json that will be passed to the script) + - Each step can also subscribe to a list of MQTT topics (which messages will be passed to the script to update its config) + +```toml +input_topics = ["te/+/+/+/+/m/+"] + +steps = [ + { script = "add_timestamp.js" }, + { script = "drop_stragglers.js", config = { max_delay = 60 } }, + { script = "te_to_c8y.js", meta_topics = ["te/+/+/+/+/m/+/meta"] } +] +``` + +- A flow script has to export at least one `process` function. + - `process(t: Timestamp, msg: Message, config: Json) -> Vec` + - This function is called for each message to be transformed + - The arguments passed to the function are: + - The current time as `{ seconds: u64, nanoseconds: u32 }` + - The message `{ topic: string, payload: string }` + - The config as read from the flow config or updated by the script + - The function is expected to return zero, one or many transformed messages `[{ topic: string, payload: string }]` + - An exception can be thrown if the input message cannot be transformed. +- A flow script can also export an `update_config` function + - This function is called on each message received on the `meta_topics` as defined in the config. + - The arguments are: + - The message to be interpreted as a config update `{ topic: string, payload: string }` + - The current config + - The returned value (an arbitrary JSON value) is then used as the new config for the flow script. +- A flow script can also export a `tick` function + - This function is called at a regular pace with the current time and config. + - The flow script can then return zero, one or many transformed messages + - By sharing an internal state between the `process` and `tick` functions, + the flow script can implement aggregations over a time window. + When messages are received they are pushed by the `process` function into that state + and the final outcome is extracted by the `tick` function at the end of the time window. + +## First release + +While the POC provides a generic mapper that is fully independent of the legacy mappers, +the plan is not to abandon the latter in favor of the former +but to revisit the legacy mappers to include the ability for users to add their own mapping rules. + +To be lovable, the first release of an extensible mapper should at least: + +- be a drop-in replacement of the current mapper (for c8y, aws, az or collect) +- feature the ability to customize MEA processing by combining builtin flow steps with user-provided functions written in JavaScript +- provide tools to create, test, monitor and debug steps and flows +- be stable enough that user-defined flow scripts will still work without changes with future releases. + +To keep things simple for the first release, the following questions are deferred: + +- Could a generic mapper let users define bridge rules as well as message transformation flows? +- Does it make sense to run such a mapper on child-devices? +- Could a flow send HTTP messages? Or could a flow step tell the runtime to send messages over HTTP? +- How to handle binary payloads on the MQTT bus? +- Could operations be managed is a similar way with user-provided functions to transform commands? +- To handle operations, would the plugins be expanded to do more complex things like HTTP calls, file-system interactions, etc.? +- What are the pros and cons to persist flow step states? +- Split a flow, forwarding transformed messages to different flows for further processing + +### API + +The POC expects the flow scripts to implement a bunch of functions. This gives a quite expressive interface +(filtering, mapping, splitting, dynamic configuration, aggregation over time windows), but at the cost of some complexity. + +- `process(t: Timestamp, msg: Message, config: Json) -> Vec` +- `tick(t: Timestamp) -> Vec` +- `update_config(msg: Message, config: Json) -> Json` + +An alternative is to let the user implement more specific functions with simpler type signatures: + +- `filter(msg: Message, config: Json) -> bool` +- `map(msg: Message, config: Json) -> Message` +- `filter_map(msg: Message, config: Json) -> Option` +- `flat_map(msg: Message, config: Json) -> Vec` + +One can also rearrange the argument order for these functions, +making life easier when a transformation does need a config or the current time +leveraging that one can pass more arguments than declared to a javascript function: + +- `process(msg: Message, config: Json, t: Timestamp) -> Vec` +- `process(msg: Message, config: Json) -> Vec` +- `process(msg: Message) -> Vec` + +One can even use a bit further the flexibility of javascript, to let the process function freely return: +- An array of message objects +- A single message object +- A null value interpreted as no messages +- A boolean + +Other ideas to explore to make the API more flexible: + +- Interaction with the entity store and tedge config. +- Allow a flow to subscribe to topics related to the device/entity it is running on +- Feed flow scripts with message excerpts as done for the workflows + +### Devops tools + +The flexibility to customize MQTT message processing with user-provided functions comes with risks: +- a step might not behave as expected, +- flows might be overlapping or conflicting, possibly sending duplicate messages or creating infinite loops +- builtin flows might be accidentally disconnected or broken +- a step might introduce a performance bottleneck. + +To help mitigating these risks, the `tedge mapping` sub-commands provide the tools to test, monitor and debug steps and flows. + +- `tedge mapping list [topic]` displays flows and steps messages received on this topic will flow through + - can be used with a set of flows not configured yet for a mapper +- `tedge mapping test [flow]` feeds a step or flow with input messages and produces the transformed output messages + - allow users to run an assertion based on the input/output of a flow + - ability to pipe `tedge mqtt sub` and `tedge mapping test` + - control of the timestamps + - test aggregation over ticks + - can be used with a set of flows not configured yet for a mapper +- `tedge mapping stats [flow]` returns statistics on the messages processed by a flow + - count message in, message out + - processing time min, median, max for each flow and step diff --git a/docs/src/references/mappers/gen-mapper.md b/docs/src/references/mappers/gen-mapper.md index 15011a388bf..d2c31210598 100644 --- a/docs/src/references/mappers/gen-mapper.md +++ b/docs/src/references/mappers/gen-mapper.md @@ -1,5 +1,5 @@ --- -title: Generic Mapper +title: Extensible mapper and user-provided Flows tags: [Reference, Mappers, Cloud] sidebar_position: 2 draft: true @@ -9,43 +9,64 @@ import ProposalBanner from '@site/src/components/ProposalBanner' -:::note -This section is actually a design document. -It includes a reference guide for the POC, but also proposes a plan toward a generic mapper. -::: +## Concepts -## Motivation - -In theory, %%te%% users can implement customized mappers to transform data published on the MQTT bus -or to interact with the cloud. In practice, they don't. -Implementing a mapper is costly while what is provided out-the-box by %%te%% already meets most requirements. -The need is not to write new mappers but to adapt existing ones. - -The aim of the generic mapper it to let users extend and adapt the mappers with their own filtering and mapping rules, +Users can extend and adapt the built-in mappers for Cumulocity, Azure and AWS +with their own filtering and message transformation rules, leveraging the core mapping rules and mapper mechanisms (bridge connections, HTTP proxies, operations). -## Vision - -The %%te%% mappers for Cumulocity, Azure, AWS and Collectd are implemented on top of a so-called generic mapper -which is used to drive all MQTT message transformations. -- Transformations are implemented by flows which consume MQTT messages, apply a sequence of transformation steps and produce MQTT messages. - - `MQTT sub| step-1 | step-2 | ... | step-n | MQTT pub` -- A flow can combine builtin and user-provided steps. -- The user can configure all the transformations used by a mapper, - editing MQTT sources, flows, steps and MQTT sinks. -- By contrast with the current implementation, where the translation of measurements from %%te%% JSON to Cumulocity JSON - is fully hard-coded, with the generic mapper a user can re-use the core of this transformation while adding customized steps: - - consuming measurement from a non-standard topic +As an example, users can now adapt to their use cases the translation of measurements from %%te%% JSON to Cumulocity JSON: + - consuming measurements from a non-standard topic - filtering out part of the measurements - normalizing units - - adding units read from some config + - adding units read from device config - producing transformed measurements on a non-standard topic. -## POC reference +The behavior of a mapper is defined by a set of *connectors*, *flows*, *steps* and transformation *scripts* +which rule how to consume, transform and produce MQTT messages. + +- A *step* function transforms one input message into zero, one or more output messages. + - Steps are effect-free functions, with no access to MQTT, HTTP or the file-system. + - The focus is on message transformation, format conversion, content extraction and completion as well as filtering and redacting. +- A *connector* is used by the mapper to consume messages from and produce messages to. + - MQTT is the primary message source and target, but overtime others can be added. + - Connectors can be seen as streams of messages all with the same shape so they can be processed by any step. +- A *flow* applies a chain of transformation *steps* to input messages producing fully processed output messages. + - The *flows* put things in motion, actually interacting with the system, consuming and producing messages. + - Messages received on a flow are passed to the first step; and the transformed messages, if any, + are pushed to the subsequent steps upto the output connector. +- A flow can combine builtin and user-provided steps. + - Builtin steps provide generic building blocks such as %%te%% JSON translation into Cumulocity JSON. + - Users can implement specific steps using JavaScript or TypeScript to refine transformations to their use cases. +- If some message transformations can be fully defined only from the input message, most require a *context*. + - What is the Cumulocity internal id of the device? What are the units used by a sensor? Does the location of the device matter? + - Such a context can only be specific and has to be built from various sources, configuration, metadata and capability messages. + - For that purpose, %%te%% maintain a context object which is + - created, cached and populated by %%te%% using configuration data, + - passed to all invocations of transformation steps, + - enriched by some flows and steps with context info extracted from metadata and capability messages, + - used by all flows and steps to adapt their behavior +- %%te%% provides some support to steps aggregating messages over time windows. + - For each aggregating step, the mapper persists a state (a JSON object) + which can be updated by the step function on each message and at regular intervals + to produce transformed messages on time-window boundaries. + +## Step API + +A transformation *scripts* is a JavaScript or TypeScript module that exports: + +- at least a function `on_message`, aimed to transform one input message into zero, one or more output messages +- possibly a function `on_interval`, called at regular intervals to produce aggregated messages. + + + + +## Flow configuration - The generic mapper loads flows and steps stored in `/etc/tedge/gen-mapper/`. - A flow is defined by a TOML file with `.toml` extension. -- A step is defined by a Javascript file with `.js` extension. +- A step is defined by a JavaScript file with `.js` extension. + - This can also be a TypeScript module with a `.ts` extension. - The definition of flows must provide a list of MQTT topics to subscribe to. - The flow will be feed with all the messages received on these topics. - A flow definition provides a list of steps. @@ -62,6 +83,8 @@ steps = [ ] ``` +## POC API + - A flow script has to export at least one `process` function. - `process(t: Timestamp, msg: Message, config: Json) -> Vec` - This function is called for each message to be transformed @@ -84,85 +107,3 @@ steps = [ the flow script can implement aggregations over a time window. When messages are received they are pushed by the `process` function into that state and the final outcome is extracted by the `tick` function at the end of the time window. - -## First release - -While the POC provides a generic mapper that is fully independent of the legacy mappers, -the plan is not to abandon the latter in favor of the former -but to revisit the legacy mappers to include the ability for users to add their own mapping rules. - -To be lovable, the first release of an extensible mapper should at least: - -- be a drop-in replacement of the current mapper (for c8y, aws, az or collect) -- feature the ability to customize MEA processing by combining builtin flow steps with user-provided functions written in JavaScript -- provide tools to create, test, monitor and debug steps and flows -- be stable enough that user-defined flow scripts will still work without changes with future releases. - -To keep things simple for the first release, the following questions are deferred: - -- Could a generic mapper let users define bridge rules as well as message transformation flows? -- Does it make sense to run such a mapper on child-devices? -- Could a flow send HTTP messages? Or could a flow step tell the runtime to send messages over HTTP? -- How to handle binary payloads on the MQTT bus? -- Could operations be managed is a similar way with user-provided functions to transform commands? -- To handle operations, would the plugins be expanded to do more complex things like HTTP calls, file-system interactions, etc.? -- What are the pros and cons to persist flow step states? -- Split a flow, forwarding transformed messages to different flows for further processing - -### API - -The POC expects the flow scripts to implement a bunch of functions. This gives a quite expressive interface -(filtering, mapping, splitting, dynamic configuration, aggregation over time windows), but at the cost of some complexity. - -- `process(t: Timestamp, msg: Message, config: Json) -> Vec` -- `tick(t: Timestamp) -> Vec` -- `update_config(msg: Message, config: Json) -> Json` - -An alternative is to let the user implement more specific functions with simpler type signatures: - -- `filter(msg: Message, config: Json) -> bool` -- `map(msg: Message, config: Json) -> Message` -- `filter_map(msg: Message, config: Json) -> Option` -- `flat_map(msg: Message, config: Json) -> Vec` - -One can also rearrange the argument order for these functions, -making life easier when a transformation does need a config or the current time -leveraging that one can pass more arguments than declared to a javascript function: - -- `process(msg: Message, config: Json, t: Timestamp) -> Vec` -- `process(msg: Message, config: Json) -> Vec` -- `process(msg: Message) -> Vec` - -One can even use a bit further the flexibility of javascript, to let the process function freely return: -- An array of message objects -- A single message object -- A null value interpreted as no messages -- A boolean - -Other ideas to explore to make the API more flexible: - -- Interaction with the entity store and tedge config. -- Allow a flow to subscribe to topics related to the device/entity it is running on -- Feed flow scripts with message excerpts as done for the workflows - -### Devops tools - -The flexibility to customize MQTT message processing with user-provided functions comes with risks: -- a step might not behave as expected, -- flows might be overlapping or conflicting, possibly sending duplicate messages or creating infinite loops -- builtin flows might be accidentally disconnected or broken -- a step might introduce a performance bottleneck. - -To help mitigating these risks, the `tedge mapping` sub-commands provide the tools to test, monitor and debug steps and flows. - -- `tedge mapping list [topic]` displays flows and steps messages received on this topic will flow through - - can be used with a set of flows not configured yet for a mapper -- `tedge mapping test [flow]` feeds a step or flow with input messages and produces the transformed output messages - - allow users to run an assertion based on the input/output of a flow - - ability to pipe `tedge mqtt sub` and `tedge mapping test` - - control of the timestamps - - test aggregation over ticks - - can be used with a set of flows not configured yet for a mapper -- `tedge mapping stats [flow]` returns statistics on the messages processed by a flow - - count message in, message out - - processing time min, median, max for each flow and step From eccec916d16c3deb68092deaa09fca0c6cb01da6 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 17 Jul 2025 10:08:28 +0200 Subject: [PATCH 50/53] zsh:1: command not found: q Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/src/js_script.rs | 69 ++++++++++++++++--- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/js_script.rs b/crates/extensions/tedge_gen_mapper/src/js_script.rs index 9f901261274..6d249259395 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_script.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_script.rs @@ -198,6 +198,7 @@ impl TryFrom for Vec { serde_json::Value::Object(map) => { Message::try_from(serde_json::Value::Object(map)).map(|message| vec![message]) } + serde_json::Value::Null => Ok(vec![]), _ => Err( anyhow::anyhow!("Flow scripts are expected to return an array of messages").into(), ), @@ -312,9 +313,23 @@ mod tests { #[tokio::test] async fn identity_script() { let js = "export function process(t,msg) { return [msg]; };"; - let mut runtime = JsRuntime::try_new().await.unwrap(); - let script = JsScript::new("id.toml".into(), 1, "id.js".into()); - runtime.load_js(script.module_name(), js).await.unwrap(); + let (runtime, script) = runtime_with(js).await; + + let input = Message::new("te/main/device///m/", "hello world"); + let output = input.clone(); + assert_eq!( + script + .process(&runtime, &DateTime::now(), &input) + .await + .unwrap(), + vec![output] + ); + } + + #[tokio::test] + async fn identity_script_no_array() { + let js = "export function process(t,msg) { return msg; };"; + let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); let output = input.clone(); @@ -327,13 +342,40 @@ mod tests { ); } + #[tokio::test] + async fn script_returning_null() { + let js = "export function process(t,msg) { return null; };"; + let (runtime, script) = runtime_with(js).await; + + let input = Message::new("te/main/device///m/", "hello world"); + assert_eq!( + script + .process(&runtime, &DateTime::now(), &input) + .await + .unwrap(), + vec![] + ); + } + + #[tokio::test] + async fn script_returning_nothing() { + let js = "export function process(t,msg) { return; };"; + let (runtime, script) = runtime_with(js).await; + + let input = Message::new("te/main/device///m/", "hello world"); + assert_eq!( + script + .process(&runtime, &DateTime::now(), &input) + .await + .unwrap(), + vec![] + ); + } + #[tokio::test] async fn error_script() { let js = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; - let mut runtime = JsRuntime::try_new().await.unwrap(); - let mut script = JsScript::new("err.toml".into(), 1, "err.js".into()); - script.no_js_process = false; - runtime.load_js(script.module_name(), js).await.unwrap(); + let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); let error = script @@ -367,10 +409,7 @@ export function process (timestamp, message, config) { }] } "#; - let mut runtime = JsRuntime::try_new().await.unwrap(); - let mut script = JsScript::new("collectd.toml".into(), 1, "collectd.js".into()); - script.no_js_process = false; - runtime.load_js(script.module_name(), js).await.unwrap(); + let (runtime, script) = runtime_with(js).await; let input = Message::new( "collectd/h/memory/percent-used", @@ -388,4 +427,12 @@ export function process (timestamp, message, config) { vec![output] ); } + + async fn runtime_with(js: &str) -> (JsRuntime, JsScript) { + let mut runtime = JsRuntime::try_new().await.unwrap(); + let mut script = JsScript::new("toml".into(), 1, "js".into()); + runtime.load_js(script.module_name(), js).await.unwrap(); + script.no_js_process = false; + (runtime, script) + } } From 6641cfc1483ec4a34b745a4abc58d168ab47f02c Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Thu, 17 Jul 2025 15:45:28 +0200 Subject: [PATCH 51/53] Set memory usage limits to flow scripts Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/src/js_runtime.rs | 62 ++++++++++++++----- .../tedge_gen_mapper/src/js_script.rs | 49 ++++++++++++++- 2 files changed, 96 insertions(+), 15 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs index 8718263f051..bccf70a99e7 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs @@ -3,10 +3,12 @@ use crate::js_script::JsonValue; use crate::LoadError; use anyhow::anyhow; use rquickjs::module::Evaluated; +use rquickjs::CaughtError; use rquickjs::Ctx; use rquickjs::Module; use std::collections::HashMap; use std::path::Path; +use std::time::Duration; use tokio::sync::mpsc; use tokio::sync::oneshot; use tracing::debug; @@ -14,14 +16,22 @@ use tracing::debug; pub struct JsRuntime { runtime: rquickjs::AsyncRuntime, worker: mpsc::Sender, + execution_timeout: Duration, } impl JsRuntime { pub async fn try_new() -> Result { let runtime = rquickjs::AsyncRuntime::new()?; + runtime.set_memory_limit(16 * 1024 * 1024).await; + runtime.set_max_stack_size(256 * 1024).await; let context = rquickjs::AsyncContext::full(&runtime).await?; let worker = JsWorker::spawn(context).await; - Ok(JsRuntime { runtime, worker }) + let execution_timeout = Duration::from_secs(5); + Ok(JsRuntime { + runtime, + worker, + execution_timeout, + }) } pub async fn load_script(&mut self, script: &mut JsScript) -> Result<(), LoadError> { @@ -55,16 +65,16 @@ impl JsRuntime { let (sender, receiver) = oneshot::channel(); let source = source.into(); let imports = vec!["process", "update_config", "tick"]; - self.worker - .send(JsRequest::LoadModule { + self.send( + receiver, + JsRequest::LoadModule { name, source, imports, sender, - }) - .await - .map_err(|err| anyhow!(err))?; - receiver.await.map_err(|err| anyhow!(err))? + }, + ) + .await? } pub async fn call_function( @@ -74,16 +84,16 @@ impl JsRuntime { args: Vec, ) -> Result { let (sender, receiver) = oneshot::channel(); - self.worker - .send(JsRequest::CallFunction { + self.send( + receiver, + JsRequest::CallFunction { module: module.to_string(), function: function.to_string(), args, sender, - }) - .await - .map_err(|err| anyhow!(err))?; - receiver.await.map_err(|err| anyhow!(err))? + }, + ) + .await? } pub async fn dump_memory_stats(&self) { @@ -97,6 +107,28 @@ impl JsRuntime { tracing::info!(target: "gen-mapper", " - string count: {}", usage.str_count); tracing::info!(target: "gen-mapper", " - atom count: {}", usage.atom_count); } + + async fn send( + &self, + mut receiver: oneshot::Receiver, + request: JsRequest, + ) -> Result { + self.worker + .send(request) + .await + .map_err(|err| anyhow!(err))?; + + // FIXME: The following timeout is not working + // - see unit test: js_script::while_loop + // - the issue is that the quickjs runtime fails to yield when executing `while(true)` + // - Using task::spawn_blocking to launch the quickjs runtime doesn't help + // - A timeout is the properly raised + // - but the JS runtime keeps executing `while(true)` and is no more responsive. + match tokio::time::timeout(self.execution_timeout, &mut receiver).await { + Ok(response) => response.map_err(|err| anyhow!(err)), + Err(_) => Err(anyhow!("Maximum processing time exceeded")), + } + } } enum JsRequest { @@ -229,8 +261,10 @@ impl<'js> JsModules<'js> { let err = anyhow::anyhow!("{ex}"); err.context("JS raised exception").into() } else { + let err = CaughtError::from_error(&ctx, err); debug!(target: "MAPPING", "execute({module_name}.{function}) => {err:?}"); - err.into() + let err = anyhow::anyhow!("{err}"); + err.context("JS runtime exception").into() } }) } diff --git a/crates/extensions/tedge_gen_mapper/src/js_script.rs b/crates/extensions/tedge_gen_mapper/src/js_script.rs index 6d249259395..3d7957783f4 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_script.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_script.rs @@ -220,7 +220,7 @@ impl<'js> IntoJs<'js> for &JsonValue { } } -impl<'a, 'js> IntoJs<'js> for JsonValueRef<'a> { +impl<'js> IntoJs<'js> for JsonValueRef<'_> { fn into_js(self, ctx: &Ctx<'js>) -> rquickjs::Result> { match self.0 { serde_json::Value::Null => Ok(Value::new_null(ctx.clone())), @@ -428,6 +428,53 @@ export function process (timestamp, message, config) { ); } + #[tokio::test] + #[ignore = "FIXME: scripts must be cancelled if running too long"] + async fn while_loop() { + let js = r#"export function process(t,msg) { while(true); };"#; + let (runtime, script) = runtime_with(js).await; + + let input = Message::new("topic", "payload"); + let error = script + .process(&runtime, &DateTime::now(), &input) + .await + .unwrap_err(); + eprintln!("{:?}", error); + assert!(error + .to_string() + .contains("Maximum processing time exceeded")); + } + + #[tokio::test] + async fn memory_eager_loop() { + let js = r#"export function process(t,msg) { var s = "foo"; while(true) { s += s; }; };"#; + let (runtime, script) = runtime_with(js).await; + + let input = Message::new("topic", "payload"); + let error = script + .process(&runtime, &DateTime::now(), &input) + .await + .unwrap_err(); + eprintln!("{:?}", error); + assert!(error.to_string().contains("out of memory")); + } + + #[tokio::test] + async fn stack_eager_loop() { + let js = r#"export function process(t,msg) { return process(t,msg); };"#; + let (runtime, script) = runtime_with(js).await; + + let input = Message::new("topic", "payload"); + let error = script + .process(&runtime, &DateTime::now(), &input) + .await + .unwrap_err(); + eprintln!("{:?}", error); + assert!(error + .to_string() + .contains("Maximum call stack size exceeded")); + } + async fn runtime_with(js: &str) -> (JsRuntime, JsScript) { let mut runtime = JsRuntime::try_new().await.unwrap(); let mut script = JsScript::new("toml".into(), 1, "js".into()); From 8ff31bfd5ecc30a3eec050fe6da67d74ab3ef353 Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 18 Jul 2025 15:15:42 +0200 Subject: [PATCH 52/53] Flow API: Use more idiomatic Javascript names - process -> onMessage - tick -> onInterval - updateConfig -> OnConfigUpdate Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/flows/add_timestamp.js | 2 +- .../tedge_gen_mapper/flows/average.js | 4 +- .../tedge_gen_mapper/flows/circuit-breaker.js | 2 +- .../tedge_gen_mapper/flows/collectd-to-te.js | 2 +- .../tedge_gen_mapper/flows/drop_stragglers.js | 2 +- .../flows/group_by_timestamp.js | 4 +- .../tedge_gen_mapper/flows/set_topic.js | 2 +- .../tedge_gen_mapper/flows/te_to_c8y.js | 4 +- .../extensions/tedge_gen_mapper/src/flow.rs | 16 ++-- .../tedge_gen_mapper/src/js_runtime.rs | 8 +- .../tedge_gen_mapper/src/js_script.rs | 86 +++++++++---------- docs/src/references/mappers/gen-mapper.md | 18 ++-- .../tedge_gen_mapper/flows/add_timestamp.js | 2 +- .../tests/tedge_gen_mapper/flows/average.js | 4 +- .../tedge_gen_mapper/flows/count-messages.js | 4 +- .../tests/tedge_gen_mapper/flows/set_topic.js | 2 +- .../tests/tedge_gen_mapper/flows/te_to_c8y.js | 4 +- 17 files changed, 83 insertions(+), 83 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/flows/add_timestamp.js b/crates/extensions/tedge_gen_mapper/flows/add_timestamp.js index f387141bfe3..1a5f1f567e4 100644 --- a/crates/extensions/tedge_gen_mapper/flows/add_timestamp.js +++ b/crates/extensions/tedge_gen_mapper/flows/add_timestamp.js @@ -1,4 +1,4 @@ -export function process (timestamp, message) { +export function onMessage (timestamp, message) { let payload = JSON.parse(message.payload) if (!payload.time) { payload.time = timestamp.seconds + (timestamp.nanoseconds / 1e9) diff --git a/crates/extensions/tedge_gen_mapper/flows/average.js b/crates/extensions/tedge_gen_mapper/flows/average.js index c9fcd1f4cf8..ea6d69acd01 100644 --- a/crates/extensions/tedge_gen_mapper/flows/average.js +++ b/crates/extensions/tedge_gen_mapper/flows/average.js @@ -7,7 +7,7 @@ class State { static agg_for_topic = {} } -export function process (timestamp, message) { +export function onMessage (timestamp, message) { let topic = message.topic let payload = JSON.parse(message.payload) let agg_payload = State.agg_for_topic[topic] @@ -74,7 +74,7 @@ export function process (timestamp, message) { return [] } -export function tick() { +export function onInterval() { let messages = [] for (let [topic, agg] of Object.entries(State.agg_for_topic)) { diff --git a/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js b/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js index 36dc8a7932a..5fc8b9043d8 100644 --- a/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js +++ b/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js @@ -15,7 +15,7 @@ class State { } -export function process (timestamp, message, config) { +export function onMessage (timestamp, message, config) { State.total += 1 State.batch[0] += 1 if (State.open) { diff --git a/crates/extensions/tedge_gen_mapper/flows/collectd-to-te.js b/crates/extensions/tedge_gen_mapper/flows/collectd-to-te.js index 5600155a0c5..11cf06cc25f 100644 --- a/crates/extensions/tedge_gen_mapper/flows/collectd-to-te.js +++ b/crates/extensions/tedge_gen_mapper/flows/collectd-to-te.js @@ -1,4 +1,4 @@ -export function process(_timestamp, message, config) { +export function onMessage(_timestamp, message, config) { let groups = message.topic.split('/') let data = message.payload.split(':') diff --git a/crates/extensions/tedge_gen_mapper/flows/drop_stragglers.js b/crates/extensions/tedge_gen_mapper/flows/drop_stragglers.js index b663ac5959c..06770ea30cf 100644 --- a/crates/extensions/tedge_gen_mapper/flows/drop_stragglers.js +++ b/crates/extensions/tedge_gen_mapper/flows/drop_stragglers.js @@ -1,5 +1,5 @@ // Reject any message that is too old, too new or with no timestamp -export function process (timestamp, message, config) { +export function onMessage (timestamp, message, config) { let payload = JSON.parse(message.payload) let msg_time = payload.time if (!msg_time) { diff --git a/crates/extensions/tedge_gen_mapper/flows/group_by_timestamp.js b/crates/extensions/tedge_gen_mapper/flows/group_by_timestamp.js index 77b061ee1e5..f93e3a4da36 100644 --- a/crates/extensions/tedge_gen_mapper/flows/group_by_timestamp.js +++ b/crates/extensions/tedge_gen_mapper/flows/group_by_timestamp.js @@ -2,12 +2,12 @@ class State { static batch = [] } -export function process (timestamp, message) { +export function onMessage (timestamp, message) { State.batch.push(message) return [] } -export function tick() { +export function onInterval() { let batch = State.batch State.batch = [] return batch diff --git a/crates/extensions/tedge_gen_mapper/flows/set_topic.js b/crates/extensions/tedge_gen_mapper/flows/set_topic.js index 1e28b490f17..302455825bc 100644 --- a/crates/extensions/tedge_gen_mapper/flows/set_topic.js +++ b/crates/extensions/tedge_gen_mapper/flows/set_topic.js @@ -1,4 +1,4 @@ -export function process (timestamp, message, config) { +export function onMessage (timestamp, message, config) { return [{ topic: config.topic || "te/error", payload: message.payload diff --git a/crates/extensions/tedge_gen_mapper/flows/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/flows/te_to_c8y.js index 5c1e6c47fa9..e57f26fd132 100644 --- a/crates/extensions/tedge_gen_mapper/flows/te_to_c8y.js +++ b/crates/extensions/tedge_gen_mapper/flows/te_to_c8y.js @@ -42,7 +42,7 @@ /// } /// } /// ``` -export function process(t, message, config) { +export function onMessage(t, message, config) { let topic_parts = message.topic.split( '/') let type = topic_parts[6] || "ThinEdgeMeasurement" let payload = JSON.parse(message.payload) @@ -116,7 +116,7 @@ export function process(t, message, config) { /// } /// } /// ``` -export function update_config(message, config) { +export function onConfigUpdate(message, config) { let type = message.topic let metadata = JSON.parse(message.payload) diff --git a/crates/extensions/tedge_gen_mapper/src/flow.rs b/crates/extensions/tedge_gen_mapper/src/flow.rs index ae4560f38cd..1920aa80dfb 100644 --- a/crates/extensions/tedge_gen_mapper/src/flow.rs +++ b/crates/extensions/tedge_gen_mapper/src/flow.rs @@ -68,7 +68,7 @@ impl Flow { ) -> Result<(), FlowError> { for step in self.steps.iter_mut() { if step.config_topics.accept_topic_name(&message.topic) { - step.script.update_config(js_runtime, message).await? + step.script.on_config_update(js_runtime, message).await? } } Ok(()) @@ -93,7 +93,7 @@ impl Flow { let mut transformed_messages = vec![]; for message in messages.iter() { let step_started_at = stats.flow_step_start(&js, "process"); - let step_output = step.script.process(js_runtime, timestamp, message).await; + let step_output = step.script.on_message(js_runtime, timestamp, message).await; match &step_output { Ok(messages) => { stats.flow_step_done(&js, "process", step_started_at, messages.len()) @@ -123,7 +123,7 @@ impl Flow { let mut transformed_messages = vec![]; for message in messages.iter() { let step_started_at = stats.flow_step_start(&js, "process"); - let step_output = step.script.process(js_runtime, timestamp, message).await; + let step_output = step.script.on_message(js_runtime, timestamp, message).await; match &step_output { Ok(messages) => { stats.flow_step_done(&js, "process", step_started_at, messages.len()) @@ -135,7 +135,7 @@ impl Flow { // Only then process the tick let step_started_at = stats.flow_step_start(&js, "tick"); - let tick_output = step.script.tick(js_runtime, timestamp).await; + let tick_output = step.script.on_interval(js_runtime, timestamp).await; match &tick_output { Ok(messages) => stats.flow_step_done(&js, "tick", step_started_at, messages.len()), Err(_) => stats.flow_step_failed(&js, "tick"), @@ -153,20 +153,20 @@ impl Flow { impl FlowStep { pub(crate) fn check(&self, flow: &Utf8Path) { let script = &self.script; - if script.no_js_process { + if script.no_js_on_message_fun { warn!(target: "MAPPING", "Flow script with no 'process' function: {}", script.path.display()); } - if script.no_js_update_config && !self.config_topics.is_empty() { + if script.no_js_on_config_update_fun && !self.config_topics.is_empty() { warn!(target: "MAPPING", "Flow script with no 'config_update' function: {}; but configured with 'config_topics' in {flow}", script.path.display()); } - if script.no_js_tick && script.tick_every_seconds != 0 { + if script.no_js_on_interval_fun && script.tick_every_seconds != 0 { warn!(target: "MAPPING", "Flow script with no 'tick' function: {}; but configured with 'tick_every_seconds' in {flow}", script.path.display()); } } pub(crate) fn fix(&mut self) { let script = &mut self.script; - if !script.no_js_tick && script.tick_every_seconds == 0 { + if !script.no_js_on_interval_fun && script.tick_every_seconds == 0 { // 0 as a default is not appropriate for a script with a tick handler script.tick_every_seconds = 1; } diff --git a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs index bccf70a99e7..329136eb405 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_runtime.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_runtime.rs @@ -38,9 +38,9 @@ impl JsRuntime { let exports = self.load_file(script.module_name(), script.path()).await?; for export in exports { match export { - "process" => script.no_js_process = false, - "update_config" => script.no_js_update_config = false, - "tick" => script.no_js_tick = false, + "onMessage" => script.no_js_on_message_fun = false, + "onConfigUpdate" => script.no_js_on_config_update_fun = false, + "onInterval" => script.no_js_on_interval_fun = false, _ => (), } } @@ -64,7 +64,7 @@ impl JsRuntime { ) -> Result, LoadError> { let (sender, receiver) = oneshot::channel(); let source = source.into(); - let imports = vec!["process", "update_config", "tick"]; + let imports = vec!["onMessage", "onConfigUpdate", "onInterval"]; self.send( receiver, JsRequest::LoadModule { diff --git a/crates/extensions/tedge_gen_mapper/src/js_script.rs b/crates/extensions/tedge_gen_mapper/src/js_script.rs index 3d7957783f4..35dec5f036b 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_script.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_script.rs @@ -18,9 +18,9 @@ pub struct JsScript { pub path: PathBuf, pub config: JsonValue, pub tick_every_seconds: u64, - pub no_js_process: bool, - pub no_js_update_config: bool, - pub no_js_tick: bool, + pub no_js_on_message_fun: bool, + pub no_js_on_config_update_fun: bool, + pub no_js_on_interval_fun: bool, } #[derive(Clone, Debug)] @@ -40,9 +40,9 @@ impl JsScript { path, config: JsonValue::default(), tick_every_seconds: 0, - no_js_process: true, - no_js_update_config: true, - no_js_tick: true, + no_js_on_message_fun: true, + no_js_on_config_update_fun: true, + no_js_on_interval_fun: true, } } @@ -76,22 +76,22 @@ impl JsScript { format!("{}", self.path.display()) } - /// Process a message returning zero, one or more messages + /// Transform an input message into zero, one or more output messages /// - /// The "process" function of the JS module is passed 3 arguments + /// The "onMessage" function of the JS module is passed 3 arguments /// - the current timestamp /// - the message to be transformed - /// - the flow step config (as configured for the flow step, possibly updated by update_config messages) + /// - the flow step config (as configured for the flow step, possibly updated by onConfigUpdate messages) /// /// The returned value is expected to be an array of messages. - pub async fn process( + pub async fn on_message( &self, js: &JsRuntime, timestamp: &DateTime, message: &Message, ) -> Result, FlowError> { - debug!(target: "MAPPING", "{}: process({timestamp:?}, {message:?})", self.module_name()); - if self.no_js_process { + debug!(target: "MAPPING", "{}: onMessage({timestamp:?}, {message:?})", self.module_name()); + if self.no_js_on_message_fun { return Ok(vec![message.clone()]); } @@ -100,7 +100,7 @@ impl JsScript { message.clone().into(), self.config.clone(), ]; - js.call_function(&self.module_name(), "process", input) + js.call_function(&self.module_name(), "onMessage", input) .await .map_err(flow::error_from_js)? .try_into() @@ -108,51 +108,51 @@ impl JsScript { /// Update the flow step config using a metadata message /// - /// The "update_config" function of the JS module is passed 2 arguments + /// The "onConfigUpdate" function of the JS module is passed 2 arguments /// - the message /// - the current flow step config /// /// The value returned by this function is used as the updated flow step config - pub async fn update_config( + pub async fn on_config_update( &mut self, js: &JsRuntime, message: &Message, ) -> Result<(), FlowError> { - debug!(target: "MAPPING", "{}: update_config({message:?})", self.module_name()); - if self.no_js_update_config { + debug!(target: "MAPPING", "{}: onConfigUpdate({message:?})", self.module_name()); + if self.no_js_on_config_update_fun { return Ok(()); } let input = vec![message.clone().into(), self.config.clone()]; let config = js - .call_function(&self.module_name(), "update_config", input) + .call_function(&self.module_name(), "onConfigUpdate", input) .await .map_err(flow::error_from_js)?; self.config = config; Ok(()) } - /// Trigger the tick function of the JS module + /// Trigger the onInterval function of the JS module /// - /// The "tick" function is passed 2 arguments + /// The "onInterval" function is passed 2 arguments /// - the current timestamp /// - the current flow step config /// /// Return zero, one or more messages - pub async fn tick( + pub async fn on_interval( &self, js: &JsRuntime, timestamp: &DateTime, ) -> Result, FlowError> { - if self.no_js_tick { + if self.no_js_on_interval_fun { return Ok(vec![]); } if !timestamp.tick_now(self.tick_every_seconds) { return Ok(vec![]); } - debug!(target: "MAPPING", "{}: tick({timestamp:?})", self.module_name()); + debug!(target: "MAPPING", "{}: onInterval({timestamp:?})", self.module_name()); let input = vec![timestamp.clone().into(), self.config.clone()]; - js.call_function(&self.module_name(), "tick", input) + js.call_function(&self.module_name(), "onInterval", input) .await .map_err(flow::error_from_js)? .try_into() @@ -312,14 +312,14 @@ mod tests { #[tokio::test] async fn identity_script() { - let js = "export function process(t,msg) { return [msg]; };"; + let js = "export function onMessage(t,msg) { return [msg]; };"; let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); let output = input.clone(); assert_eq!( script - .process(&runtime, &DateTime::now(), &input) + .on_message(&runtime, &DateTime::now(), &input) .await .unwrap(), vec![output] @@ -328,14 +328,14 @@ mod tests { #[tokio::test] async fn identity_script_no_array() { - let js = "export function process(t,msg) { return msg; };"; + let js = "export function onMessage(t,msg) { return msg; };"; let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); let output = input.clone(); assert_eq!( script - .process(&runtime, &DateTime::now(), &input) + .on_message(&runtime, &DateTime::now(), &input) .await .unwrap(), vec![output] @@ -344,13 +344,13 @@ mod tests { #[tokio::test] async fn script_returning_null() { - let js = "export function process(t,msg) { return null; };"; + let js = "export function onMessage(t,msg) { return null; };"; let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); assert_eq!( script - .process(&runtime, &DateTime::now(), &input) + .on_message(&runtime, &DateTime::now(), &input) .await .unwrap(), vec![] @@ -359,13 +359,13 @@ mod tests { #[tokio::test] async fn script_returning_nothing() { - let js = "export function process(t,msg) { return; };"; + let js = "export function onMessage(t,msg) { return; };"; let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); assert_eq!( script - .process(&runtime, &DateTime::now(), &input) + .on_message(&runtime, &DateTime::now(), &input) .await .unwrap(), vec![] @@ -374,12 +374,12 @@ mod tests { #[tokio::test] async fn error_script() { - let js = r#"export function process(t,msg) { throw new Error("Cannot process that message"); };"#; + let js = r#"export function onMessage(t,msg) { throw new Error("Cannot process that message"); };"#; let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); let error = script - .process(&runtime, &DateTime::now(), &input) + .on_message(&runtime, &DateTime::now(), &input) .await .unwrap_err(); eprintln!("{:?}", error); @@ -389,7 +389,7 @@ mod tests { #[tokio::test] async fn collectd_script() { let js = r#" -export function process (timestamp, message, config) { +export function onMessage(timestamp, message, config) { let groups = message.topic.split( '/') let data = message.payload.split(':') @@ -421,7 +421,7 @@ export function process (timestamp, message, config) { ); assert_eq!( script - .process(&runtime, &DateTime::now(), &input) + .on_message(&runtime, &DateTime::now(), &input) .await .unwrap(), vec![output] @@ -431,12 +431,12 @@ export function process (timestamp, message, config) { #[tokio::test] #[ignore = "FIXME: scripts must be cancelled if running too long"] async fn while_loop() { - let js = r#"export function process(t,msg) { while(true); };"#; + let js = r#"export function onMessage(t,msg) { while(true); };"#; let (runtime, script) = runtime_with(js).await; let input = Message::new("topic", "payload"); let error = script - .process(&runtime, &DateTime::now(), &input) + .on_message(&runtime, &DateTime::now(), &input) .await .unwrap_err(); eprintln!("{:?}", error); @@ -447,12 +447,12 @@ export function process (timestamp, message, config) { #[tokio::test] async fn memory_eager_loop() { - let js = r#"export function process(t,msg) { var s = "foo"; while(true) { s += s; }; };"#; + let js = r#"export function onMessage(t,msg) { var s = "foo"; while(true) { s += s; }; };"#; let (runtime, script) = runtime_with(js).await; let input = Message::new("topic", "payload"); let error = script - .process(&runtime, &DateTime::now(), &input) + .on_message(&runtime, &DateTime::now(), &input) .await .unwrap_err(); eprintln!("{:?}", error); @@ -461,12 +461,12 @@ export function process (timestamp, message, config) { #[tokio::test] async fn stack_eager_loop() { - let js = r#"export function process(t,msg) { return process(t,msg); };"#; + let js = r#"export function onMessage(t,msg) { return onMessage(t,msg); };"#; let (runtime, script) = runtime_with(js).await; let input = Message::new("topic", "payload"); let error = script - .process(&runtime, &DateTime::now(), &input) + .on_message(&runtime, &DateTime::now(), &input) .await .unwrap_err(); eprintln!("{:?}", error); @@ -479,7 +479,7 @@ export function process (timestamp, message, config) { let mut runtime = JsRuntime::try_new().await.unwrap(); let mut script = JsScript::new("toml".into(), 1, "js".into()); runtime.load_js(script.module_name(), js).await.unwrap(); - script.no_js_process = false; + script.no_js_on_message_fun = false; (runtime, script) } } diff --git a/docs/src/references/mappers/gen-mapper.md b/docs/src/references/mappers/gen-mapper.md index d2c31210598..02ef988daed 100644 --- a/docs/src/references/mappers/gen-mapper.md +++ b/docs/src/references/mappers/gen-mapper.md @@ -55,8 +55,8 @@ which rule how to consume, transform and produce MQTT messages. A transformation *scripts* is a JavaScript or TypeScript module that exports: -- at least a function `on_message`, aimed to transform one input message into zero, one or more output messages -- possibly a function `on_interval`, called at regular intervals to produce aggregated messages. +- at least a function `onMessage`, aimed to transform one input message into zero, one or more output messages +- possibly a function `onInterval`, called at regular intervals to produce aggregated messages. @@ -85,8 +85,8 @@ steps = [ ## POC API -- A flow script has to export at least one `process` function. - - `process(t: Timestamp, msg: Message, config: Json) -> Vec` +- A flow script has to export at least one `onMessage` function. + - `onMessage(t: Timestamp, msg: Message, config: Json) -> Vec` - This function is called for each message to be transformed - The arguments passed to the function are: - The current time as `{ seconds: u64, nanoseconds: u32 }` @@ -94,16 +94,16 @@ steps = [ - The config as read from the flow config or updated by the script - The function is expected to return zero, one or many transformed messages `[{ topic: string, payload: string }]` - An exception can be thrown if the input message cannot be transformed. -- A flow script can also export an `update_config` function +- A flow script can also export an `onConfigUpdate` function - This function is called on each message received on the `meta_topics` as defined in the config. - The arguments are: - The message to be interpreted as a config update `{ topic: string, payload: string }` - The current config - The returned value (an arbitrary JSON value) is then used as the new config for the flow script. -- A flow script can also export a `tick` function +- A flow script can also export a `onInterval` function - This function is called at a regular pace with the current time and config. - The flow script can then return zero, one or many transformed messages - - By sharing an internal state between the `process` and `tick` functions, + - By sharing an internal state between the `onMessage` and `onInterval` functions, the flow script can implement aggregations over a time window. - When messages are received they are pushed by the `process` function into that state - and the final outcome is extracted by the `tick` function at the end of the time window. + When messages are received they are pushed by the `onMessage` function into that state + and the final outcome is extracted by the `onInterval` function at the end of the time window. diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/add_timestamp.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/add_timestamp.js index f387141bfe3..1a5f1f567e4 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/add_timestamp.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/add_timestamp.js @@ -1,4 +1,4 @@ -export function process (timestamp, message) { +export function onMessage (timestamp, message) { let payload = JSON.parse(message.payload) if (!payload.time) { payload.time = timestamp.seconds + (timestamp.nanoseconds / 1e9) diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/average.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/average.js index c9fcd1f4cf8..ea6d69acd01 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/average.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/average.js @@ -7,7 +7,7 @@ class State { static agg_for_topic = {} } -export function process (timestamp, message) { +export function onMessage (timestamp, message) { let topic = message.topic let payload = JSON.parse(message.payload) let agg_payload = State.agg_for_topic[topic] @@ -74,7 +74,7 @@ export function process (timestamp, message) { return [] } -export function tick() { +export function onInterval() { let messages = [] for (let [topic, agg] of Object.entries(State.agg_for_topic)) { diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.js index 0c4e1bba81c..6f6a8784e76 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.js @@ -2,7 +2,7 @@ class State { static count_per_topic = {} } -export function process (timestamp, message) { +export function onMessage (timestamp, message) { let topic = message.topic let count = State.count_per_topic[topic] || 0 State.count_per_topic[topic] = count + 1 @@ -11,7 +11,7 @@ export function process (timestamp, message) { return [] } -export function tick(timestamp, config) { +export function onInterval(timestamp, config) { let message = { topic: config.topic || "te/error", payload: JSON.stringify(State.count_per_topic) diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/set_topic.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/set_topic.js index 1e28b490f17..302455825bc 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/set_topic.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/set_topic.js @@ -1,4 +1,4 @@ -export function process (timestamp, message, config) { +export function onMessage (timestamp, message, config) { return [{ topic: config.topic || "te/error", payload: message.payload diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/te_to_c8y.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/te_to_c8y.js index 5c1e6c47fa9..e57f26fd132 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/te_to_c8y.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/te_to_c8y.js @@ -42,7 +42,7 @@ /// } /// } /// ``` -export function process(t, message, config) { +export function onMessage(t, message, config) { let topic_parts = message.topic.split( '/') let type = topic_parts[6] || "ThinEdgeMeasurement" let payload = JSON.parse(message.payload) @@ -116,7 +116,7 @@ export function process(t, message, config) { /// } /// } /// ``` -export function update_config(message, config) { +export function onConfigUpdate(message, config) { let type = message.topic let metadata = JSON.parse(message.payload) From a45942d574c2881da121f20dcbc3e17387dc22ed Mon Sep 17 00:00:00 2001 From: Didier Wenzek Date: Fri, 18 Jul 2025 15:54:55 +0200 Subject: [PATCH 53/53] Pass processing time along the message Add timestamp to message object Signed-off-by: Didier Wenzek --- .../tedge_gen_mapper/flows/add_timestamp.js | 8 ++--- .../tedge_gen_mapper/flows/average.js | 2 +- .../tedge_gen_mapper/flows/circuit-breaker.js | 2 +- .../tedge_gen_mapper/flows/collectd-to-te.js | 2 +- .../tedge_gen_mapper/flows/drop_stragglers.js | 3 +- .../flows/group_by_timestamp.js | 2 +- .../tedge_gen_mapper/flows/set_topic.js | 2 +- .../tedge_gen_mapper/flows/te_to_c8y.js | 2 +- .../extensions/tedge_gen_mapper/src/flow.rs | 14 +++++++-- .../tedge_gen_mapper/src/js_script.rs | 31 ++++++++++--------- docs/src/references/mappers/gen-mapper.md | 5 ++- .../tedge_gen_mapper/flows/add_timestamp.js | 3 +- .../tests/tedge_gen_mapper/flows/average.js | 2 +- .../tedge_gen_mapper/flows/count-messages.js | 2 +- .../tests/tedge_gen_mapper/flows/set_topic.js | 2 +- .../tests/tedge_gen_mapper/flows/te_to_c8y.js | 2 +- 16 files changed, 48 insertions(+), 36 deletions(-) diff --git a/crates/extensions/tedge_gen_mapper/flows/add_timestamp.js b/crates/extensions/tedge_gen_mapper/flows/add_timestamp.js index 1a5f1f567e4..bb674f4f04c 100644 --- a/crates/extensions/tedge_gen_mapper/flows/add_timestamp.js +++ b/crates/extensions/tedge_gen_mapper/flows/add_timestamp.js @@ -1,11 +1,11 @@ -export function onMessage (timestamp, message) { +export function onMessage (message) { let payload = JSON.parse(message.payload) if (!payload.time) { - payload.time = timestamp.seconds + (timestamp.nanoseconds / 1e9) + payload.time = message.timestamp.seconds + (message.timestamp.nanoseconds / 1e9) } - return [{ + return { topic: message.topic, payload: JSON.stringify(payload) - }] + } } diff --git a/crates/extensions/tedge_gen_mapper/flows/average.js b/crates/extensions/tedge_gen_mapper/flows/average.js index ea6d69acd01..ff9d177f9d3 100644 --- a/crates/extensions/tedge_gen_mapper/flows/average.js +++ b/crates/extensions/tedge_gen_mapper/flows/average.js @@ -7,7 +7,7 @@ class State { static agg_for_topic = {} } -export function onMessage (timestamp, message) { +export function onMessage (message) { let topic = message.topic let payload = JSON.parse(message.payload) let agg_payload = State.agg_for_topic[topic] diff --git a/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js b/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js index 5fc8b9043d8..02eb96d0d80 100644 --- a/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js +++ b/crates/extensions/tedge_gen_mapper/flows/circuit-breaker.js @@ -15,7 +15,7 @@ class State { } -export function onMessage (timestamp, message, config) { +export function onMessage (message, config) { State.total += 1 State.batch[0] += 1 if (State.open) { diff --git a/crates/extensions/tedge_gen_mapper/flows/collectd-to-te.js b/crates/extensions/tedge_gen_mapper/flows/collectd-to-te.js index 11cf06cc25f..50568c15c08 100644 --- a/crates/extensions/tedge_gen_mapper/flows/collectd-to-te.js +++ b/crates/extensions/tedge_gen_mapper/flows/collectd-to-te.js @@ -1,4 +1,4 @@ -export function onMessage(_timestamp, message, config) { +export function onMessage(message, config) { let groups = message.topic.split('/') let data = message.payload.split(':') diff --git a/crates/extensions/tedge_gen_mapper/flows/drop_stragglers.js b/crates/extensions/tedge_gen_mapper/flows/drop_stragglers.js index 06770ea30cf..9ee746ec1f1 100644 --- a/crates/extensions/tedge_gen_mapper/flows/drop_stragglers.js +++ b/crates/extensions/tedge_gen_mapper/flows/drop_stragglers.js @@ -1,5 +1,5 @@ // Reject any message that is too old, too new or with no timestamp -export function onMessage (timestamp, message, config) { +export function onMessage (message, config) { let payload = JSON.parse(message.payload) let msg_time = payload.time if (!msg_time) { @@ -11,6 +11,7 @@ export function onMessage (timestamp, message, config) { msg_timestamp = Date.parse(msg_time) / 1e3 } + let timestamp = message.timestamp let time = timestamp.seconds + (timestamp.nanoseconds / 1e9) let max = time + (config.max_advance || 1); let min = time - (config.max_delay || 10); diff --git a/crates/extensions/tedge_gen_mapper/flows/group_by_timestamp.js b/crates/extensions/tedge_gen_mapper/flows/group_by_timestamp.js index f93e3a4da36..7337ce5bf4e 100644 --- a/crates/extensions/tedge_gen_mapper/flows/group_by_timestamp.js +++ b/crates/extensions/tedge_gen_mapper/flows/group_by_timestamp.js @@ -2,7 +2,7 @@ class State { static batch = [] } -export function onMessage (timestamp, message) { +export function onMessage (message) { State.batch.push(message) return [] } diff --git a/crates/extensions/tedge_gen_mapper/flows/set_topic.js b/crates/extensions/tedge_gen_mapper/flows/set_topic.js index 302455825bc..c02654b7453 100644 --- a/crates/extensions/tedge_gen_mapper/flows/set_topic.js +++ b/crates/extensions/tedge_gen_mapper/flows/set_topic.js @@ -1,4 +1,4 @@ -export function onMessage (timestamp, message, config) { +export function onMessage (message, config) { return [{ topic: config.topic || "te/error", payload: message.payload diff --git a/crates/extensions/tedge_gen_mapper/flows/te_to_c8y.js b/crates/extensions/tedge_gen_mapper/flows/te_to_c8y.js index e57f26fd132..21679d5f9b1 100644 --- a/crates/extensions/tedge_gen_mapper/flows/te_to_c8y.js +++ b/crates/extensions/tedge_gen_mapper/flows/te_to_c8y.js @@ -42,7 +42,7 @@ /// } /// } /// ``` -export function onMessage(t, message, config) { +export function onMessage(message, config) { let topic_parts = message.topic.split( '/') let type = topic_parts[6] || "ThinEdgeMeasurement" let payload = JSON.parse(message.payload) diff --git a/crates/extensions/tedge_gen_mapper/src/flow.rs b/crates/extensions/tedge_gen_mapper/src/flow.rs index 1920aa80dfb..5bba59ade7f 100644 --- a/crates/extensions/tedge_gen_mapper/src/flow.rs +++ b/crates/extensions/tedge_gen_mapper/src/flow.rs @@ -38,6 +38,7 @@ pub struct DateTime { pub struct Message { pub topic: String, pub payload: String, + pub timestamp: Option, } #[derive(thiserror::Error, Debug)] @@ -208,11 +209,16 @@ impl Message { Message { topic: topic.to_string(), payload: payload.to_string(), + timestamp: Some(DateTime::now()), } } pub fn json(&self) -> Value { - json!({"topic": self.topic, "payload": self.payload}) + if let Some(timestamp) = &self.timestamp { + json!({"topic": self.topic, "payload": self.payload, "timestamp": timestamp.json()}) + } else { + json!({"topic": self.topic, "payload": self.payload, "timestamp": null}) + } } } @@ -225,7 +231,11 @@ impl TryFrom for Message { .payload_str() .map_err(|_| FlowError::UnsupportedMessage("Not an UTF8 payload".to_string()))? .to_string(); - Ok(Message { topic, payload }) + Ok(Message { + topic, + payload, + timestamp: None, + }) } } diff --git a/crates/extensions/tedge_gen_mapper/src/js_script.rs b/crates/extensions/tedge_gen_mapper/src/js_script.rs index 35dec5f036b..af90a867b78 100644 --- a/crates/extensions/tedge_gen_mapper/src/js_script.rs +++ b/crates/extensions/tedge_gen_mapper/src/js_script.rs @@ -95,11 +95,11 @@ impl JsScript { return Ok(vec![message.clone()]); } - let input = vec![ - timestamp.clone().into(), - message.clone().into(), - self.config.clone(), - ]; + let mut message = message.clone(); + if message.timestamp.is_none() { + message.timestamp = Some(timestamp.clone()); + } + let input = vec![message.into(), self.config.clone()]; js.call_function(&self.module_name(), "onMessage", input) .await .map_err(flow::error_from_js)? @@ -312,7 +312,7 @@ mod tests { #[tokio::test] async fn identity_script() { - let js = "export function onMessage(t,msg) { return [msg]; };"; + let js = "export function onMessage(msg) { return [msg]; };"; let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); @@ -328,7 +328,7 @@ mod tests { #[tokio::test] async fn identity_script_no_array() { - let js = "export function onMessage(t,msg) { return msg; };"; + let js = "export function onMessage(msg) { return msg; };"; let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); @@ -344,7 +344,7 @@ mod tests { #[tokio::test] async fn script_returning_null() { - let js = "export function onMessage(t,msg) { return null; };"; + let js = "export function onMessage(msg) { return null; };"; let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); @@ -359,7 +359,7 @@ mod tests { #[tokio::test] async fn script_returning_nothing() { - let js = "export function onMessage(t,msg) { return; };"; + let js = "export function onMessage(msg) { return; };"; let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); @@ -374,7 +374,7 @@ mod tests { #[tokio::test] async fn error_script() { - let js = r#"export function onMessage(t,msg) { throw new Error("Cannot process that message"); };"#; + let js = r#"export function onMessage(msg) { throw new Error("Cannot process that message"); };"#; let (runtime, script) = runtime_with(js).await; let input = Message::new("te/main/device///m/", "hello world"); @@ -389,7 +389,7 @@ mod tests { #[tokio::test] async fn collectd_script() { let js = r#" -export function onMessage(timestamp, message, config) { +export function onMessage(message, config) { let groups = message.topic.split( '/') let data = message.payload.split(':') @@ -415,10 +415,11 @@ export function onMessage(timestamp, message, config) { "collectd/h/memory/percent-used", "1748440192.104:19.9289468288182", ); - let output = Message::new( + let mut output = Message::new( "te/device/main///m/collectd", r#"{"time": 1748440192.104, "memory": {"percent-used": 19.9289468288182}}"#, ); + output.timestamp = None; assert_eq!( script .on_message(&runtime, &DateTime::now(), &input) @@ -431,7 +432,7 @@ export function onMessage(timestamp, message, config) { #[tokio::test] #[ignore = "FIXME: scripts must be cancelled if running too long"] async fn while_loop() { - let js = r#"export function onMessage(t,msg) { while(true); };"#; + let js = r#"export function onMessage(msg) { while(true); };"#; let (runtime, script) = runtime_with(js).await; let input = Message::new("topic", "payload"); @@ -447,7 +448,7 @@ export function onMessage(timestamp, message, config) { #[tokio::test] async fn memory_eager_loop() { - let js = r#"export function onMessage(t,msg) { var s = "foo"; while(true) { s += s; }; };"#; + let js = r#"export function onMessage(msg) { var s = "foo"; while(true) { s += s; }; };"#; let (runtime, script) = runtime_with(js).await; let input = Message::new("topic", "payload"); @@ -461,7 +462,7 @@ export function onMessage(timestamp, message, config) { #[tokio::test] async fn stack_eager_loop() { - let js = r#"export function onMessage(t,msg) { return onMessage(t,msg); };"#; + let js = r#"export function onMessage(msg) { return onMessage(msg); };"#; let (runtime, script) = runtime_with(js).await; let input = Message::new("topic", "payload"); diff --git a/docs/src/references/mappers/gen-mapper.md b/docs/src/references/mappers/gen-mapper.md index 02ef988daed..6bea006c991 100644 --- a/docs/src/references/mappers/gen-mapper.md +++ b/docs/src/references/mappers/gen-mapper.md @@ -86,11 +86,10 @@ steps = [ ## POC API - A flow script has to export at least one `onMessage` function. - - `onMessage(t: Timestamp, msg: Message, config: Json) -> Vec` + - `onMessage(msg: Message, config: Json) -> Vec` - This function is called for each message to be transformed - The arguments passed to the function are: - - The current time as `{ seconds: u64, nanoseconds: u32 }` - - The message `{ topic: string, payload: string }` + - The message `{ topic: string, payload: string, timestamp: { seconds: u64, nanoseconds: u32 } }` - The config as read from the flow config or updated by the script - The function is expected to return zero, one or many transformed messages `[{ topic: string, payload: string }]` - An exception can be thrown if the input message cannot be transformed. diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/add_timestamp.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/add_timestamp.js index 1a5f1f567e4..d0ef32bb702 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/add_timestamp.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/add_timestamp.js @@ -1,6 +1,7 @@ -export function onMessage (timestamp, message) { +export function onMessage (message) { let payload = JSON.parse(message.payload) if (!payload.time) { + let timestamp = message.timestamp payload.time = timestamp.seconds + (timestamp.nanoseconds / 1e9) } diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/average.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/average.js index ea6d69acd01..ff9d177f9d3 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/average.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/average.js @@ -7,7 +7,7 @@ class State { static agg_for_topic = {} } -export function onMessage (timestamp, message) { +export function onMessage (message) { let topic = message.topic let payload = JSON.parse(message.payload) let agg_payload = State.agg_for_topic[topic] diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.js index 6f6a8784e76..ba4d84e9cdf 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/count-messages.js @@ -2,7 +2,7 @@ class State { static count_per_topic = {} } -export function onMessage (timestamp, message) { +export function onMessage (message) { let topic = message.topic let count = State.count_per_topic[topic] || 0 State.count_per_topic[topic] = count + 1 diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/set_topic.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/set_topic.js index 302455825bc..c02654b7453 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/set_topic.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/set_topic.js @@ -1,4 +1,4 @@ -export function onMessage (timestamp, message, config) { +export function onMessage (message, config) { return [{ topic: config.topic || "te/error", payload: message.payload diff --git a/tests/RobotFramework/tests/tedge_gen_mapper/flows/te_to_c8y.js b/tests/RobotFramework/tests/tedge_gen_mapper/flows/te_to_c8y.js index e57f26fd132..21679d5f9b1 100644 --- a/tests/RobotFramework/tests/tedge_gen_mapper/flows/te_to_c8y.js +++ b/tests/RobotFramework/tests/tedge_gen_mapper/flows/te_to_c8y.js @@ -42,7 +42,7 @@ /// } /// } /// ``` -export function onMessage(t, message, config) { +export function onMessage(message, config) { let topic_parts = message.topic.split( '/') let type = topic_parts[6] || "ThinEdgeMeasurement" let payload = JSON.parse(message.payload)