feat(logging): use separate file per spawned node

ant-logging/Cargo.toml

@@ -21,7 +21,7 @@ serde = { version = "1.0.133", features = ["derive", "rc"] }
 serde_json = { version = "1.0" }
 sysinfo = { version = "0.30.8", default-features = false, optional = true }
 thiserror = "1.0.23"
-tokio = { version = "1.43.1", optional = true }
+tokio = { version = "1.43.1", features = ["rt", "rt-multi-thread", "macros", "time"], optional = true }
 tracing = { version = "~0.1.26" }
 tracing-appender = "~0.2.0"
 tracing-core = "0.1.30"
@@ -31,6 +31,8 @@ tracing-subscriber = { version = "0.3.16", features = ["json"] }
 [dev-dependencies]
 color-eyre = "0.6.3"
 tracing-test = "0.2.4"
+tokio = { version = "1.43.1", features = ["rt", "macros", "rt-multi-thread", "time"] }
+tempfile = "3.8"
 
 [features]
 otlp = [

ant-logging/src/layers.rs

@@ -28,9 +28,9 @@ use tracing_subscriber::{
     Layer, Registry,
 };
 
-const MAX_LOG_SIZE: usize = 20 * 1024 * 1024;
-const MAX_UNCOMPRESSED_LOG_FILES: usize = 10;
-const MAX_LOG_FILES: usize = 1000;
+pub(crate) const MAX_LOG_SIZE: usize = 20 * 1024 * 1024;
+pub(crate) const MAX_UNCOMPRESSED_LOG_FILES: usize = 10;
+pub(crate) const MAX_LOG_FILES: usize = 1000;
 // Everything is logged by default
 const ALL_ANT_LOGS: &str = "all";
 // Trace at nodes, clients, debug at networking layer
@@ -59,7 +59,7 @@ impl ReloadHandle {
 
 #[derive(Default)]
 /// Tracing log formatter setup for easier span viewing
-pub(crate) struct LogFormatter;
+pub struct LogFormatter;
 
 impl<S, N> FormatEvent<S, N> for LogFormatter
 where
@@ -239,7 +239,7 @@ impl TracingLayers {
 /// `export ANT_LOG = libp2p=DEBUG, tokio=INFO, all, sn_client=ERROR`
 /// Custom keywords will take less precedence if the same target has been manually specified in the CSV.
 /// `sn_client=ERROR` in the above example will be used instead of the TRACE level set by "all" keyword.
-fn get_logging_targets(logging_env_value: &str) -> Result<Vec<(String, Level)>> {
+pub(crate) fn get_logging_targets(logging_env_value: &str) -> Result<Vec<(String, Level)>> {
     let mut targets = BTreeMap::new();
     let mut contains_keyword_all_sn_logs = false;
     let mut contains_keyword_verbose_sn_logs = false;

ant-logging/src/lib.rs

@@ -11,19 +11,28 @@ mod error;
 mod layers;
 #[cfg(feature = "process-metrics")]
 pub mod metrics;
+mod multi_node;
 
 use crate::error::Result;
 use layers::TracingLayers;
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 use tracing::info;
 use tracing_core::dispatcher::DefaultGuard;
-use tracing_subscriber::{prelude::__tracing_subscriber_SubscriberExt, util::SubscriberInitExt};
+use tracing_subscriber::{
+    prelude::__tracing_subscriber_SubscriberExt, util::SubscriberInitExt, Layer,
+};
 
 pub use error::Error;
 pub use layers::ReloadHandle;
 pub use tracing_appender::non_blocking::WorkerGuard;
 
+// Export for testing - hidden from docs but available for integration tests
+#[doc(hidden)]
+pub use layers::LogFormatter;
+#[doc(hidden)]
+pub use multi_node::NodeSpecificFormatter;
+
 // re-exporting the tracing crate's Level as it is used in our public API
 pub use tracing_core::Level;
 
@@ -188,6 +197,127 @@ impl LogBuilder {
         Ok((reload_handle, layers.log_appender_guard))
     }
 
+    /// Initialize multi-node logging with per-node log files.
+    /// Each node gets its own log directory and rotation.
+    ///
+    /// # Arguments
+    /// * `node_count` - Number of nodes to create logging for
+    ///
+    /// # Returns
+    /// * `ReloadHandle` - Handle to modify log levels
+    /// * `Vec<WorkerGuard>` - Guards for all node appenders
+    pub fn initialize_with_multi_node_logging(
+        self,
+        node_count: usize,
+    ) -> Result<(ReloadHandle, Vec<WorkerGuard>)> {
+        use crate::appender;
+        use crate::layers::TracingLayers;
+        use crate::multi_node::NodeRoutingLayer;
+
+        if node_count == 1 {
+            // Fall back to existing single-node implementation
+            let (handle, guard) = self.initialize()?;
+            return Ok((
+                handle,
+                vec![guard.unwrap_or_else(|| {
+                    // Create a dummy guard if none exists
+                    let (_, guard) = tracing_appender::non_blocking(std::io::sink());
+                    guard
+                })],
+            ));
+        }
+
+        // Multi-node logging requires file output
+        let base_log_dir = match &self.output_dest {
+            LogOutputDest::Path(path) => path.clone(),
+            _ => {
+                return Err(Error::LoggingConfiguration(
+                    "Multi-node logging requires file output".to_string(),
+                ))
+            }
+        };
+
+        // Get logging targets
+        let targets = match std::env::var("ANT_LOG") {
+            Ok(sn_log_val) => {
+                if self.print_updates_to_stdout {
+                    println!("Using ANT_LOG={sn_log_val}");
+                }
+                crate::layers::get_logging_targets(&sn_log_val)?
+            }
+            Err(_) => self.default_logging_targets.clone(),
+        };
+
+        // Create NodeRoutingLayer and set up per-node appenders
+        let mut routing_layer = NodeRoutingLayer::new(targets.clone());
+        let mut guards = Vec::new();
+
+        for i in 1..=node_count {
+            let node_name = format!("node_{i}");
+
+            let node_log_dir = base_log_dir.join(&node_name);
+            std::fs::create_dir_all(&node_log_dir)?;
+
+            if self.print_updates_to_stdout {
+                println!("Logging for {node_name} to directory: {node_log_dir:?}");
+            }
+
+            let (appender, guard) = appender::file_rotater(
+                &node_log_dir,
+                crate::layers::MAX_LOG_SIZE,
+                self.max_log_files
+                    .unwrap_or(crate::layers::MAX_UNCOMPRESSED_LOG_FILES),
+                self.max_archived_log_files
+                    .map(|max_archived| {
+                        max_archived
+                            + self
+                                .max_log_files
+                                .unwrap_or(crate::layers::MAX_UNCOMPRESSED_LOG_FILES)
+                    })
+                    .unwrap_or(crate::layers::MAX_LOG_FILES),
+            );
+
+            routing_layer.add_node_writer(node_name, appender);
+            guards.push(guard);
+        }
+
+        // Create reload handle for log level changes
+        let targets_filter: Box<
+            dyn tracing_subscriber::layer::Filter<tracing_subscriber::Registry> + Send + Sync,
+        > = Box::new(tracing_subscriber::filter::Targets::new().with_targets(targets));
+        let (filter, reload_handle) = tracing_subscriber::reload::Layer::new(targets_filter);
+        let reload_handle = ReloadHandle(reload_handle);
+
+        // Apply the filter to the routing layer
+        let filtered_routing_layer = routing_layer.with_filter(filter);
+
+        let mut layers = TracingLayers::default();
+        layers.layers.push(Box::new(filtered_routing_layer));
+
+        #[cfg(feature = "otlp")]
+        {
+            match std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT") {
+                Ok(_) => layers.otlp_layer(self.default_logging_targets)?,
+                Err(_) => println!(
+                    "The OTLP feature is enabled but the OTEL_EXPORTER_OTLP_ENDPOINT variable is not \
+                    set, so traces will not be submitted."
+                ),
+            }
+        }
+
+        if tracing_subscriber::registry()
+            .with(layers.layers)
+            .try_init()
+            .is_err()
+        {
+            return Err(Error::LoggingConfiguration(
+                "Global subscriber already initialized".to_string(),
+            ));
+        }
+
+        Ok((reload_handle, guards))
+    }
+
     /// Logs to the data_dir with per-test log files. Should be called from a single threaded tokio/non-tokio context.
     /// Each test gets its own log file based on the test name.
     ///