OTel integration performance impact

doc/LOGGING.md

@@ -77,3 +77,102 @@ See below an example of what the emitted metrics may look like in the logs.
 
 We recommend using the metrics only for debugging at this time.
 Metrics are currently output in an unstructured format and are subject to change in future releases.
+
+### Publishing Metrics
+
+Additionally, metrics can be published in OpenTelemetry format to a specified endpoint. This allows for integration with OpenTelemetry collectors and agents. For example, the [CloudWatch Agent](#cloudwatch-agent-installation) can listen at port 4318 for metrics in OpenTelemetry format and forward them to CloudWatch and Prometheus.
+
+Note that publishing metrics to CloudWatch may incur additional AWS costs depending on the volume of metrics and your AWS account's billing tier. Publishing metrics is entirely optional, and you can still use Mountpoint without enabling this feature.
+
+To opt-in, use the `--log-metrics-otlp <ENDPOINT>` command-line argument and provide an endpoint as a parameter. To optionally specify a time interval for Mountpoint to collect and export metrics, use the `--log-metrics-otlp-interval <SECONDS>` command-line argument. Metrics will be collected by Mountpoint and exported to the endpoint every 5 seconds by default, or every specified seconds.
+
+#### Example Command
+
+```bash
+# Mount an S3 bucket and publish metrics to a local CloudWatch agent
+mount-s3 amzn-s3-demo-bucket /mnt/s3 --log-metrics-otlp http://localhost:4318 --log-metrics-otlp-interval 10
+```
+
+#### CloudWatch Agent Installation 
+
+Follow [CloudWatch Agent installation guide](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/install-CloudWatch-Agent-commandline-fleet.html). For more information about the CloudWatch agent, see the [CloudWatch Agent Installation Guide](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Install-CloudWatch-Agent.html) and [Using the CloudWatch agent to collect OpenTelemetry metrics](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-OpenTelemetry-metrics.html). If installing on EC2 instance, ensure that you attach the CloudWatchAgentServerPolicy to the IAM role that is attached to your instance.
+
+#### CloudWatch Agent Configuration
+
+You can give the CloudWatch Agent configuration file any name and location. However, for simplicity in troubleshooting, AWS recommends using `/opt/aws/amazon-cloudwatch-agent/etc/cloudwatch-agent.json` on Linux. After creating the file, you can copy it to other servers where you want to install the agent.
+
+Basic configuration for forwarding metrics to CloudWatch:
+
+```json
+{
+    "agent": {
+        "metrics_collection_interval": 60,
+        "run_as_user": "cwagent"
+    },
+    "metrics": {
+        "namespace": "Mountpoint",
+        "metrics_collected": {
+            "otlp": {
+                "http_endpoint": "127.0.0.1:4318"
+            }
+        }
+    }
+}
+```
+
+This configuration will forward metrics that are received at the otlp endpoint/s to CloudWatch. (If no destination is provided, the default of CloudWatch is used.) 
+
+Configuration to publish to both CloudWatch and Prometheus:
+
+```json
+{
+    "agent": {
+        "metrics_collection_interval": 60,
+        "run_as_user": "cwagent"
+    },
+    "metrics": {
+        "namespace": "Mountpoint",
+        "metrics_destinations": {
+           "cloudwatch": {},
+           "amp": {
+             "workspace_id": "ws-abcd1234-ef56-7890-ab12-example"
+            }
+        },
+        "metrics_collected": {
+            "otlp": {
+                "http_endpoint": "127.0.0.1:4318"
+            }
+        }
+    }
+}
+```
+
+For more detailed configurations, follow [CloudWatch Agent Configuration File Details](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html).
+
+#### Viewing Metrics in CloudWatch
+
+Starting CloudWatch Agent with the configured json file: 
+
+```
+sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/cloudwatch-agent.json
+```
+
+Then run mountpoint with the CLI flag:
+
+```
+mount-s3 amzn-s3-demo-bucket /mnt/s3 --log-metrics-otlp http://localhost:4318 --log-metrics-otlp-interval 10
+```
+
+After running the command, metrics will be visible in your CloudWatch console.
+
+Stop the CloudWatch Agent:
+
+```
+sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop
+```
+
+#### Troubleshooting CloudWatch Agent
+
+For troubleshooting the CloudWatch Agent, refer to the [AWS CloudWatch Agent Troubleshooting Guide](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/troubleshooting-CloudWatch-Agent.html#CloudWatch-Agent-options-help).
+
+After configuring the CloudWatch Agent and starting Mountpoint with the `--log-metrics-otlp` flag, metrics should appear in your CloudWatch console within a few minutes. If metrics are not appearing, verify that the CloudWatch Agent is running and properly configured.

mountpoint-s3-fs/Cargo.toml

@@ -50,12 +50,17 @@ time = { version = "0.3.41", features = ["macros", "formatting", "serde-well-kno
 tracing = { version = "0.1.41", features = ["log"] }
 tracing-log = "0.2.0"
 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
+opentelemetry = { version = "0.30.0", features = ["metrics"] }
+opentelemetry_sdk = { version = "0.30.0", features = ["metrics", "rt-tokio"] }
+opentelemetry-otlp = { version = "0.30.0", features = ["metrics", "http-proto"] }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { version = "0.17.0", default-features = false }
 
 [dev-dependencies]
 mountpoint-s3-client = { path = "../mountpoint-s3-client", features = ["mock"] }
+opentelemetry-proto = { version = "0.30.0", features = ["metrics", "gen-tonic"] }
+prost = "0.12.0"
 
 assert_cmd = "2.0.17"
 assert_fs = "1.1.3"

mountpoint-s3-fs/examples/mount_from_config.rs

@@ -193,7 +193,9 @@ fn process_manifests(config: &ConfigOptions, database_directory: &Path) -> Resul
 
 fn setup_logging(config: &ConfigOptions) -> Result<(LoggingHandle, MetricsSinkHandle)> {
     let logging = init_logging(config.build_logging_config())?;
-    let metrics = metrics::install();
+
+    let metrics = metrics::install(None).map_err(|e| anyhow!("Failed to initialize metrics: {}", e))?;
+
     Ok((logging, metrics))
 }
 

mountpoint-s3-fs/examples/prefetch_benchmark.rs

@@ -159,7 +159,7 @@ impl CliArgs {
 
 fn main() -> anyhow::Result<()> {
     init_tracing_subscriber();
-    let _metrics_handle = mountpoint_s3_fs::metrics::install();
+    let _metrics_handle = mountpoint_s3_fs::metrics::install(None);
 
     let args = CliArgs::parse();
 

mountpoint-s3-fs/examples/upload_benchmark.rs

@@ -90,9 +90,8 @@ struct UploadBenchmarkArgs {
 
 fn main() {
     init_tracing_subscriber();
-    let _metrics_handle = mountpoint_s3_fs::metrics::install();
-
     let args = UploadBenchmarkArgs::parse();
+
     println!("starting upload benchmark with {:?}", &args);
 
     let mut endpoint_config = EndpointConfig::new(&args.region);

mountpoint-s3-fs/src/lib.rs

@@ -14,6 +14,7 @@ pub mod mem_limiter;
 pub mod memory;
 pub mod metablock;
 pub mod metrics;
+pub mod metrics_otel;
 pub mod object;
 pub mod prefetch;
 pub mod prefix;