otel collector failed to export logs

[axzsd]

Describe the bug
I failed to export logs and traces in Rust project。I'm a novice in Rust and have just started learning this language. Could you please help me check if there are any areas for improvement in my code and if my dependencies are bloated，Thank you！

Steps to reproduce (if applicable)
Steps to reproduce the behavior:
My rust project dependencies

[dependencies]
actix-web = "4.11.0"
argon2 = "0.5.3"
chrono = {version = "0.4.41", features = ["serde"] }
mimalloc = "0.1.47"
opentelemetry = "0.30.0"
opentelemetry-appender-tracing = "0.30.1"
opentelemetry-otlp = { version = "0.30.0" , features = ["grpc-tonic", "trace", "logs"] }
opentelemetry_sdk = { version = "0.30.0" , features = ["rt-tokio", "trace", "logs"] }
prometheus = "0.14.0"
prometheus_exporter = "0.8.5"
rand = "0.9.1"
serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.140"
serde_yaml = "0.9.34"
thiserror = "2.0.12"
tokio = {version = "1.45.1", features = ["full"] }
tonic = "0.13.1"
tracing = "0.1.41"
tracing-core = "0.1.34"
tracing-log = "0.2.0"
tracing-subscriber = {version = "0.3.19", features = ["env-filter", "tracing-log"] }

main.rs

use opentelemetry::trace::Tracer;
use tracing::instrument;
use tracing_log::log;

mod telemetry;

#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    // 初始化 OpenTelemetry 和 Tracing
    telemetry::init_telemetry("test-service")?;
    let tracer = opentelemetry::global::tracer("main");
    let _span = tracer.start("main_span");
    tracer.in_span("main_span", |_c| {
        log::info!("Test Service is starting...");
        test_log();
    });
    
    tokio::time::sleep(std::time::Duration::from_secs(10)).await;
    Ok(())
}

#[instrument]
 pub fn test_log() {
    log::info!("This is a info log");
    log::warn!("This is a warn log");
    log::error!("This is a error log");
}

telemetry.rs

use opentelemetry::{KeyValue, global};
use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
use opentelemetry_otlp::{LogExporter, WithExportConfig, WithTonicConfig};
use opentelemetry_sdk::{
    Resource,
    logs::SdkLoggerProvider,
    trace::{RandomIdGenerator, Sampler},
};
use std::time::Duration;
use tonic::metadata::*;
use tracing_subscriber::{Layer, filter::EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
// 初始化 OpenTelemetry 和 Tracing
pub fn init_telemetry(service_name: &str) -> Result<(), Box<dyn std::error::Error>> {
    let mut map = MetadataMap::with_capacity(3);

    map.insert("x-host", "example.com".parse().unwrap());
    map.insert("x-number", "123".parse().unwrap());
  
    let exporter = opentelemetry_otlp::SpanExporter::builder()
        .with_tonic()
        .with_endpoint("http://127.0.0.1:4317")
        .with_timeout(Duration::from_secs(3))
        .with_metadata(map.clone())
        .build()?;

    let tracer_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
        .with_batch_exporter(exporter)
        .with_sampler(Sampler::AlwaysOn)
        .with_id_generator(RandomIdGenerator::default())
        .with_max_events_per_span(64)
        .with_max_attributes_per_span(16)
        .with_resource(
            Resource::builder_empty()
                .with_attributes([KeyValue::new("service.name", service_name.to_string())])
                .build(),
        )
        .build();
    global::set_tracer_provider(tracer_provider.clone());

    // 初始化日志导出器
    let log_exporter = LogExporter::builder()
        .with_tonic()
        .with_endpoint("http://127.0.0.1:4317")
        .with_timeout(Duration::from_secs(3))
        .with_metadata(map.clone())
        .build()?;

    let logger_provider = SdkLoggerProvider::builder()
        .with_batch_exporter(log_exporter)
        .with_resource(
            Resource::builder_empty()
                .with_attributes([KeyValue::new("service.name", service_name.to_string())])
                .build(),
        )
        .build();

    let otel_layer = OpenTelemetryTracingBridge::new(&logger_provider);

    // 设置日志过滤规则
    let filter_otel = EnvFilter::new("info")
        .add_directive("hyper=off".parse().unwrap())
        .add_directive("tonic=off".parse().unwrap())
        .add_directive("h2=off".parse().unwrap())
        .add_directive("reqwest=off".parse().unwrap());
    let otel_layer = otel_layer.with_filter(filter_otel);

    let filter_fmt = EnvFilter::new("info");
    let fmt_layer = tracing_subscriber::fmt::layer()
        .with_thread_names(true)
        .with_filter(filter_fmt);

    // 初始化 tracing 订阅者
    tracing_subscriber::registry()
        .with(otel_layer)
        .with(fmt_layer)
        .init();

    Ok(())
}

opentelemetry-collector-contrib print log:
2025-06-26T09:02:28.639Z warn otlpexporter@v0.128.0/otlp.go:136 Partial success response {"resource": {"service.instance.id": "09a2953f-ed2b-426a-9b9a-fb00997fe2cd", "service.name": "otelcol-contrib", "service.version": "0.128.0"}, "otelcol.component.id": "otlp/quickwit", "otelcol.component.kind": "exporter", "otelcol.signal": "logs", "message": "", "dropped_log_records": 4}

Expected behavior
I want to export the data correctly

Configuration:
quickwit version:
Quickwit 0.8.2 (x86_64-unknown-linux-gnu 2024-09-03T11:26:51Z 0f28194)

index:
Default --> otel-logs-v0_7

otel-collector-config.yaml

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    send_batch_size: 1000
    timeout: 10s

exporters:
  otlp/quickwit:
    endpoint: quickwit:7281
    tls:
      insecure: true
    # By default, traces are sent to the otel-traces-v0_7.
    # You can customize the index ID By setting this header.
    # headers:
    #   qw-otel-traces-index: otel-traces-v0_7
  
service:
  pipelines:
    logs:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp/quickwit]
    
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp/quickwit]

    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp/quickwit]

[axzsd]

The image version number of otel-collector is otel/opentelemetry-collector-contrib:0.128.0

[fmassot]

Thanks for reporting the issue @axzsd

The log from the otel collector shows it drops 4 log records. It may come from a mismatch between the otel collector log record mapping and quickwit otel log index mapping.

Could you print the log sent to Quickwit so we can see if there is a problem with the format of the log?

You can also use a curl command to test if a log record can be indexed by quickwit, you can check this doc page.

[axzsd]

@fmassot
Ok，I created a new index and added the main fields I was concerned about. When I put the log, it still failed. This is the log output by quickwit, and there is very little information. Must my log input exactly match the field definitions of the index, including the type and the number of fields?

2025-07-01T08:23:17.476Z ERROR ingest_logs:parse_logs: quickwit_opentelemetry::otlp::logs: skipping record
2025-07-01T08:23:17.476Z ERROR ingest_logs:parse_logs: quickwit_opentelemetry::otlp::logs: skipping record
2025-07-01T08:23:17.476Z ERROR ingest_logs:parse_logs: quickwit_opentelemetry::otlp::logs: skipping record
2025-07-01T08:23:17.476Z ERROR ingest_logs:parse_logs: quickwit_opentelemetry::otlp::logs: skipping record
2025-07-01T08:23:17.476Z ERROR ingest_logs:parse_logs: quickwit_opentelemetry::otlp::logs: skipping record
2025-07-01T08:23:17.476Z ERROR ingest_logs:parse_logs: quickwit_opentelemetry::otlp::logs: skipping record
2025-07-01T08:23:17.476Z ERROR ingest_logs:parse_logs: quickwit_opentelemetry::otlp::logs: skipping record

[axzsd]

@fmassot This is my input log

curl -XPOST "http://localhost:4318/v1/logs" -H "Content-Type: application/json" \
--data-binary @- << EOF
{
    "resourceLogs": [
        {
            "resource": {
                "attributes": [
                    {
                        "key": "service.name",
                        "value": {
                            "stringValue": "user-service"
                        }
                    }
                ]
            },
            "scopeLogs": [
                {
                    "scope": {
                        "name": "log"
                    },
                    "logRecords": [
                        {
                            "observedTimeUnixNano": "1750927392046590000",
                            "severityNumber": 9,
                            "severityText": "INFO",
                            "body": {
                                "stringValue": "User Service is starting..."
                            },
                            "attributes": [
                                {
                                    "key": "log.target",
                                    "value": {
                                        "stringValue": "user_service"
                                    }
                                },
                                {
                                    "key": "log.module_path",
                                    "value": {
                                        "stringValue": "user_service"
                                    }
                                },
                                {
                                    "key": "log.file",
                                    "value": {
                                        "stringValue": "src/main.rs"
                                    }
                                },
                                {
                                    "key": "log.line",
                                    "value": {
                                        "intValue": "14"
                                    }
                                }
                            ],
                            "traceId": "",
                            "spanId": "",
                            "eventName": "log event"
                        },
                        {
                            "observedTimeUnixNano": "1750927392046794000",
                            "severityNumber": 9,
                            "severityText": "INFO",
                            "body": {
                                "stringValue": "This is a info log"
                            },
                            "attributes": [
                                {
                                    "key": "log.target",
                                    "value": {
                                        "stringValue": "user_service"
                                    }
                                },
                                {
                                    "key": "log.module_path",
                                    "value": {
                                        "stringValue": "user_service"
                                    }
                                },
                                {
                                    "key": "log.file",
                                    "value": {
                                        "stringValue": "src/main.rs"
                                    }
                                },
                                {
                                    "key": "log.line",
                                    "value": {
                                        "intValue": "22"
                                    }
                                }
                            ],
                            "traceId": "",
                            "spanId": "",
                            "eventName": "log event"
                        },
                        {
                            "observedTimeUnixNano": "1750927392046842000",
                            "severityNumber": 13,
                            "severityText": "WARN",
                            "body": {
                                "stringValue": "This is a warn log"
                            },
                            "attributes": [
                                {
                                    "key": "log.target",
                                    "value": {
                                        "stringValue": "user_service"
                                    }
                                },
                                {
                                    "key": "log.module_path",
                                    "value": {
                                        "stringValue": "user_service"
                                    }
                                },
                                {
                                    "key": "log.file",
                                    "value": {
                                        "stringValue": "src/main.rs"
                                    }
                                },
                                {
                                    "key": "log.line",
                                    "value": {
                                        "intValue": "23"
                                    }
                                }
                            ],
                            "traceId": "",
                            "spanId": "",
                            "eventName": "log event"
                        },
                        {
                            "observedTimeUnixNano": "1750927392046875000",
                            "severityNumber": 17,
                            "severityText": "ERROR",
                            "body": {
                                "stringValue": "This is a error log"
                            },
                            "attributes": [
                                {
                                    "key": "log.target",
                                    "value": {
                                        "stringValue": "user_service"
                                    }
                                },
                                {
                                    "key": "log.module_path",
                                    "value": {
                                        "stringValue": "user_service"
                                    }
                                },
                                {
                                    "key": "log.file",
                                    "value": {
                                        "stringValue": "src/main.rs"
                                    }
                                },
                                {
                                    "key": "log.line",
                                    "value": {
                                        "intValue": "24"
                                    }
                                }
                            ],
                            "traceId": "",
                            "spanId": "",
                            "eventName": "log event"
                        }
                    ]
                }
            ]
        }
    ]
}
EOF

[axzsd]

@fmassot This is my index config

{
  "field_mappings": [
    {
      "name": "observedTimeUnixNano",
      "type": "datetime",
      "description": "日志时间戳（纳秒级Unix时间戳）",
      "fast": true,
      "fast_precision": "seconds",
      "indexed": true,
      "input_formats": [
        "unix_timestamp"
      ],
      "output_format": "unix_timestamp_nanos",
      "stored": true
    },
    {
      "name": "severityNumber",
      "type": "u64",
      "coerce": true,
      "fast": true,
      "indexed": true,
      "output_format": "number",
      "stored": true
    },
    {
      "name": "severityText",
      "type": "text",
      "fast": {
        "normalizer": "raw"
      },
      "fieldnorms": false,
      "indexed": true,
      "record": "basic",
      "stored": true,
      "tokenizer": "lowercase"
    },
    {
      "name": "body",
      "type": "text",
      "fast": false,
      "fieldnorms": false,
      "indexed": true,
      "record": "position",
      "stored": true,
      "tokenizer": "default"
    },
    {
      "name": "resource.attributes.service.name",
      "type": "text",
      "fast": {
        "normalizer": "raw"
      },
      "fieldnorms": false,
      "indexed": true,
      "record": "basic",
      "stored": true,
      "tokenizer": "lowercase"
    },
    {
      "name": "attributes.log.target",
      "type": "text",
      "fast": false,
      "fieldnorms": false,
      "indexed": true,
      "record": "basic",
      "stored": true,
      "tokenizer": "lowercase"
    },
    {
      "name": "attributes.log.line",
      "type": "u64",
      "coerce": true,
      "fast": true,
      "indexed": true,
      "output_format": "number",
      "stored": true
    }
  ],
  "tag_fields": [],
  "store_source": false,
  "index_field_presence": false,
  "timestamp_field": "observedTimeUnixNano",
  "mode": "strict",
  "max_num_partitions": 200,
  "tokenizers": []
}

[jvdwrf]

Did you find a solution to this problem? I'm running into the same issue using opentelemetry_sdk from rust together with the OpenTelemetry collector. Running quickwit with RUST_LOG=debug does not provide any more details sadly.

EDIT: I think the problem is that timeUnixNanos is not set, but only observedTimeUnixNano. When this field is not set, Quickwit throws an error and skips the entry.

A workaround:

      transform/set_time_unix_nano:
        log_statements:
          # conditions:
          #   - log.time_unix_nano == nil
          - statements:
              - set(log.time_unix_nano, log.observed_time_unix_nano)