Skip to content

Update metric docs #5844

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: revamp-storage-metrics
Choose a base branch
from
Open

Update metric docs #5844

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
922a15d
Update metrics doc
rdettai Jul 16, 2025
a7f12de
Make in-flight storage get requests a memory metric
rdettai Jul 16, 2025
ad3dd11
Add precisions to job_assigned_total
rdettai Jul 17, 2025
b339041
Update example Grafana dashboards
rdettai Jul 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
197 changes: 157 additions & 40 deletions docs/reference/metrics.md
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions quickwit/quickwit-common/src/metrics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,7 @@ pub struct InFlightDataGauges {
pub doc_processor_mailbox: IntGauge,
pub indexer_mailbox: IntGauge,
pub index_writer: IntGauge,
pub get_object: IntGauge,
in_flight_gauge_vec: IntGaugeVec<1>,
}

Expand All @@ -365,6 +366,7 @@ impl Default for InFlightDataGauges {
doc_processor_mailbox: in_flight_gauge_vec.with_label_values(["doc_processor_mailbox"]),
indexer_mailbox: in_flight_gauge_vec.with_label_values(["indexer_mailbox"]),
index_writer: in_flight_gauge_vec.with_label_values(["index_writer"]),
get_object: in_flight_gauge_vec.with_label_values(["get_object"]),
in_flight_gauge_vec: in_flight_gauge_vec.clone(),
}
}
Expand Down
17 changes: 16 additions & 1 deletion quickwit/quickwit-ingest/src/ingest_v2/metrics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ use mrecordlog::ResourceUsage;
use once_cell::sync::Lazy;
use quickwit_common::metrics::{
Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, exponential_buckets,
linear_buckets, new_counter_vec, new_gauge, new_gauge_vec, new_histogram, new_histogram_vec,
linear_buckets, new_counter, new_counter_vec, new_gauge, new_gauge_vec, new_histogram,
new_histogram_vec,
};

// Counter vec counting the different outcomes of ingest requests as
Expand Down Expand Up @@ -82,6 +83,8 @@ pub(super) struct IngestV2Metrics {
pub wal_disk_used_bytes: IntGauge,
pub wal_memory_used_bytes: IntGauge,
pub ingest_results: IngestResultMetrics,
pub replicated_num_bytes_total: IntCounter,
pub replicated_num_docs_total: IntCounter,
}

impl Default for IngestV2Metrics {
Expand Down Expand Up @@ -146,6 +149,18 @@ impl Default for IngestV2Metrics {
"ingest",
&[],
),
replicated_num_bytes_total: new_counter(
"replicated_num_bytes_total",
"Total size in bytes of the replicated docs.",
"ingest",
&[],
),
replicated_num_docs_total: new_counter(
"replicated_num_docs_total",
"Total number of docs replicated.",
"ingest",
&[],
),
}
}
}
Expand Down
6 changes: 3 additions & 3 deletions quickwit/quickwit-ingest/src/ingest_v2/replication.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ use super::metrics::report_wal_usage;
use super::models::IngesterShard;
use super::mrecordlog_utils::check_enough_capacity;
use super::state::IngesterState;
use crate::ingest_v2::metrics::INGEST_V2_METRICS;
use crate::ingest_v2::mrecordlog_utils::{AppendDocBatchError, append_non_empty_doc_batch};
use crate::metrics::INGEST_METRICS;
use crate::{estimate_size, with_lock_metrics};

pub(super) const SYN_REPLICATION_STREAM_CAPACITY: usize = 5;
Expand Down Expand Up @@ -667,10 +667,10 @@ impl ReplicationTask {
.expect("replica shard should be initialized")
.set_replication_position_inclusive(current_position_inclusive.clone(), now);

INGEST_METRICS
INGEST_V2_METRICS
.replicated_num_bytes_total
.inc_by(batch_num_bytes);
INGEST_METRICS
INGEST_V2_METRICS
.replicated_num_docs_total
.inc_by(batch_num_docs);

Expand Down
28 changes: 3 additions & 25 deletions quickwit/quickwit-ingest/src/metrics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,14 @@
// limitations under the License.

use once_cell::sync::Lazy;
use quickwit_common::metrics::{IntCounter, IntGauge, new_counter, new_counter_vec, new_gauge};
use quickwit_common::metrics::{IntCounter, new_counter_vec};

pub struct IngestMetrics {
// With ingest V1 all ingested documents are considered valid
pub ingested_docs_bytes_valid: IntCounter,
pub ingested_docs_valid: IntCounter,
pub ingested_docs_bytes_invalid: IntCounter,
pub ingested_docs_invalid: IntCounter,
pub ingested_docs_valid: IntCounter,

pub replicated_num_bytes_total: IntCounter,
pub replicated_num_docs_total: IntCounter,
Comment on lines -24 to -25
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

moved this to be part of of ingest_v2 metrics

#[allow(dead_code)] // this really shouldn't be dead, it needs to be used somewhere
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this has been dead for a year

pub queue_count: IntGauge,
}

impl Default for IngestMetrics {
Expand Down Expand Up @@ -56,24 +52,6 @@ impl Default for IngestMetrics {
ingested_docs_bytes_invalid,
ingested_docs_valid,
ingested_docs_invalid,
replicated_num_bytes_total: new_counter(
"replicated_num_bytes_total",
"Total size in bytes of the replicated docs.",
"ingest",
&[],
),
replicated_num_docs_total: new_counter(
"replicated_num_docs_total",
"Total number of docs replicated.",
"ingest",
&[],
),
queue_count: new_gauge(
"queue_count",
"Number of queues currently active",
"ingest",
&[],
),
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions quickwit/quickwit-serve/src/metrics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,14 @@ impl Default for ServeMetrics {
),
ongoing_requests: new_gauge_vec(
"ongoing_requests",
"Number of ongoing requests.",
"Number of ongoing requests on specific endpoint groups",
"",
&[],
["endpoint_group"],
),
pending_requests: new_gauge_vec(
"pending_requests",
"Number of pending requests.",
"Number of pending requests on specific endpoint groups",
"",
&[],
["endpoint_group"],
Expand Down
35 changes: 8 additions & 27 deletions quickwit/quickwit-storage/src/metrics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

use once_cell::sync::Lazy;
use quickwit_common::metrics::{
GaugeGuard, HistogramVec, IntCounter, IntCounterVec, IntGauge, new_counter, new_counter_vec,
new_gauge, new_histogram_vec,
GaugeGuard, HistogramVec, IntCounter, IntCounterVec, IntGauge, MEMORY_METRICS, new_counter,
new_counter_vec, new_gauge, new_histogram_vec,
};

/// Counters associated to storage operations.
Expand All @@ -32,8 +32,6 @@ pub struct StorageMetrics {
pub get_slice_timeout_all_timeouts: IntCounter,
pub object_storage_requests_total: IntCounterVec<2>,
pub object_storage_request_duration: HistogramVec<2>,
pub object_storage_get_slice_in_flight_count: IntGauge,
pub object_storage_get_slice_in_flight_num_bytes: IntGauge,
Comment on lines -35 to -36
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • I created this metric to make sure that this wasn't the reason for OOMs on search and I confirmed it's not the case
  • I think it makes sense to keep tracking the memory usage, but move it to quickwit_memory_in_flight where it is more discoverable and actionable

pub object_storage_download_num_bytes: IntCounterVec<1>,
pub object_storage_download_errors: IntCounterVec<1>,
pub object_storage_upload_num_bytes: IntCounterVec<1>,
Expand Down Expand Up @@ -84,19 +82,6 @@ impl Default for StorageMetrics {
["action", "status"],
vec![0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0],
),
object_storage_get_slice_in_flight_count: new_gauge(
"object_storage_get_slice_in_flight_count",
"Number of get_object for which the memory was allocated but the download is \
still in progress.",
"storage",
&[],
),
object_storage_get_slice_in_flight_num_bytes: new_gauge(
"object_storage_get_slice_in_flight_num_bytes",
"Memory allocated for get_object requests that are still in progress.",
"storage",
&[],
),
object_storage_download_num_bytes: new_counter_vec(
"object_storage_download_num_bytes",
"Amount of data downloaded from object storage.",
Expand Down Expand Up @@ -199,15 +184,11 @@ pub static STORAGE_METRICS: Lazy<StorageMetrics> = Lazy::new(StorageMetrics::def
pub static CACHE_METRICS_FOR_TESTS: Lazy<CacheMetrics> =
Lazy::new(|| CacheMetrics::for_component("fortest"));

pub fn object_storage_get_slice_in_flight_guards(
get_request_size: usize,
) -> (GaugeGuard<'static>, GaugeGuard<'static>) {
let mut bytes_guard = GaugeGuard::from_gauge(
&crate::STORAGE_METRICS.object_storage_get_slice_in_flight_num_bytes,
);
/// Helps tracking pre-allocated memory for downloads that are still in progress.
///
/// This is actually recorded as a memory metric and not a storage metric.
pub fn object_storage_get_slice_in_flight_guards(get_request_size: usize) -> GaugeGuard<'static> {
let mut bytes_guard = GaugeGuard::from_gauge(&MEMORY_METRICS.in_flight.get_object);
bytes_guard.add(get_request_size as i64);
let mut count_guard =
GaugeGuard::from_gauge(&crate::STORAGE_METRICS.object_storage_get_slice_in_flight_count);
count_guard.add(1);
(bytes_guard, count_guard)
bytes_guard
}