Skip to content

Commit 1ddf718

Browse files
jkmuriithijkmuriithi
authored
SOLR-17628: Add query quantiles metrics to prometheus endpoint (#3164)
Export metric timers via `wt=prometheus` as Prometheus summaries. This introduces count, sum, and quantiles.
1 parent 236ac8d commit 1ddf718

File tree

6 files changed

+96
-40
lines changed

6 files changed

+96
-40
lines changed

‎solr/CHANGES.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,8 @@ Improvements
287287

288288
* SOLR-17746: Provide long form --jettyconfig option to go with -j in bin/solr scripts. (Eric Pugh, Rahul Goswami)
289289

290+
* SOLR-17628: Export metric timers via `wt=prometheus` as Prometheus summaries. (Jude Muriithi, Matthew Biscocho)
291+
290292
Optimizations
291293
---------------------
292294
* SOLR-17578: Remove ZkController internal core supplier, for slightly faster reconnection after Zookeeper session loss. (Pierre Salagnac)

‎solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusFormatter.java

Lines changed: 64 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,24 @@
1818

1919
import com.codahale.metrics.Meter;
2020
import com.codahale.metrics.Metric;
21+
import com.codahale.metrics.Snapshot;
2122
import com.codahale.metrics.Timer;
2223
import io.prometheus.metrics.model.snapshots.CounterSnapshot;
24+
import io.prometheus.metrics.model.snapshots.Exemplars;
2325
import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
2426
import io.prometheus.metrics.model.snapshots.Labels;
2527
import io.prometheus.metrics.model.snapshots.MetricMetadata;
2628
import io.prometheus.metrics.model.snapshots.MetricSnapshot;
2729
import io.prometheus.metrics.model.snapshots.MetricSnapshots;
30+
import io.prometheus.metrics.model.snapshots.Quantile;
31+
import io.prometheus.metrics.model.snapshots.Quantiles;
32+
import io.prometheus.metrics.model.snapshots.SummarySnapshot;
2833
import java.util.ArrayList;
34+
import java.util.Arrays;
2935
import java.util.HashMap;
3036
import java.util.List;
3137
import java.util.Map;
38+
import org.apache.solr.util.stats.MetricUtils;
3239

3340
/**
3441
* Base class for all {@link SolrPrometheusFormatter} holding {@link MetricSnapshot}s. Can export
@@ -38,10 +45,12 @@
3845
public abstract class SolrPrometheusFormatter {
3946
protected final Map<String, List<CounterSnapshot.CounterDataPointSnapshot>> metricCounters;
4047
protected final Map<String, List<GaugeSnapshot.GaugeDataPointSnapshot>> metricGauges;
48+
protected final Map<String, List<SummarySnapshot.SummaryDataPointSnapshot>> metricSummaries;
4149

4250
public SolrPrometheusFormatter() {
4351
this.metricCounters = new HashMap<>();
4452
this.metricGauges = new HashMap<>();
53+
this.metricSummaries = new HashMap<>();
4554
}
4655

4756
/**
@@ -93,18 +102,36 @@ public void exportCounter(
93102
}
94103

95104
/**
96-
* Export {@link Timer} ands its mean rate to {@link
97-
* io.prometheus.metrics.model.snapshots.GaugeSnapshot.GaugeDataPointSnapshot} and collect
105+
* Export {@link Timer} ands its quantile data to {@link
106+
* io.prometheus.metrics.model.snapshots.SummarySnapshot.SummaryDataPointSnapshot} and collect
98107
* datapoint
99108
*
100109
* @param metricName name of prometheus metric
101110
* @param dropwizardMetric the {@link Timer} to be exported
102111
* @param labels label names and values to record
103112
*/
104113
public void exportTimer(String metricName, Timer dropwizardMetric, Labels labels) {
105-
GaugeSnapshot.GaugeDataPointSnapshot dataPoint =
106-
createGaugeDatapoint(dropwizardMetric.getSnapshot().getMean(), labels);
107-
collectGaugeDatapoint(metricName, dataPoint);
114+
Snapshot snapshot = dropwizardMetric.getSnapshot();
115+
116+
long count = snapshot.size();
117+
double sum =
118+
Arrays.stream(snapshot.getValues())
119+
.asDoubleStream()
120+
.map(num -> MetricUtils.nsToMs(num))
121+
.sum();
122+
123+
Quantiles quantiles =
124+
Quantiles.of(
125+
List.of(
126+
new Quantile(0.50, MetricUtils.nsToMs(snapshot.getMedian())),
127+
new Quantile(0.75, MetricUtils.nsToMs(snapshot.get75thPercentile())),
128+
new Quantile(0.99, MetricUtils.nsToMs(snapshot.get99thPercentile())),
129+
new Quantile(0.999, MetricUtils.nsToMs(snapshot.get999thPercentile()))));
130+
131+
var summary =
132+
new SummarySnapshot.SummaryDataPointSnapshot(
133+
count, sum, quantiles, labels, Exemplars.EMPTY, 0L);
134+
collectSummaryDatapoint(metricName, summary);
108135
}
109136

110137
/**
@@ -206,20 +233,44 @@ public void collectGaugeDatapoint(
206233
metricGauges.get(metricName).add(dataPoint);
207234
}
208235

236+
/**
237+
* Collects {@link io.prometheus.metrics.model.snapshots.SummarySnapshot.SummaryDataPointSnapshot}
238+
* and appends to existing metric or create new metric if name does not exist
239+
*
240+
* @param metricName Name of metric
241+
* @param dataPoint Gauge datapoint to be collected
242+
*/
243+
public void collectSummaryDatapoint(
244+
String metricName, SummarySnapshot.SummaryDataPointSnapshot dataPoint) {
245+
metricSummaries.computeIfAbsent(metricName, k -> new ArrayList<>()).add(dataPoint);
246+
}
247+
209248
/**
210249
* Returns an immutable {@link MetricSnapshots} from the {@link
211250
* io.prometheus.metrics.model.snapshots.DataPointSnapshot}s collected from the registry
212251
*/
213252
public MetricSnapshots collect() {
214253
ArrayList<MetricSnapshot> snapshots = new ArrayList<>();
215-
for (String metricName : metricCounters.keySet()) {
216-
snapshots.add(
217-
new CounterSnapshot(new MetricMetadata(metricName), metricCounters.get(metricName)));
218-
}
219-
for (String metricName : metricGauges.keySet()) {
220-
snapshots.add(
221-
new GaugeSnapshot(new MetricMetadata(metricName), metricGauges.get(metricName)));
222-
}
254+
255+
metricCounters
256+
.entrySet()
257+
.forEach(
258+
entry ->
259+
snapshots.add(
260+
new CounterSnapshot(new MetricMetadata(entry.getKey()), entry.getValue())));
261+
metricGauges
262+
.entrySet()
263+
.forEach(
264+
entry ->
265+
snapshots.add(
266+
new GaugeSnapshot(new MetricMetadata(entry.getKey()), entry.getValue())));
267+
metricSummaries
268+
.entrySet()
269+
.forEach(
270+
entry ->
271+
snapshots.add(
272+
new SummarySnapshot(new MetricMetadata(entry.getKey()), entry.getValue())));
273+
223274
return new MetricSnapshots(snapshots);
224275
}
225276
}

‎solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHandlerMetric.java

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,7 @@
2727
public class SolrCoreHandlerMetric extends SolrCoreMetric {
2828
public static final String CORE_REQUESTS_TOTAL = "solr_metrics_core_requests";
2929
public static final String CORE_REQUESTS_UPDATE_HANDLER = "solr_metrics_core_update_handler";
30-
public static final String CORE_REQUESTS_TOTAL_TIME = "solr_metrics_core_requests_time";
31-
public static final String CORE_REQUEST_TIMES = "solr_metrics_core_average_request_time";
30+
public static final String CORE_REQUEST_TIMES = "solr_metrics_core_request_time_ms";
3231

3332
public SolrCoreHandlerMetric(Metric dropwizardMetric, String metricName) {
3433
super(dropwizardMetric, metricName);
@@ -58,10 +57,6 @@ public void toPrometheus(SolrPrometheusFormatter formatter) {
5857
} else if (dropwizardMetric instanceof Counter) {
5958
if (metricName.endsWith("requests")) {
6059
formatter.exportCounter(CORE_REQUESTS_TOTAL, (Counter) dropwizardMetric, getLabels());
61-
} else if (metricName.endsWith("totalTime")) {
62-
// Do not need type label for total time
63-
labels.remove("type");
64-
formatter.exportCounter(CORE_REQUESTS_TOTAL_TIME, (Counter) dropwizardMetric, getLabels());
6560
}
6661
} else if (dropwizardMetric instanceof Gauge) {
6762
if (!metricName.endsWith("handlerStart")) {

‎solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreSearcherMetric.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
/** Dropwizard metrics of name SEARCHER.* */
2828
public class SolrCoreSearcherMetric extends SolrCoreMetric {
2929
public static final String CORE_SEARCHER_METRICS = "solr_metrics_core_searcher_documents";
30-
public static final String CORE_SEARCHER_TIMES = "solr_metrics_core_average_searcher_warmup_time";
30+
public static final String CORE_SEARCHER_TIMES = "solr_metrics_core_searcher_warmup_time_ms";
3131

3232
public SolrCoreSearcherMetric(Metric dropwizardMetric, String metricName) {
3333
super(dropwizardMetric, metricName);

‎solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import io.prometheus.metrics.model.snapshots.Labels;
2626
import io.prometheus.metrics.model.snapshots.MetricSnapshot;
2727
import io.prometheus.metrics.model.snapshots.MetricSnapshots;
28+
import io.prometheus.metrics.model.snapshots.SummarySnapshot;
2829
import java.util.Arrays;
2930
import java.util.HashMap;
3031
import java.util.List;
@@ -730,12 +731,12 @@ public void testPrometheusMetricsCore() throws Exception {
730731
assertNotNull(actualSnapshots);
731732

732733
MetricSnapshot actualSnapshot =
733-
getMetricSnapshot(actualSnapshots, "solr_metrics_core_average_request_time");
734-
GaugeSnapshot.GaugeDataPointSnapshot actualGaugeDataPoint =
735-
getGaugeDatapointSnapshot(
734+
getMetricSnapshot(actualSnapshots, "solr_metrics_core_request_time_ms");
735+
SummarySnapshot.SummaryDataPointSnapshot actualSummaryDataPoint =
736+
getSummaryDataPointSnapshot(
736737
actualSnapshot,
737738
Labels.of("category", "QUERY", "core", "collection1", "handler", "/select[shard]"));
738-
assertEquals(0, actualGaugeDataPoint.getValue(), 0);
739+
assertEquals(0, (float) actualSummaryDataPoint.getCount(), 0);
739740

740741
actualSnapshot = getMetricSnapshot(actualSnapshots, "solr_metrics_core_requests");
741742
CounterSnapshot.CounterDataPointSnapshot actualCounterDataPoint =
@@ -753,7 +754,7 @@ public void testPrometheusMetricsCore() throws Exception {
753754
assertEquals(0, actualCounterDataPoint.getValue(), 0);
754755

755756
actualSnapshot = getMetricSnapshot(actualSnapshots, "solr_metrics_core_cache");
756-
actualGaugeDataPoint =
757+
GaugeSnapshot.GaugeDataPointSnapshot actualGaugeDataPoint =
757758
getGaugeDatapointSnapshot(
758759
actualSnapshot,
759760
Labels.of("cacheType", "fieldValueCache", "core", "collection1", "item", "hits"));
@@ -766,13 +767,6 @@ public void testPrometheusMetricsCore() throws Exception {
766767
Labels.of("item", "default", "core", "collection1", "type", "SolrFragmenter"));
767768
assertEquals(0, actualCounterDataPoint.getValue(), 0);
768769

769-
actualSnapshot = getMetricSnapshot(actualSnapshots, "solr_metrics_core_requests_time");
770-
actualCounterDataPoint =
771-
getCounterDatapointSnapshot(
772-
actualSnapshot,
773-
Labels.of("category", "QUERY", "core", "collection1", "handler", "/select[shard]"));
774-
assertEquals(0, actualCounterDataPoint.getValue(), 0);
775-
776770
actualSnapshot = getMetricSnapshot(actualSnapshots, "solr_metrics_core_searcher_documents");
777771
actualGaugeDataPoint =
778772
getGaugeDatapointSnapshot(
@@ -795,11 +789,11 @@ public void testPrometheusMetricsCore() throws Exception {
795789
assertEquals(0, actualGaugeDataPoint.getValue(), 0);
796790

797791
actualSnapshot =
798-
getMetricSnapshot(actualSnapshots, "solr_metrics_core_average_searcher_warmup_time");
799-
actualGaugeDataPoint =
800-
getGaugeDatapointSnapshot(
792+
getMetricSnapshot(actualSnapshots, "solr_metrics_core_searcher_warmup_time_ms");
793+
actualSummaryDataPoint =
794+
getSummaryDataPointSnapshot(
801795
actualSnapshot, Labels.of("core", "collection1", "type", "warmup"));
802-
assertEquals(0, actualGaugeDataPoint.getValue(), 0);
796+
assertEquals(0, (float) actualSummaryDataPoint.getCount(), 0);
803797

804798
handler.close();
805799
}
@@ -1183,6 +1177,15 @@ private CounterSnapshot.CounterDataPointSnapshot getCounterDatapointSnapshot(
11831177
.get();
11841178
}
11851179

1180+
private SummarySnapshot.SummaryDataPointSnapshot getSummaryDataPointSnapshot(
1181+
MetricSnapshot snapshot, Labels labels) {
1182+
return (SummarySnapshot.SummaryDataPointSnapshot)
1183+
snapshot.getDataPoints().stream()
1184+
.filter(ss -> ss.getLabels().hasSameValues(labels))
1185+
.findAny()
1186+
.get();
1187+
}
1188+
11861189
static class RefreshablePluginHolder extends PluginBag.PluginHolder<SolrRequestHandler> {
11871190

11881191
private DumpRequestHandler rh;

‎solr/core/src/test/org/apache/solr/metrics/SolrPrometheusFormatterTest.java

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import io.prometheus.metrics.model.snapshots.CounterSnapshot;
2929
import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
3030
import io.prometheus.metrics.model.snapshots.Labels;
31+
import io.prometheus.metrics.model.snapshots.SummarySnapshot;
3132
import java.util.HashMap;
3233
import java.util.List;
3334
import java.util.Map;
@@ -84,11 +85,11 @@ public void testExportTimer() throws InterruptedException {
8485

8586
Labels expectedLabels = Labels.of("test", "test-value");
8687
testFormatter.exportTimer(expectedMetricName, metric, expectedLabels);
87-
assertTrue(testFormatter.getMetricGauges().containsKey(expectedMetricName));
88+
assertTrue(testFormatter.getMetricSummaries().containsKey(expectedMetricName));
8889

89-
GaugeSnapshot.GaugeDataPointSnapshot actual =
90-
testFormatter.getMetricGauges().get("test_metric").get(0);
91-
assertEquals(5000000000L, actual.getValue(), 500000000L);
90+
SummarySnapshot.SummaryDataPointSnapshot actual =
91+
testFormatter.getMetricSummaries().get("test_metric").get(0);
92+
assertEquals(5000L, actual.getSum(), 500L);
9293
assertEquals(expectedLabels, actual.getLabels());
9394
}
9495

@@ -199,5 +200,9 @@ public Map<String, List<CounterSnapshot.CounterDataPointSnapshot>> getMetricCoun
199200
public Map<String, List<GaugeSnapshot.GaugeDataPointSnapshot>> getMetricGauges() {
200201
return metricGauges;
201202
}
203+
204+
public Map<String, List<SummarySnapshot.SummaryDataPointSnapshot>> getMetricSummaries() {
205+
return metricSummaries;
206+
}
202207
}
203208
}

0 commit comments

Comments
 (0)