diff --git a/NOTICE b/NOTICE index b5846b48e..c920ec3fe 100644 --- a/NOTICE +++ b/NOTICE @@ -6,3 +6,6 @@ Boxever Ltd. (http://www.boxever.com/). This product includes software developed at SoundCloud Ltd. (http://soundcloud.com/). + +This product includes software developed as part of the +Ocelli project by Netflix Inc. (https://github.com/Netflix/ocelli/). diff --git a/simpleclient/src/main/java/io/prometheus/client/CKMSQuantiles.java b/simpleclient/src/main/java/io/prometheus/client/CKMSQuantiles.java new file mode 100644 index 000000000..763e8a003 --- /dev/null +++ b/simpleclient/src/main/java/io/prometheus/client/CKMSQuantiles.java @@ -0,0 +1,293 @@ +package io.prometheus.client; + +// Copied from https://raw.githubusercontent.com/Netflix/ocelli/master/ocelli-core/src/main/java/netflix/ocelli/stats/CKMSQuantiles.java +// Revision d0357b8bf5c17a173ce94d6b26823775b3f999f6 from Jan 21, 2015. +// +// This is the original code except for the following modifications: +// +// - Changed the type of the observed values from int to double. +// - Removed the Quantiles interface and corresponding @Override annotations. +// - Changed the package name. +// - Make get() return NaN when no sample was observed. +// - Make class package private + +/* + Copyright 2012 Andrew Wang (andrew@umbrant.com) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.ListIterator; + +/** + * Implementation of the Cormode, Korn, Muthukrishnan, and Srivastava algorithm + * for streaming calculation of targeted high-percentile epsilon-approximate + * quantiles. + * + * This is a generalization of the earlier work by Greenwald and Khanna (GK), + * which essentially allows different error bounds on the targeted quantiles, + * which allows for far more efficient calculation of high-percentiles. + * + * + * See: Cormode, Korn, Muthukrishnan, and Srivastava + * "Effective Computation of Biased Quantiles over Data Streams" in ICDE 2005 + * + * Greenwald and Khanna, + * "Space-efficient online computation of quantile summaries" in SIGMOD 2001 + * + */ +class CKMSQuantiles { + /** + * Total number of items in stream. + */ + private int count = 0; + + /** + * Used for tracking incremental compression. + */ + private int compressIdx = 0; + + /** + * Current list of sampled items, maintained in sorted order with error + * bounds. + */ + protected LinkedList sample; + + /** + * Buffers incoming items to be inserted in batch. + */ + private double[] buffer = new double[500]; + + private int bufferCount = 0; + + /** + * Array of Quantiles that we care about, along with desired error. + */ + private final Quantile quantiles[]; + + public CKMSQuantiles(Quantile[] quantiles) { + this.quantiles = quantiles; + this.sample = new LinkedList(); + } + + /** + * Add a new value from the stream. + * + * @param value + */ + public synchronized void insert(double value) { + buffer[bufferCount] = value; + bufferCount++; + + if (bufferCount == buffer.length) { + insertBatch(); + compress(); + } + } + + /** + * Get the estimated value at the specified quantile. + * + * @param q + * Queried quantile, e.g. 0.50 or 0.99. + * @return Estimated value at that quantile. + */ + public synchronized double get(double q) { + // clear the buffer + insertBatch(); + compress(); + + if (sample.size() == 0) { + return Double.NaN; + } + + int rankMin = 0; + int desired = (int) (q * count); + + ListIterator it = sample.listIterator(); + Item prev, cur; + cur = it.next(); + while (it.hasNext()) { + prev = cur; + cur = it.next(); + + rankMin += prev.g; + + if (rankMin + cur.g + cur.delta > desired + + (allowableError(desired) / 2)) { + return prev.value; + } + } + + // edge case of wanting max value + return sample.getLast().value; + } + + /** + * Specifies the allowable error for this rank, depending on which quantiles + * are being targeted. + * + * This is the f(r_i, n) function from the CKMS paper. It's basically how + * wide the range of this rank can be. + * + * @param rank + * the index in the list of samples + */ + private double allowableError(int rank) { + // NOTE: according to CKMS, this should be count, not size, but this + // leads + // to error larger than the error bounds. Leaving it like this is + // essentially a HACK, and blows up memory, but does "work". + // int size = count; + int size = sample.size(); + double minError = size + 1; + + for (Quantile q : quantiles) { + double error; + if (rank <= q.quantile * size) { + error = q.u * (size - rank); + } else { + error = q.v * rank; + } + if (error < minError) { + minError = error; + } + } + + return minError; + } + + private boolean insertBatch() { + if (bufferCount == 0) { + return false; + } + + Arrays.sort(buffer, 0, bufferCount); + + // Base case: no samples + int start = 0; + if (sample.size() == 0) { + Item newItem = new Item(buffer[0], 1, 0); + sample.add(newItem); + start++; + count++; + } + + ListIterator it = sample.listIterator(); + Item item = it.next(); + + for (int i = start; i < bufferCount; i++) { + double v = buffer[i]; + while (it.nextIndex() < sample.size() && item.value < v) { + item = it.next(); + } + + // If we found that bigger item, back up so we insert ourselves + // before it + if (item.value > v) { + it.previous(); + } + + // We use different indexes for the edge comparisons, because of the + // above + // if statement that adjusts the iterator + int delta; + if (it.previousIndex() == 0 || it.nextIndex() == sample.size()) { + delta = 0; + } + else { + delta = ((int) Math.floor(allowableError(it.nextIndex()))) - 1; + } + + Item newItem = new Item(v, 1, delta); + it.add(newItem); + count++; + item = newItem; + } + + bufferCount = 0; + return true; + } + + /** + * Try to remove extraneous items from the set of sampled items. This checks + * if an item is unnecessary based on the desired error bounds, and merges + * it with the adjacent item if it is. + */ + private void compress() { + if (sample.size() < 2) { + return; + } + + ListIterator it = sample.listIterator(); + int removed = 0; + + Item prev = null; + Item next = it.next(); + + while (it.hasNext()) { + prev = next; + next = it.next(); + + if (prev.g + next.g + next.delta <= allowableError(it.previousIndex())) { + next.g += prev.g; + // Remove prev. it.remove() kills the last thing returned. + it.previous(); + it.previous(); + it.remove(); + // it.next() is now equal to next, skip it back forward again + it.next(); + removed++; + } + } + } + + private class Item { + public final double value; + public int g; + public final int delta; + + public Item(double value, int lower_delta, int delta) { + this.value = value; + this.g = lower_delta; + this.delta = delta; + } + + @Override + public String toString() { + return String.format("%d, %d, %d", value, g, delta); + } + } + + public static class Quantile { + public final double quantile; + public final double error; + public final double u; + public final double v; + + public Quantile(double quantile, double error) { + this.quantile = quantile; + this.error = error; + u = 2.0 * error / (1.0 - quantile); + v = 2.0 * error / quantile; + } + + @Override + public String toString() { + return String.format("Q{q=%.3f, eps=%.3f})", quantile, error); + } + } + +} diff --git a/simpleclient/src/main/java/io/prometheus/client/Collector.java b/simpleclient/src/main/java/io/prometheus/client/Collector.java index 4e6edb3da..b398c28f7 100644 --- a/simpleclient/src/main/java/io/prometheus/client/Collector.java +++ b/simpleclient/src/main/java/io/prometheus/client/Collector.java @@ -186,6 +186,9 @@ public static String doubleToGoString(double d) { if (d == Double.NEGATIVE_INFINITY) { return "-Inf"; } + if (Double.isNaN(d)) { + return "NaN"; + } return Double.toString(d); } } diff --git a/simpleclient/src/main/java/io/prometheus/client/Summary.java b/simpleclient/src/main/java/io/prometheus/client/Summary.java index 952be6a0c..a52114098 100644 --- a/simpleclient/src/main/java/io/prometheus/client/Summary.java +++ b/simpleclient/src/main/java/io/prometheus/client/Summary.java @@ -1,8 +1,13 @@ package io.prometheus.client; +import io.prometheus.client.CKMSQuantiles.Quantile; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; /** * Summary metric, to track the size of events. @@ -12,7 +17,7 @@ *
  • Response latency
    • *
  • Request size
    • * - * + * *

    * Example Summaries: * 

    @@ -23,7 +28,7 @@
  *     static final Summary requestLatency = Summary.build()
  *         .name("requests_latency_seconds").help("Request latency in seconds.").register();
  *
- *     void processRequest(Request req) {  
+ *     void processRequest(Request req) {
  *        Summary.Timer requestTimer = requestLatency.startTimer();
  *        try {
  *          // Your code here.
@@ -36,16 +41,87 @@
  * }
  *
    * This would allow you to track request rate, average latency and average request size. + * + *

    + * How to add custom quantiles: + * 

    + * {@code
+ *     static final Summary myMetric = Summary.build()
+ *             .quantile(0.5, 0.05)   // Add 50th percentile (= median) with 5% tolerated error
+ *             .quantile(0.9, 0.01)   // Add 90th percentile with 1% tolerated error
+ *             .quantile(0.99, 0.001) // Add 99th percentile with 0.1% tolerated error
+ *             .name("requests_size_bytes")
+ *             .help("Request size in bytes.")
+ *             .register();
+ * }
+ *
    + * + * The quantiles are calculated over a sliding window of time. There are two options to configure this time window: + *
      + *
    • maxAgeSeconds(long): Set the duration of the time window is, i.e. how long observations are kept before they are discarded. + * Default is 10 minutes. + *
    • ageBuckets(int): Set the number of buckets used to implement the sliding time window. If your time window is 10 minutes, and you have ageBuckets=5, + * buckets will be switched every 2 minutes. The value is a trade-off between resources (memory and cpu for maintaining the bucket) + * and how smooth the time window is moved. Default value is 5. + *
    + * + * See https://prometheus.io/docs/practices/histograms/ for more info on quantiles. */ public class Summary extends SimpleCollector { + final List quantiles; // Can be empty, but can never be null. + final long maxAgeSeconds; + final int ageBuckets; + Summary(Builder b) { super(b); + quantiles = Collections.unmodifiableList(new ArrayList(b.quantiles)); + this.maxAgeSeconds = b.maxAgeSeconds; + this.ageBuckets = b.ageBuckets; + initializeNoLabelsChild(); } public static class Builder extends SimpleCollector.Builder { + + private List quantiles = new ArrayList(); + private long maxAgeSeconds = TimeUnit.MINUTES.toSeconds(10); + private int ageBuckets = 5; + + public Builder quantile(double quantile, double error) { + if (quantile < 0.0 || quantile > 1.0) { + throw new IllegalArgumentException("Quantile " + quantile + " invalid: Expected number between 0.0 and 1.0."); + } + if (error < 0.0 || error > 1.0) { + throw new IllegalArgumentException("Error " + error + " invalid: Expected number between 0.0 and 1.0."); + } + quantiles.add(new Quantile(quantile, error)); + return this; + } + + public Builder maxAgeSeconds(long maxAgeSeconds) { + if (maxAgeSeconds <= 0) { + throw new IllegalArgumentException("maxAgeSeconds cannot be " + maxAgeSeconds); + } + this.maxAgeSeconds = maxAgeSeconds; + return this; + } + + public Builder ageBuckets(int ageBuckets) { + if (ageBuckets <= 0) { + throw new IllegalArgumentException("ageBuckets cannot be " + ageBuckets); + } + this.ageBuckets = ageBuckets; + return this; + } + @Override public Summary create() { + for (String label : labelNames) { + if (label.equals("quantile")) { + throw new IllegalStateException("Summary cannot have a label named 'quantile'."); + } + } + dontInitializeNoLabelsChild = true; return new Summary(this); } } @@ -59,7 +135,7 @@ public static Builder build() { @Override protected Child newChild() { - return new Child(); + return new Child(quantiles, maxAgeSeconds, ageBuckets); } /** @@ -90,13 +166,23 @@ public double observeDuration() { * {@link SimpleCollector#remove} or {@link SimpleCollector#clear}. */ public static class Child { - public static class Value { + private static class Value { public final double count; public final double sum; + public final SortedMap quantiles; - private Value(double count, double sum) { + private Value(double count, double sum, List quantiles, TimeWindowQuantiles quantileValues) { this.count = count; this.sum = sum; + this.quantiles = Collections.unmodifiableSortedMap(snapshot(quantiles, quantileValues)); + } + + private SortedMap snapshot(List quantiles, TimeWindowQuantiles quantileValues) { + SortedMap result = new TreeMap(); + for (Quantile q : quantiles) { + result.put(q.quantile, quantileValues.get(q.quantile)); + } + return result; } } @@ -106,14 +192,29 @@ private Value(double count, double sum) { // This should be reevaluated in the future. private final DoubleAdder count = new DoubleAdder(); private final DoubleAdder sum = new DoubleAdder(); + private final List quantiles; + private final TimeWindowQuantiles quantileValues; static TimeProvider timeProvider = new TimeProvider(); + + private Child(List quantiles, long maxAgeSeconds, int ageBuckets) { + this.quantiles = quantiles; + if (quantiles.size() > 0) { + quantileValues = new TimeWindowQuantiles(quantiles.toArray(new Quantile[]{}), maxAgeSeconds, ageBuckets); + } else { + quantileValues = null; + } + } + /** * Observe the given amount. */ public void observe(double amt) { count.add(1); sum.add(amt); + if (quantileValues != null) { + quantileValues.insert(amt); + } } /** * Start a timer to track a duration. @@ -129,7 +230,7 @@ public Timer startTimer() { * Warning: The definition of {@link Value} is subject to change. */ public Value get() { - return new Value(count.sum(), sum.sum()); + return new Value(count.sum(), sum.sum(), quantiles, quantileValues); } } @@ -154,6 +255,13 @@ public List collect() { List samples = new ArrayList(); for(Map.Entry, Child> c: children.entrySet()) { Child.Value v = c.getValue().get(); + List labelNamesWithQuantile = new ArrayList(labelNames); + labelNamesWithQuantile.add("quantile"); + for(Map.Entry q : v.quantiles.entrySet()) { + List labelValuesWithQuantile = new ArrayList(c.getKey()); + labelValuesWithQuantile.add(doubleToGoString(q.getKey())); + samples.add(new MetricFamilySamples.Sample(fullname, labelNamesWithQuantile, labelValuesWithQuantile, q.getValue())); + } samples.add(new MetricFamilySamples.Sample(fullname + "_count", labelNames, c.getKey(), v.count)); samples.add(new MetricFamilySamples.Sample(fullname + "_sum", labelNames, c.getKey(), v.sum)); } diff --git a/simpleclient/src/main/java/io/prometheus/client/TimeWindowQuantiles.java b/simpleclient/src/main/java/io/prometheus/client/TimeWindowQuantiles.java new file mode 100644 index 000000000..a61acd305 --- /dev/null +++ b/simpleclient/src/main/java/io/prometheus/client/TimeWindowQuantiles.java @@ -0,0 +1,54 @@ +package io.prometheus.client; + +import io.prometheus.client.CKMSQuantiles.Quantile; +import java.util.concurrent.TimeUnit; + +/** + * Wrapper around CKMSQuantiles. + * + * Maintains a ring buffer of CKMSQuantiles to provide quantiles over a sliding windows of time. + */ +class TimeWindowQuantiles { + + private final Quantile[] quantiles; + private final CKMSQuantiles[] ringBuffer; + private int currentBucket; + private long lastRotateTimestampMillis; + private final long durationBetweenRotatesMillis; + + public TimeWindowQuantiles(Quantile[] quantiles, long maxAgeSeconds, int ageBuckets) { + this.quantiles = quantiles; + this.ringBuffer = new CKMSQuantiles[ageBuckets]; + for (int i = 0; i < ageBuckets; i++) { + this.ringBuffer[i] = new CKMSQuantiles(quantiles); + } + this.currentBucket = 0; + this.lastRotateTimestampMillis = System.currentTimeMillis(); + this.durationBetweenRotatesMillis = TimeUnit.SECONDS.toMillis(maxAgeSeconds) / ageBuckets; + } + + public double get(double q) { + CKMSQuantiles currentBucket = rotate(); + return currentBucket.get(q); + } + + public void insert(double value) { + rotate(); + for (int i=0; i durationBetweenRotatesMillis) { + ringBuffer[currentBucket] = new CKMSQuantiles(quantiles); + if (++currentBucket >= ringBuffer.length) { + currentBucket = 0; + } + timeSinceLastRotateMillis -= durationBetweenRotatesMillis; + lastRotateTimestampMillis += durationBetweenRotatesMillis; + } + return ringBuffer[currentBucket]; + } +} diff --git a/simpleclient/src/test/java/io/prometheus/client/SummaryTest.java b/simpleclient/src/test/java/io/prometheus/client/SummaryTest.java index 48f18902e..013f70873 100644 --- a/simpleclient/src/test/java/io/prometheus/client/SummaryTest.java +++ b/simpleclient/src/test/java/io/prometheus/client/SummaryTest.java @@ -1,24 +1,37 @@ package io.prometheus.client; -import static org.junit.Assert.assertEquals; - -import java.util.ArrayList; -import java.util.List; import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; + public class SummaryTest { CollectorRegistry registry; - Summary noLabels, labels; + Summary noLabels, labels, labelsAndQuantiles, noLabelsAndQuantiles; @Before public void setUp() { registry = new CollectorRegistry(); noLabels = Summary.build().name("nolabels").help("help").register(registry); labels = Summary.build().name("labels").help("help").labelNames("l").register(registry); + noLabelsAndQuantiles = Summary.build() + .quantile(0.5, 0.05) + .quantile(0.9, 0.01) + .quantile(0.99, 0.001) + .name("no_labels_and_quantiles").help("help").register(registry); + labelsAndQuantiles = Summary.build() + .quantile(0.5, 0.05) + .quantile(0.9, 0.01) + .quantile(0.99, 0.001) + .labelNames("l") + .name("labels_and_quantiles").help("help").register(registry); } @After @@ -32,7 +45,13 @@ private double getCount() { private double getSum() { return registry.getSampleValue("nolabels_sum").doubleValue(); } - + private double getNoLabelQuantile(double q) { + return registry.getSampleValue("no_labels_and_quantiles", new String[]{"quantile"}, new String[]{Collector.doubleToGoString(q)}).doubleValue(); + } + private double getLabeledQuantile(String labelValue, double q) { + return registry.getSampleValue("labels_and_quantiles", new String[]{"l", "quantile"}, new String[]{labelValue, Collector.doubleToGoString(q)}).doubleValue(); + } + @Test public void testObserve() { noLabels.observe(2); @@ -43,6 +62,43 @@ public void testObserve() { assertEquals(6.0, getSum(), .001); } + @Test + public void testQuantiles() { + int nSamples = 1000000; // simulate one million samples + + for (int i=1; i<=nSamples; i++) { + // In this test, we observe the numbers from 1 to nSamples, + // because that makes it easy to verify if the quantiles are correct. + labelsAndQuantiles.labels("a").observe(i); + noLabelsAndQuantiles.observe(i); + } + assertEquals(getNoLabelQuantile(0.5), 0.5 * nSamples, 0.05 * nSamples); + assertEquals(getNoLabelQuantile(0.9), 0.9 * nSamples, 0.01 * nSamples); + assertEquals(getNoLabelQuantile(0.99), 0.99 * nSamples, 0.001 * nSamples); + + assertEquals(getLabeledQuantile("a", 0.5), 0.5 * nSamples, 0.05 * nSamples); + assertEquals(getLabeledQuantile("a", 0.9), 0.9 * nSamples, 0.01 * nSamples); + assertEquals(getLabeledQuantile("a", 0.99), 0.99 * nSamples, 0.001 * nSamples); + } + + @Test + public void testMaxAge() throws InterruptedException { + Summary summary = Summary.build() + .quantile(0.99, 0.001) + .maxAgeSeconds(1) // After 1s, all observations will be discarded. + .ageBuckets(2) // We got 2 buckets, so we discard one bucket every 500ms. + .name("short_attention_span").help("help").register(registry); + summary.observe(8.0); + double val = registry.getSampleValue("short_attention_span", new String[]{"quantile"}, new String[]{Collector.doubleToGoString(0.99)}).doubleValue(); + assertEquals(8.0, val, 0.0); // From bucket 1. + Thread.sleep(600); + val = registry.getSampleValue("short_attention_span", new String[]{"quantile"}, new String[]{Collector.doubleToGoString(0.99)}).doubleValue(); + assertEquals(8.0, val, 0.0); // From bucket 2. + Thread.sleep(600); + val = registry.getSampleValue("short_attention_span", new String[]{"quantile"}, new String[]{Collector.doubleToGoString(0.99)}).doubleValue(); + assertEquals(Double.NaN, val, 0.0); // Bucket 1 again, now it is empty. + } + @Test public void testTimer() { Summary.Child.timeProvider = new Summary.TimeProvider() { @@ -94,7 +150,7 @@ public void testLabels() { public void testCollect() { labels.labels("a").observe(2); List mfs = labels.collect(); - + ArrayList samples = new ArrayList(); ArrayList labelNames = new ArrayList(); labelNames.add("l"); @@ -108,4 +164,20 @@ public void testCollect() { assertEquals(mfsFixture, mfs.get(0)); } + @Test + public void testCollectWithQuantiles() { + labelsAndQuantiles.labels("a").observe(2); + List mfs = labelsAndQuantiles.collect(); + + ArrayList samples = new ArrayList(); + samples.add(new Collector.MetricFamilySamples.Sample("labels_and_quantiles", asList("l", "quantile"), asList("a", "0.5"), 2.0)); + samples.add(new Collector.MetricFamilySamples.Sample("labels_and_quantiles", asList("l", "quantile"), asList("a", "0.9"), 2.0)); + samples.add(new Collector.MetricFamilySamples.Sample("labels_and_quantiles", asList("l", "quantile"), asList("a", "0.99"), 2.0)); + samples.add(new Collector.MetricFamilySamples.Sample("labels_and_quantiles_count", asList("l"), asList("a"), 1.0)); + samples.add(new Collector.MetricFamilySamples.Sample("labels_and_quantiles_sum", asList("l"), asList("a"), 2.0)); + Collector.MetricFamilySamples mfsFixture = new Collector.MetricFamilySamples("labels_and_quantiles", Collector.Type.SUMMARY, "help", samples); + + assertEquals(1, mfs.size()); + assertEquals(mfsFixture, mfs.get(0)); + } } diff --git a/simpleclient_common/src/test/java/io/prometheus/client/exporter/common/TextFormatTest.java b/simpleclient_common/src/test/java/io/prometheus/client/exporter/common/TextFormatTest.java index bfbc663f4..441d3fb46 100644 --- a/simpleclient_common/src/test/java/io/prometheus/client/exporter/common/TextFormatTest.java +++ b/simpleclient_common/src/test/java/io/prometheus/client/exporter/common/TextFormatTest.java @@ -65,6 +65,23 @@ public void testSummaryOutput() throws IOException { + "nolabels_sum 2.0\n", writer.toString()); } + @Test + public void testSummaryOutputWithQuantiles() throws IOException { + Summary labelsAndQuantiles = Summary.build() + .quantile(0.5, 0.05).quantile(0.9, 0.01).quantile(0.99, 0.001) + .labelNames("l").name("labelsAndQuantiles").help("help").register(registry); + labelsAndQuantiles.labels("a").observe(2); + writer = new StringWriter(); + TextFormat.write004(writer, registry.metricFamilySamples()); + assertEquals("# HELP labelsAndQuantiles help\n" + + "# TYPE labelsAndQuantiles summary\n" + + "labelsAndQuantiles{l=\"a\",quantile=\"0.5\",} 2.0\n" + + "labelsAndQuantiles{l=\"a\",quantile=\"0.9\",} 2.0\n" + + "labelsAndQuantiles{l=\"a\",quantile=\"0.99\",} 2.0\n" + + "labelsAndQuantiles_count{l=\"a\",} 1.0\n" + + "labelsAndQuantiles_sum{l=\"a\",} 2.0\n", writer.toString()); + } + @Test public void testLabelsOutput() throws IOException { Gauge labels = (Gauge) Gauge.build().name("labels").help("help").labelNames("l").register(registry);