diff --git a/CHANGELOG.md b/CHANGELOG.md index 33919806b73..39aec2f15fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 24.1.2-SNAPSHOT ### Breaking Changes +- Following the OpenMetrics convention, the updated Prometheus client adds the `_total` suffix to every metrics of type counter, with the effect that some existing metrics have been renamed to have this suffix. If you are using the official Besu Grafana dashboard [(available here)](https://grafana.com/grafana/dashboards/16455-besu-full/), just update it to the latest revision, that accepts the old and the new name of the affected metrics. If you have a custom dashboards or use the metrics in other ways, then you need to manually update it to support the new naming. - The `trace-filter` method in JSON-RPC API now has a default block range limit of 1000, adjustable with `--rpc-max-trace-filter-range` (thanks @alyokaz) [#6446](https://github.com/hyperledger/besu/pull/6446) - Requesting the Ethereum Node Record (ENR) to acquire the fork id from bonded peers is now enabled by default, so the following change has been made [#5628](https://github.com/hyperledger/besu/pull/5628): - `--Xfilter-on-enr-fork-id` has been removed. To disable the feature use `--filter-on-enr-fork-id=false`. @@ -12,6 +13,7 @@ ### Deprecations ### Additions and Improvements +- Upgrade Prometheus and Opentelemetry dependencies [#6422](https://github.com/hyperledger/besu/pull/6422) - Add `OperationTracer.tracePrepareTransaction`, where the sender account has not yet been altered[#6453](https://github.com/hyperledger/besu/pull/6453) - Improve the high spec flag by limiting it to a few column families [#6354](https://github.com/hyperledger/besu/pull/6354) - Log blob count when importing a block via Engine API [#6466](https://github.com/hyperledger/besu/pull/6466) @@ -30,7 +32,7 @@ - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343) - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 (thanks @h4l) [#6360](https://github.com/hyperledger/besu/pull/6360) - The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411) -- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis, this to prevent possible DoS in case a single transaction is taking too long to execute, and to have a stable block production rate, but it could be a breaking change if an existing network used to have transactions that takes more time to executed that the newly introduced limit, if it is mandatory for these network to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. [#6423](https://github.com/hyperledger/besu/pull/6423) +- The time that can be spent selecting transactions during block creation is not capped at 5 seconds for PoS and PoW networks, and for PoA networks, at 75% of the block period specified in the genesis. This is to prevent possible DoS attacks in case a single transaction is taking too long to execute, and to have a stable block production rate. This could be a breaking change if an existing network needs to accept transactions that take more time to executed than the newly introduced limit. If it is mandatory for these networks to keep processing these long processing transaction, then the default value of `block-txs-selection-max-time` or `poa-block-txs-selection-max-time` needs to be tuned accordingly. [#6423](https://github.com/hyperledger/besu/pull/6423) ### Deprecations @@ -63,6 +65,7 @@ Note, due to a CI race with the release job, the initial published version of 24 ~~https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.zip / sha256 b6b64f939e0bb4937ce90fc647e0a7073ce3e359c10352b502059955070a60c6 https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/24.1.1/besu-24.1.1.tar.gz / sha256 cfcae04c30769bf338b0740ac65870f9346d3469931bb46cdba3b2f65d311e7a~~ + ## 24.1.0 ### Breaking Changes diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 673ac2df400..9a8866b5f1e 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1032,16 +1032,16 @@ - - - - - + + + + + @@ -1055,11 +1055,6 @@ - - - - - @@ -1068,6 +1063,11 @@ + + + + + @@ -1089,16 +1089,16 @@ - - - - - + + + + + @@ -2112,212 +2112,223 @@ - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + + + + - - + + - - + + + + + + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + + + + @@ -2423,25 +2434,20 @@ - - - - - - - - + + + - - + + - - - + + + - - + + @@ -2452,12 +2458,12 @@ - - - + + + - - + + @@ -2468,12 +2474,41 @@ - - - + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + @@ -2816,19 +2851,6 @@ - - - - - - - - - - - - - diff --git a/gradle/versions.gradle b/gradle/versions.gradle index 43fe774c463..23731c443d8 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -80,16 +80,16 @@ dependencyManagement { dependency group: 'io.netty', name: 'netty-transport-native-kqueue', version:'4.1.104.Final', classifier: 'osx-x86_64' dependency 'io.netty:netty-transport-native-unix-common:4.1.104.Final' - dependency 'io.opentelemetry:opentelemetry-api:1.24.0' - dependency 'io.opentelemetry:opentelemetry-exporter-otlp:1.24.0' - dependency 'io.opentelemetry:opentelemetry-extension-trace-propagators:1.24.0' - dependency 'io.opentelemetry.proto:opentelemetry-proto:0.19.0-alpha' - dependency 'io.opentelemetry:opentelemetry-sdk-metrics:1.24.0' - dependency 'io.opentelemetry:opentelemetry-sdk-trace:1.24.0' - dependency 'io.opentelemetry:opentelemetry-sdk:1.24.0' - dependency 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.24.0-alpha' - dependency 'io.opentelemetry:opentelemetry-semconv:1.24.0-alpha' - dependency 'io.opentelemetry.instrumentation:opentelemetry-okhttp-3.0:1.24.0-alpha' + dependency 'io.opentelemetry:opentelemetry-api:1.33.0' + dependency 'io.opentelemetry:opentelemetry-exporter-otlp:1.33.0' + dependency 'io.opentelemetry:opentelemetry-extension-trace-propagators:1.33.0' + dependency 'io.opentelemetry:opentelemetry-sdk-metrics:1.33.0' + dependency 'io.opentelemetry:opentelemetry-sdk-trace:1.33.0' + dependency 'io.opentelemetry:opentelemetry-sdk:1.33.0' + dependency 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.33.0' + dependency 'io.opentelemetry.instrumentation:opentelemetry-okhttp-3.0:1.32.0-alpha' + dependency 'io.opentelemetry.proto:opentelemetry-proto:1.0.0-alpha' + dependency 'io.opentelemetry.semconv:opentelemetry-semconv:1.23.1-alpha' dependency 'io.opentracing.contrib:opentracing-okhttp3:3.0.0' dependency 'io.opentracing:opentracing-api:0.33.0' @@ -97,11 +97,13 @@ dependencyManagement { dependency 'io.pkts:pkts-core:3.0.10' - dependency 'io.prometheus:simpleclient:0.9.0' - dependency 'io.prometheus:simpleclient_common:0.9.0' - dependency 'io.prometheus:simpleclient_hotspot:0.9.0' - dependency 'io.prometheus:simpleclient_pushgateway:0.9.0' - dependency 'io.prometheus:simpleclient_guava:0.16.0' + dependencySet(group: 'io.prometheus', version: '0.16.0') { + entry 'simpleclient' + entry 'simpleclient_common' + entry 'simpleclient_hotspot' + entry 'simpleclient_pushgateway' + entry 'simpleclient_guava' + } dependency 'io.reactivex.rxjava2:rxjava:2.2.21' diff --git a/metrics/core/build.gradle b/metrics/core/build.gradle index 295cdada976..bc92cb6b5fb 100644 --- a/metrics/core/build.gradle +++ b/metrics/core/build.gradle @@ -48,11 +48,11 @@ dependencies { implementation 'io.netty:netty-all' implementation 'io.opentelemetry:opentelemetry-api' implementation 'io.opentelemetry:opentelemetry-sdk' - implementation 'io.opentelemetry:opentelemetry-semconv' implementation 'io.opentelemetry:opentelemetry-sdk-trace' implementation 'io.opentelemetry:opentelemetry-sdk-metrics' implementation 'io.opentelemetry:opentelemetry-exporter-otlp' implementation 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure' + implementation 'io.opentelemetry.semconv:opentelemetry-semconv' implementation 'io.prometheus:simpleclient' implementation 'io.prometheus:simpleclient_common' diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/DebugMetricReader.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/DebugMetricReader.java index ea81e91aff9..a44ad1c1cee 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/DebugMetricReader.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/DebugMetricReader.java @@ -22,7 +22,6 @@ import io.opentelemetry.sdk.metrics.data.MetricData; import io.opentelemetry.sdk.metrics.export.CollectionRegistration; import io.opentelemetry.sdk.metrics.export.MetricReader; -import io.opentelemetry.sdk.metrics.internal.export.MetricProducer; import org.jetbrains.annotations.NotNull; class DebugMetricReader implements MetricReader { @@ -31,7 +30,7 @@ class DebugMetricReader implements MetricReader { public DebugMetricReader() {} public Collection getAllMetrics() { - return MetricProducer.asMetricProducer(this.registration).collectAllMetrics(); + return registration.collectAllMetrics(); } @Override diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java index 2c43eb8668e..c52dd037f76 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java @@ -50,6 +50,7 @@ import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.data.DoublePointData; @@ -61,7 +62,7 @@ import io.opentelemetry.sdk.metrics.data.SummaryPointData; import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; +import io.opentelemetry.semconv.ResourceAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,15 +111,18 @@ public OpenTelemetrySystem( .merge( Resource.create( Attributes.builder().put(ResourceAttributes.SERVICE_NAME, jobName).build())); - AutoConfiguredOpenTelemetrySdk autoSdk = + AutoConfiguredOpenTelemetrySdkBuilder autoSdkBuilder = AutoConfiguredOpenTelemetrySdk.builder() .addMeterProviderCustomizer( (provider, config) -> provider.setResource(resource).registerMetricReader(debugMetricReader)) - .addTracerProviderCustomizer((provider, config) -> provider.setResource(resource)) - .setResultAsGlobal(setAsGlobal) - .build(); - OpenTelemetrySdk sdk = autoSdk.getOpenTelemetrySdk(); + .addTracerProviderCustomizer((provider, config) -> provider.setResource(resource)); + + if (setAsGlobal) { + autoSdkBuilder.setResultAsGlobal(); + } + + OpenTelemetrySdk sdk = autoSdkBuilder.build().getOpenTelemetrySdk(); this.sdkMeterProvider = sdk.getSdkMeterProvider(); this.sdkTracerProvider = sdk.getSdkTracerProvider(); } diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java index fa07670521a..59211ee08a7 100644 --- a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java +++ b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java @@ -32,7 +32,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.DoubleSupplier; import java.util.function.Supplier; -import java.util.stream.Collectors; import java.util.stream.Stream; import com.google.common.collect.ImmutableSet; @@ -48,6 +47,7 @@ import io.prometheus.client.hotspot.MemoryPoolsExports; import io.prometheus.client.hotspot.StandardExports; import io.prometheus.client.hotspot.ThreadExports; +import io.vertx.core.impl.ConcurrentHashSet; /** The Prometheus metrics system. */ public class PrometheusMetricsSystem implements ObservableMetricsSystem { @@ -58,6 +58,7 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem { cachedCounters = new ConcurrentHashMap<>(); private final Map> cachedTimers = new ConcurrentHashMap<>(); + private final Set totalSuffixedCounters = new ConcurrentHashSet<>(); private final Set enabledCategories; private final boolean timersEnabled; @@ -95,7 +96,7 @@ public LabelledMetric crea final String name, final String help, final String... labelNames) { - final String metricName = convertToPrometheusName(category, name); + final String metricName = convertToPrometheusCounterName(category, name); return cachedCounters.computeIfAbsent( metricName, (k) -> { @@ -185,9 +186,7 @@ private void addCollectorUnchecked(final MetricCategory category, final Collecto category, key -> Collections.newSetFromMap(new ConcurrentHashMap<>())); final List newSamples = - metric.collect().stream() - .map(metricFamilySamples -> metricFamilySamples.name) - .collect(Collectors.toList()); + metric.collect().stream().map(metricFamilySamples -> metricFamilySamples.name).toList(); metrics.stream() .filter( @@ -230,6 +229,9 @@ private Observation createObservationFromSample( if (familySamples.type == Collector.Type.SUMMARY) { return convertSummarySampleNamesToLabels(category, sample, familySamples); } + if (familySamples.type == Collector.Type.COUNTER) { + return convertCounterNamesToLabels(category, sample, familySamples); + } return new Observation( category, convertFromPrometheusName(category, sample.name), @@ -237,6 +239,20 @@ private Observation createObservationFromSample( sample.labelValues); } + private Observation convertCounterNamesToLabels( + final MetricCategory category, final Sample sample, final MetricFamilySamples familySamples) { + final List labelValues = new ArrayList<>(sample.labelValues); + if (sample.name.endsWith("_created")) { + labelValues.add("created"); + } + + return new Observation( + category, + convertFromPrometheusCounterName(category, familySamples.name), + sample.value, + labelValues); + } + private Observation convertHistogramSampleNamesToLabels( final MetricCategory category, final Sample sample, final MetricFamilySamples familySamples) { final List labelValues = new ArrayList<>(sample.labelValues); @@ -259,6 +275,8 @@ private Observation convertSummarySampleNamesToLabels( labelValues.add("sum"); } else if (sample.name.endsWith("_count")) { labelValues.add("count"); + } else if (sample.name.endsWith("_created")) { + labelValues.add("created"); } else { labelValues.add(labelValues.size() - 1, "quantile"); } @@ -280,11 +298,34 @@ public String convertToPrometheusName(final MetricCategory category, final Strin return prometheusPrefix(category) + name; } + /** + * Convert to prometheus counter name. Prometheus adds a _total suffix to the name if not present, + * so we remember if the original name already has it, to be able to covert back correctly + * + * @param category the category + * @param name the name + * @return the name as string + */ + public String convertToPrometheusCounterName(final MetricCategory category, final String name) { + if (name.endsWith("_total")) { + totalSuffixedCounters.add(name); + } + return convertToPrometheusName(category, name); + } + private String convertFromPrometheusName(final MetricCategory category, final String metricName) { final String prefix = prometheusPrefix(category); return metricName.startsWith(prefix) ? metricName.substring(prefix.length()) : metricName; } + private String convertFromPrometheusCounterName( + final MetricCategory category, final String metricName) { + final String unPrefixedName = convertFromPrometheusName(category, metricName); + return totalSuffixedCounters.contains(unPrefixedName + "_total") + ? unPrefixedName + "_total" + : unPrefixedName; + } + private String prometheusPrefix(final MetricCategory category) { return category.getApplicationPrefix().orElse("") + category.getName() + "_"; } diff --git a/metrics/core/src/test/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystemTest.java b/metrics/core/src/test/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystemTest.java index 7c62f1f0b6c..2ddf86d347b 100644 --- a/metrics/core/src/test/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystemTest.java +++ b/metrics/core/src/test/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystemTest.java @@ -17,6 +17,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static java.util.function.Predicate.not; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hyperledger.besu.metrics.BesuMetricCategory.DEFAULT_METRIC_CATEGORIES; @@ -51,6 +52,11 @@ public class PrometheusMetricsSystemTest { Comparator.comparing(observation -> observation.getCategory().getName()) .thenComparing(Observation::getMetricName) .thenComparing((o1, o2) -> o1.getLabels().equals(o2.getLabels()) ? 0 : 1); + private static final Comparator WITH_VALUES = + Comparator.comparing(observation -> observation.getCategory().getName()) + .thenComparing(Observation::getMetricName) + .thenComparing((o1, o2) -> o1.getLabels().equals(o2.getLabels()) ? 0 : 1) + .thenComparing((o1, o2) -> o1.getValue().equals(o2.getValue()) ? 0 : 1); @BeforeEach public void resetGlobalOpenTelemetry() { @@ -66,11 +72,17 @@ public void shouldCreateObservationFromCounter() { counter.inc(); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 1.0, emptyList())); + .usingElementComparator(this::compareCounters) + .containsExactlyInAnyOrder( + new Observation(PEERS, "connected", 1.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); counter.inc(); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 2.0, emptyList())); + .usingElementComparator(this::compareCounters) + .containsExactly( + new Observation(PEERS, "connected", 2.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); } @Test @@ -83,26 +95,36 @@ public void shouldHandleDuplicateCounterCreation() { counter1.labels().inc(); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 1.0, emptyList())); + .usingElementComparator(this::compareCounters) + .containsExactly( + new Observation(PEERS, "connected", 1.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); counter2.labels().inc(); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 2.0, emptyList())); + .usingElementComparator(this::compareCounters) + .containsExactly( + new Observation(PEERS, "connected", 2.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); } @Test public void shouldCreateSeparateObservationsForEachCounterLabelValue() { final LabelledMetric counter = - metricsSystem.createLabelledCounter(PEERS, "connected", "Some help string", "labelName"); + metricsSystem.createLabelledCounter( + PEERS, "connected_total", "Some help string", "labelName"); counter.labels("value1").inc(); counter.labels("value2").inc(); counter.labels("value1").inc(); assertThat(metricsSystem.streamObservations()) + .usingElementComparator(this::compareCounters) .containsExactlyInAnyOrder( - new Observation(PEERS, "connected", 2.0, singletonList("value1")), - new Observation(PEERS, "connected", 1.0, singletonList("value2"))); + new Observation(PEERS, "connected_total", 2.0, singletonList("value1")), + new Observation(PEERS, "connected_total", 1.0, singletonList("value2")), + new Observation(PEERS, "connected_total", null, List.of("value1", "created")), + new Observation(PEERS, "connected_total", null, List.of("value2", "created"))); } @Test @@ -138,11 +160,18 @@ public void shouldIncrementCounterBySpecifiedAmount() { counter.inc(5); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 5.0, emptyList())); + .usingElementComparator(this::compareCounters) + .containsExactly( + new Observation(PEERS, "connected", 5.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); counter.inc(6); assertThat(metricsSystem.streamObservations()) - .containsExactly(new Observation(PEERS, "connected", 11.0, emptyList())); + .usingDefaultElementComparator() + .usingElementComparator(this::compareCounters) + .containsExactly( + new Observation(PEERS, "connected", 11.0, emptyList()), + new Observation(PEERS, "connected", null, List.of("created"))); } @Test @@ -162,7 +191,8 @@ public void shouldCreateObservationsFromTimer() { new Observation(RPC, "request", null, asList("quantile", "0.99")), new Observation(RPC, "request", null, asList("quantile", "1.0")), new Observation(RPC, "request", null, singletonList("sum")), - new Observation(RPC, "request", null, singletonList("count"))); + new Observation(RPC, "request", null, singletonList("count")), + new Observation(RPC, "request", null, singletonList("created"))); } @Test @@ -192,7 +222,8 @@ public void shouldCreateObservationsFromTimerWithLabels() { new Observation(RPC, "request", null, asList("method", "quantile", "0.99")), new Observation(RPC, "request", null, asList("method", "quantile", "1.0")), new Observation(RPC, "request", null, asList("method", "sum")), - new Observation(RPC, "request", null, asList("method", "count"))); + new Observation(RPC, "request", null, asList("method", "count")), + new Observation(RPC, "request", null, asList("method", "created"))); } @Test @@ -251,6 +282,8 @@ public void shouldOnlyObserveEnabledMetrics() { counterR.labels("op").inc(); assertThat(localMetricSystem.streamObservations()) + .usingRecursiveFieldByFieldElementComparator() + .filteredOn(not(this::isCreatedSample)) .containsExactly(new Observation(RPC, "name", 1.0, singletonList("op"))); } @@ -280,4 +313,18 @@ public void returnsNoOpMetricsWhenPushEnabled() { assertThat(localMetricSystem).isInstanceOf(PrometheusMetricsSystem.class); } + + private boolean isCreatedSample(final Observation obs) { + // Simple client 0.10.0 add a _created sample to every counter, histogram and summary, that we + // may want to ignore + return obs.getLabels().contains("created"); + } + + private int compareCounters(final Observation obs1, final Observation obs2) { + // for created samples ignore values + if (obs1.getLabels().contains("created") && obs2.getLabels().contains("created")) { + return IGNORE_VALUES.compare(obs1, obs2); + } + return WITH_VALUES.compare(obs1, obs2); + } }