From 47c2d39c7feeab6b861829850c32c2220f45c184 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 25 Mar 2021 12:07:33 -0700 Subject: [PATCH] Fix sampling rate recorded for dependencies (#1582) --- .../instrumentation/sdk/BytecodeUtilImpl.java | 34 ++++--- .../propagator/DelegatingPropagator.java | 82 ++++++++++++++++- .../agent/internal/sampling/AiSampler.java | 26 +++--- .../agent/internal/sampling/Samplers.java | 2 + .../internal/sampling/SamplingOverrides.java | 90 ++++++++++++++++--- .../internal/RpConfigurationPollingTest.java | 62 ++++++++++--- .../applicationinsights/agent/Exporter.java | 70 +++++++++++---- test/smoke/testApps/Sampling/build.gradle | 2 + .../ajl/simplecalc/SimpleSamplingServlet.java | 8 +- .../smoketest/SamplingTest.java | 39 ++++++-- 10 files changed, 346 insertions(+), 69 deletions(-) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/instrumentation/sdk/BytecodeUtilImpl.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/instrumentation/sdk/BytecodeUtilImpl.java index 6f793143755..0385107874a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/instrumentation/sdk/BytecodeUtilImpl.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/instrumentation/sdk/BytecodeUtilImpl.java @@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.base.Strings; +import com.microsoft.applicationinsights.agent.Exporter; import com.microsoft.applicationinsights.agent.bootstrap.BytecodeUtil.BytecodeUtilDelegate; import com.microsoft.applicationinsights.agent.internal.Global; import com.microsoft.applicationinsights.agent.internal.sampling.SamplingScoreGeneratorV2; @@ -220,20 +221,31 @@ public void logErrorOnce(Throwable t) { private static void track(Telemetry telemetry) { SpanContext context = Span.current().getSpanContext(); + double samplingPercentage; if (context.isValid()) { - String traceId = context.getTraceId(); - String spanId = context.getSpanId(); - telemetry.getContext().getOperation().setId(traceId); - telemetry.getContext().getOperation().setParentId(spanId); - } - double samplingPercentage = Global.getSamplingPercentage(); - if (sample(telemetry, samplingPercentage)) { - if (telemetry instanceof SupportSampling && samplingPercentage != 100) { - ((SupportSampling) telemetry).setSamplingPercentage(samplingPercentage); + if (!context.isSampled()) { + // sampled out + return; } - // this is not null because sdk instrumentation is not added until Global.setTelemetryClient() is called - checkNotNull(Global.getTelemetryClient()).track(telemetry); + telemetry.getContext().getOperation().setId(context.getTraceId()); + telemetry.getContext().getOperation().setParentId(context.getSpanId()); + samplingPercentage = + Exporter.getSamplingPercentage(context.getTraceState(), Global.getSamplingPercentage(), false); + } else { + // sampling is done using the configured sampling percentage + samplingPercentage = Global.getSamplingPercentage(); + if (!sample(telemetry, samplingPercentage)) { + // sampled out + return; + } + } + // sampled in + + if (telemetry instanceof SupportSampling && samplingPercentage != 100) { + ((SupportSampling) telemetry).setSamplingPercentage(samplingPercentage); } + // this is not null because sdk instrumentation is not added until Global.setTelemetryClient() is called + checkNotNull(Global.getTelemetryClient()).track(telemetry); } private static boolean sample(Telemetry telemetry, double samplingPercentage) { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/propagator/DelegatingPropagator.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/propagator/DelegatingPropagator.java index c4fcd634124..8e3b443805b 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/propagator/DelegatingPropagator.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/propagator/DelegatingPropagator.java @@ -3,6 +3,11 @@ import java.util.Collection; import javax.annotation.Nullable; +import com.microsoft.applicationinsights.agent.Exporter; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; import io.opentelemetry.context.Context; import io.opentelemetry.context.propagation.TextMapGetter; @@ -21,8 +26,13 @@ public static DelegatingPropagator getInstance() { } public void setUpStandardDelegate() { + + // TODO when should we enable baggage propagation? + // currently using modified W3CTraceContextPropagator because "ai-internal-sp" trace state + // shouldn't be sent over the wire (at least not yet, and not with that name) + // important that W3CTraceContextPropagator is last, so it will take precedence if both sets of headers are present - delegate = TextMapPropagator.composite(AiLegacyPropagator.getInstance(), W3CTraceContextPropagator.getInstance()); + delegate = TextMapPropagator.composite(AiLegacyPropagator.getInstance(), new ModifiedW3CTraceContextPropagator()); } @Override @@ -39,4 +49,74 @@ public void inject(Context context, @Nullable C carrier, TextMapSetter se public Context extract(Context context, @Nullable C carrier, TextMapGetter getter) { return delegate.extract(context, carrier, getter); } + + private static class ModifiedW3CTraceContextPropagator implements TextMapPropagator { + + private final TextMapPropagator delegate = W3CTraceContextPropagator.getInstance(); + + @Override + public Collection fields() { + return delegate.fields(); + } + + @Override + public void inject(Context context, @Nullable C carrier, TextMapSetter setter) { + // do not propagate sampling percentage downstream YET + SpanContext spanContext = Span.fromContext(context).getSpanContext(); + // sampling percentage should always be present, so no need to optimize with checking if present + TraceState traceState = spanContext.getTraceState(); + TraceState updatedTraceState; + if (traceState.size() == 1 && traceState.get(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE) != null) { + // this is a common case, worth optimizing + updatedTraceState = TraceState.getDefault(); + } else { + updatedTraceState = traceState.toBuilder() + .remove(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE) + .build(); + } + SpanContext updatedSpanContext = new ModifiedSpanContext(spanContext, updatedTraceState); + delegate.inject(Context.root().with(Span.wrap(updatedSpanContext)), carrier, setter); + } + + @Override + public Context extract(Context context, @Nullable C carrier, TextMapGetter getter) { + return delegate.extract(context, carrier, getter); + } + } + + private static class ModifiedSpanContext implements SpanContext { + + private final SpanContext delegate; + private final TraceState traceState; + + private ModifiedSpanContext(SpanContext delegate, TraceState traceState) { + this.delegate = delegate; + this.traceState = traceState; + } + + @Override + public String getTraceId() { + return delegate.getTraceId(); + } + + @Override + public String getSpanId() { + return delegate.getSpanId(); + } + + @Override + public TraceFlags getTraceFlags() { + return delegate.getTraceFlags(); + } + + @Override + public TraceState getTraceState() { + return traceState; + } + + @Override + public boolean isRemote() { + return delegate.isRemote(); + } + } } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java index 22628450445..0adf635c5aa 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java @@ -1,7 +1,6 @@ package com.microsoft.applicationinsights.agent.internal.sampling; import java.util.List; -import javax.annotation.Nullable; import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides.MatcherGroup; import io.opentelemetry.api.common.Attributes; @@ -27,7 +26,7 @@ class AiSampler implements Sampler { // // failure to follow this pattern can result in unexpected / incorrect computation of values in the portal private final double defaultSamplingPercentage; - private final SamplingResult defaultRecordAndSampleResult; + private final SamplingResult recordAndSampleAndAddTraceStateIfMissing; private final SamplingOverrides samplingOverrides; @@ -38,12 +37,13 @@ class AiSampler implements Sampler { // samplingPercentage is still used in BehaviorIfNoMatchingOverrides.RECORD_AND_SAMPLE // to set an approximate value for the span attribute "applicationinsights.internal.sampling_percentage" // - // in the future the sampling percentage (or its invert "count") will be + // in the future the sampling percentage (or its inverse "count") will be // carried down by trace state to set the accurate value AiSampler(double samplingPercentage, SamplingOverrides samplingOverrides, BehaviorIfNoMatchingOverrides behaviorIfNoMatchingOverrides) { this.defaultSamplingPercentage = samplingPercentage; - defaultRecordAndSampleResult = SamplingOverrides.getRecordAndSampleResult(defaultSamplingPercentage); + recordAndSampleAndAddTraceStateIfMissing = + SamplingOverrides.getRecordAndSampleAndAddTraceStateIfMissing(samplingPercentage); this.samplingOverrides = samplingOverrides; @@ -53,7 +53,7 @@ class AiSampler implements Sampler { } @Override - public SamplingResult shouldSample(@Nullable Context parentContext, + public SamplingResult shouldSample(Context parentContext, String traceId, String name, SpanKind spanKind, @@ -63,23 +63,27 @@ public SamplingResult shouldSample(@Nullable Context parentContext, MatcherGroup override = samplingOverrides.getOverride(attributes); if (override != null) { - return getSamplingResult(override.getPercentage(), override.getRecordAndSampleResult(), traceId, name); + return getSamplingResult(override.getPercentage(), override.getRecordAndSampleAndOverwriteTraceState(), traceId, name); } switch (behaviorIfNoMatchingOverrides) { case RECORD_AND_SAMPLE: - return defaultRecordAndSampleResult; + // this is used for localParentSampled and remoteParentSampled + // (note: currently sampling percentage portion of trace state is not propagated, + // so it will always be missing in the remoteParentSampled case) + return recordAndSampleAndAddTraceStateIfMissing; case USE_DEFAULT_SAMPLING_PERCENTAGE: - return getSamplingResult(defaultSamplingPercentage, defaultRecordAndSampleResult, traceId, name); + // this is used for root sampler + return getSamplingResult(defaultSamplingPercentage, recordAndSampleAndAddTraceStateIfMissing, traceId, name); default: throw new IllegalStateException("Unexpected BehaviorIfNoMatchingOverrides: " + behaviorIfNoMatchingOverrides); } } - private SamplingResult getSamplingResult(double percentage, SamplingResult recordAndSampleResult, String traceId, String name) { + private SamplingResult getSamplingResult(double percentage, SamplingResult sampledSamplingResult, String traceId, String name) { if (percentage == 100) { // optimization, no need to calculate score in this case - return recordAndSampleResult; + return sampledSamplingResult; } if (percentage == 0) { // optimization, no need to calculate score in this case @@ -89,7 +93,7 @@ private SamplingResult getSamplingResult(double percentage, SamplingResult recor logger.debug("Item {} sampled out", name); return dropDecision; } - return recordAndSampleResult; + return sampledSamplingResult; } @Override diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java index a47e4ec4f75..637a4ce3ade 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java @@ -13,6 +13,8 @@ public static Sampler getSampler(double samplingPercentage, Configuration config AiSampler.BehaviorIfNoMatchingOverrides.RECORD_AND_SAMPLE); // ignoreRemoteParentNotSampled is currently needed // because .NET SDK always propagates trace flags "00" (not sampled) + // NOTE: once we start propagating sampling percentage over the wire, we can use that to know that we can + // respect upstream decision for remoteParentNotSampled Sampler remoteParentNotSampled = config.preview.ignoreRemoteParentNotSampled ? rootSampler : Sampler.alwaysOff(); return Sampler.parentBasedBuilder(rootSampler) .setRemoteParentNotSampled(remoteParentNotSampled) diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java index 2aa57db60c9..16283ecb2b0 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java @@ -1,5 +1,7 @@ package com.microsoft.applicationinsights.agent.internal.sampling; +import java.math.BigDecimal; +import java.math.MathContext; import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; @@ -11,6 +13,7 @@ import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.SamplingOverrideAttribute; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; import io.opentelemetry.sdk.trace.samplers.SamplingResult; @@ -35,21 +38,84 @@ MatcherGroup getOverride(Attributes attributes) { return null; } - static SamplingResult getRecordAndSampleResult(double percentage) { - Attributes alwaysOnAttributes; - if (percentage != 100) { - alwaysOnAttributes = Attributes.of(Exporter.AI_SAMPLING_PERCENTAGE_KEY, percentage); - } else { - // the exporter assumes 100 when the AI_SAMPLING_PERCENTAGE_KEY attribute is not present - alwaysOnAttributes = Attributes.empty(); + static SamplingResult getRecordAndSampleAndOverwriteTraceState(double samplingPercentage) { + return new TraceStateUpdatingSamplingResult(SamplingDecision.RECORD_AND_SAMPLE, toRoundedString(samplingPercentage), true); + } + + static SamplingResult getRecordAndSampleAndAddTraceStateIfMissing(double samplingPercentage) { + return new TraceStateUpdatingSamplingResult(SamplingDecision.RECORD_AND_SAMPLE, toRoundedString(samplingPercentage), false); + } + + // TODO write test for + // * 33.33333333333 + // * 66.66666666666 + // * 1.123456 + // * 50.0 + // * 1.0 + // * 0 + // * 0.001 + // * 0.000001 + // 5 digit of precision, and remove any trailing zeros beyond the decimal point + private static String toRoundedString(double percentage) { + BigDecimal bigDecimal = new BigDecimal(percentage); + bigDecimal = bigDecimal.round(new MathContext(5)); + String formatted = bigDecimal.toString(); + double dv = bigDecimal.doubleValue(); + if (dv > 0 && dv < 1) { + while (formatted.endsWith("0")) { + formatted = formatted.substring(0, formatted.length() - 1); + } + } + return formatted; + } + + private static final class TraceStateUpdatingSamplingResult implements SamplingResult { + + private final SamplingDecision decision; + private final String samplingPercentage; + private final TraceState traceState; + private final boolean overwriteExisting; + + private TraceStateUpdatingSamplingResult(SamplingDecision decision, String samplingPercentage, + boolean overwriteExisting) { + this.decision = decision; + this.samplingPercentage = samplingPercentage; + this.overwriteExisting = overwriteExisting; + traceState = TraceState.builder().put(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE, samplingPercentage).build(); + } + + @Override + public SamplingDecision getDecision() { + return decision; + } + + @Override + public Attributes getAttributes() { + return Attributes.empty(); + } + + @Override + public TraceState getUpdatedTraceState(TraceState parentTraceState) { + if (parentTraceState.isEmpty()) { + return traceState; + } + String existingSamplingPercentage = parentTraceState.get(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE); + if (samplingPercentage.equals(existingSamplingPercentage)) { + return parentTraceState; + } + if (existingSamplingPercentage != null && !overwriteExisting) { + return parentTraceState; + } + return parentTraceState.toBuilder() + .put(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE, samplingPercentage) + .build(); } - return SamplingResult.create(SamplingDecision.RECORD_AND_SAMPLE, alwaysOnAttributes); } static class MatcherGroup { private final List> predicates; private final double percentage; - private final SamplingResult recordAndSampleResult; + private final SamplingResult recordAndSampleAndOverwriteTraceState; private MatcherGroup(SamplingOverride override) { predicates = new ArrayList<>(); @@ -57,15 +123,15 @@ private MatcherGroup(SamplingOverride override) { predicates.add(toPredicate(attribute)); } percentage = override.percentage; - recordAndSampleResult = SamplingOverrides.getRecordAndSampleResult(percentage); + recordAndSampleAndOverwriteTraceState = SamplingOverrides.getRecordAndSampleAndOverwriteTraceState(percentage); } double getPercentage() { return percentage; } - SamplingResult getRecordAndSampleResult() { - return recordAndSampleResult; + SamplingResult getRecordAndSampleAndOverwriteTraceState() { + return recordAndSampleAndOverwriteTraceState; } private boolean matches(Attributes attributes) { diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/RpConfigurationPollingTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/RpConfigurationPollingTest.java index 4afb36e88da..10e61677dee 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/RpConfigurationPollingTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/RpConfigurationPollingTest.java @@ -26,10 +26,19 @@ import com.google.common.io.Resources; import com.microsoft.applicationinsights.TelemetryConfiguration; +import com.microsoft.applicationinsights.agent.Exporter; import com.microsoft.applicationinsights.agent.internal.sampling.DelegatingSampler; +import com.microsoft.applicationinsights.agent.internal.sampling.Samplers; import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration; -import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.ConfigurationBuilder; import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.RpConfiguration; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; import org.junit.*; import org.junit.contrib.java.lang.system.*; @@ -40,11 +49,17 @@ public class RpConfigurationPollingTest { @Rule public EnvironmentVariables envVars = new EnvironmentVariables(); - @AfterClass - public static void tearDown() { + @Before + public void beforeEach() { + // default sampler at startup is "Sampler.alwaysOff()", and this test relies on real sampler + DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(100, new Configuration())); + } + + @After + public void afterEach() { // need to reset trace config back to default (with default sampler) // otherwise tests run after this can fail - DelegatingSampler.getInstance().setAlwaysOnDelegate(); + DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(100, new Configuration())); } @Test @@ -56,19 +71,25 @@ public void shouldUpdate() { rpConfiguration.configPath = new File(Resources.getResource("applicationinsights-rp.json").getPath()).toPath(); rpConfiguration.lastModifiedTime = 0; - TelemetryConfiguration.getActive().setConnectionString(rpConfiguration.connectionString); - Global.setSamplingPercentage(ConfigurationBuilder.roundToNearest(rpConfiguration.sampling.percentage)); + TelemetryConfiguration.getActive().setConnectionString("InstrumentationKey=00000000-0000-0000-0000-000000000000"); + Global.setSamplingPercentage(100); + + // pre-check + assertEquals("InstrumentationKey=00000000-0000-0000-0000-000000000000", TelemetryConfiguration.getActive().getConnectionString()); + assertEquals(100, Global.getSamplingPercentage(), 0); + assertEquals(100, getCurrentSamplingPercentage(), 0); // when new RpConfigurationPolling(rpConfiguration, new Configuration()).run(); // then assertEquals("InstrumentationKey=00000000-0000-0000-0000-000000000000", TelemetryConfiguration.getActive().getConnectionString()); - assertEquals(Global.getSamplingPercentage(), 10, 0); + assertEquals(10, Global.getSamplingPercentage(), 0); + assertEquals(10, getCurrentSamplingPercentage(), 0); } @Test - public void shouldStillUpdate() { + public void shouldUpdateEvenOverEnvVars() { // given RpConfiguration rpConfiguration = new RpConfiguration(); rpConfiguration.connectionString = "InstrumentationKey=11111111-1111-1111-1111-111111111111"; @@ -77,15 +98,36 @@ public void shouldStillUpdate() { rpConfiguration.lastModifiedTime = 0; TelemetryConfiguration.getActive().setConnectionString("InstrumentationKey=00000000-0000-0000-0000-000000000000"); - Global.setSamplingPercentage(ConfigurationBuilder.roundToNearest(90)); + Global.setSamplingPercentage(100); + envVars.set("APPLICATIONINSIGHTS_CONNECTION_STRING", "InstrumentationKey=00000000-0000-0000-0000-000000000000"); envVars.set("APPLICATIONINSIGHTS_SAMPLING_PERCENTAGE", "90"); + // pre-check + assertEquals("InstrumentationKey=00000000-0000-0000-0000-000000000000", TelemetryConfiguration.getActive().getConnectionString()); + assertEquals(100, Global.getSamplingPercentage(), 0); + assertEquals(100, getCurrentSamplingPercentage(), 0); + // when new RpConfigurationPolling(rpConfiguration, new Configuration()).run(); // then assertEquals("InstrumentationKey=00000000-0000-0000-0000-000000000000", TelemetryConfiguration.getActive().getConnectionString()); - assertEquals(Global.getSamplingPercentage(), 10, 0); + assertEquals(10, Global.getSamplingPercentage(), 0); + assertEquals(10, getCurrentSamplingPercentage(), 0); + } + + private double getCurrentSamplingPercentage() { + SpanContext spanContext = SpanContext.create( + "12341234123412341234123412341234", + "1234123412341234", + TraceFlags.getSampled(), + TraceState.getDefault()); + Context parentContext = Context.root().with(Span.wrap(spanContext)); + SamplingResult samplingResult = + DelegatingSampler.getInstance().shouldSample(parentContext, "12341234123412341234123412341234", "my span name", + SpanKind.SERVER, Attributes.empty(), Collections.emptyList()); + TraceState traceState = samplingResult.getUpdatedTraceState(TraceState.getDefault()); + return Double.parseDouble(traceState.get(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE)); } } diff --git a/agent/exporter/src/main/java/com/microsoft/applicationinsights/agent/Exporter.java b/agent/exporter/src/main/java/com/microsoft/applicationinsights/agent/Exporter.java index 84af728543d..fa38023fb89 100644 --- a/agent/exporter/src/main/java/com/microsoft/applicationinsights/agent/Exporter.java +++ b/agent/exporter/src/main/java/com/microsoft/applicationinsights/agent/Exporter.java @@ -22,19 +22,16 @@ import java.net.URI; import java.net.URISyntaxException; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.google.common.base.Joiner; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableSet; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.microsoft.applicationinsights.TelemetryClient; import com.microsoft.applicationinsights.TelemetryConfiguration; import com.microsoft.applicationinsights.telemetry.Duration; @@ -52,6 +49,7 @@ import io.opentelemetry.api.trace.SpanId; import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.instrumentation.api.aisdk.AiAppId; +import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.trace.data.EventData; import io.opentelemetry.sdk.trace.data.LinkData; @@ -110,7 +108,7 @@ public class Exporter implements SpanExporter { private static final Joiner JOINER = Joiner.on(", "); - public static final AttributeKey AI_SAMPLING_PERCENTAGE_KEY = AttributeKey.doubleKey("applicationinsights.internal.sampling_percentage"); + public static final String SAMPLING_PERCENTAGE_TRACE_STATE = "ai-internal-sp"; private static final AttributeKey AI_LOG_KEY = AttributeKey.booleanKey("applicationinsights.internal.log"); @@ -125,6 +123,9 @@ public class Exporter implements SpanExporter { private static final AttributeKey AI_LOGGER_NAME_KEY = AttributeKey.stringKey("applicationinsights.internal.logger_name"); private static final AttributeKey AI_LOG_ERROR_STACK_KEY = AttributeKey.stringKey("applicationinsights.internal.log_error_stack"); + private static final AtomicBoolean alreadyLoggedSamplingPercentageMissing = new AtomicBoolean(); + private static final AtomicBoolean alreadyLoggedSamplingPercentageParseError = new AtomicBoolean(); + private final TelemetryClient telemetryClient; public Exporter(TelemetryClient telemetryClient) { @@ -228,11 +229,52 @@ private void exportRemoteDependency(SpanData span, boolean inProc) { setExtraAttributes(remoteDependencyData, attributes); - Double samplingPercentage = attributes.get(AI_SAMPLING_PERCENTAGE_KEY); + double samplingPercentage = getSamplingPercentage(span.getSpanContext().getTraceState()); track(remoteDependencyData, samplingPercentage); exportEvents(span, samplingPercentage); } + private static double getSamplingPercentage(TraceState traceState) { + return getSamplingPercentage(traceState, 100, true); + } + + // for use by 2.x SDK telemetry, see BytecodeUtilImpl + public static double getSamplingPercentage(TraceState traceState, double defaultValue, boolean warnOnMissing) { + String samplingPercentageStr = traceState.get(SAMPLING_PERCENTAGE_TRACE_STATE); + if (samplingPercentageStr == null) { + if (warnOnMissing && !alreadyLoggedSamplingPercentageMissing.getAndSet(true)) { + // sampler should have set the trace state + logger.warn("did not find sampling percentage in trace state: {}", traceState); + } + return defaultValue; + } + try { + return parseSamplingPercentage(samplingPercentageStr).orElse(defaultValue); + } catch (ExecutionException e) { + // this shouldn't happen + logger.debug(e.getMessage(), e); + return defaultValue; + } + } + + private static final Cache parsedSamplingPercentageCache = + CacheBuilder.newBuilder() + .maximumSize(100) + .build(); + + public static OptionalDouble parseSamplingPercentage(String samplingPercentageStr) throws ExecutionException { + return parsedSamplingPercentageCache.get(samplingPercentageStr, () -> { + try { + return OptionalDouble.of(Double.parseDouble(samplingPercentageStr)); + } catch (NumberFormatException e) { + if (!alreadyLoggedSamplingPercentageParseError.getAndSet(true)) { + logger.warn("error parsing sampling percentage trace state: {}", samplingPercentageStr, e); + } + return OptionalDouble.empty(); + } + }); + } + private void applySemanticConventions(Attributes attributes, RemoteDependencyTelemetry remoteDependencyData, SpanKind spanKind) { String httpMethod = attributes.get(SemanticAttributes.HTTP_METHOD); if (httpMethod != null) { @@ -282,8 +324,7 @@ private void trackTrace(SpanData span) { setExtraAttributes(telemetry, attributes); telemetry.setTimestamp(new Date(NANOSECONDS.toMillis(span.getStartEpochNanos()))); - Double samplingPercentage = attributes.get(AI_SAMPLING_PERCENTAGE_KEY); - track(telemetry, samplingPercentage); + track(telemetry, getSamplingPercentage(span.getSpanContext().getTraceState())); } private void trackTraceAsException(SpanData span, String errorStack) { @@ -307,8 +348,7 @@ private void trackTraceAsException(SpanData span, String errorStack) { setExtraAttributes(telemetry, attributes); telemetry.setTimestamp(new Date(NANOSECONDS.toMillis(span.getStartEpochNanos()))); - Double samplingPercentage = attributes.get(AI_SAMPLING_PERCENTAGE_KEY); - track(telemetry, samplingPercentage); + track(telemetry, getSamplingPercentage(span.getSpanContext().getTraceState())); } private void track(Telemetry telemetry, Double samplingPercentage) { @@ -559,7 +599,7 @@ private void exportRequest(SpanData span) { setExtraAttributes(requestData, attributes); - Double samplingPercentage = attributes.get(AI_SAMPLING_PERCENTAGE_KEY); + double samplingPercentage = getSamplingPercentage(span.getSpanContext().getTraceState()); track(requestData, samplingPercentage); exportEvents(span, samplingPercentage); } diff --git a/test/smoke/testApps/Sampling/build.gradle b/test/smoke/testApps/Sampling/build.gradle index 8fb18d86d40..9d2cb5c4e82 100644 --- a/test/smoke/testApps/Sampling/build.gradle +++ b/test/smoke/testApps/Sampling/build.gradle @@ -13,6 +13,8 @@ dependencies { testCompile 'com.google.guava:guava:23.0' // VSCODE intellisense bug workaround testCompile group:'org.hamcrest', name:'hamcrest-library', version:'1.3' + + compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.7' } tasks.withType(JavaCompile) { diff --git a/test/smoke/testApps/Sampling/src/main/java/com/microsoft/ajl/simplecalc/SimpleSamplingServlet.java b/test/smoke/testApps/Sampling/src/main/java/com/microsoft/ajl/simplecalc/SimpleSamplingServlet.java index 8772ec5d3b6..794967dc024 100644 --- a/test/smoke/testApps/Sampling/src/main/java/com/microsoft/ajl/simplecalc/SimpleSamplingServlet.java +++ b/test/smoke/testApps/Sampling/src/main/java/com/microsoft/ajl/simplecalc/SimpleSamplingServlet.java @@ -10,6 +10,9 @@ import com.microsoft.applicationinsights.TelemetryClient; import com.microsoft.applicationinsights.telemetry.EventTelemetry; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; @WebServlet(description = "sends 100 event telemetry items with different op ids", urlPatterns = { "/sampling" }) public class SimpleSamplingServlet extends HttpServlet { @@ -21,7 +24,10 @@ public class SimpleSamplingServlet extends HttpServlet { protected void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { ServletFuncs.getRenderedHtml(request, response); - client.trackTrace("Trace Test."); + + CloseableHttpClient httpClient = HttpClientBuilder.create().disableAutomaticRetries().build(); + httpClient.execute(new HttpGet("https://www.bing.com")).close(); + client.trackEvent(new EventTelemetry("Event Test " + count.getAndIncrement())); } } diff --git a/test/smoke/testApps/Sampling/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingTest.java b/test/smoke/testApps/Sampling/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingTest.java index b08b205ba9f..6d25b755000 100644 --- a/test/smoke/testApps/Sampling/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingTest.java +++ b/test/smoke/testApps/Sampling/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/SamplingTest.java @@ -1,6 +1,9 @@ package com.microsoft.applicationinsights.smoketest; +import java.util.List; + import com.google.common.base.Stopwatch; +import com.microsoft.applicationinsights.internal.schemav2.Envelope; import org.junit.*; import static java.util.concurrent.TimeUnit.SECONDS; @@ -17,15 +20,35 @@ public void testSampling() throws Exception { Stopwatch stopwatch = Stopwatch.createStarted(); while (mockedIngestion.getCountForType("RequestData") < 25 && stopwatch.elapsed(SECONDS) < 10) { } - // wait ten more seconds to before checking that we didn't receive too many + // wait ten more seconds before checking that we didn't receive too many Thread.sleep(SECONDS.toMillis(10)); - int requestCount = mockedIngestion.getCountForType("RequestData"); - int eventCount = mockedIngestion.getCountForType("EventData"); - // super super low chance that number of sampled requests/events + + List requestEnvelopes = mockedIngestion.getItemsEnvelopeDataType("RequestData"); + List dependencyEnvelopes = mockedIngestion.getItemsEnvelopeDataType("RemoteDependencyData"); + List eventEnvelopes = mockedIngestion.getItemsEnvelopeDataType("EventData"); + // super super low chance that number of sampled requests/dependencies/events // is less than 25 or greater than 75 - assertThat(requestCount, greaterThanOrEqualTo(25)); - assertThat(eventCount, greaterThanOrEqualTo(25)); - assertThat(requestCount, lessThanOrEqualTo(75)); - assertThat(eventCount, lessThanOrEqualTo(75)); + assertThat(requestEnvelopes.size(), greaterThanOrEqualTo(25)); + assertThat(requestEnvelopes.size(), lessThanOrEqualTo(75)); + assertThat(dependencyEnvelopes.size(), greaterThanOrEqualTo(25)); + assertThat(dependencyEnvelopes.size(), lessThanOrEqualTo(75)); + assertThat(eventEnvelopes.size(), greaterThanOrEqualTo(25)); + assertThat(eventEnvelopes.size(), lessThanOrEqualTo(75)); + + for (Envelope requestEnvelope : requestEnvelopes) { + assertEquals(50, requestEnvelope.getSampleRate(), 0); + } + for (Envelope dependencyEnvelope : dependencyEnvelopes) { + assertEquals(50, dependencyEnvelope.getSampleRate(), 0); + } + for (Envelope eventEnvelope : eventEnvelopes) { + assertEquals(50, eventEnvelope.getSampleRate(), 0); + } + + for (Envelope requestEnvelope : requestEnvelopes) { + String operationId = requestEnvelope.getTags().get("ai.operation.id"); + mockedIngestion.waitForItemsInOperation("RemoteDependencyData", 1, operationId); + mockedIngestion.waitForItemsInOperation("EventData", 1, operationId); + } } }