From 18d5a0daa8027d129f1e9b74a61b071edc77079b Mon Sep 17 00:00:00 2001 From: Helen Y <56097766+heyams@users.noreply.github.com> Date: Tue, 31 Aug 2021 19:49:04 -0700 Subject: [PATCH] Update Statsbeat (#1859) * Send endpoint and host as part of network statsbeat * Add unit test for getHost * Update statsbeat smoke test * Remove a fixme * Send network statbeat per url * Reuse createMetricTelemetry in tests * Send correct host when redirect happens * Fix a warning * Add unit test * Refactor * Update unit tests * Fix spotless * Add two Statsbeat configs * Fix spotless * Rename Statsbeat intervals * Send attach and azure metadata service on start * Fix spotless * Send instrumentation as a FeatureStatsbeat * Update tests * Fix spotless * Fix statsbeat smoke test * Track non-essential Statsbeat disablement * Fix smoke test failures * Fix smoke test failures * Don't send statsbeat on redirect in tests * Fix a typo * Address feedback * Remove unused method * Use uppercase enums * Init telemetry client when disabled is on * Inject StatsbeatModule to RedirectPolicy * Use initial delay * Remove null check and use Mockito.doNothing() * Move disabled to preview * Change initiaDelay to 5 seconds and revert appServers * Address feedback * Track network counter per ikey * Update tests * Handle redirect * Fix spotlessApply * Fix nullpointexception at runtime * Fix spotlessApply * Null check outside sychronize * Rename * Refactor * Fix a warning methodcanbestatic * fix test failure * Fix spotlessApply * suggestions in progress * Retrieve endpoint from redirect policy cache * Fix tests * Comment out code * Remove unused methods * Fix runtime nullpointerexception * Skip statsbeat ikey in the ikey cache map * Handle redirect * Remove null check * Remove the ikey once it's sent on redirect * Fix style violations * Remove arg duplication * More injection, less global * Comment * Remove unneeded conditions * Remove redirct for now * Remove StatsbeatModule.get() * Remove used method * Remove setinstance * Refactor * Add a todo * Replace regex * Refactor Co-authored-by: Trask Stalnaker --- .../internal/configuration/Configuration.java | 15 +- .../agent/internal/exporter/Exporter.java | 6 +- .../internal/httpclient/LazyHttpClient.java | 8 +- .../internal/httpclient/RedirectPolicy.java | 28 +- .../internal/init/AiComponentInstaller.java | 13 +- .../profiler/ProfilerServiceInitializer.java | 2 +- .../agent/internal/quickpulse/QuickPulse.java | 2 +- .../statsbeat/AzureMetadataService.java | 2 +- .../agent/internal/statsbeat/Feature.java | 3 +- .../internal/statsbeat/FeatureStatsbeat.java | 59 +++- .../agent/internal/statsbeat/FeatureType.java | 27 ++ .../internal/statsbeat/NetworkStatsbeat.java | 255 +++++++++++------- .../internal/statsbeat/StatsbeatModule.java | 59 ++-- .../internal/telemetry/TelemetryChannel.java | 52 ++-- .../internal/telemetry/TelemetryClient.java | 42 ++- .../agent/internal/common/TestUtils.java | 74 +++++ .../localstorage/IntegrationTests.java | 49 +--- .../statsbeat/FeatureStatsbeatTest.java | 19 +- .../statsbeat/NetworkStatsbeatTest.java | 109 ++++---- .../telemetry/TelemetryChannelTest.java | 130 +++------ .../faststatsbeat_applicationinsights.json | 10 +- .../smoketest/StatsbeatSmokeTest.java | 30 ++- 22 files changed, 612 insertions(+), 382 deletions(-) create mode 100644 agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureType.java create mode 100644 agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/common/TestUtils.java diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java index a389bac18d1..1db206d028d 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java @@ -162,12 +162,16 @@ public static class Heartbeat { } public static class Statsbeat { + // disabledAll is used internally as an emergency kill-switch to turn off Statsbeat completely + // when something goes wrong. + public boolean disabledAll = false; + public String instrumentationKey = "c4a29126-a7cb-47e5-b348-11414998b11e"; // workspace-aistatsbeat public String endpoint = DefaultEndpoints.INGESTION_ENDPOINT; // this supports the government cloud - public long intervalSeconds = MINUTES.toSeconds(15); // default to 15 minutes - public long featureIntervalSeconds = DAYS.toSeconds(1); // default to daily + public long shortIntervalSeconds = MINUTES.toSeconds(15); // default to 15 minutes + public long longIntervalSeconds = DAYS.toSeconds(1); // default to daily } public static class Proxy { @@ -201,6 +205,7 @@ public static class PreviewConfiguration { public ProfilerConfiguration profiler = new ProfilerConfiguration(); public GcEventConfiguration gcEvents = new GcEventConfiguration(); public AadAuthentication authentication = new AadAuthentication(); + public PreviewStatsbeat statsbeat = new PreviewStatsbeat(); } public static class InheritedAttribute { @@ -279,6 +284,12 @@ public static class PreviewInstrumentation { new DisabledByDefaultInstrumentation(); } + public static class PreviewStatsbeat { + // disabled is used by customer to turn off non-essential Statsbeat, e.g. disk persistence + // operation status, optional network statsbeat, other endpoints except Breeze, etc. + public boolean disabled = false; + } + public static class EnabledByDefaultInstrumentation { public boolean enabled = true; } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/Exporter.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/Exporter.java index 61f466328e8..2293d49e2f4 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/Exporter.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/Exporter.java @@ -35,7 +35,6 @@ import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryExceptionData; import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryExceptionDetails; import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; -import com.microsoft.applicationinsights.agent.internal.statsbeat.StatsbeatModule; import com.microsoft.applicationinsights.agent.internal.telemetry.FormattedDuration; import com.microsoft.applicationinsights.agent.internal.telemetry.FormattedTime; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; @@ -192,7 +191,10 @@ public CompletableResultCode shutdown() { private void internalExport(SpanData span) { SpanKind kind = span.getKind(); String instrumentationName = span.getInstrumentationLibraryInfo().getName(); - StatsbeatModule.get().getNetworkStatsbeat().addInstrumentation(instrumentationName); + telemetryClient + .getStatsbeatModule() + .getInstrumentationStatsbeat() + .addInstrumentation(instrumentationName); if (kind == SpanKind.INTERNAL) { Boolean isLog = span.getAttributes().get(AI_LOG_KEY); if (isLog != null && isLog) { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/LazyHttpClient.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/LazyHttpClient.java index e6bc44594d6..8a08a30553e 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/LazyHttpClient.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/LazyHttpClient.java @@ -39,6 +39,7 @@ import com.azure.identity.VisualStudioCodeCredential; import com.azure.identity.VisualStudioCodeCredentialBuilder; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; +import io.opentelemetry.instrumentation.api.caching.Cache; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; @@ -120,12 +121,13 @@ ProxyOptions.Type.HTTP, new InetSocketAddress(proxyHost, proxyPortNumber))) return new NettyAsyncHttpClientBuilder().connectionProvider(connectionProvider).build(); } + // pass non-null ikeyRedirectCache if you want to use ikey-specific redirect policy public static HttpPipeline newHttpPipeLine( @Nullable Configuration.AadAuthentication aadConfiguration, - boolean followInstrumentationKeyForRedirect) { + @Nullable Cache ikeyRedirectCache) { List policies = new ArrayList<>(); - // Redirect policy to to handle v2.1/track redirects (and other redirects too, e.g. profiler) - policies.add(new RedirectPolicy(followInstrumentationKeyForRedirect)); + // Redirect policy to handle v2.1/track redirects (and other redirects too, e.g. profiler) + policies.add(new RedirectPolicy(ikeyRedirectCache)); if (aadConfiguration != null && aadConfiguration.enabled) { policies.add(getAuthenticationPolicy(aadConfiguration)); } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/RedirectPolicy.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/RedirectPolicy.java index 23e2706d092..fca5069937a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/RedirectPolicy.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/RedirectPolicy.java @@ -36,8 +36,6 @@ // This is mostly a copy from Azure Monitor Open Telemetry Exporter SDK AzureMonitorRedirectPolicy public final class RedirectPolicy implements HttpPipelinePolicy { - private final boolean followInstrumentationKeyForRedirect; - // use this only when followInstrumentationKeyForRedirect is true and instrumentation key is null private static final int PERMANENT_REDIRECT_STATUS_CODE = 308; private static final int TEMP_REDIRECT_STATUS_CODE = 307; // Based on Stamp specific redirects design doc @@ -47,11 +45,12 @@ public final class RedirectPolicy implements HttpPipelinePolicy { private final Cache redirectMappings = Cache.newBuilder().setMaximumSize(100).build(); - private final Cache instrumentationKeyMappings = - Cache.newBuilder().setMaximumSize(100).build(); - public RedirectPolicy(boolean followInstrumentationKeyForRedirect) { - this.followInstrumentationKeyForRedirect = followInstrumentationKeyForRedirect; + @Nullable private final Cache ikeyRedirectCache; + + // pass non-null ikeyRedirectCache if you want to use ikey-specific redirect policy + public RedirectPolicy(@Nullable Cache ikeyRedirectCache) { + this.ikeyRedirectCache = ikeyRedirectCache; } @Override @@ -92,24 +91,27 @@ private Mono attemptRetry( } private void cacheRedirectUrl(String redirectUrl, String instrumentationKey, URL originalUrl) { - if (!followInstrumentationKeyForRedirect) { + if (ikeyRedirectCache == null) { redirectMappings.put(originalUrl, redirectUrl); return; } - if (instrumentationKey != null) { - instrumentationKeyMappings.put(instrumentationKey, redirectUrl); + if (instrumentationKey == null) { + throw new IllegalArgumentException( + "instrumentationKey must be non-null when using ikey redirect policy"); } + ikeyRedirectCache.put(instrumentationKey, redirectUrl); } @Nullable private String getCachedRedirectUrl(String instrumentationKey, URL originalUrl) { - if (!followInstrumentationKeyForRedirect) { + if (ikeyRedirectCache == null) { return redirectMappings.get(originalUrl); } - if (instrumentationKey != null) { - return instrumentationKeyMappings.get(instrumentationKey); + if (instrumentationKey == null) { + throw new IllegalArgumentException( + "instrumentationKey must be non-null when using ikey redirect policy"); } - return null; + return ikeyRedirectCache.get(instrumentationKey); } /** diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiComponentInstaller.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiComponentInstaller.java index 7aade38390a..26c830f0593 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiComponentInstaller.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiComponentInstaller.java @@ -57,6 +57,7 @@ import com.microsoft.applicationinsights.profiler.config.ServiceProfilerServiceConfig; import io.opentelemetry.instrumentation.api.aisdk.AiAppId; import io.opentelemetry.instrumentation.api.aisdk.AiLazyConfiguration; +import io.opentelemetry.instrumentation.api.caching.Cache; import io.opentelemetry.instrumentation.api.config.Config; import io.opentelemetry.javaagent.extension.AgentListener; import io.opentelemetry.sdk.common.CompletableResultCode; @@ -166,8 +167,16 @@ private void start(Instrumentation instrumentation) { .map(MetricFilter::new) .collect(Collectors.toList()); + Cache ikeyEndpointMap = Cache.newBuilder().setMaximumSize(100).build(); + StatsbeatModule statsbeatModule = new StatsbeatModule(ikeyEndpointMap); + // TODO (heya) apply Builder design pattern to TelemetryClient TelemetryClient telemetryClient = - new TelemetryClient(config.customDimensions, metricFilters, config.preview.authentication); + new TelemetryClient( + config.customDimensions, + metricFilters, + ikeyEndpointMap, + statsbeatModule, + config.preview.authentication); TelemetryClientInitializer.initialize(telemetryClient, config); TelemetryClient.setActive(telemetryClient); @@ -211,7 +220,7 @@ private void start(Instrumentation instrumentation) { } // initialize StatsbeatModule - StatsbeatModule.get().start(telemetryClient, config); + statsbeatModule.start(telemetryClient, config); } private static GcEventMonitor.GcEventMonitorConfiguration formGcEventMonitorConfiguration( diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerServiceInitializer.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerServiceInitializer.java index e7a86d5233f..82c79c90904 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerServiceInitializer.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerServiceInitializer.java @@ -75,7 +75,7 @@ public static synchronized void initialize( GcEventMonitor.GcEventMonitorConfiguration gcEventMonitorConfiguration) { HttpPipeline httpPipeline = - LazyHttpClient.newHttpPipeLine(telemetryClient.getAadAuthentication(), false); + LazyHttpClient.newHttpPipeLine(telemetryClient.getAadAuthentication(), null); initialize( appIdSupplier, diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/quickpulse/QuickPulse.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/quickpulse/QuickPulse.java index 860f9185353..6ac5e5b60e5 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/quickpulse/QuickPulse.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/quickpulse/QuickPulse.java @@ -69,7 +69,7 @@ private void initializeSync(CountDownLatch latch, TelemetryClient telemetryClien initialized = true; String quickPulseId = UUID.randomUUID().toString().replace("-", ""); HttpPipeline httpPipeline = - LazyHttpClient.newHttpPipeLine(telemetryClient.getAadAuthentication(), false); + LazyHttpClient.newHttpPipeLine(telemetryClient.getAadAuthentication(), null); ArrayBlockingQueue sendQueue = new ArrayBlockingQueue<>(256, true); QuickPulseDataSender quickPulseDataSender = diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/AzureMetadataService.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/AzureMetadataService.java index b8aeb4ca523..6d84e182f46 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/AzureMetadataService.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/AzureMetadataService.java @@ -68,7 +68,7 @@ void scheduleWithFixedDelay(long interval) { // Querying Azure Metadata Service is required for every 15 mins since VM id will get updated // frequently. // Starting and restarting a VM will generate a new VM id each time. - scheduledExecutor.scheduleWithFixedDelay(this, interval, interval, TimeUnit.SECONDS); + scheduledExecutor.scheduleWithFixedDelay(this, 60, interval, TimeUnit.SECONDS); } // only used by tests diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/Feature.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/Feature.java index 6fea74b1da8..4f81d2f3e7a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/Feature.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/Feature.java @@ -48,7 +48,8 @@ enum Feature { RABBITMQ_DISABLED(18), SPRING_INTEGRATION_DISABLED(19), LEGACY_PROPAGATION_DISABLED(20), - GRIZZLY_DISABLED(21); // preview instrumentation + GRIZZLY_DISABLED(21), // preview instrumentation + STATSBEAT_DISABLED(22); // disable non-essential statsbeat private static final Map javaVendorFeatureMap; diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureStatsbeat.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureStatsbeat.java index 50ad621ef71..cdc34ea739b 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureStatsbeat.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureStatsbeat.java @@ -25,18 +25,25 @@ import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryUtil; -import java.util.HashSet; +import java.util.Collections; +import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; -class FeatureStatsbeat extends BaseStatsbeat { +public class FeatureStatsbeat extends BaseStatsbeat { private static final String FEATURE_METRIC_NAME = "Feature"; + private static final String INSTRUMENTATION_METRIC_NAME = "Instrumentation"; - private final Set featureList = new HashSet<>(64); + private final Set featureList = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Set instrumentationList = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final FeatureType type; - FeatureStatsbeat(CustomDimensions customDimensions) { + FeatureStatsbeat(CustomDimensions customDimensions, FeatureType type) { // track java distribution super(customDimensions); + this.type = type; String javaVendor = System.getProperty("java.vendor"); featureList.add(Feature.fromJavaVendor(javaVendor)); } @@ -46,13 +53,42 @@ long getFeature() { return Feature.encode(featureList); } + /** + * Returns a long that represents a list of instrumentations. Each bitfield maps to an + * instrumentation. + */ + long getInstrumentation() { + return Instrumentations.encode(instrumentationList); + } + + // this is used by Exporter + public void addInstrumentation(String instrumentation) { + instrumentationList.add(instrumentation); + } + @Override protected void send(TelemetryClient telemetryClient) { - TelemetryItem statsbeatTelemetry = - createStatsbeatTelemetry(telemetryClient, FEATURE_METRIC_NAME, 0); - TelemetryUtil.getProperties(statsbeatTelemetry.getData().getBaseData()) - .put("feature", String.valueOf(getFeature())); - telemetryClient.trackStatsbeatAsync(statsbeatTelemetry); + String metricName; + long encodedLong; + String featureType; + + if (type == FeatureType.FEATURE) { + metricName = FEATURE_METRIC_NAME; + encodedLong = getFeature(); + featureType = "feature"; + } else { + metricName = INSTRUMENTATION_METRIC_NAME; + encodedLong = getInstrumentation(); + featureType = "instrumentation"; + } + + TelemetryItem telemetryItem = createStatsbeatTelemetry(telemetryClient, metricName, 0); + Map properties = + TelemetryUtil.getProperties(telemetryItem.getData().getBaseData()); + properties.put("feature", String.valueOf(encodedLong)); + properties.put("type", featureType); + + telemetryClient.trackStatsbeatAsync(telemetryItem); } void trackConfigurationOptions(Configuration config) { @@ -102,5 +138,10 @@ void trackConfigurationOptions(Configuration config) { if (!config.preview.instrumentation.springIntegration.enabled) { featureList.add(Feature.SPRING_INTEGRATION_DISABLED); } + + // Statsbeat + if (config.preview.statsbeat.disabled) { + featureList.add(Feature.STATSBEAT_DISABLED); + } } } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureType.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureType.java new file mode 100644 index 00000000000..b7bea230dee --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureType.java @@ -0,0 +1,27 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.statsbeat; + +enum FeatureType { + FEATURE, + INSTRUMENTATION +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeat.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeat.java index e164833a934..fa531721a97 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeat.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeat.java @@ -21,13 +21,16 @@ package com.microsoft.applicationinsights.agent.internal.statsbeat; +import com.microsoft.applicationinsights.agent.internal.common.Strings; import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryUtil; -import java.util.Collections; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; +import io.opentelemetry.instrumentation.api.caching.Cache; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import org.checkerframework.checker.lock.qual.GuardedBy; public class NetworkStatsbeat extends BaseStatsbeat { @@ -37,35 +40,134 @@ public class NetworkStatsbeat extends BaseStatsbeat { private static final String RETRY_COUNT_METRIC_NAME = "Retry Count"; private static final String THROTTLE_COUNT_METRIC_NAME = "Throttle Count"; private static final String EXCEPTION_COUNT_METRIC_NAME = "Exception Count"; + private static final String BREEZE_ENDPOINT = "breeze"; - private static final String INSTRUMENTATION_CUSTOM_DIMENSION = "instrumentation"; + private final Object lock = new Object(); + private final Cache ikeyEndpointMap; - private volatile IntervalMetrics current; + @GuardedBy("lock") + private final Map instrumentationKeyCounterMap = new HashMap<>(); - private final Object lock = new Object(); + // only used by tests + public NetworkStatsbeat() { + super(new CustomDimensions()); + this.ikeyEndpointMap = Cache.newBuilder().build(); + } - NetworkStatsbeat(CustomDimensions customDimensions) { + public NetworkStatsbeat( + CustomDimensions customDimensions, Cache ikeyEndpointMap) { super(customDimensions); - current = new IntervalMetrics(); + this.ikeyEndpointMap = ikeyEndpointMap; } @Override protected void send(TelemetryClient telemetryClient) { - IntervalMetrics local; + Map local; synchronized (lock) { - local = current; - current = new IntervalMetrics(); + local = new HashMap<>(instrumentationKeyCounterMap); + instrumentationKeyCounterMap.clear(); + } + + for (Map.Entry entry : local.entrySet()) { + String ikey = entry.getKey(); + String endpointUrl = ikeyEndpointMap.get(ikey); + if (Strings.isNullOrEmpty(endpointUrl)) { + endpointUrl = telemetryClient.getEndpointProvider().getIngestionEndpointUrl().toString(); + } + + sendIntervalMetric(telemetryClient, ikey, entry.getValue(), getHost(endpointUrl)); } + } + + public void incrementRequestSuccessCount(long duration, String ikey) { + doWithIntervalMetrics( + ikey, + intervalMetrics -> { + intervalMetrics.requestSuccessCount.incrementAndGet(); + intervalMetrics.totalRequestDuration.getAndAdd(duration); + }); + } - // send instrumentation as an UTF-8 string - String instrumentation = String.valueOf(Instrumentations.encode(local.instrumentationList)); + public void incrementRequestFailureCount(String ikey) { + doWithIntervalMetrics( + ikey, intervalMetrics -> intervalMetrics.requestFailureCount.incrementAndGet()); + } + + public void incrementRetryCount(String ikey) { + doWithIntervalMetrics(ikey, intervalMetrics -> intervalMetrics.retryCount.incrementAndGet()); + } + public void incrementThrottlingCount(String ikey) { + doWithIntervalMetrics( + ikey, intervalMetrics -> intervalMetrics.throttlingCount.incrementAndGet()); + } + + void incrementExceptionCount(String ikey) { + doWithIntervalMetrics( + ikey, intervalMetrics -> intervalMetrics.exceptionCount.incrementAndGet()); + } + + // only used by tests + long getRequestSuccessCount(String ikey) { + synchronized (lock) { + IntervalMetrics intervalMetrics = instrumentationKeyCounterMap.get(ikey); + return intervalMetrics == null ? 0L : intervalMetrics.requestSuccessCount.get(); + } + } + + // only used by tests + long getRequestFailureCount(String ikey) { + synchronized (lock) { + IntervalMetrics intervalMetrics = instrumentationKeyCounterMap.get(ikey); + return intervalMetrics == null ? 0L : intervalMetrics.requestFailureCount.get(); + } + } + + // only used by tests + double getRequestDurationAvg(String ikey) { + synchronized (lock) { + IntervalMetrics intervalMetrics = instrumentationKeyCounterMap.get(ikey); + return intervalMetrics == null ? 0L : intervalMetrics.getRequestDurationAvg(); + } + } + + // only used by tests + long getRetryCount(String ikey) { + synchronized (lock) { + IntervalMetrics intervalMetrics = instrumentationKeyCounterMap.get(ikey); + return intervalMetrics == null ? 0L : intervalMetrics.retryCount.get(); + } + } + + // only used by tests + long getThrottlingCount(String ikey) { + synchronized (lock) { + IntervalMetrics intervalMetrics = instrumentationKeyCounterMap.get(ikey); + return intervalMetrics == null ? 0L : intervalMetrics.throttlingCount.get(); + } + } + + // only used by tests + long getExceptionCount(String ikey) { + synchronized (lock) { + IntervalMetrics intervalMetrics = instrumentationKeyCounterMap.get(ikey); + return intervalMetrics == null ? 0L : intervalMetrics.exceptionCount.get(); + } + } + + private void doWithIntervalMetrics(String ikey, Consumer update) { + synchronized (lock) { + update.accept(instrumentationKeyCounterMap.computeIfAbsent(ikey, k -> new IntervalMetrics())); + } + } + + private void sendIntervalMetric( + TelemetryClient telemetryClient, String ikey, IntervalMetrics local, String host) { if (local.requestSuccessCount.get() != 0) { TelemetryItem requestSuccessCountSt = createStatsbeatTelemetry( telemetryClient, REQUEST_SUCCESS_COUNT_METRIC_NAME, local.requestSuccessCount.get()); - TelemetryUtil.getProperties(requestSuccessCountSt.getData().getBaseData()) - .put(INSTRUMENTATION_CUSTOM_DIMENSION, instrumentation); + addCommonProperties(requestSuccessCountSt, ikey, host); telemetryClient.trackStatsbeatAsync(requestSuccessCountSt); } @@ -73,8 +175,7 @@ protected void send(TelemetryClient telemetryClient) { TelemetryItem requestFailureCountSt = createStatsbeatTelemetry( telemetryClient, REQUEST_FAILURE_COUNT_METRIC_NAME, local.requestFailureCount.get()); - TelemetryUtil.getProperties(requestFailureCountSt.getData().getBaseData()) - .put(INSTRUMENTATION_CUSTOM_DIMENSION, instrumentation); + addCommonProperties(requestFailureCountSt, ikey, host); telemetryClient.trackStatsbeatAsync(requestFailureCountSt); } @@ -82,8 +183,7 @@ protected void send(TelemetryClient telemetryClient) { if (durationAvg != 0) { TelemetryItem requestDurationSt = createStatsbeatTelemetry(telemetryClient, REQUEST_DURATION_METRIC_NAME, durationAvg); - TelemetryUtil.getProperties(requestDurationSt.getData().getBaseData()) - .put(INSTRUMENTATION_CUSTOM_DIMENSION, instrumentation); + addCommonProperties(requestDurationSt, ikey, host); telemetryClient.trackStatsbeatAsync(requestDurationSt); } @@ -91,8 +191,7 @@ protected void send(TelemetryClient telemetryClient) { TelemetryItem retryCountSt = createStatsbeatTelemetry( telemetryClient, RETRY_COUNT_METRIC_NAME, local.retryCount.get()); - TelemetryUtil.getProperties(retryCountSt.getData().getBaseData()) - .put(INSTRUMENTATION_CUSTOM_DIMENSION, instrumentation); + addCommonProperties(retryCountSt, ikey, host); telemetryClient.trackStatsbeatAsync(retryCountSt); } @@ -100,8 +199,7 @@ protected void send(TelemetryClient telemetryClient) { TelemetryItem throttleCountSt = createStatsbeatTelemetry( telemetryClient, THROTTLE_COUNT_METRIC_NAME, local.throttlingCount.get()); - TelemetryUtil.getProperties(throttleCountSt.getData().getBaseData()) - .put(INSTRUMENTATION_CUSTOM_DIMENSION, instrumentation); + addCommonProperties(throttleCountSt, ikey, host); telemetryClient.trackStatsbeatAsync(throttleCountSt); } @@ -109,93 +207,20 @@ protected void send(TelemetryClient telemetryClient) { TelemetryItem exceptionCountSt = createStatsbeatTelemetry( telemetryClient, EXCEPTION_COUNT_METRIC_NAME, local.exceptionCount.get()); - TelemetryUtil.getProperties(exceptionCountSt.getData().getBaseData()) - .put(INSTRUMENTATION_CUSTOM_DIMENSION, instrumentation); + addCommonProperties(exceptionCountSt, ikey, host); telemetryClient.trackStatsbeatAsync(exceptionCountSt); } } - // this is used by Exporter - public void addInstrumentation(String instrumentation) { - synchronized (lock) { - current.instrumentationList.add(instrumentation); - } - } - - public void incrementRequestSuccessCount(long duration) { - synchronized (lock) { - current.requestSuccessCount.incrementAndGet(); - current.totalRequestDuration.getAndAdd(duration); - } - } - - public void incrementRequestFailureCount() { - synchronized (lock) { - current.requestFailureCount.incrementAndGet(); - } - } - - public void incrementRetryCount() { - synchronized (lock) { - current.retryCount.incrementAndGet(); - } - } - - public void incrementThrottlingCount() { - synchronized (lock) { - current.throttlingCount.incrementAndGet(); - } - } - - void incrementExceptionCount() { - synchronized (lock) { - current.exceptionCount.incrementAndGet(); - } - } - - // only used by tests - long getInstrumentation() { - return Instrumentations.encode(current.instrumentationList); - } - - // only used by tests - long getRequestSuccessCount() { - return current.requestSuccessCount.get(); - } - - // only used by tests - long getRequestFailureCount() { - return current.requestFailureCount.get(); - } - - // only used by tests - double getRequestDurationAvg() { - return current.getRequestDurationAvg(); - } - - // only used by tests - long getRetryCount() { - return current.retryCount.get(); - } - - // only used by tests - long getThrottlingCount() { - return current.throttlingCount.get(); - } - - // only used by tests - long getExceptionCount() { - return current.exceptionCount.get(); - } - - // only used by tests - Set getInstrumentationList() { - return current.instrumentationList; + private static void addCommonProperties(TelemetryItem telemetryItem, String ikey, String host) { + Map properties = + TelemetryUtil.getProperties(telemetryItem.getData().getBaseData()); + properties.put("endpoint", BREEZE_ENDPOINT); + properties.put("cikey", ikey); + properties.put("host", host); } private static class IntervalMetrics { - private final Set instrumentationList = - Collections.newSetFromMap(new ConcurrentHashMap<>()); private final AtomicLong requestSuccessCount = new AtomicLong(); private final AtomicLong requestFailureCount = new AtomicLong(); // request duration count only counts request success. @@ -213,4 +238,28 @@ private double getRequestDurationAvg() { return sum; } } + + /** + * e.g. endpointUrl 'https://westus-0.in.applicationinsights.azure.com/v2.1/track' host will + * return 'westus-0.in.applicationinsights.azure.com' + */ + static String getHost(String endpointUrl) { + assert (endpointUrl != null && !endpointUrl.isEmpty()); + int start = endpointUrl.indexOf("://"); + if (start != -1) { + int end = endpointUrl.indexOf("/", start + 3); + if (end != -1) { + return endpointUrl.substring(start + 3, end); + } + + return endpointUrl.substring(start + 3); + } + + int end = endpointUrl.indexOf("/"); + if (end != -1) { + return endpointUrl.substring(0, end); + } + + return endpointUrl; + } } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/StatsbeatModule.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/StatsbeatModule.java index c9c931faa29..5589a713b91 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/StatsbeatModule.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/StatsbeatModule.java @@ -24,6 +24,7 @@ import com.microsoft.applicationinsights.agent.internal.common.ThreadPoolUtils; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; +import io.opentelemetry.instrumentation.api.caching.Cache; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -35,8 +36,6 @@ public class StatsbeatModule { private static final Logger logger = LoggerFactory.getLogger(BaseStatsbeat.class); - private static final StatsbeatModule instance = new StatsbeatModule(); - private static final ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor( ThreadPoolUtils.createDaemonThreadFactory(BaseStatsbeat.class)); @@ -46,14 +45,16 @@ public class StatsbeatModule { private final NetworkStatsbeat networkStatsbeat; private final AttachStatsbeat attachStatsbeat; private final FeatureStatsbeat featureStatsbeat; + private final FeatureStatsbeat instrumentationStatsbeat; private final AtomicBoolean started = new AtomicBoolean(); - private StatsbeatModule() { + public StatsbeatModule(Cache ikeyEndpointMap) { customDimensions = new CustomDimensions(); - networkStatsbeat = new NetworkStatsbeat(customDimensions); + networkStatsbeat = new NetworkStatsbeat(customDimensions, ikeyEndpointMap); attachStatsbeat = new AttachStatsbeat(customDimensions); - featureStatsbeat = new FeatureStatsbeat(customDimensions); + featureStatsbeat = new FeatureStatsbeat(customDimensions, FeatureType.FEATURE); + instrumentationStatsbeat = new FeatureStatsbeat(customDimensions, FeatureType.INSTRUMENTATION); } public void start(TelemetryClient telemetryClient, Configuration config) { @@ -61,46 +62,64 @@ public void start(TelemetryClient telemetryClient, Configuration config) { throw new IllegalStateException("initialize already called"); } - long intervalSeconds = config.internal.statsbeat.intervalSeconds; - long featureIntervalSeconds = config.internal.statsbeat.featureIntervalSeconds; + if (config.internal.statsbeat.disabledAll) { + // disabledAll is an internal emergency kill-switch to turn off Statsbeat completely when + // something goes wrong. + // this happens rarely. + return; + } + + long shortIntervalSeconds = config.internal.statsbeat.shortIntervalSeconds; + long longIntervalSeconds = config.internal.statsbeat.longIntervalSeconds; scheduledExecutor.scheduleWithFixedDelay( new StatsbeatSender(networkStatsbeat, telemetryClient), - intervalSeconds, - intervalSeconds, + shortIntervalSeconds, + shortIntervalSeconds, TimeUnit.SECONDS); scheduledExecutor.scheduleWithFixedDelay( new StatsbeatSender(attachStatsbeat, telemetryClient), - intervalSeconds, - intervalSeconds, + 60, + longIntervalSeconds, TimeUnit.SECONDS); scheduledExecutor.scheduleWithFixedDelay( new StatsbeatSender(featureStatsbeat, telemetryClient), - featureIntervalSeconds, - featureIntervalSeconds, + longIntervalSeconds, + longIntervalSeconds, + TimeUnit.SECONDS); + scheduledExecutor.scheduleWithFixedDelay( + new StatsbeatSender(instrumentationStatsbeat, telemetryClient), + longIntervalSeconds, + longIntervalSeconds, TimeUnit.SECONDS); ResourceProvider rp = customDimensions.getResourceProvider(); // only turn on AzureMetadataService when the resource provider is VM or UNKNOWN. - // FIXME (heya) Need to figure out why AzureMetadataService is not reachable from a function app - // and it's not necessary to make this call. if (rp == ResourceProvider.RP_VM || rp == ResourceProvider.UNKNOWN) { // will only reach here the first time, after instance has been instantiated - new AzureMetadataService(attachStatsbeat, customDimensions) - .scheduleWithFixedDelay(intervalSeconds); + AzureMetadataService metadataService = + new AzureMetadataService(attachStatsbeat, customDimensions); + metadataService.scheduleWithFixedDelay(longIntervalSeconds); } featureStatsbeat.trackConfigurationOptions(config); - } - public static StatsbeatModule get() { - return instance; + if (config.preview.statsbeat.disabled) { + // disabled will disable non-essentials Statsbeat, such as tracking failure or success of disk + // persistence operations, optional network statsbeat, live metric, + // azure metadata service failure, profile endpoint, etc. + // TODO stop sending non-essential Statsbeat when applicable + } } public NetworkStatsbeat getNetworkStatsbeat() { return networkStatsbeat; } + public FeatureStatsbeat getInstrumentationStatsbeat() { + return instrumentationStatsbeat; + } + /** Runnable which is responsible for calling the send method to transmit Statsbeat telemetry. */ private static class StatsbeatSender implements Runnable { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannel.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannel.java index 6bade6d4547..06ea7fc1db4 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannel.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannel.java @@ -39,7 +39,8 @@ import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient; import com.microsoft.applicationinsights.agent.internal.httpclient.RedirectPolicy; import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileWriter; -import com.microsoft.applicationinsights.agent.internal.statsbeat.StatsbeatModule; +import com.microsoft.applicationinsights.agent.internal.statsbeat.NetworkStatsbeat; +import io.opentelemetry.instrumentation.api.caching.Cache; import io.opentelemetry.sdk.common.CompletableResultCode; import java.io.IOException; import java.net.URL; @@ -83,17 +84,17 @@ public class TelemetryChannel { private final HttpPipeline pipeline; private final URL endpointUrl; private final LocalFileWriter localFileWriter; + // this is null for the statsbeat channel + @Nullable private final NetworkStatsbeat networkStatsbeat; public static TelemetryChannel create( URL endpointUrl, LocalFileWriter localFileWriter, - Configuration.AadAuthentication aadAuthentication) { - HttpPipeline httpPipeline = LazyHttpClient.newHttpPipeLine(aadAuthentication, true); - return new TelemetryChannel(httpPipeline, endpointUrl, localFileWriter); - } - - public static TelemetryChannel create(URL endpointUrl, LocalFileWriter localFileWriter) { - return create(endpointUrl, localFileWriter, null); + Cache ikeyEndpointMap, + @Nullable NetworkStatsbeat networkStatsbeat, + @Nullable Configuration.AadAuthentication aadAuthentication) { + HttpPipeline httpPipeline = LazyHttpClient.newHttpPipeLine(aadAuthentication, ikeyEndpointMap); + return new TelemetryChannel(httpPipeline, endpointUrl, localFileWriter, networkStatsbeat); } public CompletableResultCode sendRawBytes(ByteBuffer buffer) { @@ -101,10 +102,15 @@ public CompletableResultCode sendRawBytes(ByteBuffer buffer) { } // used by tests only - public TelemetryChannel(HttpPipeline pipeline, URL endpointUrl, LocalFileWriter localFileWriter) { + public TelemetryChannel( + HttpPipeline pipeline, + URL endpointUrl, + LocalFileWriter localFileWriter, + @Nullable NetworkStatsbeat networkStatsbeat) { this.pipeline = pipeline; this.endpointUrl = endpointUrl; this.localFileWriter = localFileWriter; + this.networkStatsbeat = networkStatsbeat; } public CompletableResultCode send(List telemetryItems) { @@ -208,21 +214,24 @@ private CompletableResultCode internalSend( .send(request, Context.of(contextKeyValues)) .subscribe( response -> { - parseResponseCode(response.getStatusCode(), byteBuffers, byteBuffers); + parseResponseCode( + response.getStatusCode(), byteBuffers, byteBuffers, instrumentationKey); LazyHttpClient.consumeResponseBody(response); }, error -> { - StatsbeatModule.get().getNetworkStatsbeat().incrementRequestFailureCount(); + if (networkStatsbeat != null) { + networkStatsbeat.incrementRequestFailureCount(instrumentationKey); + } ExceptionUtils.parseError( error, endpointUrl.toString(), friendlyExceptionThrown, logger); writeToDiskOnFailure(byteBuffers, byteBuffers); result.fail(); }, () -> { - StatsbeatModule.get() - .getNetworkStatsbeat() - .incrementRequestSuccessCount(System.currentTimeMillis() - startTime); - + if (networkStatsbeat != null) { + networkStatsbeat.incrementRequestSuccessCount( + System.currentTimeMillis() - startTime, instrumentationKey); + } if (byteBuffers != null) { byteBufferPool.offer(byteBuffers); } @@ -247,7 +256,10 @@ private void writeToDiskOnFailure( } private void parseResponseCode( - int statusCode, List byteBuffers, List finalByteBuffers) { + int statusCode, + List byteBuffers, + List finalByteBuffers, + String instrumentationKey) { switch (statusCode) { case 401: // UNAUTHORIZED case 403: // FORBIDDEN @@ -263,7 +275,9 @@ private void parseResponseCode( case 439: // Breeze-specific: THROTTLED OVER EXTENDED TIME // TODO handle throttling // TODO (heya) track throttling count via Statsbeat - StatsbeatModule.get().getNetworkStatsbeat().incrementThrottlingCount(); + if (networkStatsbeat != null) { + networkStatsbeat.incrementThrottlingCount(instrumentationKey); + } break; case 200: // SUCCESS operationLogger.recordSuccess(); @@ -274,7 +288,9 @@ private void parseResponseCode( case 0: // client-side exception // TODO exponential backoff and retry to a limit // TODO (heya) track failure count via Statsbeat - StatsbeatModule.get().getNetworkStatsbeat().incrementRetryCount(); + if (networkStatsbeat != null) { + networkStatsbeat.incrementRetryCount(instrumentationKey); + } break; default: // ok diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java index 875bdedecf9..83a19a4cf3f 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java @@ -45,6 +45,8 @@ import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileSender; import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileWriter; import com.microsoft.applicationinsights.agent.internal.quickpulse.QuickPulseDataCollector; +import com.microsoft.applicationinsights.agent.internal.statsbeat.StatsbeatModule; +import io.opentelemetry.instrumentation.api.caching.Cache; import io.opentelemetry.sdk.common.CompletableResultCode; import java.io.File; import java.util.ArrayList; @@ -54,9 +56,9 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.apache.commons.text.StringSubstitutor; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; public class TelemetryClient { @@ -101,7 +103,10 @@ public class TelemetryClient { private final List metricFilters; - private final Configuration.AadAuthentication aadAuthentication; + private final Cache ikeyEndpointMap; + private final StatsbeatModule statsbeatModule; + + @Nullable private final Configuration.AadAuthentication aadAuthentication; private final Object channelInitLock = new Object(); private volatile @MonotonicNonNull BatchSpanProcessor channelBatcher; @@ -109,13 +114,20 @@ public class TelemetryClient { // only used by tests public TelemetryClient() { - this(new HashMap<>(), new ArrayList<>(), null); + this( + new HashMap<>(), + new ArrayList<>(), + Cache.newBuilder().build(), + new StatsbeatModule(null), + null); } public TelemetryClient( Map customDimensions, List metricFilters, - Configuration.AadAuthentication aadAuthentication) { + Cache ikeyEndpointMap, + StatsbeatModule statsbeatModule, + @Nullable Configuration.AadAuthentication aadAuthentication) { StringSubstitutor substitutor = new StringSubstitutor(System.getenv()); Map globalProperties = new HashMap<>(); Map globalTags = new HashMap<>(); @@ -136,6 +148,8 @@ public TelemetryClient( this.globalProperties = globalProperties; this.globalTags = globalTags; this.metricFilters = metricFilters; + this.ikeyEndpointMap = ikeyEndpointMap; + this.statsbeatModule = statsbeatModule; this.aadAuthentication = aadAuthentication; } @@ -221,7 +235,11 @@ public BatchSpanProcessor getChannelBatcher() { LocalFileWriter localFileWriter = new LocalFileWriter(localFileCache, telemetryFolder); TelemetryChannel channel = TelemetryChannel.create( - endpointProvider.getIngestionEndpointUrl(), localFileWriter, aadAuthentication); + endpointProvider.getIngestionEndpointUrl(), + localFileWriter, + ikeyEndpointMap, + statsbeatModule.getNetworkStatsbeat(), + aadAuthentication); LocalFileSender.start(localFileLoader, channel); channelBatcher = BatchSpanProcessor.builder(channel).build(); } @@ -239,7 +257,12 @@ public BatchSpanProcessor getStatsbeatChannelBatcher() { LocalFileLoader localFileLoader = new LocalFileLoader(localFileCache, statsbeatFolder); LocalFileWriter localFileWriter = new LocalFileWriter(localFileCache, statsbeatFolder); TelemetryChannel channel = - TelemetryChannel.create(endpointProvider.getStatsbeatEndpointUrl(), localFileWriter); + TelemetryChannel.create( + endpointProvider.getStatsbeatEndpointUrl(), + localFileWriter, + ikeyEndpointMap, + null, + null); LocalFileSender.start(localFileLoader, channel); statsbeatChannelBatcher = BatchSpanProcessor.builder(channel).build(); } @@ -272,7 +295,8 @@ public void setStatsbeatInstrumentationKey(String key) { statsbeatInstrumentationKey = key; } - public @Nullable String getRoleName() { + @Nullable + public String getRoleName() { return roleName; } @@ -306,6 +330,10 @@ public Configuration.AadAuthentication getAadAuthentication() { return aadAuthentication; } + public StatsbeatModule getStatsbeatModule() { + return statsbeatModule; + } + public void addNonFilterableMetricNames(String... metricNames) { nonFilterableMetricNames.addAll(asList(metricNames)); } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/common/TestUtils.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/common/TestUtils.java new file mode 100644 index 00000000000..26cc269df61 --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/common/TestUtils.java @@ -0,0 +1,74 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.common; + +import com.microsoft.applicationinsights.agent.internal.exporter.models.DataPointType; +import com.microsoft.applicationinsights.agent.internal.exporter.models.MetricDataPoint; +import com.microsoft.applicationinsights.agent.internal.exporter.models.MetricsData; +import com.microsoft.applicationinsights.agent.internal.exporter.models.MonitorBase; +import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; +import com.microsoft.applicationinsights.agent.internal.telemetry.FormattedTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public final class TestUtils { + + public static TelemetryItem createMetricTelemetry( + String name, int value, String instrumentationKey) { + TelemetryItem telemetry = new TelemetryItem(); + telemetry.setVersion(1); + telemetry.setName("Metric"); + telemetry.setInstrumentationKey(instrumentationKey); + Map tags = new HashMap<>(); + tags.put("ai.internal.sdkVersion", "test_version"); + tags.put("ai.internal.nodeName", "test_role_name"); + tags.put("ai.cloud.roleInstance", "test_cloud_name"); + telemetry.setTags(tags); + + MetricsData data = new MetricsData(); + List dataPoints = new ArrayList<>(); + MetricDataPoint dataPoint = new MetricDataPoint(); + dataPoint.setDataPointType(DataPointType.MEASUREMENT); + dataPoint.setName(name); + dataPoint.setValue(value); + dataPoint.setCount(1); + dataPoints.add(dataPoint); + + Map properties = new HashMap<>(); + properties.put("state", "blocked"); + + data.setMetrics(dataPoints); + data.setProperties(properties); + + MonitorBase monitorBase = new MonitorBase(); + monitorBase.setBaseType("MetricData"); + monitorBase.setBaseData(data); + telemetry.setData(monitorBase); + telemetry.setTime(FormattedTime.offSetDateTimeFromNow()); + + return telemetry; + } + + private TestUtils() {} +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/IntegrationTests.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/IntegrationTests.java index c2671ba2c92..2487df30549 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/IntegrationTests.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/IntegrationTests.java @@ -30,21 +30,15 @@ import com.azure.core.http.HttpPipelineBuilder; import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; -import com.microsoft.applicationinsights.agent.internal.exporter.models.DataPointType; -import com.microsoft.applicationinsights.agent.internal.exporter.models.MetricDataPoint; -import com.microsoft.applicationinsights.agent.internal.exporter.models.MetricsData; -import com.microsoft.applicationinsights.agent.internal.exporter.models.MonitorBase; +import com.microsoft.applicationinsights.agent.internal.common.TestUtils; import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; -import com.microsoft.applicationinsights.agent.internal.telemetry.FormattedTime; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryChannel; import io.opentelemetry.sdk.common.CompletableResultCode; import java.io.File; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -74,14 +68,16 @@ public void setup() throws MalformedURLException { new TelemetryChannel( pipelineBuilder.build(), new URL("http://foo.bar"), - new LocalFileWriter(localFileCache, tempFolder)); + new LocalFileWriter(localFileCache, tempFolder), + null); localFileLoader = new LocalFileLoader(localFileCache, tempFolder); } @Test public void integrationTest() throws InterruptedException { List telemetryItems = new ArrayList<>(); - telemetryItems.add(createMetricTelemetry("metric" + 1, 1)); + telemetryItems.add( + TestUtils.createMetricTelemetry("metric" + 1, 1, "00000000-0000-0000-0000-0FEEDDADBEEF")); ExecutorService executorService = Executors.newFixedThreadPool(10); for (int i = 0; i < 10; i++) { @@ -107,39 +103,4 @@ public void integrationTest() throws InterruptedException { assertThat(localFileCache.getPersistedFilesCache().size()).isEqualTo(0); } - - private static TelemetryItem createMetricTelemetry(String name, int value) { - TelemetryItem telemetry = new TelemetryItem(); - telemetry.setVersion(1); - telemetry.setName("Metric"); - telemetry.setInstrumentationKey("00000000-0000-0000-0000-0FEEDDADBEEF"); - Map tags = new HashMap<>(); - tags.put("ai.internal.sdkVersion", "test_version"); - tags.put("ai.internal.nodeName", "test_role_name"); - tags.put("ai.cloud.roleInstance", "test_cloud_name"); - telemetry.setTags(tags); - - MetricsData data = new MetricsData(); - List dataPoints = new ArrayList<>(); - MetricDataPoint dataPoint = new MetricDataPoint(); - dataPoint.setDataPointType(DataPointType.MEASUREMENT); - dataPoint.setName(name); - dataPoint.setValue(value); - dataPoint.setCount(1); - dataPoints.add(dataPoint); - - Map properties = new HashMap<>(); - properties.put("state", "blocked"); - - data.setMetrics(dataPoints); - data.setProperties(properties); - - MonitorBase monitorBase = new MonitorBase(); - monitorBase.setBaseType("MetricData"); - monitorBase.setBaseData(data); - telemetry.setData(monitorBase); - telemetry.setTime(FormattedTime.offSetDateTimeFromNow()); - - return telemetry; - } } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureStatsbeatTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureStatsbeatTest.java index 4a7532681b2..872102fdf54 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureStatsbeatTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/FeatureStatsbeatTest.java @@ -97,6 +97,22 @@ public void testSpringSchedulingEnabled() { Feature.SPRING_SCHEDULING_DISABLED); } + @Test + public void testAddInstrumentation() { + FeatureStatsbeat instrumentationStatsbeat = + new FeatureStatsbeat(new CustomDimensions(), FeatureType.INSTRUMENTATION); + instrumentationStatsbeat.addInstrumentation("io.opentelemetry.jdbc"); + instrumentationStatsbeat.addInstrumentation("io.opentelemetry.tomcat-7.0"); + instrumentationStatsbeat.addInstrumentation("io.opentelemetry.http-url-connection"); + assertThat(instrumentationStatsbeat.getInstrumentation()) + .isEqualTo( + (long) + (Math.pow(2, 13) + + Math.pow(2, 21) + + Math.pow( + 2, 57))); // Exponents are keys from StatsbeatHelper.INSTRUMENTATION_MAP + } + private static void testFeatureTrackingEnablement( BiConsumer init, Feature feature) { testFeature(init, feature, false, false); @@ -115,7 +131,8 @@ private static void testFeature( boolean configValue, boolean featureValue) { // given - FeatureStatsbeat featureStatsbeat = new FeatureStatsbeat(new CustomDimensions()); + FeatureStatsbeat featureStatsbeat = + new FeatureStatsbeat(new CustomDimensions(), FeatureType.FEATURE); Configuration config = newConfiguration(); init.accept(config, configValue); diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatTest.java index fc03ba4024d..35af80c9e7d 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatTest.java @@ -26,92 +26,75 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; public class NetworkStatsbeatTest { private NetworkStatsbeat networkStatsbeat; + private static final String IKEY = "00000000-0000-0000-0000-0FEEDDADBEEF"; @BeforeEach public void init() { - networkStatsbeat = new NetworkStatsbeat(new CustomDimensions()); - } - - @Test - public void testAddInstrumentation() { - networkStatsbeat.addInstrumentation("io.opentelemetry.jdbc"); - networkStatsbeat.addInstrumentation("io.opentelemetry.tomcat-7.0"); - networkStatsbeat.addInstrumentation("io.opentelemetry.http-url-connection"); - assertThat(networkStatsbeat.getInstrumentation()) - .isEqualTo( - (long) - (Math.pow(2, 13) - + Math.pow(2, 21) - + Math.pow( - 2, 57))); // Exponents are keys from StatsbeatHelper.INSTRUMENTATION_MAP + networkStatsbeat = new NetworkStatsbeat(); } @Test public void testIncrementRequestSuccessCount() { - assertThat(networkStatsbeat.getRequestSuccessCount()).isEqualTo(0); - assertThat(networkStatsbeat.getRequestDurationAvg()).isEqualTo(0); - networkStatsbeat.incrementRequestSuccessCount(1000); - networkStatsbeat.incrementRequestSuccessCount(3000); - assertThat(networkStatsbeat.getRequestSuccessCount()).isEqualTo(2); - assertThat(networkStatsbeat.getRequestDurationAvg()).isEqualTo(2000.0); + assertThat(networkStatsbeat.getRequestSuccessCount(IKEY)).isEqualTo(0); + assertThat(networkStatsbeat.getRequestDurationAvg(IKEY)).isEqualTo(0); + networkStatsbeat.incrementRequestSuccessCount(1000, IKEY); + networkStatsbeat.incrementRequestSuccessCount(3000, IKEY); + assertThat(networkStatsbeat.getRequestSuccessCount(IKEY)).isEqualTo(2); + assertThat(networkStatsbeat.getRequestDurationAvg(IKEY)).isEqualTo(2000.0); } @Test public void testIncrementRequestFailureCount() { - assertThat(networkStatsbeat.getRequestFailureCount()).isEqualTo(0); - networkStatsbeat.incrementRequestFailureCount(); - networkStatsbeat.incrementRequestFailureCount(); - assertThat(networkStatsbeat.getRequestFailureCount()).isEqualTo(2); + assertThat(networkStatsbeat.getRequestFailureCount(IKEY)).isEqualTo(0); + networkStatsbeat.incrementRequestFailureCount(IKEY); + networkStatsbeat.incrementRequestFailureCount(IKEY); + assertThat(networkStatsbeat.getRequestFailureCount(IKEY)).isEqualTo(2); } @Test public void testIncrementRetryCount() { - assertThat(networkStatsbeat.getRetryCount()).isEqualTo(0); - networkStatsbeat.incrementRetryCount(); - networkStatsbeat.incrementRetryCount(); - assertThat(networkStatsbeat.getRetryCount()).isEqualTo(2); + assertThat(networkStatsbeat.getRetryCount(IKEY)).isEqualTo(0); + networkStatsbeat.incrementRetryCount(IKEY); + networkStatsbeat.incrementRetryCount(IKEY); + assertThat(networkStatsbeat.getRetryCount(IKEY)).isEqualTo(2); } @Test public void testIncrementThrottlingCount() { - assertThat(networkStatsbeat.getThrottlingCount()).isEqualTo(0); - networkStatsbeat.incrementThrottlingCount(); - networkStatsbeat.incrementThrottlingCount(); - assertThat(networkStatsbeat.getThrottlingCount()).isEqualTo(2); + assertThat(networkStatsbeat.getThrottlingCount(IKEY)).isEqualTo(0); + networkStatsbeat.incrementThrottlingCount(IKEY); + networkStatsbeat.incrementThrottlingCount(IKEY); + assertThat(networkStatsbeat.getThrottlingCount(IKEY)).isEqualTo(2); } @Test public void testIncrementExceptionCount() { - assertThat(networkStatsbeat.getExceptionCount()).isEqualTo(0); - networkStatsbeat.incrementExceptionCount(); - networkStatsbeat.incrementExceptionCount(); - assertThat(networkStatsbeat.getExceptionCount()).isEqualTo(2); + assertThat(networkStatsbeat.getExceptionCount(IKEY)).isEqualTo(0); + networkStatsbeat.incrementExceptionCount(IKEY); + networkStatsbeat.incrementExceptionCount(IKEY); + assertThat(networkStatsbeat.getExceptionCount(IKEY)).isEqualTo(2); } @Test public void testRaceCondition() throws InterruptedException { ExecutorService executorService = Executors.newFixedThreadPool(100); - AtomicInteger instrumentationCounter = new AtomicInteger(); for (int i = 0; i < 100; i++) { executorService.execute( new Runnable() { @Override public void run() { for (int j = 0; j < 1000; j++) { - networkStatsbeat.incrementRequestSuccessCount(j % 2 == 0 ? 5 : 10); - networkStatsbeat.incrementRequestFailureCount(); - networkStatsbeat.incrementRetryCount(); - networkStatsbeat.incrementThrottlingCount(); - networkStatsbeat.incrementExceptionCount(); - networkStatsbeat.addInstrumentation( - "instrumentation" + instrumentationCounter.getAndDecrement()); + networkStatsbeat.incrementRequestSuccessCount(j % 2 == 0 ? 5 : 10, IKEY); + networkStatsbeat.incrementRequestFailureCount(IKEY); + networkStatsbeat.incrementRetryCount(IKEY); + networkStatsbeat.incrementThrottlingCount(IKEY); + networkStatsbeat.incrementExceptionCount(IKEY); } } }); @@ -119,12 +102,32 @@ public void run() { executorService.shutdown(); executorService.awaitTermination(10, TimeUnit.MINUTES); - assertThat(networkStatsbeat.getRequestSuccessCount()).isEqualTo(100000); - assertThat(networkStatsbeat.getRequestFailureCount()).isEqualTo(100000); - assertThat(networkStatsbeat.getRetryCount()).isEqualTo(100000); - assertThat(networkStatsbeat.getThrottlingCount()).isEqualTo(100000); - assertThat(networkStatsbeat.getExceptionCount()).isEqualTo(100000); - assertThat(networkStatsbeat.getRequestDurationAvg()).isEqualTo(7.5); - assertThat(networkStatsbeat.getInstrumentationList().size()).isEqualTo(100000); + assertThat(networkStatsbeat.getRequestSuccessCount(IKEY)).isEqualTo(100000); + assertThat(networkStatsbeat.getRequestFailureCount(IKEY)).isEqualTo(100000); + assertThat(networkStatsbeat.getRetryCount(IKEY)).isEqualTo(100000); + assertThat(networkStatsbeat.getThrottlingCount(IKEY)).isEqualTo(100000); + assertThat(networkStatsbeat.getExceptionCount(IKEY)).isEqualTo(100000); + assertThat(networkStatsbeat.getRequestDurationAvg(IKEY)).isEqualTo(7.5); + } + + @Test + public void testGetHost() { + String url = "https://fake-host.applicationinsights.azure.com/v2.1/track"; + assertThat(NetworkStatsbeat.getHost(url)).isEqualTo("fake-host.applicationinsights.azure.com"); + + url = "http://fake-host.example.com/v2/track"; + assertThat(NetworkStatsbeat.getHost(url)).isEqualTo("fake-host.example.com"); + + url = "http://www.fake-host.com/v2/track"; + assertThat(NetworkStatsbeat.getHost(url)).isEqualTo("www.fake-host.com"); + + url = "www.fake-host.com/"; + assertThat(NetworkStatsbeat.getHost(url)).isEqualTo("www.fake-host.com"); + + url = "http://fake-host.com"; + assertThat(NetworkStatsbeat.getHost(url)).isEqualTo("fake-host.com"); + + url = "http://fake-host.com/"; + assertThat(NetworkStatsbeat.getHost(url)).isEqualTo("fake-host.com"); } } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannelTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannelTest.java index 0add01c55f0..ea7f33d21cb 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannelTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannelTest.java @@ -31,14 +31,12 @@ import com.azure.core.http.policy.HttpPipelinePolicy; import com.azure.core.util.FluxUtil; import com.microsoft.applicationinsights.agent.internal.MockHttpResponse; -import com.microsoft.applicationinsights.agent.internal.exporter.models.DataPointType; -import com.microsoft.applicationinsights.agent.internal.exporter.models.MetricDataPoint; -import com.microsoft.applicationinsights.agent.internal.exporter.models.MetricsData; -import com.microsoft.applicationinsights.agent.internal.exporter.models.MonitorBase; +import com.microsoft.applicationinsights.agent.internal.common.TestUtils; import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; import com.microsoft.applicationinsights.agent.internal.httpclient.RedirectPolicy; import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileCache; import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileWriter; +import io.opentelemetry.instrumentation.api.caching.Cache; import io.opentelemetry.sdk.common.CompletableResultCode; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -56,7 +54,6 @@ import java.util.function.Function; import java.util.zip.GZIPInputStream; import javax.annotation.Nullable; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -65,7 +62,6 @@ public class TelemetryChannelTest { RecordingHttpClient recordingHttpClient; - private static final AtomicInteger requestCount = new AtomicInteger(); private static final String INSTRUMENTATION_KEY = "00000000-0000-0000-0000-0FEEDDADBEEF"; private static final String REDIRECT_INSTRUMENTATION_KEY = "00000000-0000-0000-0000-0FEEDDADBEEE"; private static final String END_POINT_URL = "http://foo.bar"; @@ -73,10 +69,10 @@ public class TelemetryChannelTest { @TempDir File tempFolder; - private TelemetryChannel getTelemetryChannel(boolean followInstrumentationKeyForRedirect) - throws MalformedURLException { + private TelemetryChannel getTelemetryChannel() throws MalformedURLException { List policies = new ArrayList<>(); - policies.add(new RedirectPolicy(followInstrumentationKeyForRedirect)); + + policies.add(new RedirectPolicy(Cache.newBuilder().setMaximumSize(5).build())); HttpPipelineBuilder pipelineBuilder = new HttpPipelineBuilder() .policies(policies.toArray(new HttpPipelinePolicy[0])) @@ -85,7 +81,8 @@ private TelemetryChannel getTelemetryChannel(boolean followInstrumentationKeyFor return new TelemetryChannel( pipelineBuilder.build(), new URL(END_POINT_URL), - new LocalFileWriter(localFileCache, tempFolder)); + new LocalFileWriter(localFileCache, tempFolder), + null); } @Nullable @@ -134,32 +131,12 @@ public void setup() { }); } - @AfterEach - public void reset() { - requestCount.set(0); - } - - @Test - public void nullIkeyTest() throws MalformedURLException { - // given - List telemetryItems = new ArrayList<>(); - telemetryItems.add(createMetricTelemetry("metric" + 1, 1, null)); - TelemetryChannel telemetryChannel = getTelemetryChannel(true); - - // when - CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); - - // then - assertThat(completableResultCode.isSuccess()).isEqualTo(true); - assertThat(recordingHttpClient.getCount()).isEqualTo(2); - } - @Test public void singleIkeyTest() throws MalformedURLException { // given List telemetryItems = new ArrayList<>(); - telemetryItems.add(createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(true); + telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); + TelemetryChannel telemetryChannel = getTelemetryChannel(); // when CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); @@ -173,9 +150,10 @@ public void singleIkeyTest() throws MalformedURLException { public void dualIkeyTest() throws MalformedURLException { // given List telemetryItems = new ArrayList<>(); - telemetryItems.add(createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 2, 2, REDIRECT_INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(true); + telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); + telemetryItems.add( + TestUtils.createMetricTelemetry("metric" + 2, 2, REDIRECT_INSTRUMENTATION_KEY)); + TelemetryChannel telemetryChannel = getTelemetryChannel(); // when CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); @@ -189,9 +167,9 @@ public void dualIkeyTest() throws MalformedURLException { public void singleIkeyBatchTest() throws MalformedURLException { // given List telemetryItems = new ArrayList<>(); - telemetryItems.add(createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 2, 2, INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(true); + telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); + telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 2, 2, INSTRUMENTATION_KEY)); + TelemetryChannel telemetryChannel = getTelemetryChannel(); // when CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); @@ -205,11 +183,13 @@ public void singleIkeyBatchTest() throws MalformedURLException { public void dualIkeyBatchTest() throws MalformedURLException { // given List telemetryItems = new ArrayList<>(); - telemetryItems.add(createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 2, 2, INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 3, 3, REDIRECT_INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 4, 4, REDIRECT_INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(true); + telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); + telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 2, 2, INSTRUMENTATION_KEY)); + telemetryItems.add( + TestUtils.createMetricTelemetry("metric" + 3, 3, REDIRECT_INSTRUMENTATION_KEY)); + telemetryItems.add( + TestUtils.createMetricTelemetry("metric" + 4, 4, REDIRECT_INSTRUMENTATION_KEY)); + TelemetryChannel telemetryChannel = getTelemetryChannel(); // when CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); @@ -223,11 +203,13 @@ public void dualIkeyBatchTest() throws MalformedURLException { public void dualIkeyBatchWithDelayTest() throws MalformedURLException { // given List telemetryItems = new ArrayList<>(); - telemetryItems.add(createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 2, 2, INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 3, 3, REDIRECT_INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 4, 4, REDIRECT_INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(true); + telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); + telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 2, 2, INSTRUMENTATION_KEY)); + telemetryItems.add( + TestUtils.createMetricTelemetry("metric" + 3, 3, REDIRECT_INSTRUMENTATION_KEY)); + telemetryItems.add( + TestUtils.createMetricTelemetry("metric" + 4, 4, REDIRECT_INSTRUMENTATION_KEY)); + TelemetryChannel telemetryChannel = getTelemetryChannel(); // when CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); @@ -248,11 +230,13 @@ public void dualIkeyBatchWithDelayTest() throws MalformedURLException { public void dualIkeyBatchWithDelayAndRedirectFlagFalseTest() throws MalformedURLException { // given List telemetryItems = new ArrayList<>(); - telemetryItems.add(createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 2, 2, INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 3, 3, REDIRECT_INSTRUMENTATION_KEY)); - telemetryItems.add(createMetricTelemetry("metric" + 4, 4, REDIRECT_INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(false); + telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); + telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 2, 2, INSTRUMENTATION_KEY)); + telemetryItems.add( + TestUtils.createMetricTelemetry("metric" + 3, 3, REDIRECT_INSTRUMENTATION_KEY)); + telemetryItems.add( + TestUtils.createMetricTelemetry("metric" + 4, 4, REDIRECT_INSTRUMENTATION_KEY)); + TelemetryChannel telemetryChannel = getTelemetryChannel(); // when CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); @@ -269,42 +253,6 @@ public void dualIkeyBatchWithDelayAndRedirectFlagFalseTest() throws MalformedURL assertThat(recordingHttpClient.getCount()).isEqualTo(5); } - private static TelemetryItem createMetricTelemetry( - String name, int value, String instrumentationKey) { - TelemetryItem telemetry = new TelemetryItem(); - telemetry.setVersion(1); - telemetry.setName("Metric"); - telemetry.setInstrumentationKey(instrumentationKey); - Map tags = new HashMap<>(); - tags.put("ai.internal.sdkVersion", "test_version"); - tags.put("ai.internal.nodeName", "test_role_name"); - tags.put("ai.cloud.roleInstance", "test_cloud_name"); - telemetry.setTags(tags); - - MetricsData data = new MetricsData(); - List dataPoints = new ArrayList<>(); - MetricDataPoint dataPoint = new MetricDataPoint(); - dataPoint.setDataPointType(DataPointType.MEASUREMENT); - dataPoint.setName(name); - dataPoint.setValue(value); - dataPoint.setCount(1); - dataPoints.add(dataPoint); - - Map properties = new HashMap<>(); - properties.put("state", "blocked"); - - data.setMetrics(dataPoints); - data.setProperties(properties); - - MonitorBase monitorBase = new MonitorBase(); - monitorBase.setBaseType("MetricData"); - monitorBase.setBaseData(data); - telemetry.setData(monitorBase); - telemetry.setTime(FormattedTime.offSetDateTimeFromNow()); - - return telemetry; - } - static class RecordingHttpClient implements HttpClient { private final AtomicInteger count = new AtomicInteger(); @@ -323,9 +271,5 @@ public Mono send(HttpRequest httpRequest) { int getCount() { return count.get(); } - - void resetCount() { - count.set(0); - } } } diff --git a/test/smoke/appServers/global-resources/faststatsbeat_applicationinsights.json b/test/smoke/appServers/global-resources/faststatsbeat_applicationinsights.json index 0d8ef344aac..018a373f4d2 100644 --- a/test/smoke/appServers/global-resources/faststatsbeat_applicationinsights.json +++ b/test/smoke/appServers/global-resources/faststatsbeat_applicationinsights.json @@ -1,11 +1,17 @@ { "connectionString": "InstrumentationKey=00000000-0000-0000-0000-0FEEDDADBEEF;IngestionEndpoint=http://host.docker.internal:6060/", + "preview": { + "statsbeat": { + "disabled": false + } + }, "internal": { "statsbeat": { + "disabledAll": false, "instrumentationKey": "00000000-0000-0000-0000-0FEEDDADBEEG", "endpoint": "http://host.docker.internal:6060/", - "intervalSeconds": 30, - "featureIntervalSeconds": 30 + "shortIntervalSeconds": 30, + "longIntervalSeconds": 30 } } } \ No newline at end of file diff --git a/test/smoke/testApps/Statsbeat/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/StatsbeatSmokeTest.java b/test/smoke/testApps/Statsbeat/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/StatsbeatSmokeTest.java index 65eb33e3277..8d372d511e1 100644 --- a/test/smoke/testApps/Statsbeat/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/StatsbeatSmokeTest.java +++ b/test/smoke/testApps/Statsbeat/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/StatsbeatSmokeTest.java @@ -50,7 +50,23 @@ public void testStatsbeat() throws Exception { assertNotNull(data.getProperties().get("language")); assertNotNull(data.getProperties().get("version")); assertNotNull(data.getProperties().get("feature")); - assertEquals(8, data.getProperties().size()); + assertNotNull(data.getProperties().get("type")); + assertEquals(9, data.getProperties().size()); + + List instrumentationMetrics = + mockedIngestion.waitForItems(getMetricPredicate("Instrumentation"), 1, 70, TimeUnit.SECONDS); + + MetricData instrumentationData = (MetricData) ((Data) instrumentationMetrics.get(0).getData()).getBaseData(); + assertNotNull(instrumentationData.getProperties().get("rp")); + assertNotNull(instrumentationData.getProperties().get("attach")); + assertNotNull(instrumentationData.getProperties().get("cikey")); + assertNotNull(instrumentationData.getProperties().get("runtimeVersion")); + assertNotNull(instrumentationData.getProperties().get("os")); + assertNotNull(instrumentationData.getProperties().get("language")); + assertNotNull(instrumentationData.getProperties().get("version")); + assertNotNull(instrumentationData.getProperties().get("feature")); + assertNotNull(instrumentationData.getProperties().get("type")); + assertEquals(9, instrumentationData.getProperties().size()); List attachMetrics = mockedIngestion.waitForItems(getMetricPredicate("Attach"), 1, 70, TimeUnit.SECONDS); @@ -64,7 +80,7 @@ public void testStatsbeat() throws Exception { assertNotNull(attachData.getProperties().get("language")); assertNotNull(attachData.getProperties().get("version")); assertNotNull(attachData.getProperties().get("rpId")); - assertEquals(8, data.getProperties().size()); + assertEquals(8, attachData.getProperties().size()); List requestSuccessCountMetrics = mockedIngestion.waitForItems( @@ -79,8 +95,9 @@ public void testStatsbeat() throws Exception { assertNotNull(requestSuccessCountData.getProperties().get("os")); assertNotNull(requestSuccessCountData.getProperties().get("language")); assertNotNull(requestSuccessCountData.getProperties().get("version")); - assertNotNull(requestSuccessCountData.getProperties().get("instrumentation")); - assertEquals(8, data.getProperties().size()); + assertNotNull(requestSuccessCountData.getProperties().get("endpoint")); + assertNotNull(requestSuccessCountData.getProperties().get("host")); + assertEquals(9, requestSuccessCountData.getProperties().size()); List requestDurationMetrics = mockedIngestion.waitForItems( @@ -95,8 +112,9 @@ public void testStatsbeat() throws Exception { assertNotNull(requestDurationData.getProperties().get("os")); assertNotNull(requestDurationData.getProperties().get("language")); assertNotNull(requestDurationData.getProperties().get("version")); - assertNotNull(requestDurationData.getProperties().get("instrumentation")); - assertEquals(8, data.getProperties().size()); + assertNotNull(requestSuccessCountData.getProperties().get("endpoint")); + assertNotNull(requestSuccessCountData.getProperties().get("host")); + assertEquals(9, requestDurationData.getProperties().size()); } private static Predicate getMetricPredicate(String name) {