diff --git a/agent/agent-tooling/build.gradle.kts b/agent/agent-tooling/build.gradle.kts index ec56da7f6c1..a27a7da633a 100644 --- a/agent/agent-tooling/build.gradle.kts +++ b/agent/agent-tooling/build.gradle.kts @@ -26,8 +26,6 @@ dependencies { implementation("net.bytebuddy:byte-buddy") implementation("commons-codec:commons-codec") - implementation("org.apache.commons:commons-lang3") - implementation("commons-io:commons-io") implementation("org.apache.commons:commons-text") // TODO (trask) this is probably still needed for above apache commons projects implementation("org.slf4j:jcl-over-slf4j") diff --git a/agent/agent-tooling/gradle.lockfile b/agent/agent-tooling/gradle.lockfile index dcff4878a20..1b2e4b2c025 100644 --- a/agent/agent-tooling/gradle.lockfile +++ b/agent/agent-tooling/gradle.lockfile @@ -38,7 +38,6 @@ com.squareup.moshi:moshi:1.11.0=runtimeClasspath com.squareup.okhttp3:okhttp:4.9.3=runtimeClasspath com.squareup.okio:okio:2.8.0=runtimeClasspath commons-codec:commons-codec:1.15=runtimeClasspath -commons-io:commons-io:2.7=runtimeClasspath io.netty:netty-bom:4.1.72.Final=runtimeClasspath io.netty:netty-buffer:4.1.72.Final=runtimeClasspath io.netty:netty-codec-dns:4.1.72.Final=runtimeClasspath diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/common/SystemInformation.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/common/SystemInformation.java index 23c78794ccd..37313a89124 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/common/SystemInformation.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/common/SystemInformation.java @@ -22,7 +22,7 @@ package com.microsoft.applicationinsights.agent.internal.common; import java.lang.management.ManagementFactory; -import org.apache.commons.lang3.SystemUtils; +import java.util.Locale; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +32,16 @@ public class SystemInformation { private static final String DEFAULT_PROCESS_NAME = "Java_Process"; + private static final boolean WINDOWS; + private static final boolean LINUX; + + static { + String osName = System.getProperty("os.name"); + String osNameLower = osName == null ? null : osName.toLowerCase(Locale.ENGLISH); + WINDOWS = osNameLower != null && osNameLower.startsWith("windows"); + LINUX = osNameLower != null && osNameLower.startsWith("linux"); + } + private static final String processId = initializeProcessId(); public static String getProcessId() { @@ -39,11 +49,11 @@ public static String getProcessId() { } public static boolean isWindows() { - return SystemUtils.IS_OS_WINDOWS; + return WINDOWS; } - public static boolean isUnix() { - return SystemUtils.IS_OS_UNIX; + public static boolean isLinux() { + return LINUX; } /** diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/heartbeat/HeartBeatProvider.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/heartbeat/HeartBeatProvider.java index afd5b411b85..ca7c7fa350d 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/heartbeat/HeartBeatProvider.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/heartbeat/HeartBeatProvider.java @@ -34,7 +34,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,24 +106,19 @@ public void initialize(TelemetryClient telemetryClient) { public boolean addHeartBeatProperty( String propertyName, String propertyValue, boolean isHealthy) { - boolean isAdded = false; - if (!StringUtils.isEmpty(propertyName)) { - if (!heartbeatProperties.containsKey(propertyName)) { - HeartBeatPropertyPayload payload = new HeartBeatPropertyPayload(); - payload.setHealthy(isHealthy); - payload.setPayloadValue(propertyValue); - heartbeatProperties.put(propertyName, payload); - isAdded = true; - logger.trace("added heartbeat property {} - {}", propertyName, propertyValue); - } else { - logger.trace( - "heartbeat property {} cannot be added twice. Please use setHeartBeatProperty instead to modify the value", - propertyName); - } - } else { - logger.warn("cannot add property without property name"); + if (heartbeatProperties.containsKey(propertyName)) { + logger.trace( + "heartbeat property {} cannot be added twice. Please use setHeartBeatProperty instead to modify the value", + propertyName); + return false; } - return isAdded; + + HeartBeatPropertyPayload payload = new HeartBeatPropertyPayload(); + payload.setHealthy(isHealthy); + payload.setPayloadValue(propertyValue); + heartbeatProperties.put(propertyName, payload); + logger.trace("added heartbeat property {} - {}", propertyName, propertyValue); + return true; } public long getHeartBeatInterval() { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/LazyHttpClient.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/LazyHttpClient.java index 395396b8f1d..1ef914e88e7 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/LazyHttpClient.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/LazyHttpClient.java @@ -21,6 +21,8 @@ package com.microsoft.applicationinsights.agent.internal.httpclient; +import static java.util.Arrays.asList; + import com.azure.core.http.HttpClient; import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpPipelineBuilder; @@ -29,9 +31,11 @@ import com.azure.core.http.ProxyOptions; import com.azure.core.http.netty.NettyAsyncHttpClientBuilder; import com.azure.core.http.policy.BearerTokenAuthenticationPolicy; +import com.azure.core.http.policy.DefaultRedirectStrategy; import com.azure.core.http.policy.HttpLogOptions; import com.azure.core.http.policy.HttpLoggingPolicy; import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.http.policy.RedirectPolicy; import com.azure.core.util.Context; import com.azure.identity.ClientSecretCredentialBuilder; import com.azure.identity.ManagedIdentityCredential; @@ -39,7 +43,6 @@ import com.azure.identity.VisualStudioCodeCredential; import com.azure.identity.VisualStudioCodeCredentialBuilder; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; -import io.opentelemetry.instrumentation.api.cache.Cache; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; @@ -125,16 +128,19 @@ private static HttpClient init() { .build(); } - // pass non-null ikeyRedirectCache if you want to use ikey-specific redirect policy + public static HttpPipeline newHttpPipeLineWithDefaultRedirect( + @Nullable Configuration.AadAuthentication aadConfiguration) { + return newHttpPipeLine(aadConfiguration, new RedirectPolicy(new DefaultRedirectStrategy())); + } + public static HttpPipeline newHttpPipeLine( @Nullable Configuration.AadAuthentication aadConfiguration, - @Nullable Cache ikeyRedirectCache) { + HttpPipelinePolicy... additionalPolicies) { List policies = new ArrayList<>(); - // Redirect policy to handle v2.1/track redirects (and other redirects too, e.g. profiler) - policies.add(new RedirectPolicy(ikeyRedirectCache)); if (aadConfiguration != null && aadConfiguration.enabled) { policies.add(getAuthenticationPolicy(aadConfiguration)); } + policies.addAll(asList(additionalPolicies)); // Add Logging Policy. Can be enabled using AZURE_LOG_LEVEL. // TODO set the logging level based on self diagnostic log level set by user policies.add(new HttpLoggingPolicy(new HttpLogOptions())); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/RedirectPolicy.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/RedirectPolicy.java deleted file mode 100644 index 3cb9736d4a9..00000000000 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/httpclient/RedirectPolicy.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * ApplicationInsights-Java - * Copyright (c) Microsoft Corporation - * All rights reserved. - * - * MIT License - * Permission is hereby granted, free of charge, to any person obtaining a copy of this - * software and associated documentation files (the ""Software""), to deal in the Software - * without restriction, including without limitation the rights to use, copy, modify, merge, - * publish, distribute, sublicense, and/or sell copies of the Software, and to permit - * persons to whom the Software is furnished to do so, subject to the following conditions: - * The above copyright notice and this permission notice shall be included in all copies or - * substantial portions of the Software. - * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, - * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR - * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE - * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -package com.microsoft.applicationinsights.agent.internal.httpclient; - -import com.azure.core.http.HttpPipelineCallContext; -import com.azure.core.http.HttpPipelineNextPolicy; -import com.azure.core.http.HttpRequest; -import com.azure.core.http.HttpResponse; -import com.azure.core.http.policy.HttpPipelinePolicy; -import io.opentelemetry.instrumentation.api.cache.Cache; -import java.net.HttpURLConnection; -import java.net.URL; -import org.checkerframework.checker.nullness.qual.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Mono; - -// This is mostly a copy from Azure Monitor Open Telemetry Exporter SDK AzureMonitorRedirectPolicy -public final class RedirectPolicy implements HttpPipelinePolicy { - private static final int PERMANENT_REDIRECT_STATUS_CODE = 308; - private static final int TEMP_REDIRECT_STATUS_CODE = 307; - // Based on Stamp specific redirects design doc - private static final int MAX_REDIRECT_RETRIES = 10; - private static final Logger logger = LoggerFactory.getLogger(RedirectPolicy.class); - public static final String INSTRUMENTATION_KEY = "instrumentationKey"; - - private final Cache redirectMappings = Cache.bounded(100); - - @Nullable private final Cache ikeyRedirectCache; - - // pass non-null ikeyRedirectCache if you want to use ikey-specific redirect policy - public RedirectPolicy(@Nullable Cache ikeyRedirectCache) { - this.ikeyRedirectCache = ikeyRedirectCache; - } - - @Override - public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { - return attemptRetry(context, next, context.getHttpRequest(), 0); - } - - /** - * Function to process through the HTTP Response received in the pipeline and retry sending the - * request with new redirect url. - */ - private Mono attemptRetry( - HttpPipelineCallContext context, - HttpPipelineNextPolicy next, - HttpRequest originalHttpRequest, - int retryCount) { - String instrumentationKey = getInstrumentationKeyFromContext(context); - String redirectUrl = getCachedRedirectUrl(instrumentationKey, originalHttpRequest.getUrl()); - if (redirectUrl != null) { - // make sure the context is not modified during retry, except for the URL - context.setHttpRequest(originalHttpRequest.copy().setUrl(redirectUrl)); - } - return next.clone() - .process() - .flatMap( - httpResponse -> { - if (shouldRetryWithRedirect(httpResponse.getStatusCode(), retryCount)) { - String responseLocation = httpResponse.getHeaderValue("Location"); - if (responseLocation != null) { - cacheRedirectUrl( - responseLocation, instrumentationKey, originalHttpRequest.getUrl()); - context.setHttpRequest(originalHttpRequest.copy().setUrl(responseLocation)); - return attemptRetry(context, next, originalHttpRequest, retryCount + 1); - } - } - return Mono.just(httpResponse); - }); - } - - private void cacheRedirectUrl(String redirectUrl, String instrumentationKey, URL originalUrl) { - if (ikeyRedirectCache == null) { - redirectMappings.put(originalUrl, redirectUrl); - return; - } - if (instrumentationKey == null) { - throw new IllegalArgumentException( - "instrumentationKey must be non-null when using ikey redirect policy"); - } - ikeyRedirectCache.put(instrumentationKey, redirectUrl); - } - - @Nullable - private String getCachedRedirectUrl(String instrumentationKey, URL originalUrl) { - if (ikeyRedirectCache == null) { - return redirectMappings.get(originalUrl); - } - if (instrumentationKey == null) { - throw new IllegalArgumentException( - "instrumentationKey must be non-null when using ikey redirect policy"); - } - return ikeyRedirectCache.get(instrumentationKey); - } - - /** - * Determines if it's a valid retry scenario based on statusCode and tryCount. - * - * @param statusCode HTTP response status code - * @param tryCount Redirect retries so far - * @return True if statusCode corresponds to HTTP redirect response codes and redirect retries is - * less than {@code MAX_REDIRECT_RETRIES}. - */ - private static boolean shouldRetryWithRedirect(int statusCode, int tryCount) { - if (tryCount >= MAX_REDIRECT_RETRIES) { - logger.warn("Max redirect retries limit reached:{}.", MAX_REDIRECT_RETRIES); - return false; - } - return statusCode == HttpURLConnection.HTTP_MOVED_TEMP - || statusCode == HttpURLConnection.HTTP_MOVED_PERM - || statusCode == PERMANENT_REDIRECT_STATUS_CODE - || statusCode == TEMP_REDIRECT_STATUS_CODE; - } - - private static String getInstrumentationKeyFromContext(HttpPipelineCallContext context) { - return (String) context.getData(INSTRUMENTATION_KEY).orElse(null); - } -} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiComponentInstaller.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiComponentInstaller.java index c3fb87c4ddd..f13a1392e6d 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiComponentInstaller.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/AiComponentInstaller.java @@ -57,7 +57,6 @@ import com.microsoft.applicationinsights.profiler.config.ServiceProfilerServiceConfig; import io.opentelemetry.instrumentation.api.aisdk.AiAppId; import io.opentelemetry.instrumentation.api.aisdk.AiLazyConfiguration; -import io.opentelemetry.instrumentation.api.cache.Cache; import io.opentelemetry.sdk.common.CompletableResultCode; import java.io.File; import java.lang.instrument.Instrumentation; @@ -164,13 +163,11 @@ private static AppIdSupplier start(Instrumentation instrumentation) { .map(MetricFilter::new) .collect(Collectors.toList()); - Cache ikeyEndpointMap = Cache.bounded(100); - StatsbeatModule statsbeatModule = new StatsbeatModule(ikeyEndpointMap); + StatsbeatModule statsbeatModule = new StatsbeatModule(); TelemetryClient telemetryClient = TelemetryClient.builder() .setCustomDimensions(config.customDimensions) .setMetricFilters(metricFilters) - .setIkeyEndpointMap(ikeyEndpointMap) .setStatsbeatModule(statsbeatModule) .setReadOnlyFileSystem(readOnlyFileSystem) .setGeneralExportQueueSize(config.preview.generalExportQueueCapacity) @@ -283,7 +280,7 @@ private static String getCodelessSdkNamePrefix() { sdkNamePrefix.append(DiagnosticsHelper.rpIntegrationChar()); if (SystemInformation.isWindows()) { sdkNamePrefix.append("w"); - } else if (SystemInformation.isUnix()) { + } else if (SystemInformation.isLinux()) { sdkNamePrefix.append("l"); } else { startupLogger.warn("could not detect os: {}", System.getProperty("os.name")); @@ -311,7 +308,7 @@ public void run() { CompletableResultCode result = new CompletableResultCode(); otelFlush.whenComplete( () -> { - CompletableResultCode batchingClientFlush = telemetryClient.flushChannelBatcher(); + CompletableResultCode batchingClientFlush = telemetryClient.forceFlush(); batchingClientFlush.whenComplete( () -> { if (otelFlush.isSuccess() && batchingClientFlush.isSuccess()) { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/legacysdk/BytecodeUtilImpl.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/legacysdk/BytecodeUtilImpl.java index 2040357bb07..bbf2878f033 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/legacysdk/BytecodeUtilImpl.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/legacysdk/BytecodeUtilImpl.java @@ -401,7 +401,7 @@ private static SeverityLevel getSeverityLevel(int value) { public void flush() { // this is not null because sdk instrumentation is not added until TelemetryClient.setActive() // is called - TelemetryClient.getActive().flushChannelBatcher().join(10, SECONDS); + TelemetryClient.getActive().forceFlush().join(10, SECONDS); } @Override diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/FileUtil.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/FileUtil.java new file mode 100644 index 00000000000..fa51a87fdae --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/FileUtil.java @@ -0,0 +1,56 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.localstorage; + +import static java.util.Arrays.asList; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +class FileUtil { + + static List listTrnFiles(File directory) { + File[] files = directory.listFiles((dir, name) -> name.endsWith(".trn")); + return files == null ? Collections.emptyList() : asList(files); + } + + static String getBaseName(File file) { + String name = file.getName(); + int index = name.lastIndexOf('.'); + return index == -1 ? name : name.substring(0, index); + } + + static void moveFile(File srcFile, File destFile) throws IOException { + if (!srcFile.renameTo(destFile)) { + throw new IOException( + "Unable to rename file '" + + srcFile.getAbsolutePath() + + "' to '" + + destFile.getAbsolutePath() + + "'"); + } + } + + private FileUtil() {} +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileCache.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileCache.java index af99d638aa8..2be589239e9 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileCache.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileCache.java @@ -27,10 +27,8 @@ import java.util.Queue; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.stream.Collectors; -import javax.annotation.Nullable; -import org.apache.commons.io.FileUtils; -public class LocalFileCache { +class LocalFileCache { /** * Track a list of active filenames persisted on disk. FIFO (First-In-First-Out) read will avoid @@ -39,35 +37,34 @@ public class LocalFileCache { * C# uses "User@processName" to identify each app, but Java can't rely on process name since it's * a system property that can be customized via the command line. */ - private final Queue persistedFilesCache = new ConcurrentLinkedDeque<>(); + private final Queue persistedFilesCache = new ConcurrentLinkedDeque<>(); - public LocalFileCache(File folder) { + LocalFileCache(File folder) { List files = sortPersistedFiles(folder); // existing files are not older than 48 hours and need to get added to the queue to be // re-processed. // this will avoid data loss in the case of app crashes and restarts. for (File file : files) { - persistedFilesCache.add(file.getName()); + persistedFilesCache.add(file); } } // Track the newly persisted filename to the concurrent hashmap. - void addPersistedFilenameToMap(String filename) { - persistedFilesCache.add(filename); + void addPersistedFile(File file) { + persistedFilesCache.add(file); } - String poll() { + File poll() { return persistedFilesCache.poll(); } // only used by tests - Queue getPersistedFilesCache() { + Queue getPersistedFilesCache() { return persistedFilesCache; } - @Nullable private static List sortPersistedFiles(File folder) { - return FileUtils.listFiles(folder, new String[] {"trn"}, false).stream() + return FileUtil.listTrnFiles(folder).stream() .sorted(Comparator.comparing(File::lastModified)) .collect(Collectors.toList()); } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileLoader.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileLoader.java index 3982d92edec..b092246b408 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileLoader.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileLoader.java @@ -32,11 +32,9 @@ import java.nio.ByteBuffer; import java.util.regex.Pattern; import javax.annotation.Nullable; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; /** This class manages loading a list of {@link ByteBuffer} from the disk. */ -public class LocalFileLoader { +class LocalFileLoader { // A regex to validate that an instrumentation key is well-formed. It's copied straight from the // Breeze repo. @@ -55,7 +53,7 @@ public class LocalFileLoader { private static final OperationLogger updateOperationLogger = new OperationLogger(LocalFileLoader.class, "Updating local telemetry on disk"); - public LocalFileLoader( + LocalFileLoader( LocalFileCache localFileCache, File telemetryFolder, @Nullable NonessentialStatsbeat nonessentialStatsbeat) { @@ -67,8 +65,8 @@ public LocalFileLoader( // Load ByteBuffer from persisted files on disk in FIFO order. @Nullable PersistedFile loadTelemetriesFromDisk() { - String filenameToBeLoaded = localFileCache.poll(); - if (filenameToBeLoaded == null) { + File fileToBeLoaded = localFileCache.poll(); + if (fileToBeLoaded == null) { return null; } @@ -79,22 +77,19 @@ PersistedFile loadTelemetriesFromDisk() { // response confirms it is sent successfully; otherwise, temp file will get renamed back to the // source file extension. File tempFile; - File sourceFile; try { - sourceFile = new File(telemetryFolder, filenameToBeLoaded); - if (!sourceFile.exists()) { + if (!fileToBeLoaded.exists()) { return null; } tempFile = new File( - telemetryFolder, - FilenameUtils.getBaseName(filenameToBeLoaded) + TEMPORARY_FILE_EXTENSION); - FileUtils.moveFile(sourceFile, tempFile); + telemetryFolder, FileUtil.getBaseName(fileToBeLoaded) + TEMPORARY_FILE_EXTENSION); + FileUtil.moveFile(fileToBeLoaded, tempFile); } catch (IOException e) { operationLogger.recordFailure( "Failed to change " - + filenameToBeLoaded + + fileToBeLoaded.getAbsolutePath() + " to have " + TEMPORARY_FILE_EXTENSION + " extension: ", @@ -179,10 +174,9 @@ public void updateProcessedFileStatus(boolean successOrNonRetryableError, File f } } else { // rename the temp file back to .trn source file extension - File sourceFile = - new File(telemetryFolder, FilenameUtils.getBaseName(file.getName()) + ".trn"); + File sourceFile = new File(telemetryFolder, FileUtil.getBaseName(file) + ".trn"); try { - FileUtils.moveFile(file, sourceFile); + FileUtil.moveFile(file, sourceFile); } catch (IOException ex) { updateOperationLogger.recordFailure( "Fail to rename " + file.getName() + " to have a .trn extension.", ex); @@ -191,7 +185,7 @@ public void updateProcessedFileStatus(boolean successOrNonRetryableError, File f updateOperationLogger.recordSuccess(); // add the source filename back to local file cache to be processed later. - localFileCache.addPersistedFilenameToMap(sourceFile.getName()); + localFileCache.addPersistedFile(sourceFile); } } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFilePurger.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFilePurger.java index ebdd9bb9d2d..3ff148dd75a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFilePurger.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFilePurger.java @@ -29,7 +29,6 @@ import java.util.Date; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +77,7 @@ public void run() { } private void purgedExpiredFiles(File folder) { - Collection files = FileUtils.listFiles(folder, new String[] {"trn"}, false); + Collection files = FileUtil.listTrnFiles(folder); int numDeleted = 0; for (File file : files) { if (expired(file.getName())) { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileSender.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileSender.java index 7c7ce889a98..d47c31c5f52 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileSender.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileSender.java @@ -21,10 +21,14 @@ package com.microsoft.applicationinsights.agent.internal.localstorage; +import static java.util.Collections.singletonList; + import com.microsoft.applicationinsights.agent.internal.common.Strings; import com.microsoft.applicationinsights.agent.internal.common.ThreadPoolUtils; -import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryChannel; +import com.microsoft.applicationinsights.agent.internal.telemetry.DiagnosticTelemetryPipelineListener; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipeline; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipelineListener; import io.opentelemetry.sdk.common.CompletableResultCode; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -32,7 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class LocalFileSender implements Runnable { +class LocalFileSender implements Runnable { private static final Logger logger = LoggerFactory.getLogger(LocalFileSender.class); @@ -43,17 +47,21 @@ public class LocalFileSender implements Runnable { ThreadPoolUtils.createDaemonThreadFactory(LocalFileLoader.class)); private final LocalFileLoader localFileLoader; - private final TelemetryChannel telemetryChannel; + private final TelemetryPipeline telemetryPipeline; + + private final TelemetryPipelineListener diagnosticListener = + new DiagnosticTelemetryPipelineListener( + "Sending telemetry to the ingestion service (retry from disk)"); - public static void start(LocalFileLoader localFileLoader, TelemetryChannel telemetryChannel) { - LocalFileSender localFileSender = new LocalFileSender(localFileLoader, telemetryChannel); + static void start(LocalFileLoader localFileLoader, TelemetryPipeline telemetryPipeline) { + LocalFileSender localFileSender = new LocalFileSender(localFileLoader, telemetryPipeline); scheduledExecutor.scheduleWithFixedDelay( localFileSender, INTERVAL_SECONDS, INTERVAL_SECONDS, TimeUnit.SECONDS); } - private LocalFileSender(LocalFileLoader localFileLoader, TelemetryChannel telemetryChannel) { + private LocalFileSender(LocalFileLoader localFileLoader, TelemetryPipeline telemetryPipeline) { this.localFileLoader = localFileLoader; - this.telemetryChannel = telemetryChannel; + this.telemetryPipeline = telemetryPipeline; } @Override @@ -67,12 +75,13 @@ public void run() { LocalFileLoader.PersistedFile persistedFile = localFileLoader.loadTelemetriesFromDisk(); if (persistedFile != null) { CompletableResultCode resultCode = - telemetryChannel.sendRawBytes( - persistedFile.rawBytes, + telemetryPipeline.send( + singletonList(persistedFile.rawBytes), persistedFile.instrumentationKey, - () -> localFileLoader.updateProcessedFileStatus(true, persistedFile.file), - retryable -> - localFileLoader.updateProcessedFileStatus(!retryable, persistedFile.file)); + TelemetryPipelineListener.composite( + diagnosticListener, + new LocalFileSenderTelemetryPipelineListener( + localFileLoader, persistedFile.file))); resultCode.join(30, TimeUnit.SECONDS); // wait max 30 seconds for request to be completed. } } catch (RuntimeException ex) { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileSenderTelemetryPipelineListener.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileSenderTelemetryPipelineListener.java new file mode 100644 index 00000000000..4d732c54b2c --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileSenderTelemetryPipelineListener.java @@ -0,0 +1,55 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.localstorage; + +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipelineListener; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipelineRequest; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipelineResponse; +import java.io.File; + +class LocalFileSenderTelemetryPipelineListener implements TelemetryPipelineListener { + + private final LocalFileLoader localFileLoader; + private final File file; + + LocalFileSenderTelemetryPipelineListener(LocalFileLoader localFileLoader, File file) { + this.localFileLoader = localFileLoader; + this.file = file; + } + + @Override + public void onResponse(TelemetryPipelineRequest request, TelemetryPipelineResponse response) { + int responseCode = response.getStatusCode(); + if (responseCode == 200) { + localFileLoader.updateProcessedFileStatus(true, file); + } else { + localFileLoader.updateProcessedFileStatus( + !LocalStorageTelemetryPipelineListener.RETRYABLE_CODES.contains(responseCode), file); + } + } + + @Override + public void onException( + TelemetryPipelineRequest request, String errorMessage, Throwable throwable) { + localFileLoader.updateProcessedFileStatus(false, file); + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileWriter.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileWriter.java index 45359b348ed..0b752198ebe 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileWriter.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileWriter.java @@ -33,11 +33,9 @@ import java.util.Collection; import java.util.List; import javax.annotation.Nullable; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; /** This class manages writing a list of {@link ByteBuffer} to the file system. */ -public final class LocalFileWriter { +final class LocalFileWriter { // 50MB per folder for all apps. private static final long MAX_FILE_SIZE_IN_BYTES = 52428800; // 50MB @@ -52,7 +50,7 @@ public final class LocalFileWriter { new OperationLogger( LocalFileWriter.class, "Writing telemetry to disk (telemetry is discarded on failure)"); - public LocalFileWriter( + LocalFileWriter( LocalFileCache localFileCache, File telemetryFolder, @Nullable NonessentialStatsbeat nonessentialStatsbeat) { @@ -61,7 +59,7 @@ public LocalFileWriter( this.nonessentialStatsbeat = nonessentialStatsbeat; } - public void writeToDisk(List buffers, String instrumentationKey) { + void writeToDisk(String instrumentationKey, List buffers) { long size = getTotalSizeOfPersistedFiles(telemetryFolder); if (size >= MAX_FILE_SIZE_IN_BYTES) { operationLogger.recordFailure( @@ -91,15 +89,13 @@ public void writeToDisk(List buffers, String instrumentationKey) { File permanentFile; try { - String filename = tempFile.getName(); - File sourceFile = new File(telemetryFolder, filename); permanentFile = - new File(telemetryFolder, FilenameUtils.getBaseName(filename) + PERMANENT_FILE_EXTENSION); - FileUtils.moveFile(sourceFile, permanentFile); + new File(telemetryFolder, FileUtil.getBaseName(tempFile) + PERMANENT_FILE_EXTENSION); + FileUtil.moveFile(tempFile, permanentFile); } catch (IOException e) { operationLogger.recordFailure( "Fail to change " - + tempFile.getName() + + tempFile.getAbsolutePath() + " to have " + PERMANENT_FILE_EXTENSION + " extension: ", @@ -108,7 +104,7 @@ public void writeToDisk(List buffers, String instrumentationKey) { return; } - localFileCache.addPersistedFilenameToMap(permanentFile.getName()); + localFileCache.addPersistedFile(permanentFile); operationLogger.recordSuccess(); } @@ -140,7 +136,7 @@ private static long getTotalSizeOfPersistedFiles(File telemetryFolder) { } long sum = 0; - Collection files = FileUtils.listFiles(telemetryFolder, new String[] {"trn"}, false); + Collection files = FileUtil.listTrnFiles(telemetryFolder); for (File file : files) { sum += file.length(); } diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalStorageSystem.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalStorageSystem.java new file mode 100644 index 00000000000..00a44e3f7b5 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalStorageSystem.java @@ -0,0 +1,53 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.localstorage; + +import com.microsoft.applicationinsights.agent.internal.statsbeat.NonessentialStatsbeat; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipeline; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipelineListener; +import java.io.File; +import org.checkerframework.checker.nullness.qual.Nullable; + +public class LocalStorageSystem { + + private final LocalFileWriter writer; + private final LocalFileLoader loader; + + public LocalStorageSystem( + File telemetryFolder, @Nullable NonessentialStatsbeat nonessentialStatsbeat) { + LocalFileCache localFileCache = new LocalFileCache(telemetryFolder); + loader = new LocalFileLoader(localFileCache, telemetryFolder, nonessentialStatsbeat); + writer = new LocalFileWriter(localFileCache, telemetryFolder, nonessentialStatsbeat); + } + + public TelemetryPipelineListener createTelemetryPipelineListener() { + return new LocalStorageTelemetryPipelineListener(writer); + } + + public void startSendingFromDisk(TelemetryPipeline pipeline) { + LocalFileSender.start(loader, pipeline); + } + + public void stop() { + // TODO (trask) this will be needed in azure-monitor-opentelemetry-exporter + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalStorageTelemetryPipelineListener.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalStorageTelemetryPipelineListener.java new file mode 100644 index 00000000000..004ea4ce1b7 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalStorageTelemetryPipelineListener.java @@ -0,0 +1,63 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.localstorage; + +import static java.util.Arrays.asList; + +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipelineListener; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipelineRequest; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipelineResponse; +import java.util.HashSet; +import java.util.Set; + +class LocalStorageTelemetryPipelineListener implements TelemetryPipelineListener { + + static final Set RETRYABLE_CODES = + new HashSet<>( + asList( + 401, + 403, + 408, // REQUEST TIMEOUT + 429, // TOO MANY REQUESTS + 500, // INTERNAL SERVER ERROR + 503 // SERVICE UNAVAILABLE + )); + + private final LocalFileWriter localFileWriter; + + LocalStorageTelemetryPipelineListener(LocalFileWriter localFileWriter) { + this.localFileWriter = localFileWriter; + } + + @Override + public void onResponse(TelemetryPipelineRequest request, TelemetryPipelineResponse response) { + if (RETRYABLE_CODES.contains(response.getStatusCode())) { + localFileWriter.writeToDisk(request.getInstrumentationKey(), request.getTelemetry()); + } + } + + @Override + public void onException( + TelemetryPipelineRequest request, String errorMessage, Throwable throwable) { + localFileWriter.writeToDisk(request.getInstrumentationKey(), request.getTelemetry()); + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerServiceInitializer.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerServiceInitializer.java index f28f5f358e5..e236414c807 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerServiceInitializer.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/ProfilerServiceInitializer.java @@ -76,7 +76,7 @@ public static synchronized void initialize( GcEventMonitor.GcEventMonitorConfiguration gcEventMonitorConfiguration) { HttpPipeline httpPipeline = - LazyHttpClient.newHttpPipeLine(telemetryClient.getAadAuthentication(), null); + LazyHttpClient.newHttpPipeLineWithDefaultRedirect(telemetryClient.getAadAuthentication()); initialize( appIdSupplier, diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/quickpulse/QuickPulse.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/quickpulse/QuickPulse.java index 6ac5e5b60e5..35102e69973 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/quickpulse/QuickPulse.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/quickpulse/QuickPulse.java @@ -69,7 +69,8 @@ private void initializeSync(CountDownLatch latch, TelemetryClient telemetryClien initialized = true; String quickPulseId = UUID.randomUUID().toString().replace("-", ""); HttpPipeline httpPipeline = - LazyHttpClient.newHttpPipeLine(telemetryClient.getAadAuthentication(), null); + LazyHttpClient.newHttpPipeLineWithDefaultRedirect( + telemetryClient.getAadAuthentication()); ArrayBlockingQueue sendQueue = new ArrayBlockingQueue<>(256, true); QuickPulseDataSender quickPulseDataSender = diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingScoreGeneratorV2.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingScoreGeneratorV2.java index 8a0c613fb07..f64773c4001 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingScoreGeneratorV2.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingScoreGeneratorV2.java @@ -22,7 +22,6 @@ package com.microsoft.applicationinsights.agent.internal.sampling; import java.util.Random; -import org.apache.commons.lang3.StringUtils; /** * This class generates the sample using the random number generator. It also contains the logic to @@ -39,23 +38,15 @@ public class SamplingScoreGeneratorV2 { * @return [0.0, 1.0) */ public static double getSamplingScore(String operationId) { - - double samplingScore; - - if (!StringUtils.isEmpty(operationId)) { - samplingScore = ((double) getSamplingHashCode(operationId) / Integer.MAX_VALUE); + if (operationId != null && !operationId.isEmpty()) { + return 100 * ((double) getSamplingHashCode(operationId) / Integer.MAX_VALUE); } else { - samplingScore = random.nextDouble(); // [0,1) + return 100 * random.nextDouble(); } - - return samplingScore * 100.0; // always < 100.0 } /** Returns value in [0, Integer.MAX_VALUE). */ - static int getSamplingHashCode(String operationId) { - if (StringUtils.isEmpty(operationId)) { - return 0; - } + private static int getSamplingHashCode(String operationId) { CharSequence opId; if (operationId.length() < 8) { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/CustomDimensions.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/CustomDimensions.java index 0bf09a6d45e..9b0194b24b0 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/CustomDimensions.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/CustomDimensions.java @@ -98,7 +98,7 @@ void populateProperties(StatsbeatTelemetryBuilder telemetryBuilder, String custo private static OperatingSystem initOperatingSystem() { if (SystemInformation.isWindows()) { return OperatingSystem.OS_WINDOWS; - } else if (SystemInformation.isUnix()) { + } else if (SystemInformation.isLinux()) { return OperatingSystem.OS_LINUX; } else { return OperatingSystem.OS_UNKNOWN; diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeat.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeat.java index c0be71a5f92..8b018adf655 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeat.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeat.java @@ -21,10 +21,8 @@ package com.microsoft.applicationinsights.agent.internal.statsbeat; -import com.microsoft.applicationinsights.agent.internal.common.Strings; import com.microsoft.applicationinsights.agent.internal.exporter.builders.StatsbeatTelemetryBuilder; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; -import io.opentelemetry.instrumentation.api.cache.Cache; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; @@ -42,7 +40,6 @@ public class NetworkStatsbeat extends BaseStatsbeat { private static final String BREEZE_ENDPOINT = "breeze"; private final Object lock = new Object(); - private final Cache ikeyEndpointMap; @GuardedBy("lock") private final Map instrumentationKeyCounterMap = new HashMap<>(); @@ -50,13 +47,10 @@ public class NetworkStatsbeat extends BaseStatsbeat { // only used by tests public NetworkStatsbeat() { super(new CustomDimensions()); - this.ikeyEndpointMap = Cache.bounded(100); } - public NetworkStatsbeat( - CustomDimensions customDimensions, Cache ikeyEndpointMap) { + public NetworkStatsbeat(CustomDimensions customDimensions) { super(customDimensions); - this.ikeyEndpointMap = ikeyEndpointMap; } @Override @@ -69,41 +63,40 @@ protected void send(TelemetryClient telemetryClient) { for (Map.Entry entry : local.entrySet()) { String ikey = entry.getKey(); - String endpointUrl = ikeyEndpointMap.get(ikey); - if (Strings.isNullOrEmpty(endpointUrl)) { - endpointUrl = telemetryClient.getEndpointProvider().getIngestionEndpointUrl().toString(); - } - - sendIntervalMetric(telemetryClient, ikey, entry.getValue(), getHost(endpointUrl)); + sendIntervalMetric(telemetryClient, ikey, entry.getValue()); } } - public void incrementRequestSuccessCount(long duration, String ikey) { + public void incrementRequestSuccessCount(long duration, String ikey, String host) { doWithIntervalMetrics( ikey, + host, intervalMetrics -> { intervalMetrics.requestSuccessCount.incrementAndGet(); intervalMetrics.totalRequestDuration.getAndAdd(duration); }); } - public void incrementRequestFailureCount(String ikey) { + public void incrementRequestFailureCount(String ikey, String host) { doWithIntervalMetrics( - ikey, intervalMetrics -> intervalMetrics.requestFailureCount.incrementAndGet()); + ikey, host, intervalMetrics -> intervalMetrics.requestFailureCount.incrementAndGet()); } - public void incrementRetryCount(String ikey) { - doWithIntervalMetrics(ikey, intervalMetrics -> intervalMetrics.retryCount.incrementAndGet()); + // TODO (heya) this is never called + public void incrementRetryCount(String ikey, String host) { + doWithIntervalMetrics( + ikey, host, intervalMetrics -> intervalMetrics.retryCount.incrementAndGet()); } - public void incrementThrottlingCount(String ikey) { + public void incrementThrottlingCount(String ikey, String host) { doWithIntervalMetrics( - ikey, intervalMetrics -> intervalMetrics.throttlingCount.incrementAndGet()); + ikey, host, intervalMetrics -> intervalMetrics.throttlingCount.incrementAndGet()); } - void incrementExceptionCount(String ikey) { + // TODO (heya) this is never called + void incrementExceptionCount(String ikey, String host) { doWithIntervalMetrics( - ikey, intervalMetrics -> intervalMetrics.exceptionCount.incrementAndGet()); + ikey, host, intervalMetrics -> intervalMetrics.exceptionCount.incrementAndGet()); } // only used by tests @@ -154,19 +147,22 @@ long getExceptionCount(String ikey) { } } - private void doWithIntervalMetrics(String ikey, Consumer update) { + private void doWithIntervalMetrics(String ikey, String host, Consumer update) { synchronized (lock) { - update.accept(instrumentationKeyCounterMap.computeIfAbsent(ikey, k -> new IntervalMetrics())); + IntervalMetrics intervalMetrics = + instrumentationKeyCounterMap.computeIfAbsent(ikey, k -> new IntervalMetrics()); + intervalMetrics.host = host; + update.accept(intervalMetrics); } } private void sendIntervalMetric( - TelemetryClient telemetryClient, String ikey, IntervalMetrics local, String host) { + TelemetryClient telemetryClient, String ikey, IntervalMetrics local) { if (local.requestSuccessCount.get() != 0) { StatsbeatTelemetryBuilder requestSuccessCountSt = createStatsbeatTelemetry( telemetryClient, REQUEST_SUCCESS_COUNT_METRIC_NAME, local.requestSuccessCount.get()); - addCommonProperties(requestSuccessCountSt, ikey, host); + addCommonProperties(requestSuccessCountSt, ikey, local.host); telemetryClient.trackStatsbeatAsync(requestSuccessCountSt.build()); } @@ -174,7 +170,7 @@ private void sendIntervalMetric( StatsbeatTelemetryBuilder requestFailureCountSt = createStatsbeatTelemetry( telemetryClient, REQUEST_FAILURE_COUNT_METRIC_NAME, local.requestFailureCount.get()); - addCommonProperties(requestFailureCountSt, ikey, host); + addCommonProperties(requestFailureCountSt, ikey, local.host); telemetryClient.trackStatsbeatAsync(requestFailureCountSt.build()); } @@ -182,7 +178,7 @@ private void sendIntervalMetric( if (durationAvg != 0) { StatsbeatTelemetryBuilder requestDurationSt = createStatsbeatTelemetry(telemetryClient, REQUEST_DURATION_METRIC_NAME, durationAvg); - addCommonProperties(requestDurationSt, ikey, host); + addCommonProperties(requestDurationSt, ikey, local.host); telemetryClient.trackStatsbeatAsync(requestDurationSt.build()); } @@ -190,7 +186,7 @@ private void sendIntervalMetric( StatsbeatTelemetryBuilder retryCountSt = createStatsbeatTelemetry( telemetryClient, RETRY_COUNT_METRIC_NAME, local.retryCount.get()); - addCommonProperties(retryCountSt, ikey, host); + addCommonProperties(retryCountSt, ikey, local.host); telemetryClient.trackStatsbeatAsync(retryCountSt.build()); } @@ -198,7 +194,7 @@ private void sendIntervalMetric( StatsbeatTelemetryBuilder throttleCountSt = createStatsbeatTelemetry( telemetryClient, THROTTLE_COUNT_METRIC_NAME, local.throttlingCount.get()); - addCommonProperties(throttleCountSt, ikey, host); + addCommonProperties(throttleCountSt, ikey, local.host); telemetryClient.trackStatsbeatAsync(throttleCountSt.build()); } @@ -206,7 +202,7 @@ private void sendIntervalMetric( StatsbeatTelemetryBuilder exceptionCountSt = createStatsbeatTelemetry( telemetryClient, EXCEPTION_COUNT_METRIC_NAME, local.exceptionCount.get()); - addCommonProperties(exceptionCountSt, ikey, host); + addCommonProperties(exceptionCountSt, ikey, local.host); telemetryClient.trackStatsbeatAsync(exceptionCountSt.build()); } } @@ -227,6 +223,8 @@ private static class IntervalMetrics { private final AtomicLong throttlingCount = new AtomicLong(); private final AtomicLong exceptionCount = new AtomicLong(); + private volatile String host; + private double getRequestDurationAvg() { double sum = totalRequestDuration.get(); if (requestSuccessCount.get() != 0) { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatHttpPipelinePolicy.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatHttpPipelinePolicy.java new file mode 100644 index 00000000000..912add74512 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatHttpPipelinePolicy.java @@ -0,0 +1,74 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.statsbeat; + +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.policy.HttpPipelinePolicy; +import java.util.concurrent.atomic.AtomicLong; +import reactor.core.publisher.Mono; + +public class NetworkStatsbeatHttpPipelinePolicy implements HttpPipelinePolicy { + + private static final String INSTRUMENTATION_KEY_DATA = "instrumentationKey"; + + private final NetworkStatsbeat networkStatsbeat; + + public NetworkStatsbeatHttpPipelinePolicy(NetworkStatsbeat networkStatsbeat) { + this.networkStatsbeat = networkStatsbeat; + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + // using AtomicLong for both mutable holder and volatile (but atomicity is not needed here) + AtomicLong startTime = new AtomicLong(); + String host = context.getHttpRequest().getUrl().getHost(); + String instrumentationKey = + context.getData(INSTRUMENTATION_KEY_DATA).orElse("unknown").toString(); + return next.process() + .doOnSubscribe(subscription -> startTime.set(System.currentTimeMillis())) + .doOnSuccess( + response -> { + int statusCode = response.getStatusCode(); + if (statusCode == 200) { + networkStatsbeat.incrementRequestSuccessCount( + System.currentTimeMillis() - startTime.get(), instrumentationKey, host); + } else if (statusCode == 301 + || statusCode == 302 + || statusCode == 307 + || statusCode == 308) { + // these are not tracked as success or failure since they are just redirects + } else if (statusCode == 439) { + networkStatsbeat.incrementThrottlingCount(instrumentationKey, host); + } else { + // note: 401 and 403 are currently tracked as failures + networkStatsbeat.incrementRequestFailureCount(instrumentationKey, host); + } + }) + .doOnError( + throwable -> { + // TODO (heya) should this be incrementExceptionCount()? + networkStatsbeat.incrementRequestFailureCount(instrumentationKey, host); + }); + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/OperatingSystem.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/OperatingSystem.java index ba2127c62d8..8db6c46b1b9 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/OperatingSystem.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/OperatingSystem.java @@ -24,6 +24,7 @@ enum OperatingSystem { OS_WINDOWS("Windows"), OS_LINUX("Linux"), + // TODO (heya) should we add Mac/OSX? OS_UNKNOWN("unknown"); private final String value; diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/StatsbeatModule.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/StatsbeatModule.java index c0289e40ab5..8be0f7eb0e3 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/StatsbeatModule.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/statsbeat/StatsbeatModule.java @@ -24,7 +24,6 @@ import com.microsoft.applicationinsights.agent.internal.common.ThreadPoolUtils; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient; -import io.opentelemetry.instrumentation.api.cache.Cache; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -50,9 +49,9 @@ public class StatsbeatModule { private final AtomicBoolean started = new AtomicBoolean(); - public StatsbeatModule(Cache ikeyEndpointMap) { + public StatsbeatModule() { customDimensions = new CustomDimensions(); - networkStatsbeat = new NetworkStatsbeat(customDimensions, ikeyEndpointMap); + networkStatsbeat = new NetworkStatsbeat(customDimensions); attachStatsbeat = new AttachStatsbeat(customDimensions); featureStatsbeat = new FeatureStatsbeat(customDimensions, FeatureType.FEATURE); instrumentationStatsbeat = new FeatureStatsbeat(customDimensions, FeatureType.INSTRUMENTATION); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchSpanProcessor.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchItemProcessor.java similarity index 80% rename from agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchSpanProcessor.java rename to agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchItemProcessor.java index 3a00622e002..b92e15255a2 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchSpanProcessor.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchItemProcessor.java @@ -37,27 +37,27 @@ import org.jctools.queues.MpscArrayQueue; // copied from io.opentelemetry.sdk.trace.export.BatchSpanProcessor -public final class BatchSpanProcessor { +public final class BatchItemProcessor { private static final String WORKER_THREAD_NAME = - BatchSpanProcessor.class.getSimpleName() + "_WorkerThread"; + BatchItemProcessor.class.getSimpleName() + "_WorkerThread"; private final Worker worker; private final AtomicBoolean isShutdown = new AtomicBoolean(false); /** - * Returns a new Builder for {@link BatchSpanProcessor}. + * Returns a new Builder for {@link BatchItemProcessor}. * - * @param spanExporter the {@code SpanExporter} to where the Spans are pushed. - * @return a new {@link BatchSpanProcessor}. - * @throws NullPointerException if the {@code spanExporter} is {@code null}. + * @param exporter the {@code TelemetryItemExporter} to where the telemetry items are pushed. + * @return a new {@link BatchItemProcessor}. + * @throws NullPointerException if the {@code exporter} is {@code null}. */ - public static BatchSpanProcessorBuilder builder(TelemetryChannel spanExporter) { - return new BatchSpanProcessorBuilder(spanExporter); + public static BatchItemProcessorBuilder builder(TelemetryItemExporter exporter) { + return new BatchItemProcessorBuilder(exporter); } - BatchSpanProcessor( - TelemetryChannel spanExporter, + BatchItemProcessor( + TelemetryItemExporter exporter, long scheduleDelayNanos, int maxQueueSize, int maxExportBatchSize, @@ -66,7 +66,7 @@ public static BatchSpanProcessorBuilder builder(TelemetryChannel spanExporter) { MpscArrayQueue queue = new MpscArrayQueue<>(maxQueueSize); this.worker = new Worker( - spanExporter, + exporter, scheduleDelayNanos, maxExportBatchSize, exporterTimeoutNanos, @@ -77,8 +77,8 @@ public static BatchSpanProcessorBuilder builder(TelemetryChannel spanExporter) { workerThread.start(); } - public void trackAsync(TelemetryItem span) { - worker.addSpan(span); + public void trackAsync(TelemetryItem item) { + worker.addItem(item); } public CompletableResultCode shutdown() { @@ -92,11 +92,11 @@ public CompletableResultCode forceFlush() { return worker.forceFlush(); } - // Worker is a thread that batches multiple spans and calls the registered SpanExporter to export - // the data. + // Worker is a thread that batches multiple items and calls the registered TelemetryItemExporter + // to export the data. private static final class Worker implements Runnable { - private final TelemetryChannel spanExporter; + private final TelemetryItemExporter exporter; private final long scheduleDelayNanos; private final int maxExportBatchSize; private final long exporterTimeoutNanos; @@ -106,30 +106,30 @@ private static final class Worker implements Runnable { private final Queue queue; private final int queueCapacity; private final String queueName; - // When waiting on the spans queue, exporter thread sets this atomic to the number of more - // spans it needs before doing an export. Writer threads would then wait for the queue to reach - // spansNeeded size before notifying the exporter thread about new entries. + // When waiting on the items queue, exporter thread sets this atomic to the number of more + // items it needs before doing an export. Writer threads would then wait for the queue to reach + // itemsNeeded size before notifying the exporter thread about new entries. // Integer.MAX_VALUE is used to imply that exporter thread is not expecting any signal. Since // exporter thread doesn't expect any signal initially, this value is initialized to // Integer.MAX_VALUE. - private final AtomicInteger spansNeeded = new AtomicInteger(Integer.MAX_VALUE); + private final AtomicInteger itemsNeeded = new AtomicInteger(Integer.MAX_VALUE); private final BlockingQueue signal; private final AtomicReference flushRequested = new AtomicReference<>(); private volatile boolean continueWork = true; private final ArrayList batch; - private static final OperationLogger queuingSpanLogger = - new OperationLogger(BatchSpanProcessor.class, "Queuing span"); + private static final OperationLogger queuingItemLogger = + new OperationLogger(BatchItemProcessor.class, "Queuing telemetry item"); private Worker( - TelemetryChannel spanExporter, + TelemetryItemExporter exporter, long scheduleDelayNanos, int maxExportBatchSize, long exporterTimeoutNanos, Queue queue, int queueCapacity, String queueName) { - this.spanExporter = spanExporter; + this.exporter = exporter; this.scheduleDelayNanos = scheduleDelayNanos; this.maxExportBatchSize = maxExportBatchSize; this.exporterTimeoutNanos = exporterTimeoutNanos; @@ -140,9 +140,9 @@ private Worker( this.batch = new ArrayList<>(this.maxExportBatchSize); } - private void addSpan(TelemetryItem span) { - if (!queue.offer(span)) { - queuingSpanLogger.recordFailure( + private void addItem(TelemetryItem item) { + if (!queue.offer(item)) { + queuingItemLogger.recordFailure( "Max " + queueName + " export queue capacity of " @@ -156,8 +156,8 @@ private void addSpan(TelemetryItem span) { + (queueCapacity * 2) + " } }"); } else { - queuingSpanLogger.recordSuccess(); - if (queue.size() >= spansNeeded.get()) { + queuingItemLogger.recordSuccess(); + if (queue.size() >= itemsNeeded.get()) { signal.offer(true); } } @@ -182,9 +182,9 @@ public void run() { try { long pollWaitTime = nextExportTime - System.nanoTime(); if (pollWaitTime > 0) { - spansNeeded.set(maxExportBatchSize - batch.size()); + itemsNeeded.set(maxExportBatchSize - batch.size()); signal.poll(pollWaitTime, TimeUnit.NANOSECONDS); - spansNeeded.set(Integer.MAX_VALUE); + itemsNeeded.set(Integer.MAX_VALUE); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -195,12 +195,12 @@ public void run() { } private void flush() { - int spansToFlush = queue.size(); - while (spansToFlush > 0) { - TelemetryItem span = queue.poll(); - assert span != null; - batch.add(span); - spansToFlush--; + int itemsToFlush = queue.size(); + while (itemsToFlush > 0) { + TelemetryItem item = queue.poll(); + assert item != null; + batch.add(item); + itemsToFlush--; if (batch.size() >= maxExportBatchSize) { exportCurrentBatch(); } @@ -251,7 +251,7 @@ private void exportCurrentBatch() { try { // batching, retry, logging, and writing to disk on failure occur downstream - CompletableResultCode result = spanExporter.send(Collections.unmodifiableList(batch)); + CompletableResultCode result = exporter.send(Collections.unmodifiableList(batch)); result.join(exporterTimeoutNanos, TimeUnit.NANOSECONDS); } finally { batch.clear(); diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchSpanProcessorBuilder.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchItemProcessorBuilder.java similarity index 74% rename from agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchSpanProcessorBuilder.java rename to agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchItemProcessorBuilder.java index da8ca1ed3a1..601e238bed5 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchSpanProcessorBuilder.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/BatchItemProcessorBuilder.java @@ -28,21 +28,21 @@ import java.util.concurrent.TimeUnit; // copied from io.opentelemetry.sdk.trace.export.BatchSpanProcessorBuilder -final class BatchSpanProcessorBuilder { +final class BatchItemProcessorBuilder { private static final long DEFAULT_SCHEDULE_DELAY_MILLIS = 5000; private static final int DEFAULT_MAX_QUEUE_SIZE = 2048; private static final int DEFAULT_MAX_EXPORT_BATCH_SIZE = 512; private static final int DEFAULT_EXPORT_TIMEOUT_MILLIS = 30_000; - private final TelemetryChannel spanExporter; + private final TelemetryItemExporter exporter; private long scheduleDelayNanos = TimeUnit.MILLISECONDS.toNanos(DEFAULT_SCHEDULE_DELAY_MILLIS); private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE; private int maxExportBatchSize = DEFAULT_MAX_EXPORT_BATCH_SIZE; private long exporterTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(DEFAULT_EXPORT_TIMEOUT_MILLIS); - BatchSpanProcessorBuilder(TelemetryChannel spanExporter) { - this.spanExporter = requireNonNull(spanExporter, "spanExporter"); + BatchItemProcessorBuilder(TelemetryItemExporter exporter) { + this.exporter = requireNonNull(exporter, "exporter"); } // TODO: Consider to add support for constant Attributes and/or Resource. @@ -51,7 +51,7 @@ final class BatchSpanProcessorBuilder { * Sets the delay interval between two consecutive exports. If unset, defaults to {@value * DEFAULT_SCHEDULE_DELAY_MILLIS}ms. */ - public BatchSpanProcessorBuilder setScheduleDelay(long delay, TimeUnit unit) { + public BatchItemProcessorBuilder setScheduleDelay(long delay, TimeUnit unit) { requireNonNull(unit, "unit"); checkArgument(delay >= 0, "delay must be non-negative"); scheduleDelayNanos = unit.toNanos(delay); @@ -62,7 +62,7 @@ public BatchSpanProcessorBuilder setScheduleDelay(long delay, TimeUnit unit) { * Sets the delay interval between two consecutive exports. If unset, defaults to {@value * DEFAULT_SCHEDULE_DELAY_MILLIS}ms. */ - public BatchSpanProcessorBuilder setScheduleDelay(Duration delay) { + public BatchItemProcessorBuilder setScheduleDelay(Duration delay) { requireNonNull(delay, "delay"); return setScheduleDelay(delay.toNanos(), TimeUnit.NANOSECONDS); } @@ -71,7 +71,7 @@ public BatchSpanProcessorBuilder setScheduleDelay(Duration delay) { * Sets the maximum time an export will be allowed to run before being cancelled. If unset, * defaults to {@value DEFAULT_EXPORT_TIMEOUT_MILLIS}ms. */ - public BatchSpanProcessorBuilder setExporterTimeout(long timeout, TimeUnit unit) { + public BatchItemProcessorBuilder setExporterTimeout(long timeout, TimeUnit unit) { requireNonNull(unit, "unit"); checkArgument(timeout >= 0, "timeout must be non-negative"); exporterTimeoutNanos = unit.toNanos(timeout); @@ -82,56 +82,56 @@ public BatchSpanProcessorBuilder setExporterTimeout(long timeout, TimeUnit unit) * Sets the maximum time an export will be allowed to run before being cancelled. If unset, * defaults to {@value DEFAULT_EXPORT_TIMEOUT_MILLIS}ms. */ - public BatchSpanProcessorBuilder setExporterTimeout(Duration timeout) { + public BatchItemProcessorBuilder setExporterTimeout(Duration timeout) { requireNonNull(timeout, "timeout"); return setExporterTimeout(timeout.toNanos(), TimeUnit.NANOSECONDS); } /** - * Sets the maximum number of Spans that are kept in the queue before start dropping. More memory + * Sets the maximum number of items that are kept in the queue before start dropping. More memory * than this value may be allocated to optimize queue access. * - *

See the BatchSampledSpansProcessor class description for a high-level design description of - * this class. + *

See the BatchItemProcessor class description for a high-level design description of this + * class. * *

Default value is {@code 2048}. * - * @param maxQueueSize the maximum number of Spans that are kept in the queue before start + * @param maxQueueSize the maximum number of items that are kept in the queue before start * dropping. * @return this. - * @see BatchSpanProcessorBuilder#DEFAULT_MAX_QUEUE_SIZE + * @see BatchItemProcessorBuilder#DEFAULT_MAX_QUEUE_SIZE */ - public BatchSpanProcessorBuilder setMaxQueueSize(int maxQueueSize) { + public BatchItemProcessorBuilder setMaxQueueSize(int maxQueueSize) { this.maxQueueSize = maxQueueSize; return this; } /** * Sets the maximum batch size for every export. This must be smaller or equal to {@code - * maxQueuedSpans}. + * maxQueuedItems}. * *

Default value is {@code 512}. * * @param maxExportBatchSize the maximum batch size for every export. * @return this. - * @see BatchSpanProcessorBuilder#DEFAULT_MAX_EXPORT_BATCH_SIZE + * @see BatchItemProcessorBuilder#DEFAULT_MAX_EXPORT_BATCH_SIZE */ - public BatchSpanProcessorBuilder setMaxExportBatchSize(int maxExportBatchSize) { + public BatchItemProcessorBuilder setMaxExportBatchSize(int maxExportBatchSize) { checkArgument(maxExportBatchSize > 0, "maxExportBatchSize must be positive."); this.maxExportBatchSize = maxExportBatchSize; return this; } /** - * Returns a new {@link io.opentelemetry.sdk.trace.export.BatchSpanProcessor} that batches, then - * converts spans to proto and forwards them to the given {@code spanExporter}. + * Returns a new {@link BatchItemProcessor} that batches, then converts items to proto and + * forwards them to the given {@code exporter}. * - * @return a new {@link io.opentelemetry.sdk.trace.export.BatchSpanProcessor}. - * @throws NullPointerException if the {@code spanExporter} is {@code null}. + * @return a new {@link BatchItemProcessor}. + * @throws NullPointerException if the {@code exporter} is {@code null}. */ - public BatchSpanProcessor build(String queueName) { - return new BatchSpanProcessor( - spanExporter, + public BatchItemProcessor build(String queueName) { + return new BatchItemProcessor( + exporter, scheduleDelayNanos, maxQueueSize, maxExportBatchSize, diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/ConnectionString.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/ConnectionString.java index 34faa91e31c..dffa229b92e 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/ConnectionString.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/ConnectionString.java @@ -26,7 +26,6 @@ import java.net.URL; import java.util.Map; import java.util.TreeMap; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -104,23 +103,26 @@ private static void mapToConnectionConfiguration( // resolve suffix String suffix = kvps.get(Keywords.ENDPOINT_SUFFIX); if (!Strings.isNullOrEmpty(suffix)) { + if (suffix.startsWith(".")) { + suffix = suffix.substring(1); + } try { telemetryClient .getEndpointProvider() .setIngestionEndpoint( - constructSecureEndpoint(EndpointPrefixes.INGESTION_ENDPOINT_PREFIX, suffix)); + new URL("https://" + EndpointPrefixes.INGESTION_ENDPOINT_PREFIX + "." + suffix)); telemetryClient .getEndpointProvider() .setLiveEndpoint( - constructSecureEndpoint(EndpointPrefixes.LIVE_ENDPOINT_PREFIX, suffix)); + new URL("https://" + EndpointPrefixes.LIVE_ENDPOINT_PREFIX + "." + suffix)); telemetryClient .getEndpointProvider() .setProfilerEndpoint( - constructSecureEndpoint(EndpointPrefixes.PROFILER_ENDPOINT_PREFIX, suffix)); + new URL("https://" + EndpointPrefixes.PROFILER_ENDPOINT_PREFIX + "." + suffix)); telemetryClient .getEndpointProvider() .setSnapshotEndpoint( - constructSecureEndpoint(EndpointPrefixes.SNAPSHOT_ENDPOINT_PREFIX, suffix)); + new URL("https://" + EndpointPrefixes.SNAPSHOT_ENDPOINT_PREFIX + "." + suffix)); } catch (MalformedURLException e) { throw new InvalidConnectionStringException( Keywords.ENDPOINT_SUFFIX + " is invalid: " + suffix, e); @@ -172,12 +174,6 @@ private static URL toUrlOrThrow(String url, String field) } } - // visible for testing - static URL constructSecureEndpoint(String prefix, String suffix) throws MalformedURLException { - return new URL( - "https://" + StringUtils.strip(prefix, ".") + "." + StringUtils.strip(suffix, ".")); - } - /** All tokens are lowercase. Parsing should be case insensitive. */ // visible for testing static class Keywords { diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/DiagnosticTelemetryPipelineListener.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/DiagnosticTelemetryPipelineListener.java new file mode 100644 index 00000000000..f0fe46af110 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/DiagnosticTelemetryPipelineListener.java @@ -0,0 +1,140 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.telemetry; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.applicationinsights.agent.internal.common.NetworkFriendlyExceptions; +import com.microsoft.applicationinsights.agent.internal.common.OperationLogger; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DiagnosticTelemetryPipelineListener implements TelemetryPipelineListener { + + private static final Class FOR_CLASS = TelemetryPipeline.class; + private static final Logger logger = LoggerFactory.getLogger(FOR_CLASS); + + private final OperationLogger operationLogger; + + private final AtomicBoolean friendlyExceptionThrown = new AtomicBoolean(); + + // e.g. "Sending telemetry to the ingestion service" + public DiagnosticTelemetryPipelineListener(String operation) { + operationLogger = new OperationLogger(FOR_CLASS, operation); + } + + @Override + public void onResponse(TelemetryPipelineRequest request, TelemetryPipelineResponse response) { + switch (response.getStatusCode()) { + case 200: // SUCCESS + operationLogger.recordSuccess(); + break; + case 206: // PARTIAL CONTENT, Breeze-specific: PARTIAL SUCCESS + operationLogger.recordFailure( + getErrorMessageFromPartialSuccessResponse(response.getBody())); + break; + case 301: + case 302: + case 307: + case 308: + operationLogger.recordFailure("Too many redirects"); + break; + case 401: // breeze returns if aad enabled and no authentication token provided + case 403: // breeze returns if aad enabled or disabled (both cases) and + // wrong/expired credentials provided + operationLogger.recordFailure( + getErrorMessageFromCredentialRelatedResponse( + response.getStatusCode(), response.getBody())); + break; + case 408: // REQUEST TIMEOUT + case 429: // TOO MANY REQUESTS + case 500: // INTERNAL SERVER ERROR + case 503: // SERVICE UNAVAILABLE + operationLogger.recordFailure( + "received response code " + + response.getStatusCode() + + " (telemetry will be stored to disk and retried later)"); + break; + case 439: // Breeze-specific: THROTTLED OVER EXTENDED TIME + // TODO handle throttling + operationLogger.recordFailure("received response code 439 (throttled over extended time)"); + break; + default: + operationLogger.recordFailure("received response code: " + response.getStatusCode()); + } + } + + @Override + public void onException(TelemetryPipelineRequest request, String reason, Throwable throwable) { + + if (!NetworkFriendlyExceptions.logSpecialOneTimeFriendlyException( + throwable, request.getUrl().toString(), friendlyExceptionThrown, logger)) { + operationLogger.recordFailure(reason, throwable); + } + + operationLogger.recordFailure(reason, throwable); + } + + private static String getErrorMessageFromPartialSuccessResponse(String body) { + JsonNode jsonNode; + try { + jsonNode = new ObjectMapper().readTree(body); + } catch (JsonProcessingException e) { + return "ingestion service returned 206, but could not parse response as json: " + body; + } + List errors = new ArrayList<>(); + jsonNode.get("errors").forEach(errors::add); + StringBuilder message = new StringBuilder(); + message.append(errors.get(0).get("message").asText()); + int moreErrors = errors.size() - 1; + if (moreErrors > 0) { + message.append(" (and ").append(moreErrors).append(" more)"); + } + return message.toString(); + } + + private static String getErrorMessageFromCredentialRelatedResponse( + int responseCode, String responseBody) { + JsonNode jsonNode; + try { + jsonNode = new ObjectMapper().readTree(responseBody); + } catch (JsonProcessingException e) { + return "ingestion service returned " + + responseCode + + ", but could not parse response as json: " + + responseBody; + } + String action = + responseCode == 401 + ? ". Please provide Azure Active Directory credentials" + : ". Please check your Azure Active Directory credentials, they might be incorrect or expired"; + List errors = new ArrayList<>(); + jsonNode.get("errors").forEach(errors::add); + return errors.get(0).get("message").asText() + + action + + " (telemetry will be stored to disk and retried later)"; + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannel.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannel.java deleted file mode 100644 index 15fd0e7c37d..00000000000 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannel.java +++ /dev/null @@ -1,426 +0,0 @@ -/* - * ApplicationInsights-Java - * Copyright (c) Microsoft Corporation - * All rights reserved. - * - * MIT License - * Permission is hereby granted, free of charge, to any person obtaining a copy of this - * software and associated documentation files (the ""Software""), to deal in the Software - * without restriction, including without limitation the rights to use, copy, modify, merge, - * publish, distribute, sublicense, and/or sell copies of the Software, and to permit - * persons to whom the Software is furnished to do so, subject to the following conditions: - * The above copyright notice and this permission notice shall be included in all copies or - * substantial portions of the Software. - * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, - * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR - * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE - * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -package com.microsoft.applicationinsights.agent.internal.telemetry; - -import static java.util.Collections.singletonList; - -import com.azure.core.http.HttpMethod; -import com.azure.core.http.HttpPipeline; -import com.azure.core.http.HttpRequest; -import com.azure.core.http.HttpResponse; -import com.azure.core.util.Context; -import com.azure.core.util.tracing.Tracer; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.io.SerializedString; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.microsoft.applicationinsights.agent.internal.common.NetworkFriendlyExceptions; -import com.microsoft.applicationinsights.agent.internal.common.OperationLogger; -import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; -import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; -import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient; -import com.microsoft.applicationinsights.agent.internal.httpclient.RedirectPolicy; -import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileWriter; -import com.microsoft.applicationinsights.agent.internal.statsbeat.StatsbeatModule; -import io.opentelemetry.instrumentation.api.cache.Cache; -import io.opentelemetry.sdk.common.CompletableResultCode; -import java.io.IOException; -import java.io.StringWriter; -import java.net.URL; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; -import java.util.zip.GZIPOutputStream; -import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -// TODO performance testing -public class TelemetryChannel { - - private static final Logger logger = LoggerFactory.getLogger(TelemetryChannel.class); - - private static final ObjectMapper mapper = createObjectMapper(); - - private static final AppInsightsByteBufferPool byteBufferPool = new AppInsightsByteBufferPool(); - - // TODO (heya) should we suppress logging statsbeat telemetry ingestion issues? - private static final OperationLogger operationLogger = - new OperationLogger(TelemetryChannel.class, "Sending telemetry to the ingestion service"); - - private static final OperationLogger retryOperationLogger = - new OperationLogger( - TelemetryChannel.class, "Sending telemetry to the ingestion service (retry)"); - - // TODO (kryalama) do we still need this AtomicBoolean, or can we use throttling built in to the - // operationLogger? - private final AtomicBoolean friendlyExceptionThrown = new AtomicBoolean(); - - @SuppressWarnings("CatchAndPrintStackTrace") - private static ObjectMapper createObjectMapper() { - ObjectMapper mapper = new ObjectMapper(); - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - // it's important to pass in the "agent class loader" since TelemetryChannel is initialized - // lazily and can be initialized via an application thread, in which case the thread context - // class loader is used to look up jsr305 module and its not found - mapper.registerModules(ObjectMapper.findModules(TelemetryChannel.class.getClassLoader())); - mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); - return mapper; - } - - private final HttpPipeline pipeline; - private final URL endpointUrl; - private final LocalFileWriter localFileWriter; - private final StatsbeatModule statsbeatModule; - private final boolean isStatsbeat; - - public static TelemetryChannel create( - URL endpointUrl, - LocalFileWriter localFileWriter, - Cache ikeyEndpointMap, - StatsbeatModule statsbeatModule, - boolean isStatsbeat, - @Nullable Configuration.AadAuthentication aadAuthentication) { - HttpPipeline httpPipeline = LazyHttpClient.newHttpPipeLine(aadAuthentication, ikeyEndpointMap); - return new TelemetryChannel( - httpPipeline, endpointUrl, localFileWriter, statsbeatModule, isStatsbeat); - } - - public CompletableResultCode sendRawBytes( - ByteBuffer buffer, - String instrumentationKey, - Runnable onSuccess, - Consumer onFailure) { - return internalSend( - singletonList(buffer), instrumentationKey, onSuccess, onFailure, retryOperationLogger); - } - - // used by tests only - public TelemetryChannel( - HttpPipeline pipeline, - URL endpointUrl, - LocalFileWriter localFileWriter, - StatsbeatModule statsbeatModule, - boolean isStatsbeat) { - this.pipeline = pipeline; - this.endpointUrl = endpointUrl; - this.localFileWriter = localFileWriter; - this.statsbeatModule = statsbeatModule; - this.isStatsbeat = isStatsbeat; - } - - public CompletableResultCode send(List telemetryItems) { - Map> instrumentationKeyMap = new HashMap<>(); - List resultCodeList = new ArrayList<>(); - for (TelemetryItem telemetryItem : telemetryItems) { - String instrumentationKey = telemetryItem.getInstrumentationKey(); - if (!instrumentationKeyMap.containsKey(instrumentationKey)) { - instrumentationKeyMap.put(instrumentationKey, new ArrayList<>()); - } - instrumentationKeyMap.get(instrumentationKey).add(telemetryItem); - } - for (String instrumentationKey : instrumentationKeyMap.keySet()) { - resultCodeList.add( - internalSendByInstrumentationKey( - instrumentationKeyMap.get(instrumentationKey), instrumentationKey)); - } - return CompletableResultCode.ofAll(resultCodeList); - } - - public CompletableResultCode internalSendByInstrumentationKey( - List telemetryItems, String instrumentationKey) { - List byteBuffers; - try { - byteBuffers = encode(telemetryItems); - } catch (Throwable t) { - operationLogger.recordFailure("Error encoding telemetry items: " + t.getMessage(), t); - return CompletableResultCode.ofFailure(); - } - try { - return internalSend( - byteBuffers, - instrumentationKey, - () -> byteBufferPool.offer(byteBuffers), - retryable -> { - localFileWriter.writeToDisk(byteBuffers, instrumentationKey); - byteBufferPool.offer(byteBuffers); - }, - operationLogger); - } catch (Throwable t) { - operationLogger.recordFailure("Error sending telemetry items: " + t.getMessage(), t); - return CompletableResultCode.ofFailure(); - } - } - - List encode(List telemetryItems) throws IOException { - - if (logger.isDebugEnabled()) { - StringWriter debug = new StringWriter(); - try (JsonGenerator jg = mapper.createGenerator(debug)) { - writeTelemetryItems(jg, telemetryItems); - } - logger.debug("sending telemetry to ingestion service:\n{}", debug); - } - - ByteBufferOutputStream out = new ByteBufferOutputStream(byteBufferPool); - - try (JsonGenerator jg = mapper.createGenerator(new GZIPOutputStream(out))) { - writeTelemetryItems(jg, telemetryItems); - } catch (IOException e) { - byteBufferPool.offer(out.getByteBuffers()); - throw e; - } - - out.close(); // closing ByteBufferOutputStream is a no-op, but this line makes LGTM happy - - List byteBuffers = out.getByteBuffers(); - for (ByteBuffer byteBuffer : byteBuffers) { - byteBuffer.flip(); - } - return byteBuffers; - } - - private static void writeTelemetryItems(JsonGenerator jg, List telemetryItems) - throws IOException { - jg.setRootValueSeparator(new SerializedString("\n")); - for (TelemetryItem telemetryItem : telemetryItems) { - mapper.writeValue(jg, telemetryItem); - } - } - - /** - * Object can be a list of {@link ByteBuffer} or a raw byte array. Regular telemetries will be - * sent as {@code List}. Persisted telemetries will be sent as byte[] - */ - private CompletableResultCode internalSend( - List byteBuffers, - String instrumentationKey, - Runnable onSuccess, - Consumer onFailure, - OperationLogger operationLogger) { - HttpRequest request = new HttpRequest(HttpMethod.POST, endpointUrl); - - request.setBody(Flux.fromIterable(byteBuffers)); - int contentLength = byteBuffers.stream().mapToInt(ByteBuffer::limit).sum(); - - request.setHeader("Content-Length", Integer.toString(contentLength)); - - // need to suppress the default User-Agent "ReactorNetty/dev", otherwise Breeze ingestion - // service will put that - // User-Agent header into the client_Browser field for all telemetry that doesn't explicitly set - // it's own - // UserAgent (ideally Breeze would only have this behavior for ingestion directly from browsers) - // TODO(trask) - // not setting User-Agent header at all would be a better option, but haven't figured out how - // to do that yet - request.setHeader("User-Agent", ""); - request.setHeader("Content-Encoding", "gzip"); - - // TODO(trask) subscribe with listener - // * retry on first failure (may not need to worry about this if retry policy in pipeline - // already, see above) - // * write to disk on second failure - CompletableResultCode result = new CompletableResultCode(); - final long startTime = System.currentTimeMillis(); - // Add instrumentation key to context to use in redirectPolicy - Map contextKeyValues = new HashMap<>(); - contextKeyValues.put(RedirectPolicy.INSTRUMENTATION_KEY, instrumentationKey); - contextKeyValues.put(Tracer.DISABLE_TRACING_KEY, true); - - pipeline - .send(request, Context.of(contextKeyValues)) - .subscribe( - responseHandler( - instrumentationKey, - startTime, - () -> { - onSuccess.run(); - result.succeed(); - }, - retryable -> { - onFailure.accept(retryable); - result.fail(); - }, - operationLogger), - errorHandler( - instrumentationKey, - retryable -> { - onFailure.accept(retryable); - result.fail(); - }, - operationLogger)); - return result; - } - - private Consumer responseHandler( - String instrumentationKey, - long startTime, - Runnable onSuccess, - Consumer onFailure, - OperationLogger operationLogger) { - - return response -> - response - .getBodyAsString() - .switchIfEmpty(Mono.just("")) - .subscribe( - body -> { - int statusCode = response.getStatusCode(); - switch (statusCode) { - case 200: // SUCCESS - operationLogger.recordSuccess(); - onSuccess.run(); - break; - case 206: // PARTIAL CONTENT, Breeze-specific: PARTIAL SUCCESS - operationLogger.recordFailure( - getErrorMessageFromPartialSuccessResponse(body)); - onFailure.accept(false); - break; - case 401: // breeze returns if aad enabled and no authentication token provided - case 403: // breeze returns if aad enabled or disabled (both cases) and - // wrong/expired credentials provided - operationLogger.recordFailure( - getErrorMessageFromCredentialRelatedResponse(statusCode, body)); - onFailure.accept(true); - break; - case 408: // REQUEST TIMEOUT - case 429: // TOO MANY REQUESTS - case 500: // INTERNAL SERVER ERROR - case 503: // SERVICE UNAVAILABLE - operationLogger.recordFailure( - "received response code " - + statusCode - + " (telemetry will be stored to disk and retried later)"); - onFailure.accept(true); - break; - case 439: // Breeze-specific: THROTTLED OVER EXTENDED TIME - // TODO handle throttling - operationLogger.recordFailure( - "received response code 439 (throttled over extended time)"); - onFailure.accept(false); - break; - default: - operationLogger.recordFailure("received response code: " + statusCode); - onFailure.accept(false); - } - if (!isStatsbeat) { - handleStatsbeatOnResponse(instrumentationKey, startTime, statusCode); - } - }, - exception -> { - operationLogger.recordFailure("exception retrieving response body", exception); - onFailure.accept(false); - }); - } - - private void handleStatsbeatOnResponse( - String instrumentationKey, long startTime, int statusCode) { - if (statusCode == 200) { - statsbeatModule - .getNetworkStatsbeat() - .incrementRequestSuccessCount(System.currentTimeMillis() - startTime, instrumentationKey); - } else { - statsbeatModule.getNetworkStatsbeat().incrementRequestFailureCount(instrumentationKey); - } - if (statusCode == 439) { - statsbeatModule.getNetworkStatsbeat().incrementThrottlingCount(instrumentationKey); - } - } - - private Consumer errorHandler( - String instrumentationKey, Consumer onFailure, OperationLogger operationLogger) { - - return error -> { - if (isStatsbeat && error instanceof UnknownHostException) { - // when sending a Statsbeat request and server returns an UnknownHostException, it's - // likely that it's using AMPLS. In that case, we use the kill-switch to turn off Statsbeat. - statsbeatModule.shutdown(); - onFailure.accept(false); - return; - } - - // TODO (trask) only log one-time friendly exception if no prior successes - if (!NetworkFriendlyExceptions.logSpecialOneTimeFriendlyException( - error, endpointUrl.toString(), friendlyExceptionThrown, logger)) { - operationLogger.recordFailure( - "Error sending telemetry items: " + error.getMessage(), error); - } - - if (!isStatsbeat) { - statsbeatModule.getNetworkStatsbeat().incrementRequestFailureCount(instrumentationKey); - } - - onFailure.accept(true); - }; - } - - private static String getErrorMessageFromPartialSuccessResponse(String body) { - JsonNode jsonNode; - try { - jsonNode = new ObjectMapper().readTree(body); - } catch (JsonProcessingException e) { - return "ingestion service returned 206, but could not parse response as json: " + body; - } - List errors = new ArrayList<>(); - jsonNode.get("errors").forEach(errors::add); - StringBuilder message = new StringBuilder(); - message.append(errors.get(0).get("message").asText()); - int moreErrors = errors.size() - 1; - if (moreErrors > 0) { - message.append(" (and ").append(moreErrors).append(" more)"); - } - return message.toString(); - } - - private static String getErrorMessageFromCredentialRelatedResponse(int statusCode, String body) { - JsonNode jsonNode; - try { - jsonNode = new ObjectMapper().readTree(body); - } catch (JsonProcessingException e) { - return "ingestion service returned " - + statusCode - + ", but could not parse response as json: " - + body; - } - String action = - statusCode == 401 - ? ". Please provide Azure Active Directory credentials" - : ". Please check your Azure Active Directory credentials, they might be incorrect or expired"; - List errors = new ArrayList<>(); - jsonNode.get("errors").forEach(errors::add); - StringBuilder message = new StringBuilder(); - message.append(errors.get(0).get("message").asText()); - message.append(action); - message.append(" (telemetry will be stored to disk and retried later)"); - return message.toString(); - } -} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java index 0c2c0e9857b..e20b64af13c 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryClient.java @@ -23,6 +23,7 @@ import static java.util.Arrays.asList; +import com.azure.core.http.HttpPipeline; import com.microsoft.applicationinsights.agent.internal.common.PropertyHelper; import com.microsoft.applicationinsights.agent.internal.common.Strings; import com.microsoft.applicationinsights.agent.internal.configuration.Configuration; @@ -39,16 +40,13 @@ import com.microsoft.applicationinsights.agent.internal.exporter.models.MetricsData; import com.microsoft.applicationinsights.agent.internal.exporter.models.MonitorDomain; import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; -import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileCache; -import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileLoader; -import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileSender; -import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileWriter; +import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient; +import com.microsoft.applicationinsights.agent.internal.localstorage.LocalStorageSystem; import com.microsoft.applicationinsights.agent.internal.localstorage.LocalStorageUtils; import com.microsoft.applicationinsights.agent.internal.quickpulse.QuickPulseDataCollector; +import com.microsoft.applicationinsights.agent.internal.statsbeat.NetworkStatsbeatHttpPipelinePolicy; import com.microsoft.applicationinsights.agent.internal.statsbeat.StatsbeatModule; -import io.opentelemetry.instrumentation.api.cache.Cache; import io.opentelemetry.sdk.common.CompletableResultCode; -import java.io.File; import java.net.MalformedURLException; import java.util.ArrayList; import java.util.HashMap; @@ -86,7 +84,6 @@ public class TelemetryClient { private final List metricFilters; - private final Cache ikeyEndpointMap; private final StatsbeatModule statsbeatModule; private final boolean readOnlyFileSystem; private final int generalExportQueueCapacity; @@ -94,10 +91,10 @@ public class TelemetryClient { @Nullable private final Configuration.AadAuthentication aadAuthentication; - private final Object channelInitLock = new Object(); - private volatile @MonotonicNonNull BatchSpanProcessor generalChannelBatcher; - private volatile @MonotonicNonNull BatchSpanProcessor metricsChannelBatcher; - private volatile @MonotonicNonNull BatchSpanProcessor statsbeatChannelBatcher; + private final Object batchItemProcessorInitLock = new Object(); + private volatile @MonotonicNonNull BatchItemProcessor generalBatchItemProcessor; + private volatile @MonotonicNonNull BatchItemProcessor metricsBatchItemProcessor; + private volatile @MonotonicNonNull BatchItemProcessor statsbeatBatchItemProcessor; public static TelemetryClient.Builder builder() { return new TelemetryClient.Builder(); @@ -108,8 +105,7 @@ public static TelemetryClient createForTest() { return builder() .setCustomDimensions(new HashMap<>()) .setMetricFilters(new ArrayList<>()) - .setIkeyEndpointMap(Cache.bounded(100)) - .setStatsbeatModule(new StatsbeatModule(null)) + .setStatsbeatModule(new StatsbeatModule()) .build(); } @@ -117,7 +113,6 @@ public TelemetryClient(Builder builder) { this.globalTags = builder.globalTags; this.globalProperties = builder.globalProperties; this.metricFilters = builder.metricFilters; - this.ikeyEndpointMap = builder.ikeyEndpointMap; this.statsbeatModule = builder.statsbeatModule; this.readOnlyFileSystem = builder.readOnlyFileSystem; this.generalExportQueueCapacity = builder.generalExportQueueCapacity; @@ -186,9 +181,9 @@ public void trackAsync(TelemetryItem telemetryItem) { // for simplicity not reporting back success/failure from this layer // only that it was successfully delivered to the next layer if (data instanceof MetricsData) { - getMetricsChannelBatcher().trackAsync(telemetryItem); + getMetricsBatchItemProcessor().trackAsync(telemetryItem); } else { - getGeneralChannelBatcher().trackAsync(telemetryItem); + getGeneralBatchItemProcessor().trackAsync(telemetryItem); } } @@ -196,107 +191,109 @@ public void trackStatsbeatAsync(TelemetryItem telemetry) { // batching, retry, throttling, and writing to disk on failure occur downstream // for simplicity not reporting back success/failure from this layer // only that it was successfully delivered to the next layer - getStatsbeatChannelBatcher().trackAsync(telemetry); + getStatsbeatBatchItemProcessor().trackAsync(telemetry); } - public CompletableResultCode flushChannelBatcher() { - if (generalChannelBatcher != null) { - return generalChannelBatcher.forceFlush(); - } else { - return CompletableResultCode.ofSuccess(); + public CompletableResultCode forceFlush() { + List resultCodes = new ArrayList<>(); + if (generalBatchItemProcessor != null) { + resultCodes.add(generalBatchItemProcessor.forceFlush()); + } + if (metricsBatchItemProcessor != null) { + resultCodes.add(metricsBatchItemProcessor.forceFlush()); } + if (statsbeatBatchItemProcessor != null) { + resultCodes.add(statsbeatBatchItemProcessor.forceFlush()); + } + return CompletableResultCode.ofAll(resultCodes); } - private BatchSpanProcessor getGeneralChannelBatcher() { - if (generalChannelBatcher == null) { - synchronized (channelInitLock) { - if (generalChannelBatcher == null) { - generalChannelBatcher = initChannelBatcher(generalExportQueueCapacity, 512, "general"); + private BatchItemProcessor getGeneralBatchItemProcessor() { + if (generalBatchItemProcessor == null) { + synchronized (batchItemProcessorInitLock) { + if (generalBatchItemProcessor == null) { + generalBatchItemProcessor = + initBatchItemProcessor(generalExportQueueCapacity, 512, "general"); } } } - return generalChannelBatcher; + return generalBatchItemProcessor; } // metrics get flooded every 60 seconds by default, so need much larger queue size to avoid // dropping telemetry (they are much smaller so a larger queue size and larger batch size are ok) - private BatchSpanProcessor getMetricsChannelBatcher() { - if (metricsChannelBatcher == null) { - synchronized (channelInitLock) { - if (metricsChannelBatcher == null) { - metricsChannelBatcher = initChannelBatcher(metricsExportQueueCapacity, 2048, "metrics"); + private BatchItemProcessor getMetricsBatchItemProcessor() { + if (metricsBatchItemProcessor == null) { + synchronized (batchItemProcessorInitLock) { + if (metricsBatchItemProcessor == null) { + metricsBatchItemProcessor = + initBatchItemProcessor(metricsExportQueueCapacity, 2048, "metrics"); } } } - return metricsChannelBatcher; + return metricsBatchItemProcessor; } - private BatchSpanProcessor initChannelBatcher( + private BatchItemProcessor initBatchItemProcessor( int exportQueueCapacity, int maxExportBatchSize, String queueName) { - LocalFileLoader localFileLoader = null; - LocalFileWriter localFileWriter = null; + LocalStorageSystem localStorageSystem = null; + TelemetryPipelineListener telemetryPipelineListener = TelemetryPipelineListener.noop(); if (!readOnlyFileSystem) { - File telemetryFolder = LocalStorageUtils.getOfflineTelemetryFolder(); - LocalFileCache localFileCache = new LocalFileCache(telemetryFolder); - localFileLoader = - new LocalFileLoader( - localFileCache, telemetryFolder, statsbeatModule.getNonessentialStatsbeat()); - localFileWriter = - new LocalFileWriter( - localFileCache, telemetryFolder, statsbeatModule.getNonessentialStatsbeat()); + localStorageSystem = + new LocalStorageSystem( + LocalStorageUtils.getOfflineTelemetryFolder(), + statsbeatModule.getNonessentialStatsbeat()); + telemetryPipelineListener = localStorageSystem.createTelemetryPipelineListener(); } - TelemetryChannel channel = - TelemetryChannel.create( - endpointProvider.getIngestionEndpointUrl(), - localFileWriter, - ikeyEndpointMap, - statsbeatModule, - false, - aadAuthentication); + HttpPipeline httpPipeline = + LazyHttpClient.newHttpPipeLine( + aadAuthentication, + new NetworkStatsbeatHttpPipelinePolicy(statsbeatModule.getNetworkStatsbeat())); + TelemetryPipeline telemetryPipeline = + new TelemetryPipeline(httpPipeline, endpointProvider.getIngestionEndpointUrl()); + + TelemetryItemExporter exporter = + new TelemetryItemExporter(telemetryPipeline, telemetryPipelineListener); if (!readOnlyFileSystem) { - LocalFileSender.start(localFileLoader, channel); + localStorageSystem.startSendingFromDisk(telemetryPipeline); } - return BatchSpanProcessor.builder(channel) + return BatchItemProcessor.builder(exporter) .setMaxQueueSize(exportQueueCapacity) .setMaxExportBatchSize(maxExportBatchSize) .build(queueName); } - public BatchSpanProcessor getStatsbeatChannelBatcher() { - if (statsbeatChannelBatcher == null) { - synchronized (channelInitLock) { - if (statsbeatChannelBatcher == null) { - File statsbeatFolder; - LocalFileLoader localFileLoader = null; - LocalFileWriter localFileWriter = null; + public BatchItemProcessor getStatsbeatBatchItemProcessor() { + if (statsbeatBatchItemProcessor == null) { + synchronized (batchItemProcessorInitLock) { + if (statsbeatBatchItemProcessor == null) { + LocalStorageSystem localStorageSystem = null; + TelemetryPipelineListener telemetryPipelineListener = TelemetryPipelineListener.noop(); if (!readOnlyFileSystem) { - statsbeatFolder = LocalStorageUtils.getOfflineStatsbeatFolder(); - LocalFileCache localFileCache = new LocalFileCache(statsbeatFolder); - localFileLoader = new LocalFileLoader(localFileCache, statsbeatFolder, null); - localFileWriter = new LocalFileWriter(localFileCache, statsbeatFolder, null); + localStorageSystem = + new LocalStorageSystem(LocalStorageUtils.getOfflineStatsbeatFolder(), null); + telemetryPipelineListener = localStorageSystem.createTelemetryPipelineListener(); } - TelemetryChannel channel = - TelemetryChannel.create( - endpointProvider.getStatsbeatEndpointUrl(), - localFileWriter, - ikeyEndpointMap, - statsbeatModule, - true, - null); + HttpPipeline httpPipeline = LazyHttpClient.newHttpPipeLine(null); + TelemetryPipeline telemetryPipeline = + new TelemetryPipeline(httpPipeline, endpointProvider.getStatsbeatEndpointUrl()); + + TelemetryItemExporter exporter = + new TelemetryItemExporter(telemetryPipeline, telemetryPipelineListener); if (!readOnlyFileSystem) { - LocalFileSender.start(localFileLoader, channel); + localStorageSystem.startSendingFromDisk(telemetryPipeline); } - statsbeatChannelBatcher = BatchSpanProcessor.builder(channel).build("statsbeat"); + statsbeatBatchItemProcessor = BatchItemProcessor.builder(exporter).build("statsbeat"); } } } - return statsbeatChannelBatcher; + return statsbeatBatchItemProcessor; } /** Gets or sets the default instrumentation key for the application. */ @@ -422,7 +419,6 @@ public static class Builder { private Map globalTags; private Map globalProperties; private List metricFilters; - private Cache ikeyEndpointMap; private StatsbeatModule statsbeatModule; private boolean readOnlyFileSystem; private int generalExportQueueCapacity; @@ -458,11 +454,6 @@ public Builder setMetricFilters(List metricFilters) { return this; } - public Builder setIkeyEndpointMap(Cache ikeyEndpointMap) { - this.ikeyEndpointMap = ikeyEndpointMap; - return this; - } - public Builder setStatsbeatModule(StatsbeatModule statsbeatModule) { this.statsbeatModule = statsbeatModule; return this; diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryItemExporter.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryItemExporter.java new file mode 100644 index 00000000000..38e4cca0849 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryItemExporter.java @@ -0,0 +1,142 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.telemetry; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.io.SerializedString; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.microsoft.applicationinsights.agent.internal.common.OperationLogger; +import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; +import io.opentelemetry.sdk.common.CompletableResultCode; +import java.io.IOException; +import java.io.StringWriter; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TelemetryItemExporter { + + private static final Logger logger = LoggerFactory.getLogger(TelemetryItemExporter.class); + + private static final ObjectMapper mapper = createObjectMapper(); + + private static final AppInsightsByteBufferPool byteBufferPool = new AppInsightsByteBufferPool(); + + private static final OperationLogger encodeBatchOperationLogger = + new OperationLogger(TelemetryItemExporter.class, "Encoding telemetry batch into json"); + + private static ObjectMapper createObjectMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + // it's important to pass in the "agent class loader" since TelemetryItemPipeline is initialized + // lazily and can be initialized via an application thread, in which case the thread context + // class loader is used to look up jsr305 module and its not found + mapper.registerModules(ObjectMapper.findModules(TelemetryItemExporter.class.getClassLoader())); + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + return mapper; + } + + private final TelemetryPipeline byteBufferPipeline; + private final TelemetryPipelineListener listener; + + // e.g. construct with diagnostic listener and local storage listener + public TelemetryItemExporter( + TelemetryPipeline byteBufferPipeline, TelemetryPipelineListener listener) { + this.byteBufferPipeline = byteBufferPipeline; + this.listener = listener; + } + + public CompletableResultCode send(List telemetryItems) { + Map> instrumentationKeyMap = new HashMap<>(); + List resultCodeList = new ArrayList<>(); + for (TelemetryItem telemetryItem : telemetryItems) { + String instrumentationKey = telemetryItem.getInstrumentationKey(); + if (!instrumentationKeyMap.containsKey(instrumentationKey)) { + instrumentationKeyMap.put(instrumentationKey, new ArrayList<>()); + } + instrumentationKeyMap.get(instrumentationKey).add(telemetryItem); + } + for (String instrumentationKey : instrumentationKeyMap.keySet()) { + resultCodeList.add( + internalSendByInstrumentationKey( + instrumentationKeyMap.get(instrumentationKey), instrumentationKey)); + } + return CompletableResultCode.ofAll(resultCodeList); + } + + CompletableResultCode internalSendByInstrumentationKey( + List telemetryItems, String instrumentationKey) { + List byteBuffers; + try { + byteBuffers = encode(telemetryItems); + encodeBatchOperationLogger.recordSuccess(); + } catch (Throwable t) { + encodeBatchOperationLogger.recordFailure( + "Error encoding telemetry items: " + t.getMessage(), t); + return CompletableResultCode.ofFailure(); + } + return byteBufferPipeline.send(byteBuffers, instrumentationKey, listener); + } + + List encode(List telemetryItems) throws IOException { + + if (logger.isDebugEnabled()) { + StringWriter debug = new StringWriter(); + try (JsonGenerator jg = mapper.createGenerator(debug)) { + writeTelemetryItems(jg, telemetryItems); + } + logger.debug("sending telemetry to ingestion service:\n{}", debug); + } + + ByteBufferOutputStream out = new ByteBufferOutputStream(byteBufferPool); + + try (JsonGenerator jg = mapper.createGenerator(new GZIPOutputStream(out))) { + writeTelemetryItems(jg, telemetryItems); + } catch (IOException e) { + byteBufferPool.offer(out.getByteBuffers()); + throw e; + } + + out.close(); // closing ByteBufferOutputStream is a no-op, but this line makes LGTM happy + + List byteBuffers = out.getByteBuffers(); + for (ByteBuffer byteBuffer : byteBuffers) { + byteBuffer.flip(); + } + return byteBuffers; + } + + private static void writeTelemetryItems(JsonGenerator jg, List telemetryItems) + throws IOException { + jg.setRootValueSeparator(new SerializedString("\n")); + for (TelemetryItem telemetryItem : telemetryItems) { + mapper.writeValue(jg, telemetryItem); + } + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipeline.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipeline.java new file mode 100644 index 00000000000..1eba45e7d2f --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipeline.java @@ -0,0 +1,155 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.telemetry; + +import static java.util.Arrays.asList; + +import com.azure.core.http.HttpPipeline; +import com.azure.core.http.HttpResponse; +import com.azure.core.util.Context; +import com.azure.core.util.tracing.Tracer; +import io.opentelemetry.sdk.common.CompletableResultCode; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import reactor.core.publisher.Mono; + +public class TelemetryPipeline { + + static final Set REDIRECT_RESPONSE_CODES = new HashSet<>(asList(301, 302, 307, 308)); + + // Based on Stamp specific redirects design doc + private static final int MAX_REDIRECTS = 10; + + private final HttpPipeline pipeline; + private final URL ingestionEndpointUrl; + + // key is instrumentationKey, value is redirectUrl + private final Map redirectCache = + Collections.synchronizedMap( + new LinkedHashMap() { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > 100; + } + }); + + public TelemetryPipeline(HttpPipeline pipeline, URL ingestionEndpointUrl) { + this.pipeline = pipeline; + this.ingestionEndpointUrl = ingestionEndpointUrl; + } + + public CompletableResultCode send( + List telemetry, String instrumentationKey, TelemetryPipelineListener listener) { + + URL url = redirectCache.getOrDefault(instrumentationKey, ingestionEndpointUrl); + TelemetryPipelineRequest request = + new TelemetryPipelineRequest(url, instrumentationKey, telemetry); + + try { + CompletableResultCode result = new CompletableResultCode(); + sendInternal(request, listener, result, MAX_REDIRECTS); + return result; + } catch (Throwable t) { + listener.onException(request, "Error sending telemetry items: " + t.getMessage(), t); + return CompletableResultCode.ofFailure(); + } + } + + private void sendInternal( + TelemetryPipelineRequest request, + TelemetryPipelineListener listener, + CompletableResultCode result, + int remainingRedirects) { + + // Add instrumentation key to context to use in StatsbeatHttpPipelinePolicy + Map contextKeyValues = new HashMap<>(); + contextKeyValues.put("instrumentationKey", request.getInstrumentationKey()); + contextKeyValues.put(Tracer.DISABLE_TRACING_KEY, true); + + pipeline + .send(request.createHttpRequest(), Context.of(contextKeyValues)) + .subscribe( + response -> + response + .getBodyAsString() + .switchIfEmpty(Mono.just("")) + .subscribe( + responseBody -> + onResponseBody( + request, + response, + responseBody, + listener, + result, + remainingRedirects), + throwable -> { + listener.onException( + request, "Error retrieving response body: " + throwable, throwable); + result.fail(); + }), + throwable -> { + listener.onException(request, "Error sending telemetry items" + throwable, throwable); + result.fail(); + }); + } + + private void onResponseBody( + TelemetryPipelineRequest request, + HttpResponse response, + String responseBody, + TelemetryPipelineListener listener, + CompletableResultCode result, + int remainingRedirects) { + + int responseCode = response.getStatusCode(); + + if (REDIRECT_RESPONSE_CODES.contains(responseCode) && remainingRedirects > 0) { + String location = response.getHeaderValue("Location"); + URL locationUrl; + try { + locationUrl = new URL(location); + } catch (MalformedURLException e) { + listener.onException(request, "Invalid redirect: " + location, e); + return; + } + redirectCache.put(request.getInstrumentationKey(), locationUrl); + request.setUrl(locationUrl); + sendInternal(request, listener, result, remainingRedirects - 1); + return; + } + + listener.onResponse(request, new TelemetryPipelineResponse(responseCode, responseBody)); + if (responseCode == 200) { + result.succeed(); + } else { + result.fail(); + } + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipelineListener.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipelineListener.java new file mode 100644 index 00000000000..de2286f795c --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipelineListener.java @@ -0,0 +1,77 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.telemetry; + +import static java.util.Arrays.asList; + +import java.util.List; + +public interface TelemetryPipelineListener { + + void onResponse(TelemetryPipelineRequest request, TelemetryPipelineResponse response); + + void onException(TelemetryPipelineRequest request, String errorMessage, Throwable throwable); + + static TelemetryPipelineListener composite(TelemetryPipelineListener... delegates) { + return new CompositeTelemetryPipelineListener(asList(delegates)); + } + + static TelemetryPipelineListener noop() { + return NoopTelemetryPipelineListener.INSTANCE; + } + + class CompositeTelemetryPipelineListener implements TelemetryPipelineListener { + + private final List delegates; + + public CompositeTelemetryPipelineListener(List delegates) { + this.delegates = delegates; + } + + @Override + public void onResponse(TelemetryPipelineRequest request, TelemetryPipelineResponse response) { + for (TelemetryPipelineListener delegate : delegates) { + delegate.onResponse(request, response); + } + } + + @Override + public void onException( + TelemetryPipelineRequest request, String errorMessage, Throwable throwable) { + for (TelemetryPipelineListener delegate : delegates) { + delegate.onException(request, errorMessage, throwable); + } + } + } + + class NoopTelemetryPipelineListener implements TelemetryPipelineListener { + + static final TelemetryPipelineListener INSTANCE = new NoopTelemetryPipelineListener(); + + @Override + public void onResponse(TelemetryPipelineRequest request, TelemetryPipelineResponse response) {} + + @Override + public void onException( + TelemetryPipelineRequest request, String errorMessage, Throwable throwable) {} + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipelineRequest.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipelineRequest.java new file mode 100644 index 00000000000..7d3d5cfb610 --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipelineRequest.java @@ -0,0 +1,77 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.telemetry; + +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.List; +import reactor.core.publisher.Flux; + +public class TelemetryPipelineRequest { + + private volatile URL url; + private final String instrumentationKey; + private final List telemetry; + private final int contentLength; + + TelemetryPipelineRequest(URL url, String instrumentationKey, List telemetry) { + this.url = url; + this.instrumentationKey = instrumentationKey; + this.telemetry = telemetry; + contentLength = telemetry.stream().mapToInt(ByteBuffer::limit).sum(); + } + + public URL getUrl() { + return url; + } + + void setUrl(URL url) { + this.url = url; + } + + public String getInstrumentationKey() { + return instrumentationKey; + } + + public List getTelemetry() { + return telemetry; + } + + HttpRequest createHttpRequest() { + HttpRequest request = new HttpRequest(HttpMethod.POST, url); + request.setBody(Flux.fromIterable(telemetry)); + request.setHeader("Content-Length", Integer.toString(contentLength)); + + // need to suppress the default User-Agent "ReactorNetty/dev", otherwise Breeze ingestionservice + // will put that User-Agent header into the client_Browser field for all telemetry that doesn't + // explicitly set it's own UserAgent (ideally Breeze would only have this behavior for ingestion + // directly from browsers) + // TODO (trask) not setting User-Agent header at all would be a better option, but haven't + // figured out how to do that yet + request.setHeader("User-Agent", ""); + request.setHeader("Content-Encoding", "gzip"); + + return request; + } +} diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipelineResponse.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipelineResponse.java new file mode 100644 index 00000000000..4a27b0465ea --- /dev/null +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryPipelineResponse.java @@ -0,0 +1,41 @@ +/* + * ApplicationInsights-Java + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the ""Software""), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +package com.microsoft.applicationinsights.agent.internal.telemetry; + +public class TelemetryPipelineResponse { + + private final int statusCode; + private final String body; + + TelemetryPipelineResponse(int statusCode, String body) { + this.statusCode = statusCode; + this.body = body; + } + + public int getStatusCode() { + return statusCode; + } + + public String getBody() { + return body; + } +} diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/common/SystemInformationTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/common/SystemInformationTest.java index f11170224d9..03fcedff9c2 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/common/SystemInformationTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/common/SystemInformationTest.java @@ -30,7 +30,7 @@ class SystemInformationTest { @Test void testOs() { assertThat( - SystemUtils.IS_OS_WINDOWS ? SystemInformation.isWindows() : SystemInformation.isUnix()) + SystemUtils.IS_OS_WINDOWS ? SystemInformation.isWindows() : SystemInformation.isLinux()) .isTrue(); } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/IntegrationTests.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/IntegrationTests.java index 9338159efb4..3911cc661ff 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/IntegrationTests.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/IntegrationTests.java @@ -35,9 +35,8 @@ import com.azure.core.util.Context; import com.microsoft.applicationinsights.agent.internal.common.TestUtils; import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; -import com.microsoft.applicationinsights.agent.internal.statsbeat.NetworkStatsbeat; -import com.microsoft.applicationinsights.agent.internal.statsbeat.StatsbeatModule; -import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryChannel; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryItemExporter; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipeline; import io.opentelemetry.sdk.common.CompletableResultCode; import java.io.ByteArrayInputStream; import java.io.File; @@ -56,18 +55,16 @@ import java.util.zip.GZIPInputStream; import okio.BufferedSource; import okio.Okio; -import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mockito; import reactor.core.publisher.Mono; public class IntegrationTests { private static final String INSTRUMENTATION_KEY = "00000000-0000-0000-0000-0FEEDDADBEEF"; private static final String PERSISTED_FILENAME = "gzipped-raw-bytes.trn"; - private TelemetryChannel telemetryChannel; + private TelemetryItemExporter telemetryItemExporter; private LocalFileCache localFileCache; private LocalFileLoader localFileLoader; @@ -98,16 +95,13 @@ public void setup() throws Exception { localFileCache = new LocalFileCache(tempFolder); localFileLoader = new LocalFileLoader(localFileCache, tempFolder, null); - StatsbeatModule statsbeatModule = Mockito.mock(StatsbeatModule.class); - when(statsbeatModule.getNetworkStatsbeat()).thenReturn(Mockito.mock(NetworkStatsbeat.class)); - - telemetryChannel = - new TelemetryChannel( - pipelineBuilder.build(), - new URL("http://foo.bar"), - new LocalFileWriter(localFileCache, tempFolder, null), - statsbeatModule, - false); + TelemetryPipeline telemetryPipeline = + new TelemetryPipeline(pipelineBuilder.build(), new URL("http://foo.bar")); + telemetryItemExporter = + new TelemetryItemExporter( + telemetryPipeline, + new LocalStorageTelemetryPipelineListener( + new LocalFileWriter(localFileCache, tempFolder, null))); } @Test @@ -124,7 +118,8 @@ public void integrationTest() throws Exception { executorService.execute( () -> { for (int j = 0; j < 10; j++) { - CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); + CompletableResultCode completableResultCode = + telemetryItemExporter.send(telemetryItems); completableResultCode.join(10, SECONDS); assertThat(completableResultCode.isSuccess()).isFalse(); } @@ -154,12 +149,12 @@ public void verifyGzipRawBytesTest() throws Exception { File sourceFile = new File(getClass().getClassLoader().getResource(PERSISTED_FILENAME).getPath()); File persistedFile = new File(tempFolder, PERSISTED_FILENAME); - FileUtils.copyFile(sourceFile, persistedFile); + Files.copy(sourceFile.toPath(), persistedFile.toPath()); assertThat(persistedFile.exists()).isTrue(); LocalFileCache localFileCache = new LocalFileCache(tempFolder); - localFileCache.addPersistedFilenameToMap(PERSISTED_FILENAME); + localFileCache.addPersistedFile(persistedFile); LocalFileLoader localFileLoader = new LocalFileLoader(localFileCache, tempFolder, null); LocalFileLoader.PersistedFile loadedPersistedFile = localFileLoader.loadTelemetriesFromDisk(); diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileCacheTests.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileCacheTests.java index 54b6fa0f2a9..9dfe6254bd7 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileCacheTests.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileCacheTests.java @@ -31,8 +31,6 @@ import java.util.List; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedDeque; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -47,7 +45,7 @@ public void setup() throws Exception { List unsortedFiles = new ArrayList<>(); for (int i = 0; i < 100; i++) { File tempFile = createTempFile(tempFolder); - File trnFile = new File(tempFolder, FilenameUtils.getBaseName(tempFile.getName()) + ".trn"); + File trnFile = new File(tempFolder, FileUtil.getBaseName(tempFile) + ".trn"); tempFile.renameTo(trnFile); unsortedFiles.add(trnFile); } @@ -57,7 +55,7 @@ public void setup() throws Exception { sortedLastModified.add(file.lastModified()); } - Collection files = FileUtils.listFiles(tempFolder, new String[] {"trn"}, false); + Collection files = FileUtil.listTrnFiles(tempFolder); assertThat(files.size()).isEqualTo(100); assertThat(files.size()).isEqualTo(sortedLastModified.size()); } @@ -65,13 +63,13 @@ public void setup() throws Exception { @Test public void testSortPersistedFiles() { LocalFileCache cache = new LocalFileCache(tempFolder); - Queue sortedPersistedFile = cache.getPersistedFilesCache(); + Queue sortedPersistedFile = cache.getPersistedFilesCache(); assertThat(sortedPersistedFile.size()).isEqualTo(sortedLastModified.size()); while (sortedPersistedFile.peek() != null && sortedLastModified.peek() != null) { - String actualFilename = sortedPersistedFile.poll(); - Long actualLastModified = new File(tempFolder, actualFilename).lastModified(); + File actualFile = sortedPersistedFile.poll(); + Long actualLastModified = actualFile.lastModified(); Long expectedLastModified = sortedLastModified.poll(); assertThat(actualLastModified).isEqualTo(expectedLastModified); } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileLoaderTests.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileLoaderTests.java index 9555f2a8b55..c8dd37b4053 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileLoaderTests.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileLoaderTests.java @@ -37,9 +37,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.applicationinsights.agent.internal.MockHttpResponse; -import com.microsoft.applicationinsights.agent.internal.statsbeat.NetworkStatsbeat; -import com.microsoft.applicationinsights.agent.internal.statsbeat.StatsbeatModule; -import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryChannel; +import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryPipeline; import io.opentelemetry.sdk.common.CompletableResultCode; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -47,16 +45,15 @@ import java.io.IOException; import java.net.URL; import java.nio.ByteBuffer; +import java.nio.file.Files; import java.util.Arrays; -import java.util.Collection; +import java.util.List; import java.util.function.Function; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mockito; import reactor.core.publisher.Mono; public class LocalFileLoaderTests { @@ -88,11 +85,11 @@ public void testPersistedFileWithoutInstrumentationKey() throws IOException { new File(getClass().getClassLoader().getResource(GZIPPED_RAW_BYTES_WITHOUT_IKEY).getPath()); File persistedFile = new File(tempFolder, GZIPPED_RAW_BYTES_WITHOUT_IKEY); - FileUtils.copyFile(sourceFile, persistedFile); + Files.copy(sourceFile.toPath(), persistedFile.toPath()); assertThat(persistedFile.exists()).isTrue(); LocalFileCache localFileCache = new LocalFileCache(tempFolder); - localFileCache.addPersistedFilenameToMap(GZIPPED_RAW_BYTES_WITHOUT_IKEY); + localFileCache.addPersistedFile(persistedFile); LocalFileLoader localFileLoader = new LocalFileLoader(localFileCache, tempFolder, null); LocalFileLoader.PersistedFile loadedPersistedFile = localFileLoader.loadTelemetriesFromDisk(); @@ -108,11 +105,11 @@ public void testLoadFile() throws IOException { File persistedFile = new File(tempFolder, BYTE_BUFFERS_TEST_FILE); - FileUtils.copyFile(sourceFile, persistedFile); + Files.copy(sourceFile.toPath(), persistedFile.toPath()); assertThat(persistedFile.exists()).isTrue(); LocalFileCache localFileCache = new LocalFileCache(tempFolder); - localFileCache.addPersistedFilenameToMap(BYTE_BUFFERS_TEST_FILE); + localFileCache.addPersistedFile(persistedFile); LocalFileLoader localFileLoader = new LocalFileLoader(localFileCache, tempFolder, null); LocalFileLoader.PersistedFile loadedPersistedFile = localFileLoader.loadTelemetriesFromDisk(); @@ -190,7 +187,7 @@ public void testWriteAndReadRandomText() { String text = "hello world"; LocalFileCache cache = new LocalFileCache(tempFolder); LocalFileWriter writer = new LocalFileWriter(cache, tempFolder, null); - writer.writeToDisk(singletonList(ByteBuffer.wrap(text.getBytes(UTF_8))), INSTRUMENTATION_KEY); + writer.writeToDisk(INSTRUMENTATION_KEY, singletonList(ByteBuffer.wrap(text.getBytes(UTF_8)))); LocalFileLoader loader = new LocalFileLoader(cache, tempFolder, null); LocalFileLoader.PersistedFile persistedFile = loader.loadTelemetriesFromDisk(); @@ -220,7 +217,7 @@ public void testWriteGzipRawByte() throws IOException { byte[] result = byteArrayOutputStream.toByteArray(); LocalFileCache cache = new LocalFileCache(tempFolder); LocalFileWriter writer = new LocalFileWriter(cache, tempFolder, null); - writer.writeToDisk(singletonList(ByteBuffer.wrap(result)), INSTRUMENTATION_KEY); + writer.writeToDisk(INSTRUMENTATION_KEY, singletonList(ByteBuffer.wrap(result))); // read gzipped byte[] from disk LocalFileLoader loader = new LocalFileLoader(cache, tempFolder, null); @@ -249,26 +246,18 @@ public void testDeleteFilePermanentlyOnSuccess() throws Exception { LocalFileWriter localFileWriter = new LocalFileWriter(localFileCache, tempFolder, null); LocalFileLoader localFileLoader = new LocalFileLoader(localFileCache, tempFolder, null); - StatsbeatModule mockedStatsbeatModule = Mockito.mock(StatsbeatModule.class); - when(mockedStatsbeatModule.getNetworkStatsbeat()) - .thenReturn(Mockito.mock(NetworkStatsbeat.class)); - TelemetryChannel telemetryChannel = - new TelemetryChannel( - pipelineBuilder.build(), - new URL("http://foo.bar"), - localFileWriter, - mockedStatsbeatModule, - false); + TelemetryPipeline telemetryPipeline = + new TelemetryPipeline(pipelineBuilder.build(), new URL("http://foo.bar")); // persist 10 files to disk for (int i = 0; i < 10; i++) { localFileWriter.writeToDisk( - singletonList(ByteBuffer.wrap("hello world".getBytes(UTF_8))), INSTRUMENTATION_KEY); + INSTRUMENTATION_KEY, singletonList(ByteBuffer.wrap("hello world".getBytes(UTF_8)))); } assertThat(localFileCache.getPersistedFilesCache().size()).isEqualTo(10); - Collection files = FileUtils.listFiles(tempFolder, new String[] {"trn"}, false); + List files = FileUtil.listTrnFiles(tempFolder); assertThat(files.size()).isEqualTo(10); int expectedCount = 10; @@ -277,16 +266,17 @@ public void testDeleteFilePermanentlyOnSuccess() throws Exception { for (int i = 0; i < 10; i++) { LocalFileLoader.PersistedFile persistedFile = localFileLoader.loadTelemetriesFromDisk(); CompletableResultCode completableResultCode = - telemetryChannel.sendRawBytes( - persistedFile.rawBytes, persistedFile.instrumentationKey, () -> {}, retryable -> {}); + telemetryPipeline.send( + singletonList(persistedFile.rawBytes), + persistedFile.instrumentationKey, + new LocalFileSenderTelemetryPipelineListener(localFileLoader, persistedFile.file)); completableResultCode.join(10, SECONDS); assertThat(completableResultCode.isSuccess()).isEqualTo(true); - localFileLoader.updateProcessedFileStatus(true, persistedFile.file); // sleep 1 second to wait for delete to complete Thread.sleep(1000); - files = FileUtils.listFiles(tempFolder, new String[] {"trn"}, false); + files = FileUtil.listTrnFiles(tempFolder); assertThat(files.size()).isEqualTo(--expectedCount); } @@ -307,26 +297,18 @@ public void testDeleteFilePermanentlyOnFailure() throws Exception { LocalFileLoader localFileLoader = new LocalFileLoader(localFileCache, tempFolder, null); LocalFileWriter localFileWriter = new LocalFileWriter(localFileCache, tempFolder, null); - StatsbeatModule mockedStatsbeatModule = Mockito.mock(StatsbeatModule.class); - when(mockedStatsbeatModule.getNetworkStatsbeat()) - .thenReturn(Mockito.mock(NetworkStatsbeat.class)); - TelemetryChannel telemetryChannel = - new TelemetryChannel( - pipelineBuilder.build(), - new URL("http://foo.bar"), - localFileWriter, - mockedStatsbeatModule, - false); + TelemetryPipeline telemetryPipeline = + new TelemetryPipeline(pipelineBuilder.build(), new URL("http://foo.bar")); // persist 10 files to disk for (int i = 0; i < 10; i++) { localFileWriter.writeToDisk( - singletonList(ByteBuffer.wrap("hello world".getBytes(UTF_8))), INSTRUMENTATION_KEY); + INSTRUMENTATION_KEY, singletonList(ByteBuffer.wrap("hello world".getBytes(UTF_8)))); } assertThat(localFileCache.getPersistedFilesCache().size()).isEqualTo(10); - Collection files = FileUtils.listFiles(tempFolder, new String[] {"trn"}, false); + List files = FileUtil.listTrnFiles(tempFolder); assertThat(files.size()).isEqualTo(10); // fail to send persisted files and expect them to be kept on disk @@ -335,14 +317,15 @@ public void testDeleteFilePermanentlyOnFailure() throws Exception { assertThat(persistedFile.instrumentationKey).isEqualTo(INSTRUMENTATION_KEY); CompletableResultCode completableResultCode = - telemetryChannel.sendRawBytes( - persistedFile.rawBytes, persistedFile.instrumentationKey, () -> {}, retryable -> {}); + telemetryPipeline.send( + singletonList(persistedFile.rawBytes), + persistedFile.instrumentationKey, + new LocalFileSenderTelemetryPipelineListener(localFileLoader, persistedFile.file)); completableResultCode.join(10, SECONDS); assertThat(completableResultCode.isSuccess()).isEqualTo(false); - localFileLoader.updateProcessedFileStatus(false, persistedFile.file); } - files = FileUtils.listFiles(tempFolder, new String[] {"trn"}, false); + files = FileUtil.listTrnFiles(tempFolder); assertThat(files.size()).isEqualTo(10); assertThat(localFileCache.getPersistedFilesCache().size()).isEqualTo(10); } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFilePurgerTests.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFilePurgerTests.java index 69e4f733c72..582083c2676 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFilePurgerTests.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFilePurgerTests.java @@ -28,7 +28,6 @@ import java.io.File; import java.nio.ByteBuffer; import java.util.Collection; -import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -48,16 +47,16 @@ public void testPurgedExpiredFiles() throws InterruptedException { // persist 100 files to disk for (int i = 0; i < 100; i++) { writer.writeToDisk( - singletonList(ByteBuffer.wrap(text.getBytes(UTF_8))), - "00000000-0000-0000-0000-0FEEDDADBEE"); + "00000000-0000-0000-0000-0FEEDDADBEE", + singletonList(ByteBuffer.wrap(text.getBytes(UTF_8)))); } - Collection files = FileUtils.listFiles(tempFolder, new String[] {"trn"}, false); + Collection files = FileUtil.listTrnFiles(tempFolder); assertThat(files.size()).isEqualTo(100); Thread.sleep(10000); // wait 10 seconds - files = FileUtils.listFiles(tempFolder, new String[] {"trn"}, false); + files = FileUtil.listTrnFiles(tempFolder); assertThat(files.size()).isEqualTo(0); } } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileWriterTests.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileWriterTests.java index 70e1287fafe..c6efcefe374 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileWriterTests.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/localstorage/LocalFileWriterTests.java @@ -92,7 +92,7 @@ public void testWriteByteBuffersList() throws IOException { assertThat(byteBuffers.size()).isEqualTo(10); LocalFileWriter writer = new LocalFileWriter(localFileCache, tempFolder, nonessentialStatsbeat); - writer.writeToDisk(byteBuffers, "00000000-0000-0000-0000-0FEEDDADBEEF"); + writer.writeToDisk("00000000-0000-0000-0000-0FEEDDADBEEF", byteBuffers); assertThat(nonessentialStatsbeat.getWriteFailureCount()).isEqualTo(0); assertThat(localFileCache.getPersistedFilesCache().size()).isEqualTo(1); } @@ -100,7 +100,7 @@ public void testWriteByteBuffersList() throws IOException { @Test public void testWriteRawByteArray() { LocalFileWriter writer = new LocalFileWriter(localFileCache, tempFolder, nonessentialStatsbeat); - writer.writeToDisk(singletonList(buffer), "00000000-0000-0000-0000-0FEEDDADBEEF"); + writer.writeToDisk("00000000-0000-0000-0000-0FEEDDADBEEF", singletonList(buffer)); assertThat(nonessentialStatsbeat.getWriteFailureCount()).isEqualTo(0); assertThat(localFileCache.getPersistedFilesCache().size()).isEqualTo(1); } @@ -118,8 +118,8 @@ public void testWriteUnderMultipleThreadsEnvironment() throws InterruptedExcepti LocalFileWriter writer = new LocalFileWriter(localFileCache, tempFolder, nonessentialStatsbeat); writer.writeToDisk( - singletonList(ByteBuffer.wrap(telemetry.getBytes(UTF_8))), - "00000000-0000-0000-0000-0FEEDDADBEEF"); + "00000000-0000-0000-0000-0FEEDDADBEEF", + singletonList(ByteBuffer.wrap(telemetry.getBytes(UTF_8)))); } }); } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/CustomDimensionsTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/CustomDimensionsTest.java index 45d85425627..a91123f2316 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/CustomDimensionsTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/CustomDimensionsTest.java @@ -48,7 +48,7 @@ public void testOperatingSystem() { OperatingSystem os = OperatingSystem.OS_UNKNOWN; if (SystemInformation.isWindows()) { os = OperatingSystem.OS_WINDOWS; - } else if (SystemInformation.isUnix()) { + } else if (SystemInformation.isLinux()) { os = OperatingSystem.OS_LINUX; } assertThat(customDimensions.getOperatingSystem()).isEqualTo(os); diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatTest.java index 35af80c9e7d..0f041b7b8ba 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/statsbeat/NetworkStatsbeatTest.java @@ -43,8 +43,8 @@ public void init() { public void testIncrementRequestSuccessCount() { assertThat(networkStatsbeat.getRequestSuccessCount(IKEY)).isEqualTo(0); assertThat(networkStatsbeat.getRequestDurationAvg(IKEY)).isEqualTo(0); - networkStatsbeat.incrementRequestSuccessCount(1000, IKEY); - networkStatsbeat.incrementRequestSuccessCount(3000, IKEY); + networkStatsbeat.incrementRequestSuccessCount(1000, IKEY, "host"); + networkStatsbeat.incrementRequestSuccessCount(3000, IKEY, "host"); assertThat(networkStatsbeat.getRequestSuccessCount(IKEY)).isEqualTo(2); assertThat(networkStatsbeat.getRequestDurationAvg(IKEY)).isEqualTo(2000.0); } @@ -52,32 +52,32 @@ public void testIncrementRequestSuccessCount() { @Test public void testIncrementRequestFailureCount() { assertThat(networkStatsbeat.getRequestFailureCount(IKEY)).isEqualTo(0); - networkStatsbeat.incrementRequestFailureCount(IKEY); - networkStatsbeat.incrementRequestFailureCount(IKEY); + networkStatsbeat.incrementRequestFailureCount(IKEY, "host"); + networkStatsbeat.incrementRequestFailureCount(IKEY, "host"); assertThat(networkStatsbeat.getRequestFailureCount(IKEY)).isEqualTo(2); } @Test public void testIncrementRetryCount() { assertThat(networkStatsbeat.getRetryCount(IKEY)).isEqualTo(0); - networkStatsbeat.incrementRetryCount(IKEY); - networkStatsbeat.incrementRetryCount(IKEY); + networkStatsbeat.incrementRetryCount(IKEY, "host"); + networkStatsbeat.incrementRetryCount(IKEY, "host"); assertThat(networkStatsbeat.getRetryCount(IKEY)).isEqualTo(2); } @Test public void testIncrementThrottlingCount() { assertThat(networkStatsbeat.getThrottlingCount(IKEY)).isEqualTo(0); - networkStatsbeat.incrementThrottlingCount(IKEY); - networkStatsbeat.incrementThrottlingCount(IKEY); + networkStatsbeat.incrementThrottlingCount(IKEY, "host"); + networkStatsbeat.incrementThrottlingCount(IKEY, "host"); assertThat(networkStatsbeat.getThrottlingCount(IKEY)).isEqualTo(2); } @Test public void testIncrementExceptionCount() { assertThat(networkStatsbeat.getExceptionCount(IKEY)).isEqualTo(0); - networkStatsbeat.incrementExceptionCount(IKEY); - networkStatsbeat.incrementExceptionCount(IKEY); + networkStatsbeat.incrementExceptionCount(IKEY, "host"); + networkStatsbeat.incrementExceptionCount(IKEY, "host"); assertThat(networkStatsbeat.getExceptionCount(IKEY)).isEqualTo(2); } @@ -90,11 +90,11 @@ public void testRaceCondition() throws InterruptedException { @Override public void run() { for (int j = 0; j < 1000; j++) { - networkStatsbeat.incrementRequestSuccessCount(j % 2 == 0 ? 5 : 10, IKEY); - networkStatsbeat.incrementRequestFailureCount(IKEY); - networkStatsbeat.incrementRetryCount(IKEY); - networkStatsbeat.incrementThrottlingCount(IKEY); - networkStatsbeat.incrementExceptionCount(IKEY); + networkStatsbeat.incrementRequestSuccessCount(j % 2 == 0 ? 5 : 10, IKEY, "host"); + networkStatsbeat.incrementRequestFailureCount(IKEY, "host"); + networkStatsbeat.incrementRetryCount(IKEY, "host"); + networkStatsbeat.incrementThrottlingCount(IKEY, "host"); + networkStatsbeat.incrementExceptionCount(IKEY, "host"); } } }); diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/ConnectionStringParsingTests.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/ConnectionStringParsingTests.java index 3dc6ba233b7..3e84a27df9b 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/ConnectionStringParsingTests.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/ConnectionStringParsingTests.java @@ -29,7 +29,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.UUID; -import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.Test; class ConnectionStringParsingTests { @@ -487,7 +486,10 @@ void invalidUrlIsInvalidConnectionString() { @Test void giantValuesAreNotAllowed() { - String bigIkey = StringUtils.repeat('0', ConnectionString.CONNECTION_STRING_MAX_LENGTH * 2); + StringBuilder bigIkey = new StringBuilder(); + for (int i = 0; i < ConnectionString.CONNECTION_STRING_MAX_LENGTH * 2; i++) { + bigIkey.append('0'); + } assertThatThrownBy( () -> ConnectionString.parseInto("InstrumentationKey=" + bigIkey, telemetryClient)) diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannelTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryItemExporterTest.java similarity index 79% rename from agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannelTest.java rename to agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryItemExporterTest.java index 2d508c64cac..f5f1b5194fd 100644 --- a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryChannelTest.java +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/telemetry/TelemetryItemExporterTest.java @@ -22,24 +22,17 @@ package com.microsoft.applicationinsights.agent.internal.telemetry; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; import com.azure.core.http.HttpClient; import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpPipelineBuilder; import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; -import com.azure.core.http.policy.HttpPipelinePolicy; import com.azure.core.util.FluxUtil; import com.microsoft.applicationinsights.agent.internal.MockHttpResponse; import com.microsoft.applicationinsights.agent.internal.common.TestUtils; import com.microsoft.applicationinsights.agent.internal.exporter.models.TelemetryItem; -import com.microsoft.applicationinsights.agent.internal.httpclient.RedirectPolicy; -import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileCache; -import com.microsoft.applicationinsights.agent.internal.localstorage.LocalFileWriter; -import com.microsoft.applicationinsights.agent.internal.statsbeat.NetworkStatsbeat; -import com.microsoft.applicationinsights.agent.internal.statsbeat.StatsbeatModule; -import io.opentelemetry.instrumentation.api.cache.Cache; +import com.microsoft.applicationinsights.agent.internal.localstorage.LocalStorageSystem; import io.opentelemetry.sdk.common.CompletableResultCode; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -60,11 +53,10 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mockito; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public class TelemetryChannelTest { +public class TelemetryItemExporterTest { RecordingHttpClient recordingHttpClient; private static final String INSTRUMENTATION_KEY = "00000000-0000-0000-0000-0FEEDDADBEEF"; private static final String REDIRECT_INSTRUMENTATION_KEY = "00000000-0000-0000-0000-0FEEDDADBEEE"; @@ -73,23 +65,14 @@ public class TelemetryChannelTest { @TempDir File tempFolder; - private TelemetryChannel getTelemetryChannel() throws MalformedURLException { - List policies = new ArrayList<>(); - - policies.add(new RedirectPolicy(Cache.bounded(5))); - HttpPipelineBuilder pipelineBuilder = - new HttpPipelineBuilder() - .policies(policies.toArray(new HttpPipelinePolicy[0])) - .httpClient(recordingHttpClient); - LocalFileCache localFileCache = new LocalFileCache(tempFolder); - StatsbeatModule mockedStatsModule = Mockito.mock(StatsbeatModule.class); - when(mockedStatsModule.getNetworkStatsbeat()).thenReturn(Mockito.mock(NetworkStatsbeat.class)); - return new TelemetryChannel( - pipelineBuilder.build(), - new URL(END_POINT_URL), - new LocalFileWriter(localFileCache, tempFolder, null), - mockedStatsModule, - false); + private TelemetryItemExporter getExporter() throws MalformedURLException { + HttpPipelineBuilder pipelineBuilder = new HttpPipelineBuilder().httpClient(recordingHttpClient); + LocalStorageSystem localStorageSystem = new LocalStorageSystem(tempFolder, null); + + TelemetryPipeline telemetryPipeline = + new TelemetryPipeline(pipelineBuilder.build(), new URL(END_POINT_URL)); + return new TelemetryItemExporter( + telemetryPipeline, localStorageSystem.createTelemetryPipelineListener()); } @Nullable @@ -143,10 +126,10 @@ public void singleIkeyTest() throws MalformedURLException { // given List telemetryItems = new ArrayList<>(); telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(); + TelemetryItemExporter exporter = getExporter(); // when - CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); + CompletableResultCode completableResultCode = exporter.send(telemetryItems); // then assertThat(completableResultCode.isSuccess()).isEqualTo(true); @@ -160,10 +143,10 @@ public void dualIkeyTest() throws MalformedURLException { telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); telemetryItems.add( TestUtils.createMetricTelemetry("metric" + 2, 2, REDIRECT_INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(); + TelemetryItemExporter exporter = getExporter(); // when - CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); + CompletableResultCode completableResultCode = exporter.send(telemetryItems); // then assertThat(completableResultCode.isSuccess()).isEqualTo(true); @@ -176,10 +159,10 @@ public void singleIkeyBatchTest() throws MalformedURLException { List telemetryItems = new ArrayList<>(); telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 1, 1, INSTRUMENTATION_KEY)); telemetryItems.add(TestUtils.createMetricTelemetry("metric" + 2, 2, INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(); + TelemetryItemExporter exporter = getExporter(); // when - CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); + CompletableResultCode completableResultCode = exporter.send(telemetryItems); // then assertThat(completableResultCode.isSuccess()).isEqualTo(true); @@ -196,10 +179,10 @@ public void dualIkeyBatchTest() throws MalformedURLException { TestUtils.createMetricTelemetry("metric" + 3, 3, REDIRECT_INSTRUMENTATION_KEY)); telemetryItems.add( TestUtils.createMetricTelemetry("metric" + 4, 4, REDIRECT_INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(); + TelemetryItemExporter exporter = getExporter(); // when - CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); + CompletableResultCode completableResultCode = exporter.send(telemetryItems); // then assertThat(completableResultCode.isSuccess()).isEqualTo(true); @@ -216,16 +199,16 @@ public void dualIkeyBatchWithDelayTest() throws MalformedURLException { TestUtils.createMetricTelemetry("metric" + 3, 3, REDIRECT_INSTRUMENTATION_KEY)); telemetryItems.add( TestUtils.createMetricTelemetry("metric" + 4, 4, REDIRECT_INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(); + TelemetryItemExporter exporter = getExporter(); // when - CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); + CompletableResultCode completableResultCode = exporter.send(telemetryItems); // then assertThat(completableResultCode.isSuccess()).isEqualTo(true); assertThat(recordingHttpClient.getCount()).isEqualTo(3); - completableResultCode = telemetryChannel.send(telemetryItems); + completableResultCode = exporter.send(telemetryItems); // then // the redirect url should be cached and should not invoke another redirect. @@ -243,16 +226,16 @@ public void dualIkeyBatchWithDelayAndRedirectFlagFalseTest() throws MalformedURL TestUtils.createMetricTelemetry("metric" + 3, 3, REDIRECT_INSTRUMENTATION_KEY)); telemetryItems.add( TestUtils.createMetricTelemetry("metric" + 4, 4, REDIRECT_INSTRUMENTATION_KEY)); - TelemetryChannel telemetryChannel = getTelemetryChannel(); + TelemetryItemExporter exporter = getExporter(); // when - CompletableResultCode completableResultCode = telemetryChannel.send(telemetryItems); + CompletableResultCode completableResultCode = exporter.send(telemetryItems); // then assertThat(completableResultCode.isSuccess()).isEqualTo(true); assertThat(recordingHttpClient.getCount()).isEqualTo(3); - completableResultCode = telemetryChannel.send(telemetryItems); + completableResultCode = exporter.send(telemetryItems); // then // the redirect url should be cached and should not invoke another redirect. diff --git a/agent/instrumentation/gradle.lockfile b/agent/instrumentation/gradle.lockfile index 9a5bbb58c24..0bdefaddf84 100644 --- a/agent/instrumentation/gradle.lockfile +++ b/agent/instrumentation/gradle.lockfile @@ -36,7 +36,6 @@ com.squareup.moshi:moshi:1.11.0=runtimeClasspath com.squareup.okhttp3:okhttp:4.9.3=runtimeClasspath com.squareup.okio:okio:2.8.0=runtimeClasspath commons-codec:commons-codec:1.15=runtimeClasspath -commons-io:commons-io:2.7=runtimeClasspath io.netty:netty-bom:4.1.72.Final=runtimeClasspath io.netty:netty-buffer:4.1.72.Final=runtimeClasspath io.netty:netty-codec-dns:4.1.72.Final=runtimeClasspath diff --git a/dependencyManagement/build.gradle.kts b/dependencyManagement/build.gradle.kts index 4b4ac56691b..55f2651de19 100644 --- a/dependencyManagement/build.gradle.kts +++ b/dependencyManagement/build.gradle.kts @@ -24,7 +24,7 @@ val DEPENDENCY_BOMS = listOf( "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha:${otelInstrumentationVersionAlpha}", "org.junit:junit-bom:5.7.2", "io.netty:netty-bom:4.1.72.Final", - ) +) // TODO consolidate to just one json library // TODO remove dependencies on apache commons @@ -95,8 +95,6 @@ val DEPENDENCIES = listOf( "com.google.auto.service:auto-service:1.0", "com.uber.nullaway:nullaway:0.9.1", "commons-codec:commons-codec:1.15", - "commons-io:commons-io:2.7", - "org.apache.commons:commons-lang3:3.7", "org.apache.commons:commons-text:1.9", "com.google.code.gson:gson:2.8.2", "com.azure:azure-core:1.21.0", diff --git a/etw/etw-testapp/build.gradle.kts b/etw/etw-testapp/build.gradle.kts index ab4ff2b4753..3e0962d3a73 100644 --- a/etw/etw-testapp/build.gradle.kts +++ b/etw/etw-testapp/build.gradle.kts @@ -13,5 +13,5 @@ dependencies { implementation("org.springframework.boot:spring-boot-starter-web:2.1.7.RELEASE") { exclude("org.springframework.boot", "spring-boot-starter-tomcat") } - implementation("org.apache.commons:commons-lang3") + implementation("org.apache.commons:commons-lang3:3.7") } diff --git a/etw/java/build.gradle b/etw/java/build.gradle index 3c0b602d9d3..b0c180de28b 100644 --- a/etw/java/build.gradle +++ b/etw/java/build.gradle @@ -41,7 +41,7 @@ dependencies { testImplementation("org.junit.jupiter:junit-jupiter") testImplementation("org.assertj:assertj-core") testImplementation("org.mockito:mockito-core") - testImplementation("org.apache.commons:commons-lang3") + testImplementation("org.apache.commons:commons-lang3:3.7") } // To rebuild naitive code with new headers, first run `gradlew :etw:java:classes -Pai.etw.native.generateHeaders` to generate new header, then update the method implementations. diff --git a/test/smoke/framework/testCore/build.gradle.kts b/test/smoke/framework/testCore/build.gradle.kts index a6d85d191e3..cb5fdb9ccd8 100644 --- a/test/smoke/framework/testCore/build.gradle.kts +++ b/test/smoke/framework/testCore/build.gradle.kts @@ -5,7 +5,7 @@ plugins { dependencies { implementation("com.google.guava:guava") implementation("junit:junit:4.13.2") - implementation("org.apache.commons:commons-lang3") + implementation("org.apache.commons:commons-lang3:3.7") implementation(project(":test:smoke:framework:utils")) implementation(project(":test:fakeIngestion:standalone")) implementation(project(":test:fakeIngestion:servlet")) diff --git a/test/smoke/framework/utils/build.gradle.kts b/test/smoke/framework/utils/build.gradle.kts index e9af046c16d..59cf131fd3f 100644 --- a/test/smoke/framework/utils/build.gradle.kts +++ b/test/smoke/framework/utils/build.gradle.kts @@ -5,7 +5,7 @@ plugins { dependencies { implementation("com.google.code.gson:gson") implementation("org.apache.httpcomponents:httpclient:4.5.13") - implementation("org.apache.commons:commons-lang3") + implementation("org.apache.commons:commons-lang3:3.7") implementation("org.hamcrest:hamcrest-library:1.3") testImplementation("org.assertj:assertj-core")