Fix custom metric on Function Consumption (#2946)

Disable profiler and heartbeat when connection string is null

agent/agent-bootstrap/src/main/java/com/microsoft/applicationinsights/agent/bootstrap/diagnostics/status/StatusFile.java

@@ -55,14 +55,19 @@ public class StatusFile {
   static final String HOME_ENV_VAR = "HOME";
 
   // visible for testing
-  static final String DEFAULT_LOGDIR = "/LogFiles";
+  static final String DEFAULT_LOGDIR = "LogFiles";
 
   // visible for testing
-  static final String DEFAULT_APPLICATIONINSIGHTS_LOGDIR = "/ApplicationInsights";
+  static final String DEFAULT_APPLICATIONINSIGHTS_LOGDIR = "ApplicationInsights";
 
   // visible for testing
   static final String WINDOWS_DEFAULT_HOME_DIR =
-      "/home" + DEFAULT_LOGDIR + DEFAULT_APPLICATIONINSIGHTS_LOGDIR;
+      File.separator
+          + "home"
+          + File.separator
+          + DEFAULT_LOGDIR
+          + File.separator
+          + DEFAULT_APPLICATIONINSIGHTS_LOGDIR;
 
   // visible for testing
   static String logDir;
@@ -110,11 +115,15 @@ static String initLogDir() {
     if (DiagnosticsHelper.isOsWindows()) {
       String siteLogDir = System.getProperty(SITE_LOGDIR_PROPERTY);
       if (siteLogDir != null && !siteLogDir.isEmpty()) {
-        return siteLogDir + DEFAULT_APPLICATIONINSIGHTS_LOGDIR;
+        return siteLogDir + File.separator + DEFAULT_APPLICATIONINSIGHTS_LOGDIR;
       }
       String homeDir = System.getenv(HOME_ENV_VAR);
       if (homeDir != null && !homeDir.isEmpty()) {
-        return homeDir + DEFAULT_LOGDIR + DEFAULT_APPLICATIONINSIGHTS_LOGDIR;
+        return homeDir
+            + File.separator
+            + DEFAULT_LOGDIR
+            + File.separator
+            + DEFAULT_APPLICATIONINSIGHTS_LOGDIR;
       }
       return WINDOWS_DEFAULT_HOME_DIR;
     }

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java

@@ -615,7 +615,7 @@ private static String getDefaultPath() {
       }
       if (DiagnosticsHelper.useAppSvcRpIntegrationLogging()
           || DiagnosticsHelper.useFunctionsRpIntegrationLogging()) {
-        return StatusFile.getLogDir() + "/" + DEFAULT_NAME;
+        return StatusFile.getLogDir() + File.separator + DEFAULT_NAME;
       }
       // azure spring cloud
       return DEFAULT_NAME;

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/LoggingConfigurator.java

@@ -115,6 +115,11 @@ private void configureFunctions() {
     Appender<ILoggingEvent> diagnosticAppender = configureConsoleAppender();
     diagnosticLogger.addAppender(diagnosticAppender);
 
+    ApplicationInsightsDiagnosticsLogFilter filter = new ApplicationInsightsDiagnosticsLogFilter();
+    filter.setContext(loggerContext);
+    filter.start();
+    diagnosticAppender.addFilter(filter);
+
     // errors reported by other loggers should also go to diagnostic log
     // (level filter for these is applied in ApplicationInsightsDiagnosticsLogFilter)
     rootLogger.addAppender(diagnosticAppender);

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RuntimeConfiguration.java

@@ -22,4 +22,8 @@ public class RuntimeConfiguration {
   public String instrumentationLoggingLevel;
 
   public String selfDiagnosticsLevel;
+
+  public boolean profilerEnabled;
+
+  public long heartbeatIntervalSeconds;
 }

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/RuntimeConfigurator.java

@@ -3,18 +3,26 @@
 
 package com.microsoft.applicationinsights.agent.internal.init;
 
+import static java.util.concurrent.TimeUnit.MINUTES;
+
 import ch.qos.logback.classic.LoggerContext;
+import com.azure.monitor.opentelemetry.exporter.implementation.heartbeat.HeartbeatExporter;
+import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem;
 import com.azure.monitor.opentelemetry.exporter.implementation.utils.Strings;
 import com.microsoft.applicationinsights.agent.internal.classicsdk.BytecodeUtilImpl;
 import com.microsoft.applicationinsights.agent.internal.configuration.Configuration;
 import com.microsoft.applicationinsights.agent.internal.exporter.AgentLogExporter;
 import com.microsoft.applicationinsights.agent.internal.legacyheaders.DelegatingPropagator;
+import com.microsoft.applicationinsights.agent.internal.profiler.ProfilingInitializer;
 import com.microsoft.applicationinsights.agent.internal.sampling.DelegatingSampler;
 import com.microsoft.applicationinsights.agent.internal.sampling.Samplers;
 import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 import javax.annotation.Nullable;
 import org.slf4j.Logger;
@@ -26,15 +34,26 @@ public class RuntimeConfigurator {
 
   private final TelemetryClient telemetryClient;
   private final Supplier<AgentLogExporter> agentLogExporter;
+  private final Configuration initialConfig;
   private volatile RuntimeConfiguration currentConfig;
+  private final Consumer<List<TelemetryItem>> heartbeatTelemetryItemsConsumer;
+  private final File tempDir;
+
+  private final AtomicBoolean profilerStarted = new AtomicBoolean();
+  private final AtomicBoolean heartbeatStarted = new AtomicBoolean();
 
   RuntimeConfigurator(
       TelemetryClient telemetryClient,
       Supplier<AgentLogExporter> agentLogExporter,
-      Configuration initialConfig) {
+      Configuration initialConfig,
+      Consumer<List<TelemetryItem>> heartbeatTelemetryItemConsumer,
+      File tempDir) {
     this.telemetryClient = telemetryClient;
     this.agentLogExporter = agentLogExporter;
+    this.initialConfig = initialConfig;
     currentConfig = captureInitialConfig(initialConfig);
+    this.heartbeatTelemetryItemsConsumer = heartbeatTelemetryItemConsumer;
+    this.tempDir = tempDir;
   }
 
   private static RuntimeConfiguration captureInitialConfig(Configuration initialConfig) {
@@ -58,6 +77,9 @@ private static RuntimeConfiguration captureInitialConfig(Configuration initialCo
 
     runtimeConfig.instrumentationLoggingLevel = initialConfig.instrumentation.logging.level;
     runtimeConfig.selfDiagnosticsLevel = initialConfig.selfDiagnostics.level;
+
+    runtimeConfig.profilerEnabled = initialConfig.preview.profiler.enabled;
+    runtimeConfig.heartbeatIntervalSeconds = initialConfig.heartbeat.intervalSeconds;
     return runtimeConfig;
   }
 
@@ -79,6 +101,9 @@ private static RuntimeConfiguration copy(RuntimeConfiguration config) {
 
     copy.instrumentationLoggingLevel = config.instrumentationLoggingLevel;
     copy.selfDiagnosticsLevel = config.selfDiagnosticsLevel;
+
+    copy.profilerEnabled = config.profilerEnabled;
+    copy.heartbeatIntervalSeconds = config.heartbeatIntervalSeconds;
     return copy;
   }
 
@@ -112,6 +137,42 @@ public void apply(RuntimeConfiguration runtimeConfig) {
       updateSampling(enabled, runtimeConfig.sampling, runtimeConfig.samplingPreview);
     }
 
+    // initialize Profiler
+    if (runtimeConfig.profilerEnabled && telemetryClient.getConnectionString() != null) {
+      // this prevents profiler being initialized more than once in Azure Spring App
+      if (!profilerStarted.getAndSet(true)) {
+        try {
+          ProfilingInitializer.initialize(
+              tempDir,
+              initialConfig.preview.profiler,
+              initialConfig.preview.gcEvents.reportingLevel,
+              runtimeConfig.role.name,
+              runtimeConfig.role.instance,
+              telemetryClient);
+        } catch (RuntimeException e) {
+          logger.warn("Failed to initialize profiler", e);
+        }
+      } else {
+        logger.debug("Profiler has already been initialized.");
+      }
+    }
+
+    // enable Heartbeat
+    if (telemetryClient.getConnectionString() != null) {
+      // this prevents heartbeat being started more than once in Azure Spring App
+      if (!heartbeatStarted.getAndSet(true)) {
+        // interval longer than 15 minutes is not allowed since we use this data for usage telemetry
+        long intervalSeconds =
+            Math.min(runtimeConfig.heartbeatIntervalSeconds, MINUTES.toSeconds(15));
+        HeartbeatExporter.start(
+            intervalSeconds, telemetryClient::populateDefaults, heartbeatTelemetryItemsConsumer);
+      } else {
+        logger.debug("Heartbeat has already started.");
+      }
+    }
+
+    // TODO (heya) enable Statsbeat and need to refactor RuntimeConfiguration
+
     updateInstrumentationLoggingLevel(runtimeConfig.instrumentationLoggingLevel);
     updateSelfDiagnosticsLevel(runtimeConfig.selfDiagnosticsLevel);
 

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java

@@ -135,9 +135,7 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) {
             .setDiskPersistenceMaxSizeMb(configuration.preview.diskPersistenceMaxSizeMb)
             .build();
 
-    // interval longer than 15 minutes is not allowed since we use this data for usage telemetry
-    long intervalSeconds = Math.min(configuration.heartbeat.intervalSeconds, MINUTES.toSeconds(15));
-    Consumer<List<TelemetryItem>> telemetryItemsConsumer =
+    Consumer<List<TelemetryItem>> heartbeatTelemetryItemConsumer =
         telemetryItems -> {
           for (TelemetryItem telemetryItem : telemetryItems) {
             TelemetryObservers.INSTANCE
@@ -146,13 +144,40 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) {
             telemetryClient.getMetricsBatchItemProcessor().trackAsync(telemetryItem);
           }
         };
-    HeartbeatExporter.start(
-        intervalSeconds, telemetryClient::populateDefaults, telemetryItemsConsumer);
+
+    if (telemetryClient.getConnectionString() != null) {
+      startupLogger.verbose("connection string is not null, start HeartbeatExporter");
+      // interval longer than 15 minutes is not allowed since we use this data for usage telemetry
+      long intervalSeconds =
+          Math.min(configuration.heartbeat.intervalSeconds, MINUTES.toSeconds(15));
+      HeartbeatExporter.start(
+          intervalSeconds, telemetryClient::populateDefaults, heartbeatTelemetryItemConsumer);
+    }
 
     TelemetryClient.setActive(telemetryClient);
 
+    if (configuration.preview.profiler.enabled && telemetryClient.getConnectionString() != null) {
+      try {
+        ProfilingInitializer.initialize(
+            tempDir,
+            configuration.preview.profiler,
+            configuration.preview.gcEvents.reportingLevel,
+            configuration.role.name,
+            configuration.role.instance,
+            telemetryClient);
+      } catch (RuntimeException e) {
+        startupLogger.warning("Failed to initialize profiler", e);
+      }
+    }
+
+    // TODO (heya) remove duplicate code in both RuntimeConfigurator and SecondEntryPoint
     RuntimeConfigurator runtimeConfigurator =
-        new RuntimeConfigurator(telemetryClient, () -> agentLogExporter, configuration);
+        new RuntimeConfigurator(
+            telemetryClient,
+            () -> agentLogExporter,
+            configuration,
+            heartbeatTelemetryItemConsumer,
+            tempDir);
 
     if (configuration.sampling.percentage != null) {
       BytecodeUtilImpl.samplingPercentage = configuration.sampling.percentage.floatValue();
@@ -164,14 +189,6 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) {
     BytecodeUtilImpl.connectionStringConfiguredAtRuntime =
         configuration.connectionStringConfiguredAtRuntime;
 
-    if (configuration.preview.profiler.enabled) {
-      try {
-        ProfilingInitializer.initialize(tempDir, configuration, telemetryClient);
-      } catch (RuntimeException e) {
-        startupLogger.warning("Failed to initialize profiler", e);
-      }
-    }
-
     if (ConfigurationBuilder.inAzureFunctionsConsumptionWorker()) {
       AzureFunctions.setup(
           () -> telemetryClient.getConnectionString() != null,

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/profiler/PerformanceMonitoringService.java

@@ -5,6 +5,7 @@
 
 import com.azure.monitor.opentelemetry.exporter.implementation.utils.ThreadPoolUtils;
 import com.microsoft.applicationinsights.agent.internal.configuration.Configuration;
+import com.microsoft.applicationinsights.agent.internal.configuration.GcReportingLevel;
 import com.microsoft.applicationinsights.agent.internal.profiler.service.ServiceProfilerClient;
 import com.microsoft.applicationinsights.agent.internal.profiler.triggers.AlertingSubsystemInit;
 import com.microsoft.applicationinsights.agent.internal.profiler.upload.UploadService;
@@ -38,7 +39,8 @@ public class PerformanceMonitoringService {
   private final String machineName;
   private final String roleName;
   private final TelemetryClient telemetryClient;
-  private final Configuration configuration;
+  private final Configuration.ProfilerConfiguration configuration;
+  private final GcReportingLevel reportingLevel;
   private final File tempDir;
 
   private boolean currentlyEnabled = false;
@@ -58,13 +60,15 @@ public PerformanceMonitoringService(
       String machineName,
       String roleName,
       TelemetryClient telemetryClient,
-      Configuration configuration,
+      Configuration.ProfilerConfiguration configuration,
+      GcReportingLevel reportingLevel,
       File tempDir) {
     this.processId = processId;
     this.machineName = machineName;
     this.roleName = roleName;
     this.telemetryClient = telemetryClient;
     this.configuration = configuration;
+    this.reportingLevel = reportingLevel;
     this.tempDir = tempDir;
   }
 
@@ -80,7 +84,7 @@ synchronized void enableProfiler(
     logger.warn("INITIALISING JFR PROFILING SUBSYSTEM THIS FEATURE IS IN BETA");
 
     diagnosticEngine = null;
-    if (configuration.preview.profiler.enableDiagnostics) {
+    if (configuration.enabled) {
       // Initialise diagnostic service
       diagnosticEngine = startDiagnosticEngine();
     }
@@ -91,11 +95,12 @@ synchronized void enableProfiler(
             ThreadPoolUtils.createDaemonThreadFactory(
                 ProfilingInitializer.class, "ServiceProfilerAlertingService"));
 
-    profiler = new Profiler(configuration.preview.profiler, tempDir);
+    profiler = new Profiler(configuration, tempDir);
 
     alerting =
         AlertingSubsystemInit.create(
             configuration,
+            reportingLevel,
             TelemetryObservers.INSTANCE,
             profiler,
             telemetryClient,