Add azure_monitor to metrics exporter for AKS

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java

@@ -264,6 +264,19 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) {
               return props;
             })
         .addPropertiesCustomizer(new AiConfigCustomizer())
+        .addPropertiesCustomizer(
+            otelConfig -> {
+              Map<String, String> props = new HashMap<>();
+              if (isAksAttach()) {
+                String metricsExporter = otelConfig.getString("otel.metrics.exporter");
+                String amle =
+                    otelConfig.getString("applicationinsights.metrics.to.loganalytics.enabled");
+                props.put(
+                    "otel.metrics.exporter",
+                    conditionallyAddAzureMonitorExporter(metricsExporter, amle));
+              }
+              return props;
+            })
         .addSpanExporterCustomizer(
             (spanExporter, configProperties) -> {
               if (spanExporter instanceof AzureMonitorSpanExporterProvider.MarkerSpanExporter) {
@@ -798,4 +811,46 @@ private static CompletableResultCode flushAll(
         });
     return overallResult;
   }
+
+  private static boolean isAksAttach() {
+    return !Strings.isNullOrEmpty(System.getenv("AKS_ARM_NAMESPACE_ID"));
+  }
+
+  // visible for tests
+  // Per spec: when amle=true, ensure azure_monitor is included; otherwise respect user's setting
+  // https://github.com/aep-health-and-standards/Telemetry-Collection-Spec/blob/main/ApplicationInsights/AutoAttach_Env_Vars.md#metrics-exporter
+  static String conditionallyAddAzureMonitorExporter(String metricsExporter, String amle) {
+
+    // Default to azure_monitor when not set
+    if (Strings.isNullOrEmpty(metricsExporter)) {
+      // Note: this won't really happen since we default otel.metrics.exporter
+      // already in the PropertiesSupplier above which runs before this
+      return AzureMonitorExporterProviderKeys.EXPORTER_NAME;
+    }
+
+    // When amle=true, ensure azure_monitor is included
+    if ("true".equals(amle)) {
+      if ("none".equals(metricsExporter)) {
+        return AzureMonitorExporterProviderKeys.EXPORTER_NAME;
+      }
+      if (!containsAzureMonitor(metricsExporter)) {
+        return metricsExporter + "," + AzureMonitorExporterProviderKeys.EXPORTER_NAME;
+      }
+    }
+
+    return metricsExporter;
+  }
+
+  // visible for tests
+  static boolean containsAzureMonitor(String metricsExporter) {
+    if (metricsExporter == null) {
+      return false;
+    }
+    for (String exporter : metricsExporter.split(",")) {
+      if (AzureMonitorExporterProviderKeys.EXPORTER_NAME.equals(exporter.trim())) {
+        return true;
+      }
+    }
+    return false;
+  }
 }

agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPointTest.java

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.microsoft.applicationinsights.agent.internal.init;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.stream.Stream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class SecondEntryPointTest {
+
+  // Test cases matching the spec table in
+  // https://github.com/aep-health-and-standards/Telemetry-Collection-Spec/blob/main/ApplicationInsights/AutoAttach_Env_Vars.md#metrics-exporter
+  //
+  // OTEL_METRICS_EXPORTER | AMLE   | azure_monitor included
+  // ----------------------|--------|------------------------
+  static Stream<Arguments> metricsExporterSpecTable() {
+    return Stream.of(
+        // AMLE unset
+        Arguments.of(null, null, true),
+        Arguments.of("none", null, false),
+        Arguments.of("azure_monitor", null, true),
+        Arguments.of("otlp,azure_monitor", null, true),
+        Arguments.of("otlp", null, false),
+        // AMLE=true (always include azure_monitor)
+        Arguments.of(null, "true", true),
+        Arguments.of("none", "true", true),
+        Arguments.of("azure_monitor", "true", true),
+        Arguments.of("otlp,azure_monitor", "true", true),
+        Arguments.of("otlp", "true", true),
+        // AMLE=false (same as unset)
+        Arguments.of(null, "false", true),
+        Arguments.of("none", "false", false),
+        Arguments.of("azure_monitor", "false", true),
+        Arguments.of("otlp,azure_monitor", "false", true),
+        Arguments.of("otlp", "false", false));
+  }
+
+  @ParameterizedTest(name = "exporter={0}, amle={1} -> included={2}")
+  @MethodSource("metricsExporterSpecTable")
+  void testUpdateMetricsExporter(String exporter, String amle, boolean expectAzureMonitor) {
+    String result = SecondEntryPoint.conditionallyAddAzureMonitorExporter(exporter, amle);
+    assertThat(SecondEntryPoint.containsAzureMonitor(result)).isEqualTo(expectAzureMonitor);
+  }
+}

smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpLogAnalyticsOnAksTest.java

@@ -0,0 +1,231 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.microsoft.applicationinsights.smoketest;
+
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11_OPENJ9;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_17;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_17_OPENJ9;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_21;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_21_OPENJ9;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_25;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_25_OPENJ9;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8;
+import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8_OPENJ9;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+import static org.mockserver.model.HttpRequest.request;
+import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
+
+import com.microsoft.applicationinsights.smoketest.schemav2.Data;
+import com.microsoft.applicationinsights.smoketest.schemav2.Envelope;
+import com.microsoft.applicationinsights.smoketest.schemav2.MetricData;
+import com.microsoft.applicationinsights.smoketest.schemav2.RequestData;
+import io.opentelemetry.proto.metrics.v1.Metric;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.mockserver.model.HttpRequest;
+import org.springframework.boot.test.context.SpringBootTest;
+
+@SpringBootTest(
+    classes = {OtlpApplication.class},
+    webEnvironment = RANDOM_PORT)
+@UseAgent
+abstract class OtlpLogAnalyticsOnAksTest {
+
+  @RegisterExtension
+  static final SmokeTestExtension testing =
+      SmokeTestExtension.builder()
+          .setEnvVar("APPLICATIONINSIGHTS_METRICS_TO_LOGANALYTICS_ENABLED", "true")
+          .setEnvVar("AKS_ARM_NAMESPACE_ID", "dummy-aks-namespace")
+          .useOtlpEndpointOnly()
+          .build();
+
+  @Test
+  @TargetUri("/ping")
+  public void testOtlpTelemetry() throws Exception {
+    // verify request sent to breeze endpoint
+    List<Envelope> rdList = testing.mockedIngestion.waitForItems("RequestData", 1);
+    Envelope rdEnvelope = rdList.get(0);
+    RequestData rd = (RequestData) ((Data<?>) rdEnvelope.getData()).getBaseData();
+    assertThat(rd.getName()).isEqualTo("GET /OtlpMetrics/ping");
+
+    // verify custom histogram metric sent to Application Insights endpoint
+    List<Envelope> metricList =
+        testing.mockedIngestion.waitForItems(
+            "MetricData", OtlpLogAnalyticsOnAksTest::isHistogramMetric, 1);
+    Envelope metricEnvelope = metricList.get(0);
+    MetricData metricData = (MetricData) ((Data<?>) metricEnvelope.getData()).getBaseData();
+    assertThat(metricData.getMetrics().get(0).getName()).isEqualTo("histogram-test-otlp-exporter");
+
+    // verify stable otel metric sent to Application Insights endpoint
+    List<Envelope> stableOtelMetrics =
+        testing.mockedIngestion.waitForItems(
+            "MetricData", OtlpLogAnalyticsOnAksTest::isStableOtelMetric, 1);
+    Envelope stableOtelMetricEnvelope = stableOtelMetrics.get(0);
+    assertThat(
+            ((MetricData) ((Data<?>) stableOtelMetricEnvelope.getData()).getBaseData())
+                .getMetrics()
+                .get(0)
+                .getName())
+        .isEqualTo("http.server.request.duration");
+
+    // verify pre-aggregated standard metric sent to Application Insights endpoint
+    List<Envelope> standardMetrics =
+        testing.mockedIngestion.waitForStandardMetricItems("requests/duration", 1);
+    Envelope standardMetricEnvelope = standardMetrics.get(0);
+    MetricData standardMetricData =
+        (MetricData) ((Data<?>) standardMetricEnvelope.getData()).getBaseData();
+    assertThat(standardMetricData.getMetrics().get(0).getName())
+        .isEqualTo("http.server.request.duration");
+    assertThat(standardMetricData.getProperties().get("_MS.IsAutocollected")).isEqualTo("True");
+
+    // verify Statsbeat sent to the breeze endpoint
+    verifyStatsbeatSentToBreezeEndpoint();
+
+    // verify custom histogram metric 'histogram-test-otlp-exporter' and otel metric
+    // 'http.server.request.duration' sent to OTLP endpoint
+    // verify Statsbeat doesn't get sent to OTLP endpoint
+    verifyMetricsSentToOtlpEndpoint();
+  }
+
+  @SuppressWarnings("PreferJavaTimeOverload") // legacy time API required for backward compatibility
+  private void verifyMetricsSentToOtlpEndpoint() {
+    await()
+        .atMost(60, SECONDS)
+        .untilAsserted(
+            () -> {
+              HttpRequest[] requests =
+                  testing
+                      .mockedOtlpIngestion
+                      .getCollectorServer()
+                      .retrieveRecordedRequests(request());
+
+              // verify metrics
+              List<Metric> metrics =
+                  testing.mockedOtlpIngestion.extractMetricsFromRequests(requests);
+              assertThat(metrics)
+                  .extracting(Metric::getName)
+                  .contains("histogram-test-otlp-exporter", "http.server.request.duration")
+                  .doesNotContain("Attach", "Feature"); // statsbeat
+            });
+  }
+
+  private static boolean isHistogramMetric(Envelope envelope) {
+    if (envelope.getData().getBaseType().equals("MetricData")) {
+      MetricData data = (MetricData) ((Data<?>) envelope.getData()).getBaseData();
+      return data.getMetrics().get(0).getName().equals("histogram-test-otlp-exporter");
+    }
+    return false;
+  }
+
+  private static boolean isStableOtelMetric(Envelope envelope) {
+    if (envelope.getData().getBaseType().equals("MetricData")) {
+      MetricData data = (MetricData) ((Data<?>) envelope.getData()).getBaseData();
+      return data.getMetrics().get(0).getName().equals("http.server.request.duration")
+          && data.getProperties().get("http.response.status_code") != null;
+    }
+    return false;
+  }
+
+  private void verifyStatsbeatSentToBreezeEndpoint() throws Exception {
+    List<Envelope> statsbeatMetricList =
+        testing.mockedIngestion.waitForItems(
+            "MetricData", OtlpLogAnalyticsOnAksTest::isAttachStatsbeat, 1);
+    Envelope statsbeatEnvelope = statsbeatMetricList.get(0);
+    MetricData statsbeatMetricData =
+        (MetricData) ((Data<?>) statsbeatEnvelope.getData()).getBaseData();
+    assertThat(statsbeatMetricData.getMetrics().get(0).getName()).isEqualTo("Attach");
+    assertThat(statsbeatMetricData.getProperties().get("rp")).isNotNull();
+    assertThat(statsbeatMetricData.getProperties().get("attach")).isEqualTo("StandaloneAuto");
+
+    List<Envelope> features =
+        testing.mockedIngestion.waitForItems(
+            "MetricData", OtlpLogAnalyticsOnAksTest::isFeatureStatsbeat, 2);
+    Envelope featureEnvelope = features.get(0);
+    MetricData featureMetricData = (MetricData) ((Data<?>) featureEnvelope.getData()).getBaseData();
+    assertThat(featureMetricData.getMetrics().get(0).getName()).isEqualTo("Feature");
+    assertThat(featureMetricData.getProperties().get("type")).isNotEmpty();
+
+    List<Envelope> requestSuccessCounts =
+        testing.mockedIngestion.waitForItems(
+            "MetricData", OtlpLogAnalyticsOnAksTest::isRequestSuccessCount, 1);
+    Envelope rscEnvelope = requestSuccessCounts.get(0);
+    MetricData rscMetricData = (MetricData) ((Data<?>) rscEnvelope.getData()).getBaseData();
+    assertThat(rscMetricData.getMetrics().get(0).getName()).isEqualTo("Request_Success_Count");
+    assertThat(rscMetricData.getProperties().get("endpoint")).isEqualTo("breeze");
+
+    List<Envelope> requestDurations =
+        testing.mockedIngestion.waitForItems(
+            "MetricData", OtlpLogAnalyticsOnAksTest::isRequestDuration, 1);
+    Envelope rdEnvelope = requestDurations.get(0);
+    MetricData rdMetricData = (MetricData) ((Data<?>) rdEnvelope.getData()).getBaseData();
+    assertThat(rdMetricData.getMetrics().get(0).getName()).isEqualTo("Request_Duration");
+    assertThat(rdMetricData.getProperties().get("endpoint")).isEqualTo("breeze");
+  }
+
+  private static boolean isAttachStatsbeat(Envelope envelope) {
+    if (envelope.getData().getBaseType().equals("MetricData")) {
+      MetricData data = (MetricData) ((Data<?>) envelope.getData()).getBaseData();
+      return data.getMetrics().get(0).getName().equals("Attach");
+    }
+    return false;
+  }
+
+  private static boolean isFeatureStatsbeat(Envelope envelope) {
+    if (envelope.getData().getBaseType().equals("MetricData")) {
+      MetricData data = (MetricData) ((Data<?>) envelope.getData()).getBaseData();
+      return data.getMetrics().get(0).getName().equals("Feature");
+    }
+    return false;
+  }
+
+  private static boolean isRequestSuccessCount(Envelope envelope) {
+    if (envelope.getData().getBaseType().equals("MetricData")) {
+      MetricData data = (MetricData) ((Data<?>) envelope.getData()).getBaseData();
+      return data.getMetrics().get(0).getName().equals("Request_Success_Count");
+    }
+    return false;
+  }
+
+  private static boolean isRequestDuration(Envelope envelope) {
+    if (envelope.getData().getBaseType().equals("MetricData")) {
+      MetricData data = (MetricData) ((Data<?>) envelope.getData()).getBaseData();
+      return data.getMetrics().get(0).getName().equals("Request_Duration");
+    }
+    return false;
+  }
+
+  @Environment(TOMCAT_8_JAVA_8)
+  static class Tomcat8Java8Test extends OtlpLogAnalyticsOnAksTest {}
+
+  @Environment(TOMCAT_8_JAVA_8_OPENJ9)
+  static class Tomcat8Java8OpenJ9Test extends OtlpLogAnalyticsOnAksTest {}
+
+  @Environment(TOMCAT_8_JAVA_11)
+  static class Tomcat8Java11Test extends OtlpLogAnalyticsOnAksTest {}
+
+  @Environment(TOMCAT_8_JAVA_11_OPENJ9)
+  static class Tomcat8Java11OpenJ9Test extends OtlpLogAnalyticsOnAksTest {}
+
+  @Environment(TOMCAT_8_JAVA_17)
+  static class Tomcat8Java17Test extends OtlpLogAnalyticsOnAksTest {}
+
+  @Environment(TOMCAT_8_JAVA_17_OPENJ9)
+  static class Tomcat8Java17OpenJ9Test extends OtlpLogAnalyticsOnAksTest {}
+
+  @Environment(TOMCAT_8_JAVA_21)
+  static class Tomcat8Java21Test extends OtlpLogAnalyticsOnAksTest {}
+
+  @Environment(TOMCAT_8_JAVA_21_OPENJ9)
+  static class Tomcat8Java21OpenJ9Test extends OtlpLogAnalyticsOnAksTest {}
+
+  @Environment(TOMCAT_8_JAVA_25)
+  static class Tomcat8Java23Test extends OtlpLogAnalyticsOnAksTest {}
+
+  @Environment(TOMCAT_8_JAVA_25_OPENJ9)
+  static class Tomcat8Java23OpenJ9Test extends OtlpLogAnalyticsOnAksTest {}
+}