feat: Add Metrics host for built in metrics (#3519)

This PR allows users to set the custom monitoring host for built in metrics.

google-cloud-spanner/clirr-ignored-differences.xml

@@ -709,6 +709,13 @@
     <method>boolean isEnableBuiltInMetrics()</method>
   </difference>
 
+  <!-- Added Monitoring host option -->
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/spanner/SpannerOptions$SpannerEnvironment</className>
+    <method>java.lang.String getMonitoringHost()</method>
+  </difference>
+
   <!-- Added ExcludeTxnFromChangeStreams -->
   <difference>
     <differenceType>7012</differenceType>

google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java

@@ -59,12 +59,14 @@ final class BuiltInOpenTelemetryMetricsProvider {
 
   private BuiltInOpenTelemetryMetricsProvider() {}
 
-  OpenTelemetry getOrCreateOpenTelemetry(String projectId, @Nullable Credentials credentials) {
+  OpenTelemetry getOrCreateOpenTelemetry(
+      String projectId, @Nullable Credentials credentials, @Nullable String monitoringHost) {
     try {
       if (this.openTelemetry == null) {
         SdkMeterProviderBuilder sdkMeterProviderBuilder = SdkMeterProvider.builder();
         BuiltInOpenTelemetryMetricsView.registerBuiltinMetrics(
-            SpannerCloudMonitoringExporter.create(projectId, credentials), sdkMeterProviderBuilder);
+            SpannerCloudMonitoringExporter.create(projectId, credentials, monitoringHost),
+            sdkMeterProviderBuilder);
         SdkMeterProvider sdkMeterProvider = sdkMeterProviderBuilder.build();
         this.openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
         Runtime.getRuntime().addShutdownHook(new Thread(sdkMeterProvider::close));

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java

@@ -29,7 +29,6 @@
 import com.google.cloud.monitoring.v3.MetricServiceClient;
 import com.google.cloud.monitoring.v3.MetricServiceSettings;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.MoreObjects;
 import com.google.common.collect.Iterables;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.monitoring.v3.CreateTimeSeriesRequest;
@@ -63,13 +62,6 @@ class SpannerCloudMonitoringExporter implements MetricExporter {
   private static final Logger logger =
       Logger.getLogger(SpannerCloudMonitoringExporter.class.getName());
 
-  // This system property can be used to override the monitoring endpoint
-  // to a different environment. It's meant for internal testing only.
-  private static final String MONITORING_ENDPOINT =
-      MoreObjects.firstNonNull(
-          System.getProperty("spanner.test-monitoring-endpoint"),
-          MetricServiceSettings.getDefaultEndpoint());
-
   // This the quota limit from Cloud Monitoring. More details in
   // https://cloud.google.com/monitoring/quotas#custom_metrics_quotas.
   private static final int EXPORT_BATCH_SIZE_LIMIT = 200;
@@ -78,7 +70,8 @@ class SpannerCloudMonitoringExporter implements MetricExporter {
   private final MetricServiceClient client;
   private final String spannerProjectId;
 
-  static SpannerCloudMonitoringExporter create(String projectId, @Nullable Credentials credentials)
+  static SpannerCloudMonitoringExporter create(
+      String projectId, @Nullable Credentials credentials, @Nullable String monitoringHost)
       throws IOException {
     MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder();
     CredentialsProvider credentialsProvider;
@@ -88,7 +81,9 @@ static SpannerCloudMonitoringExporter create(String projectId, @Nullable Credent
       credentialsProvider = FixedCredentialsProvider.create(credentials);
     }
     settingsBuilder.setCredentialsProvider(credentialsProvider);
-    settingsBuilder.setEndpoint(MONITORING_ENDPOINT);
+    if (monitoringHost != null) {
+      settingsBuilder.setEndpoint(monitoringHost);
+    }
 
     org.threeten.bp.Duration timeout = Duration.ofMinutes(1);
     // TODO: createServiceTimeSeries needs special handling if the request failed. Leaving

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java

@@ -164,6 +164,7 @@ public class SpannerOptions extends ServiceOptions<Spanner, SpannerOptions> {
   private final boolean enableBuiltInMetrics;
   private final boolean enableExtendedTracing;
   private final boolean enableEndToEndTracing;
+  private final String monitoringHost;
 
   enum TracingFramework {
     OPEN_CENSUS,
@@ -672,6 +673,7 @@ protected SpannerOptions(Builder builder) {
     enableExtendedTracing = builder.enableExtendedTracing;
     enableBuiltInMetrics = builder.enableBuiltInMetrics;
     enableEndToEndTracing = builder.enableEndToEndTracing;
+    monitoringHost = builder.monitoringHost;
   }
 
   /**
@@ -712,6 +714,10 @@ default boolean isEnableBuiltInMetrics() {
     default boolean isEnableEndToEndTracing() {
       return false;
     }
+
+    default String getMonitoringHost() {
+      return null;
+    }
   }
 
   /**
@@ -728,6 +734,7 @@ private static class SpannerEnvironmentImpl implements SpannerEnvironment {
     private static final String SPANNER_ENABLE_END_TO_END_TRACING =
         "SPANNER_ENABLE_END_TO_END_TRACING";
     private static final String SPANNER_DISABLE_BUILTIN_METRICS = "SPANNER_DISABLE_BUILTIN_METRICS";
+    private static final String SPANNER_MONITORING_HOST = "SPANNER_MONITORING_HOST";
 
     private SpannerEnvironmentImpl() {}
 
@@ -763,6 +770,11 @@ public boolean isEnableBuiltInMetrics() {
     public boolean isEnableEndToEndTracing() {
       return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_END_TO_END_TRACING));
     }
+
+    @Override
+    public String getMonitoringHost() {
+      return System.getenv(SPANNER_MONITORING_HOST);
+    }
   }
 
   /** Builder for {@link SpannerOptions} instances. */
@@ -828,6 +840,7 @@ public static class Builder
     private boolean enableExtendedTracing = SpannerOptions.environment.isEnableExtendedTracing();
     private boolean enableEndToEndTracing = SpannerOptions.environment.isEnableEndToEndTracing();
     private boolean enableBuiltInMetrics = SpannerOptions.environment.isEnableBuiltInMetrics();
+    private String monitoringHost = SpannerOptions.environment.getMonitoringHost();
 
     private static String createCustomClientLibToken(String token) {
       return token + " " + ServiceOptions.getGoogApiClientLibName();
@@ -895,6 +908,7 @@ protected Builder() {
       this.enableExtendedTracing = options.enableExtendedTracing;
       this.enableBuiltInMetrics = options.enableBuiltInMetrics;
       this.enableEndToEndTracing = options.enableEndToEndTracing;
+      this.monitoringHost = options.monitoringHost;
     }
 
     @Override
@@ -1417,6 +1431,12 @@ public Builder setBuiltInMetricsEnabled(boolean enableBuiltInMetrics) {
       return this;
     }
 
+    /** Sets the monitoring host to be used for Built-in client side metrics */
+    public Builder setMonitoringHost(String monitoringHost) {
+      this.monitoringHost = monitoringHost;
+      return this;
+    }
+
     /**
      * Sets whether to enable extended OpenTelemetry tracing. Enabling this option will add the
      * following additional attributes to the traces that are generated by the client:
@@ -1727,7 +1747,7 @@ private ApiTracerFactory getDefaultApiTracerFactory() {
   private ApiTracerFactory createMetricsApiTracerFactory() {
     OpenTelemetry openTelemetry =
         this.builtInOpenTelemetryMetricsProvider.getOrCreateOpenTelemetry(
-            this.getProjectId(), getCredentials());
+            this.getProjectId(), getCredentials(), this.monitoringHost);
 
     return openTelemetry != null
         ? new MetricsTracerFactory(
@@ -1754,6 +1774,11 @@ public boolean isEnableBuiltInMetrics() {
     return enableBuiltInMetrics;
   }
 
+  /** Returns the override metrics Host. */
+  String getMonitoringHost() {
+    return monitoringHost;
+  }
+
   @BetaApi
   public boolean isUseVirtualThreads() {
     return useVirtualThreads;

google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerCloudMonitoringExporterTest.java

@@ -337,7 +337,7 @@ public void testExportingSumDataInBatches() {
   @Test
   public void getAggregationTemporality() throws IOException {
     SpannerCloudMonitoringExporter actualExporter =
-        SpannerCloudMonitoringExporter.create(projectId, null);
+        SpannerCloudMonitoringExporter.create(projectId, null, null);
     assertThat(actualExporter.getAggregationTemporality(InstrumentType.COUNTER))
         .isEqualTo(AggregationTemporality.CUMULATIVE);
   }
@@ -348,7 +348,7 @@ public void testSkipExportingDataIfMissingInstanceId() throws IOException {
         Attributes.builder().putAll(attributes).remove(INSTANCE_ID_KEY).build();
 
     SpannerCloudMonitoringExporter actualExporter =
-        SpannerCloudMonitoringExporter.create(projectId, null);
+        SpannerCloudMonitoringExporter.create(projectId, null, null);
     assertThat(actualExporter.getAggregationTemporality(InstrumentType.COUNTER))
         .isEqualTo(AggregationTemporality.CUMULATIVE);
     ArgumentCaptor<CreateTimeSeriesRequest> argumentCaptor =

google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpannerOptionsTest.java

@@ -754,6 +754,19 @@ public void testEndToEndTracingEnablement() {
             .isEndToEndTracingEnabled());
   }
 
+  @Test
+  public void testmonitoringHost() {
+    String metricsEndpoint = "test-endpoint:443";
+    assertNull(SpannerOptions.newBuilder().setProjectId("p").build().getMonitoringHost());
+    assertThat(
+            SpannerOptions.newBuilder()
+                .setProjectId("p")
+                .setMonitoringHost(metricsEndpoint)
+                .build()
+                .getMonitoringHost())
+        .isEqualTo(metricsEndpoint);
+  }
+
   @Test
   public void testSetDirectedReadOptions() {
     final DirectedReadOptions directedReadOptions =