HDDS-11642. MutableQuantiles should be stopped (apache#7392)

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java

@@ -30,6 +30,7 @@
 import org.apache.hadoop.metrics2.lib.MutableQuantiles;
 import org.apache.hadoop.metrics2.lib.MutableRate;
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.util.MetricUtil;
 
 import java.util.Map;
 import java.util.UUID;
@@ -98,6 +99,7 @@ public static synchronized void release() {
     }
     referenceCount--;
     if (referenceCount == 0) {
+      instance.stop();
       DefaultMetricsSystem.instance().unregisterSource(
           SOURCE_NAME + instanceCount);
       instance = null;
@@ -156,6 +158,17 @@ private ContainerClientMetrics() {
     }
   }
 
+  public void stop() {
+    MetricUtil.stop(listBlockLatency);
+    MetricUtil.stop(getBlockLatency);
+    MetricUtil.stop(getCommittedBlockLengthLatency);
+    MetricUtil.stop(readChunkLatency);
+    MetricUtil.stop(getSmallFileLatency);
+    MetricUtil.stop(hsyncLatencyNs);
+    MetricUtil.stop(omHsyncLatencyNs);
+    MetricUtil.stop(datanodeHsyncLatencyNs);
+  }
+
   public void recordWriteChunk(Pipeline pipeline, long chunkSizeBytes) {
     writeChunkCallsByPipeline.computeIfAbsent(pipeline.getId(),
         pipelineID -> registry.newCounter(

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java

@@ -21,6 +21,7 @@
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.hadoop.metrics2.MetricsCollector;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.metrics2.MetricsSource;
@@ -129,6 +130,7 @@ public void reset() {
   }
 
   public void unRegister() {
+    IOUtils.closeQuietly(containerOpsLatency.values());
     MetricsSystem ms = DefaultMetricsSystem.instance();
     ms.unregisterSource(SOURCE_NAME);
   }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.metrics2.lib.MutableRate;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.util.MetricUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -100,6 +101,8 @@ public static synchronized GrpcMetrics create(Configuration conf) {
    */
   public void unRegister() {
     DefaultMetricsSystem.instance().unregisterSource(SOURCE_NAME);
+    MetricUtil.stop(grpcProcessingTimeMillisQuantiles);
+    MetricUtil.stop(grpcQueueTimeMillisQuantiles);
   }
 
   @Override

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/MetricUtil.java

@@ -110,4 +110,20 @@ public static List<MutableQuantiles> createQuantiles(MetricsRegistry registry,
           sampleName, valueName, interval);
     }).collect(Collectors.toList());
   }
+
+  public static void stop(MutableQuantiles... quantiles) {
+    if (quantiles != null) {
+      stop(Arrays.asList(quantiles));
+    }
+  }
+
+  public static void stop(Iterable<MutableQuantiles> quantiles) {
+    if (quantiles != null) {
+      for (MutableQuantiles q : quantiles) {
+        if (q != null) {
+          q.stop();
+        }
+      }
+    }
+  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetrics.java

@@ -22,15 +22,17 @@
 import org.apache.hadoop.metrics2.lib.MutableQuantiles;
 import org.apache.hadoop.metrics2.lib.MutableStat;
 
+import java.io.Closeable;
 import java.util.List;
+import java.util.Map;
 
 /**
  * The {@code PerformanceMetrics} class encapsulates a collection of related
  * metrics including a MutableStat, MutableQuantiles, and a MutableMinMax.
  * This class provides methods to update these metrics and to
  * snapshot their values for reporting.
  */
-public class PerformanceMetrics {
+public class PerformanceMetrics implements Closeable {
   private final MutableStat stat;
   private final List<MutableQuantiles> quantiles;
   private final MutableMinMax minMax;
@@ -43,12 +45,13 @@ public class PerformanceMetrics {
    * @param intervals the intervals for quantiles computation. Note, each
    *        interval in 'intervals' increases memory usage, as it corresponds
    *        to a separate quantile calculator.
+   * @return {@link PerformanceMetrics} instances created, mapped by field name
    */
-  public static synchronized <T> void initializeMetrics(T source,
+  public static synchronized <T> Map<String, PerformanceMetrics> initializeMetrics(T source,
       MetricsRegistry registry, String sampleName, String valueName,
       int[] intervals) {
     try {
-      PerformanceMetricsInitializer.initialize(
+      return PerformanceMetricsInitializer.initialize(
           source, registry, sampleName, valueName, intervals);
     } catch (IllegalAccessException e) {
       throw new RuntimeException("Failed to initialize PerformanceMetrics", e);
@@ -73,6 +76,11 @@ public PerformanceMetrics(
     minMax = new MutableMinMax(registry, name, description, valueName);
   }
 
+  @Override
+  public void close() {
+    MetricUtil.stop(quantiles);
+  }
+
   /**
    * Adds a value to all the aggregated metrics.
    *
@@ -95,6 +103,5 @@ public void snapshot(MetricsRecordBuilder recordBuilder, boolean all) {
     this.quantiles.forEach(quantile -> quantile.snapshot(recordBuilder, all));
     this.minMax.snapshot(recordBuilder, all);
   }
-
 }
 

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/PerformanceMetricsInitializer.java

@@ -21,6 +21,8 @@
 import org.apache.hadoop.metrics2.lib.MetricsRegistry;
 
 import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * Utility class for initializing PerformanceMetrics in a MetricsSource.
@@ -36,11 +38,13 @@ private PerformanceMetricsInitializer() { }
    * @param sampleName sample name
    * @param valueName value name
    * @param intervals intervals for quantiles
+   * @return {@link PerformanceMetrics} instances created, mapped by field name
    * @throws IllegalAccessException if unable to access the field
    */
-  public static <T> void initialize(T source, MetricsRegistry registry,
+  public static <T> Map<String, PerformanceMetrics> initialize(T source, MetricsRegistry registry,
       String sampleName, String valueName, int[] intervals)
       throws IllegalAccessException {
+    Map<String, PerformanceMetrics> instances = new HashMap<>();
     Field[] fields = source.getClass().getDeclaredFields();
 
     for (Field field : fields) {
@@ -54,8 +58,11 @@ public static <T> void initialize(T source, MetricsRegistry registry,
                   sampleName, valueName, intervals);
           field.setAccessible(true);
           field.set(source, performanceMetrics);
+          instances.put(name, performanceMetrics);
         }
       }
     }
+
+    return instances;
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java

@@ -30,7 +30,9 @@
 import org.apache.hadoop.metrics2.lib.MutableCounterLong;
 import org.apache.hadoop.metrics2.lib.MutableQuantiles;
 import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.ozone.util.MetricUtil;
 
+import java.io.Closeable;
 import java.util.EnumMap;
 
 /**
@@ -47,7 +49,7 @@
  */
 @InterfaceAudience.Private
 @Metrics(about = "Storage Container DataNode Metrics", context = "dfs")
-public class ContainerMetrics {
+public class ContainerMetrics implements Closeable {
   public static final String STORAGE_CONTAINER_METRICS =
       "StorageContainerMetrics";
   @Metric private MutableCounterLong numOps;
@@ -105,6 +107,11 @@ public static void remove() {
     ms.unregisterSource(STORAGE_CONTAINER_METRICS);
   }
 
+  @Override
+  public void close() {
+    opsLatQuantiles.values().forEach(MetricUtil::stop);
+  }
+
   public void incContainerOpsMetrics(ContainerProtos.Type type) {
     numOps.incr();
     numOpsArray.get(type).incr();

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java

@@ -34,6 +34,7 @@
 import org.apache.hadoop.hdds.security.token.TokenVerifier;
 import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
+import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.hadoop.ozone.HddsDatanodeService;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
 import org.apache.hadoop.ozone.container.common.impl.BlockDeletingService;
@@ -512,6 +513,7 @@ public void stop() {
     }
     blockDeletingService.shutdown();
     recoveringContainerScrubbingService.shutdown();
+    IOUtils.closeQuietly(metrics);
     ContainerMetrics.remove();
   }
 

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/metrics/S3GatewayMetrics.java

@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.hadoop.metrics2.MetricsCollector;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.metrics2.MetricsSource;
@@ -33,12 +34,15 @@
 import org.apache.hadoop.ozone.util.PerformanceMetrics;
 import org.apache.hadoop.util.Time;
 
+import java.io.Closeable;
+import java.util.Map;
+
 /**
  * This class maintains S3 Gateway related metrics.
  */
 @InterfaceAudience.Private
 @Metrics(about = "S3 Gateway Metrics", context = OzoneConsts.OZONE)
-public final class S3GatewayMetrics implements MetricsSource {
+public final class S3GatewayMetrics implements Closeable, MetricsSource {
 
   public static final String SOURCE_NAME =
       S3GatewayMetrics.class.getSimpleName();
@@ -242,17 +246,24 @@ public final class S3GatewayMetrics implements MetricsSource {
   @Metric(about = "Latency for copy metadata of an key in nanoseconds")
   private PerformanceMetrics copyKeyMetadataLatencyNs;
 
+  private final Map<String, PerformanceMetrics> performanceMetrics;
+
   /**
    * Private constructor.
    */
   private S3GatewayMetrics(OzoneConfiguration conf) {
     this.registry = new MetricsRegistry(SOURCE_NAME);
     int[] intervals = conf.getInts(S3GatewayConfigKeys
         .OZONE_S3G_METRICS_PERCENTILES_INTERVALS_SECONDS_KEY);
-    PerformanceMetrics.initializeMetrics(
+    performanceMetrics = PerformanceMetrics.initializeMetrics(
         this, registry, "Ops", "Time", intervals);
   }
 
+  @Override
+  public void close() {
+    IOUtils.closeQuietly(performanceMetrics.values());
+  }
+
   /**
    * Create and returns S3 Gateway Metrics instance.
    *
@@ -272,6 +283,7 @@ public static synchronized S3GatewayMetrics create(OzoneConfiguration conf) {
    * Unregister the metrics instance.
    */
   public static void unRegister() {
+    IOUtils.closeQuietly(instance);
     instance = null;
     MetricsSystem ms = DefaultMetricsSystem.instance();
     ms.unregisterSource(SOURCE_NAME);

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java

@@ -75,7 +75,8 @@
         DatanodeSimulator.class,
         OmMetadataGenerator.class,
         DNRPCLoadGenerator.class,
-        HsyncGenerator.class
+        HsyncGenerator.class,
+        OzoneClientCreator.class,
     },
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true)

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientCreator.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import com.codahale.metrics.Timer;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Creates and closes Ozone clients.
+ */
+@CommandLine.Command(name = "occ",
+    aliases = "ozone-client-creator",
+    description = "Create and close Ozone clients without doing anything useful",
+    versionProvider = HddsVersionProvider.class,
+    mixinStandardHelpOptions = true,
+    showDefaultValues = true)
+public class OzoneClientCreator extends BaseFreonGenerator implements Callable<Void> {
+
+  @CommandLine.Option(names = "--om-service-id",
+      description = "OM Service ID"
+  )
+  private String omServiceID;
+
+  private Timer timer;
+  private OzoneConfiguration conf;
+
+  @Override
+  public Void call() {
+    init();
+    conf = createOzoneConfiguration();
+    timer = getMetrics().timer("client-create");
+    runTests(this::createClient);
+    return null;
+  }
+
+  private void createClient(long step) {
+    timer.time(this::createClientSafely);
+  }
+
+  private void createClientSafely() {
+    try {
+      createOzoneClient(omServiceID, conf).close();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}