HDDS-12442. Add latency metrics for OM deletion services (apache#7986)

Author: Tejaskriya

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMPerformanceMetrics.java

@@ -21,6 +21,7 @@
 import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MutableGaugeFloat;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
 import org.apache.hadoop.metrics2.lib.MutableRate;
 
 /**
@@ -151,6 +152,15 @@ public static void unregister() {
   @Metric(about = "ACLs check in getObjectTagging")
   private MutableRate getObjectTaggingAclCheckLatencyNs;
 
+  @Metric(about = "Latency of each iteration of DirectoryDeletingService in ms")
+  private MutableGaugeLong directoryDeletingServiceLatencyMs;
+
+  @Metric(about = "Latency of each iteration of KeyDeletingService in ms")
+  private MutableGaugeLong keyDeletingServiceLatencyMs;
+
+  @Metric(about = "Latency of each iteration of OpenKeyCleanupService in ms")
+  private MutableGaugeLong openKeyCleanupServiceLatencyMs;
+
   public void addLookupLatency(long latencyInNs) {
     lookupLatencyNs.add(latencyInNs);
   }
@@ -299,4 +309,16 @@ public MutableRate getGetObjectTaggingAclCheckLatencyNs() {
   public void addGetObjectTaggingLatencyNs(long latencyInNs) {
     getObjectTaggingAclCheckLatencyNs.add(latencyInNs);
   }
+
+  public void setDirectoryDeletingServiceLatencyMs(long latencyInMs) {
+    directoryDeletingServiceLatencyMs.set(latencyInMs);
+  }
+
+  public void setKeyDeletingServiceLatencyMs(long latencyInMs) {
+    keyDeletingServiceLatencyMs.set(latencyInMs);
+  }
+
+  public void setOpenKeyCleanupServiceLatencyMs(long latencyInMs) {
+    openKeyCleanupServiceLatencyMs.set(latencyInMs);
+  }
 }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java

@@ -47,6 +47,7 @@
 import org.apache.hadoop.ozone.om.KeyManager;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
 import org.apache.hadoop.ozone.om.OzoneManager;
 import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -73,6 +74,7 @@ public abstract class AbstractKeyDeletingService extends BackgroundService
 
   private final OzoneManager ozoneManager;
   private final DeletingServiceMetrics metrics;
+  private final OMPerformanceMetrics perfMetrics;
   private final ScmBlockLocationProtocol scmClient;
   private final ClientId clientId = ClientId.randomId();
   private final AtomicLong deletedDirsCount;
@@ -94,6 +96,7 @@ public AbstractKeyDeletingService(String serviceName, long interval,
     this.movedFilesCount = new AtomicLong(0);
     this.runCount = new AtomicLong(0);
     this.metrics = ozoneManager.getDeletionMetrics();
+    this.perfMetrics = ozoneManager.getPerfMetrics();
   }
 
   protected int processKeyDeletes(List<BlockGroup> keyBlocksList,
@@ -119,14 +122,15 @@ protected int processKeyDeletes(List<BlockGroup> keyBlocksList,
     LOG.info("{} BlockGroup deletion are acked by SCM in {} ms",
         keyBlocksList.size(), Time.monotonicNow() - startTime);
     if (blockDeletionResults != null) {
-      startTime = Time.monotonicNow();
+      long purgeStartTime = Time.monotonicNow();
       delCount = submitPurgeKeysRequest(blockDeletionResults,
           keysToModify, snapTableKey, expectedPreviousSnapshotId);
       int limit = ozoneManager.getConfiguration().getInt(OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK,
           OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT);
       LOG.info("Blocks for {} (out of {}) keys are deleted from DB in {} ms. Limit per task is {}.",
-          delCount, blockDeletionResults.size(), Time.monotonicNow() - startTime, limit);
+          delCount, blockDeletionResults.size(), Time.monotonicNow() - purgeStartTime, limit);
     }
+    perfMetrics.setKeyDeletingServiceLatencyMs(Time.monotonicNow() - startTime);
     return delCount;
   }
 
@@ -416,6 +420,7 @@ public void optimizeDirDeletesAndSubmitRequest(
           dirNum, subdirDelNum, subFileNum, (subDirNum - subdirDelNum),
           timeTakenInIteration, rnCnt);
       metrics.incrementDirectoryDeletionTotalMetrics(dirNum + subdirDelNum, subDirNum, subFileNum);
+      perfMetrics.setDirectoryDeletingServiceLatencyMs(timeTakenInIteration);
     }
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/OpenKeyCleanupService.java

@@ -237,9 +237,11 @@ public BackgroundTaskResult call() throws Exception {
         });
       }
 
+      long timeTaken = Time.monotonicNow() - startTime;
       LOG.info("Number of expired open keys submitted for deletion: {},"
               + " for commit: {}, cleanupLimit: {}, elapsed time: {}ms",
-          numOpenKeys, numHsyncKeys, cleanupLimitPerTask, Time.monotonicNow() - startTime);
+          numOpenKeys, numHsyncKeys, cleanupLimitPerTask, timeTaken);
+      ozoneManager.getPerfMetrics().setOpenKeyCleanupServiceLatencyMs(timeTaken);
       final int numKeys = numOpenKeys + numHsyncKeys;
       submittedOpenKeyCount.addAndGet(numKeys);
       return () -> numKeys;