HDDS-11747. CLI retrieve full key path for matching containers

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java

@@ -33,10 +33,13 @@
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneTestUtils;
 import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneKeyDetails;
 import org.apache.hadoop.ozone.client.OzoneSnapshot;
 import org.apache.hadoop.ozone.debug.DBScanner;
+import org.apache.hadoop.ozone.debug.KeyPathRetriever;
 import org.apache.hadoop.ozone.debug.OzoneDebug;
 import org.apache.hadoop.ozone.debug.RDBParser;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
@@ -74,6 +77,8 @@
 import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Test Ozone Debug shell.
@@ -175,6 +180,37 @@ public void testLdbCliForOzoneSnapshot() throws Exception {
     assertThat(cmdOut).contains(keyName);
   }
 
+  @Test
+  public void testKeyPathRetriever() throws Exception {
+    final String volumeName = UUID.randomUUID().toString();
+    final String bucketName = UUID.randomUUID().toString();
+    StringWriter stdout = new StringWriter();
+    PrintWriter pstdout = new PrintWriter(stdout);
+    String[] args;
+    CommandLine cmd;
+    try (OzoneClient client = OzoneClientFactory.getRpcClient(conf)) {
+      TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED);
+      OzoneBucket bucket = client.getObjectStore().getVolume(volumeName).getBucket(bucketName);
+      bucket.createDirectory("dir1");
+      TestDataUtil.createKey(bucket, "dir1/file1", "test");
+      TestDataUtil.createKey(bucket, "dir1/file2", "test");
+      TestDataUtil.createKey(bucket, "dir1/file3", "test");
+      OzoneKeyDetails key1 = bucket.getKey("dir1/file1");
+      OzoneKeyDetails key3 = bucket.getKey("dir1/file2");
+      StringBuilder sb = new StringBuilder();
+      key1.getOzoneKeyLocations().forEach(e -> sb.append(e.getContainerID()));
+      key3.getOzoneKeyLocations().forEach(e -> sb.append(",").append(e.getContainerID()));
+      String dbFile = OMStorage.getOmDbDir(conf).getPath() + "/om.db";
+      cmd = new CommandLine(new OzoneDebug()).addSubcommand(new CommandLine(new KeyPathRetriever())).setOut(pstdout);
+      args = new String[] {"retrieve-key-fullpath", "--db", dbFile, "--containers", sb.toString()};
+    }
+    int exitCode = cmd.execute(args);
+    assertEquals(0, exitCode);
+    String keyPrefix = volumeName + "/" + bucketName + "/dir1";
+    assertTrue(stdout.toString().contains(keyPrefix + "file1") && stdout.toString().contains(keyPrefix + "file2"));
+    assertFalse(stdout.toString().contains(keyPrefix + "file3"));
+  }
+
   private static String getSnapshotDBPath(String checkPointDir) {
     return OMStorage.getOmDbDir(conf) +
         OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX +

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java

@@ -392,14 +392,14 @@ public static OmMetadataManagerImpl createCheckpointMetadataManager(
   }
 
   /**
-   * Metadata constructor for checkpoints.
+   * Metadata constructor for checkpoints and readonly db.
    *
    * @param conf - Ozone conf.
-   * @param dir - Checkpoint parent directory.
-   * @param name - Checkpoint directory name.
+   * @param dir - parent directory.
+   * @param name - directory name.
    * @throws IOException
    */
-  private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
+  public OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
       throws IOException {
     lock = new OmReadOnlyLock();
     omEpoch = 0;

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/KeyPathRetriever.java

@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.cli.SubcommandWithParent;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.kohsuke.MetaInfServices;
+import picocli.CommandLine;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+
+/**
+ * Tool that retrieve key full path from OM db file table with pattern match.
+ */
+@CommandLine.Command(
+    name = "retrieve-key-fullpath",
+    description = "retrieve full key path with matching criteria")
+@MetaInfServices(SubcommandWithParent.class)
+public class KeyPathRetriever implements Callable<Void>, SubcommandWithParent {
+  @CommandLine.Spec
+  private static CommandLine.Model.CommandSpec spec;
+
+  @CommandLine.Option(names = {"--db"},
+      required = true,
+      description = "Database File")
+  private String dbFile;
+
+  @CommandLine.Option(names = {"--containers"},
+      required = false,
+      description = "Comma separated Container Ids")
+  private String strContainerIds;
+
+  @CommandLine.Option(names = {"--container-file"},
+      required = false,
+      description = "file with container id in new line")
+  private String containerFileName;
+
+  @CommandLine.Option(names = {"--out", "-o"},
+      required = false,
+      description = "out file name")
+  private String fileName;
+
+  @Override
+  public Class<?> getParentType() {
+    return OzoneDebug.class;
+  }
+
+  @Override
+  public Void call() throws Exception {
+    // get containerIds filter
+    Set<Long> containerIds = new HashSet<>();
+    if (!StringUtils.isEmpty(strContainerIds)) {
+      if (!StringUtils.isEmpty(strContainerIds)) {
+        String[] split = strContainerIds.split(",");
+        for (String id : split) {
+          containerIds.add(Long.parseLong(id));
+        }
+      }
+    } else if (!StringUtils.isEmpty(containerFileName)) {
+      try (Stream<String> stream = Files.lines(Paths.get(containerFileName))) {
+        stream.forEach(e -> containerIds.add(Long.parseLong(e)));
+      }
+    }
+    if (containerIds.isEmpty()) {
+      System.out.println("No containers to be filtered");
+      return null;
+    }
+    // out stream
+    PrintWriter writer;
+    if (StringUtils.isEmpty(fileName)) {
+      writer = out();
+    } else {
+      writer = new PrintWriter(new BufferedWriter(new PrintWriter(fileName, UTF_8.name())));
+    }
+
+    // db handler
+    OMMetadataManager metadataManager = getOmMetadataManager(dbFile);
+    if (metadataManager == null) {
+      writer.close();
+      return null;
+    }
+
+    try {
+      retrieve(metadataManager, writer, containerIds);
+    } finally {
+      writer.close();
+      metadataManager.stop();
+    }
+    return null;
+  }
+
+  public void retrieve(
+      OMMetadataManager metadataManager, PrintWriter writer, Set<Long> containerIds) {
+    // build dir tree
+    Map<Long, Pair<Long, String>> dirObjNameParentMap = new HashMap<>();
+    try {
+      prepareDirIdTree(metadataManager, dirObjNameParentMap);
+    } catch (Exception e) {
+      System.out.println("Exception occurred reading directory Table, " + e);
+      return;
+    }
+
+    // iterate file table and filter for container
+    try (TableIterator<String, ? extends KeyValue<String, OmKeyInfo>> fileItr
+            = metadataManager.getFileTable().iterator()) {
+      while (fileItr.hasNext()) {
+        KeyValue<String, OmKeyInfo> next = fileItr.next();
+        boolean found = next.getValue().getKeyLocationVersions().stream().anyMatch(
+            e -> e.getLocationList().stream().anyMatch(
+                blk -> containerIds.contains(blk.getBlockID().getContainerID())));
+        if (found) {
+          StringBuilder sb = new StringBuilder(next.getValue().getKeyName());
+          Long prvParent = next.getValue().getParentObjectID();
+          while (dirObjNameParentMap.containsKey(prvParent)) {
+            Pair<Long, String> nameParentPair = dirObjNameParentMap.get(prvParent);
+            sb.insert(0, nameParentPair.getValue() + OM_KEY_PREFIX);
+            prvParent = nameParentPair.getKey();
+            if (null == prvParent) {
+              // add to output as reached till volume
+              writer.println(sb);
+              break;
+            }
+          }
+        }
+      }
+    } catch (Exception e) {
+      System.out.println("Exception occurred reading file Table, " + e);
+    }
+  }
+
+  private static OMMetadataManager getOmMetadataManager(String db) throws IOException {
+    if (!Files.exists(Paths.get(db))) {
+      System.out.println("DB with path not exist:" + db);
+      return null;
+    }
+    System.out.println("Db Path is:" + db);
+    File file = new File(db);
+
+    OzoneConfiguration conf = new OzoneConfiguration();
+    return new OmMetadataManagerImpl(conf, file.getParentFile(), file.getName());
+  }
+
+  private static void prepareDirIdTree(
+      OMMetadataManager metadataManager, Map<Long, Pair<Long, String>> dirObjNameParentMap) throws IOException {
+    // add bucket volume tree
+    try (TableIterator<String, ? extends KeyValue<String, OmBucketInfo>> bucItr
+             = metadataManager.getBucketTable().iterator()) {
+      while (bucItr.hasNext()) {
+        KeyValue<String, OmBucketInfo> next = bucItr.next();
+        dirObjNameParentMap.put(next.getValue().getObjectID(),
+            Pair.of(metadataManager.getVolumeId(next.getValue().getVolumeName()), next.getValue().getBucketName()));
+        dirObjNameParentMap.putIfAbsent(metadataManager.getVolumeId(next.getValue().getVolumeName()),
+            Pair.of(null, next.getValue().getVolumeName()));
+      }
+    }
+    // add dir tree
+    try (TableIterator<String, ? extends KeyValue<String, OmDirectoryInfo>> dirItr
+             = metadataManager.getDirectoryTable().iterator()) {
+      while (dirItr.hasNext()) {
+        KeyValue<String, OmDirectoryInfo> next = dirItr.next();
+        dirObjNameParentMap.put(next.getValue().getObjectID(), Pair.of(next.getValue().getParentObjectID(),
+            next.getValue().getName()));
+      }
+    }
+  }
+
+  private static PrintWriter out() {
+    return spec.commandLine().getOut();
+  }
+}