HDDS-11746. Support arbitrary schemas as an option in ozone debug ldb.

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java

@@ -26,9 +26,12 @@
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.utils.db.DBDefinition;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
 import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
 import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -80,6 +83,8 @@ public class TestLDBCli {
   private static final String KEY_TABLE = "keyTable";
   private static final String BLOCK_DATA = "block_data";
   public static final String PIPELINES = "pipelines";
+  public static final String DUMMY_DB = "anotherKeyTable";
+  public static final String ANOTHER_KEY_TABLE_NAME = "anotherKeyTable";
   private static final ObjectMapper MAPPER = new ObjectMapper();
   private OzoneConfiguration conf;
   private DBStore dbStore;
@@ -410,6 +415,40 @@ void testSchemaCommand() throws IOException {
     assertEquals("", stderr.toString());
   }
 
+  @Test
+  void testCommandsWithDBDefOverride() throws IOException {
+    // Prepare dummy table
+    prepareTable(DUMMY_DB, true);
+
+    // Prepare args for value-schema command
+    List<String> completeScanArgs = new ArrayList<>(Arrays.asList(
+        "--db", dbStore.getDbLocation().getAbsolutePath(),
+        "--schema", DummyDBDefinition.class.getName(),
+        "value-schema",
+        "--column-family", ANOTHER_KEY_TABLE_NAME));
+
+    int exitCode = cmd.execute(completeScanArgs.toArray(new String[0]));
+    assertEquals(0, exitCode, stderr.toString());
+    Pattern p = Pattern.compile(".*String.*String.*", Pattern.MULTILINE);
+    Matcher m = p.matcher(stdout.toString());
+    assertTrue(m.find());
+    assertEquals("", stderr.toString());
+
+    // Prepare args for scan command
+    completeScanArgs = new ArrayList<>(Arrays.asList(
+        "--db", dbStore.getDbLocation().getAbsolutePath(),
+        "--schema", DummyDBDefinition.class.getName(),
+        "scan",
+        "--column-family", ANOTHER_KEY_TABLE_NAME));
+
+    exitCode = cmd.execute(completeScanArgs.toArray(new String[0]));
+    assertEquals(0, exitCode, stderr.toString());
+    p = Pattern.compile(".*random-key.*random-value.*", Pattern.MULTILINE);
+    m = p.matcher(stdout.toString());
+    assertTrue(m.find());
+    assertEquals("", stderr.toString());
+  }
+
   /**
    * Converts String input to a Map and compares to the given Map input.
    * @param expected expected result Map
@@ -473,11 +512,23 @@ private void prepareTable(String tableName, boolean schemaV3)
         }
       }
       break;
+
     case PIPELINES:
       // Empty table
       dbStore = DBStoreBuilder.newBuilder(conf).setName("scm.db")
           .setPath(tempDir.toPath()).addTable(PIPELINES).build();
       break;
+
+    case DUMMY_DB:
+      dbStore = DBStoreBuilder.newBuilder(new OzoneConfiguration())
+          .setName("another.db")
+          .setPath(tempDir.toPath())
+          .addTable(ANOTHER_KEY_TABLE_NAME)
+          .build();
+      dbStore.getTable(ANOTHER_KEY_TABLE_NAME, String.class, String.class)
+          .put("random-key", "random-value");
+      break;
+
     default:
       throw new IllegalArgumentException("Unsupported table: " + tableName);
     }
@@ -512,4 +563,25 @@ private static Map<String, Object> toMap(Object obj) throws IOException {
     return MAPPER.readValue(json, new TypeReference<Map<String, Object>>() { });
   }
 
+  /**
+   * New DBDefinition to test arbitrary schemas for ldb commands.
+   */
+  public static class DummyDBDefinition extends DBDefinition.WithMap {
+    public static final DBColumnFamilyDefinition<String, String> ANOTHER_KEY_TABLE
+        = new DBColumnFamilyDefinition<>(ANOTHER_KEY_TABLE_NAME, StringCodec.get(), StringCodec.get());
+    private static final Map<String, DBColumnFamilyDefinition<?, ?>>
+        COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(ANOTHER_KEY_TABLE);
+    protected DummyDBDefinition() {
+      super(COLUMN_FAMILIES);
+    }
+    @Override
+    public String getName() {
+      return "another.db";
+    }
+    @Override
+    public String getLocationConfigKey() {
+      return "";
+    }
+  }
+
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java

@@ -18,10 +18,13 @@
 
 package org.apache.hadoop.ozone.debug;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
 import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -39,6 +42,7 @@
 
 import com.amazonaws.services.kms.model.InvalidArnException;
 import com.google.common.base.Preconditions;
+import org.apache.ratis.util.function.CheckedSupplier;
 
 import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
 import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB;
@@ -98,6 +102,56 @@ public static DBDefinition getDefinition(Path dbPath,
     return getDefinition(dbName);
   }
 
+  public static List<CheckedSupplier<DBDefinition, Exception>> getFactories(
+      Class<?> clazz, String dbPathString, ConfigurationSource config) {
+    return Arrays.asList(
+        () -> {
+          Method factoryMethod = clazz.getDeclaredMethod("get");
+          factoryMethod.setAccessible(true);
+          return (DBDefinition) factoryMethod.invoke(clazz);
+        },
+        () -> {
+          Constructor<?> constructor = clazz.getDeclaredConstructor();
+          constructor.setAccessible(true);
+          return (DBDefinition) constructor.newInstance();
+        },
+        () -> {
+          Constructor<?> constructor = clazz.getDeclaredConstructor(String.class);
+          constructor.setAccessible(true);
+          return (DBDefinition) constructor.newInstance(dbPathString);
+        },
+        () -> {
+          Constructor<?> constructor = clazz.getDeclaredConstructor(String.class, ConfigurationSource.class);
+          constructor.setAccessible(true);
+          return (DBDefinition) constructor.newInstance(dbPathString, config);
+        }
+    );
+  }
+
+  public static DBDefinition getDefinition(Path dbPath, ConfigurationSource config, String overrideDBDef) {
+    if (overrideDBDef == null) {
+      return getDefinition(dbPath, config);
+    }
+    try {
+      Class<?> clazz = Class.forName(overrideDBDef);
+      if (DBDefinition.class.isAssignableFrom(clazz)) {
+        String dbPathString = dbPath.toAbsolutePath().toString();
+        for (CheckedSupplier<DBDefinition, Exception> factory : getFactories(clazz, dbPathString, config)) {
+          try {
+            return factory.get();
+          } catch (Exception ignored) {
+          }
+        }
+        throw new IllegalArgumentException("Could not get instance of " + overrideDBDef);
+      } else {
+        System.err.println("Class does not implement DBDefinition: " + overrideDBDef);
+      }
+    } catch (ClassNotFoundException e) {
+      System.err.println("Class not found: " + overrideDBDef);
+    }
+    throw new IllegalArgumentException("Incorrect DBDefinition class.");
+  }
+
   private static DBDefinition getReconDBDefinition(String dbName) {
     if (dbName.startsWith(RECON_CONTAINER_KEY_DB)) {
       return new ReconDBDefinition(dbName);

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/DBScanner.java

@@ -581,7 +581,7 @@ private boolean printTable(List<ColumnFamilyHandle> columnFamilyHandleList,
     dbPath = removeTrailingSlashIfNeeded(dbPath);
     DBDefinitionFactory.setDnDBSchemaVersion(dnDBSchemaVersion);
     DBDefinition dbDefinition = DBDefinitionFactory.getDefinition(
-        Paths.get(dbPath), new OzoneConfiguration());
+        Paths.get(dbPath), new OzoneConfiguration(), parent.getDbDefinition());
     if (dbDefinition == null) {
       err().println("Error: Incorrect DB Path");
       return false;

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/RDBParser.java

@@ -43,11 +43,19 @@ public class RDBParser implements DebugSubcommand {
       description = "Database File Path")
   private String dbPath;
 
+  @CommandLine.Option(names = {"--schema"},
+      description = "DBDefinition of the database")
+  private String dbDefinition;
+
   public String getDbPath() {
     return dbPath;
   }
 
   public void setDbPath(String dbPath) {
     this.dbPath = dbPath;
   }
+
+  public String getDbDefinition() {
+    return dbDefinition;
+  }
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ldb/ValueSchema.java

@@ -86,7 +86,7 @@ public Void call() throws Exception {
 
     String dbPath = parent.getDbPath();
     Map<String, Object> fields = new HashMap<>();
-    success = getValueFields(dbPath, fields, depth, tableName, dnDBSchemaVersion);
+    success = getValueFields(dbPath, fields, depth, tableName, dnDBSchemaVersion, parent.getDbDefinition());
 
     out().println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(fields));
 
@@ -100,11 +100,11 @@ public Void call() throws Exception {
   }
 
   public static boolean getValueFields(String dbPath, Map<String, Object> valueSchema, int d, String table,
-                                       String dnDBSchemaVersion) {
+                                       String dnDBSchemaVersion, String dbDef) {
 
     dbPath = removeTrailingSlashIfNeeded(dbPath);
     DBDefinitionFactory.setDnDBSchemaVersion(dnDBSchemaVersion);
-    DBDefinition dbDefinition = DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration());
+    DBDefinition dbDefinition = DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration(), dbDef);
     if (dbDefinition == null) {
       err().println("Error: Incorrect DB Path");
       return false;

hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java

@@ -21,6 +21,10 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
 import org.apache.hadoop.hdds.utils.db.DBDefinition;
@@ -34,8 +38,13 @@
 import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB;
 import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_SNAPSHOT_DB;
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import org.apache.ratis.util.function.CheckedSupplier;
 import org.junit.jupiter.api.Test;
+import org.reflections.Reflections;
 
 /**
  * Simple factory unit test.
@@ -75,4 +84,48 @@ public void testGetDefinition() {
     definition = DBDefinitionFactory.getDefinition(dbPath, conf);
     assertInstanceOf(DatanodeSchemaThreeDBDefinition.class, definition);
   }
+
+  @Test
+  public void testGetDefinitionWithOverride() {
+    final OzoneConfiguration conf = new OzoneConfiguration();
+    Path dbPath = Paths.get("another.db");
+    DBDefinition definition = DBDefinitionFactory.getDefinition(dbPath, conf, OMDBDefinition.class.getName());
+    assertInstanceOf(OMDBDefinition.class, definition);
+  }
+
+  /*
+   * Test to ensure that any DBDefinition has a default constructor or a constructor with 1 parameter.
+   * This is needed for ldb tools to run with arbitrary DB definitions.
+   */
+  @Test
+  public void testAllDBDefinitionsHaveCorrectConstructor() {
+    Set<Class<?>> subclasses = new HashSet<>();
+    try {
+      Reflections reflections = new Reflections("org.apache.hadoop");
+      subclasses.addAll(reflections.getSubTypesOf(DBDefinition.class));
+      subclasses.remove(Class.forName("org.apache.hadoop.hdds.utils.db.DBDefinition$WithMap"));
+      subclasses.remove(Class.forName("org.apache.hadoop.hdds.utils.db.DBDefinition$WithMapInterface"));
+      subclasses.remove(Class.forName(
+          "org.apache.hadoop.ozone.container.metadata.AbstractDatanodeDBDefinition"));
+    } catch (Exception e) {
+      fail("Error while finding subclasses: " + e.getMessage());
+    }
+    assertFalse(subclasses.isEmpty(), "No classes found extending DBDefinition");
+
+    for (Class<?> clazz : subclasses) {
+      List<CheckedSupplier<DBDefinition, Exception>> factories =
+          DBDefinitionFactory.getFactories(clazz, "testDbPath", new OzoneConfiguration());
+      boolean hasValidFactory = factories.stream().anyMatch(factory -> {
+        try {
+          factory.get();
+          return true;
+        } catch (Exception e) {
+          return false;
+        }
+      });
+
+      assertTrue(hasValidFactory,
+          "Class " + clazz.getName() + " does not have a valid constructor or factory method.");
+    }
+  }
 }