Add tests to check arbitrary schema option

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java

@@ -26,9 +26,12 @@
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.utils.db.DBDefinition;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
 import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
 import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.ozone.ClientVersion;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -80,6 +83,7 @@ public class TestLDBCli {
   private static final String KEY_TABLE = "keyTable";
   private static final String BLOCK_DATA = "block_data";
   public static final String PIPELINES = "pipelines";
+  public static final String DUMMY_DB = "anotherKeyTable";
   private static final ObjectMapper MAPPER = new ObjectMapper();
   private OzoneConfiguration conf;
   private DBStore dbStore;
@@ -410,6 +414,40 @@ void testSchemaCommand() throws IOException {
     assertEquals("", stderr.toString());
   }
 
+  @Test
+  void testCommandsWithDBDefOverride() throws IOException {
+    // Prepare dummy table
+    prepareTable(DUMMY_DB, true);
+
+    // Prepare args for value-schema command
+    List<String> completeScanArgs = new ArrayList<>(Arrays.asList(
+        "--db", dbStore.getDbLocation().getAbsolutePath(),
+        "--schema", DummyDBDefinition.class.getName(),
+        "value-schema",
+        "--column-family", DummyDBDefinition.ANOTHER_KEY_TABLE_NAME));
+
+    int exitCode = cmd.execute(completeScanArgs.toArray(new String[0]));
+    assertEquals(0, exitCode, stderr.toString());
+    Pattern p = Pattern.compile(".*String.*String.*", Pattern.MULTILINE);
+    Matcher m = p.matcher(stdout.toString());
+    assertTrue(m.find());
+    assertEquals("", stderr.toString());
+
+    // Prepare args for scan command
+    completeScanArgs = new ArrayList<>(Arrays.asList(
+        "--db", dbStore.getDbLocation().getAbsolutePath(),
+        "--schema", DummyDBDefinition.class.getName(),
+        "scan",
+        "--column-family", DummyDBDefinition.ANOTHER_KEY_TABLE_NAME));
+
+    exitCode = cmd.execute(completeScanArgs.toArray(new String[0]));
+    assertEquals(0, exitCode, stderr.toString());
+    p = Pattern.compile(".*random-key.*random-value.*", Pattern.MULTILINE);
+    m = p.matcher(stdout.toString());
+    assertTrue(m.find());
+    assertEquals("", stderr.toString());
+  }
+
   /**
    * Converts String input to a Map and compares to the given Map input.
    * @param expected expected result Map
@@ -473,11 +511,23 @@ private void prepareTable(String tableName, boolean schemaV3)
         }
       }
       break;
+
     case PIPELINES:
       // Empty table
       dbStore = DBStoreBuilder.newBuilder(conf).setName("scm.db")
           .setPath(tempDir.toPath()).addTable(PIPELINES).build();
       break;
+
+    case DUMMY_DB:
+      dbStore = DBStoreBuilder.newBuilder(new OzoneConfiguration())
+          .setName("another.db")
+          .setPath(tempDir.toPath())
+          .addTable(DummyDBDefinition.ANOTHER_KEY_TABLE_NAME)
+          .build();
+      dbStore.getTable(DummyDBDefinition.ANOTHER_KEY_TABLE_NAME, String.class, String.class)
+          .put("random-key", "random-value");
+      break;
+
     default:
       throw new IllegalArgumentException("Unsupported table: " + tableName);
     }
@@ -512,4 +562,23 @@ private static Map<String, Object> toMap(Object obj) throws IOException {
     return MAPPER.readValue(json, new TypeReference<Map<String, Object>>() { });
   }
 
+  public static class DummyDBDefinition extends DBDefinition.WithMap {
+    public static String ANOTHER_KEY_TABLE_NAME = "anotherKeyTable";
+    public static final DBColumnFamilyDefinition<String, String> ANOTHER_KEY_TABLE
+        = new DBColumnFamilyDefinition<>(ANOTHER_KEY_TABLE_NAME, StringCodec.get(), StringCodec.get());
+    private static final Map<String, DBColumnFamilyDefinition<?, ?>>
+        COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(ANOTHER_KEY_TABLE);
+    protected DummyDBDefinition() {
+      super(COLUMN_FAMILIES);
+    }
+    @Override
+    public String getName() {
+      return "another.db";
+    }
+    @Override
+    public String getLocationConfigKey() {
+      return "";
+    }
+  }
+
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java

@@ -99,9 +99,9 @@ public static DBDefinition getDefinition(Path dbPath,
     return getDefinition(dbName);
   }
 
-  public static DBDefinition getDefinition(Path dbName, ConfigurationSource config, String overrideDBDef) {
+  public static DBDefinition getDefinition(Path dbPath, ConfigurationSource config, String overrideDBDef) {
     if (overrideDBDef == null) {
-      return getDefinition(dbName, config);
+      return getDefinition(dbPath, config);
     }
     try {
       Class<?> clazz = Class.forName(overrideDBDef);
@@ -114,7 +114,7 @@ public static DBDefinition getDefinition(Path dbName, ConfigurationSource config
         } catch (NoSuchMethodException e) {
           Constructor<?> stringParamConstructor = clazz.getDeclaredConstructor(String.class);
           stringParamConstructor.setAccessible(true);
-          instance = (DBDefinition) stringParamConstructor.newInstance(dbName.toAbsolutePath().toString());
+          instance = (DBDefinition) stringParamConstructor.newInstance(dbPath.toAbsolutePath().toString());
         }
         return instance;
       } else {

hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java

@@ -18,9 +18,13 @@
 
 package org.apache.hadoop.ozone.debug;
 
+import java.lang.reflect.Constructor;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
+import java.util.HashSet;
+import java.util.Set;
+
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
 import org.apache.hadoop.hdds.utils.db.DBDefinition;
@@ -34,8 +38,12 @@
 import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB;
 import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_SNAPSHOT_DB;
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.junit.jupiter.api.Test;
+import org.reflections.Reflections;
 
 /**
  * Simple factory unit test.
@@ -75,4 +83,46 @@ public void testGetDefinition() {
     definition = DBDefinitionFactory.getDefinition(dbPath, conf);
     assertInstanceOf(DatanodeSchemaThreeDBDefinition.class, definition);
   }
+
+  @Test
+  public void testGetDefinitionWithOverride() {
+    final OzoneConfiguration conf = new OzoneConfiguration();
+    Path dbPath = Paths.get("another.db");
+    DBDefinition definition = DBDefinitionFactory.getDefinition(dbPath, conf, OMDBDefinition.class.getName());
+    assertInstanceOf(OMDBDefinition.class, definition);
+  }
+
+  /*
+   * Test to ensure that any DBDefinition has a default constructor or a constructor with 1 parameter.
+   * This is needed for ldb tools to run with arbitrary DB definitions.
+   */
+  @Test
+  public void testAllDBDefinitionsHaveCorrectConstructor() {
+    Set<Class<? extends DBDefinition.WithMap>> withMapSubclasses = new HashSet<>();
+    try {
+      Reflections reflections = new Reflections("org.apache.hadoop");
+      withMapSubclasses = reflections.getSubTypesOf(DBDefinition.WithMap.class);
+    } catch (Exception e) {
+      fail("Error while finding subclasses: " + e.getMessage());
+    }
+    assertFalse(withMapSubclasses.isEmpty(), "No classes found extending DBDefinition.WithMap");
+    System.out.println(withMapSubclasses);
+
+    // Check constructors for each subclass
+    for (Class<?> clazz : withMapSubclasses) {
+      Constructor<?>[] constructors = clazz.getDeclaredConstructors();
+      boolean hasValidConstructor = false;
+
+      for (Constructor<?> constructor : constructors) {
+        int paramCount = constructor.getParameterCount();
+        if (paramCount == 0 || paramCount == 1) {
+          hasValidConstructor = true;
+          break;
+        }
+      }
+
+      assertTrue(hasValidConstructor,
+          "Class " + clazz.getName() + " does not have a valid constructor (default or single parameter)");
+    }
+  }
 }