diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index 0624c0e56d1..53f48388a86 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java @@ -54,9 +54,12 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneTestUtils; import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneSnapshot; +import org.apache.hadoop.ozone.debug.KeyPathRetriever; import org.apache.hadoop.ozone.debug.OzoneDebug; import org.apache.hadoop.ozone.debug.ldb.RDBParser; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -173,6 +176,37 @@ public void testLdbCliForOzoneSnapshot() throws Exception { assertThat(cmdOut).contains(keyName); } + @Test + public void testKeyPathRetriever() throws Exception { + final String volumeName = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + StringWriter stdout = new StringWriter(); + PrintWriter pstdout = new PrintWriter(stdout); + String[] args; + CommandLine cmd; + try (OzoneClient client = OzoneClientFactory.getRpcClient(conf)) { + TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED); + OzoneBucket bucket = client.getObjectStore().getVolume(volumeName).getBucket(bucketName); + bucket.createDirectory("dir1"); + TestDataUtil.createKey(bucket, "dir1/file1", "test"); + TestDataUtil.createKey(bucket, "dir1/file2", "test"); + TestDataUtil.createKey(bucket, "dir1/file3", "test"); + OzoneKeyDetails key1 = bucket.getKey("dir1/file1"); + OzoneKeyDetails key3 = bucket.getKey("dir1/file2"); + StringBuilder sb = new StringBuilder(); + key1.getOzoneKeyLocations().forEach(e -> sb.append(e.getContainerID())); + key3.getOzoneKeyLocations().forEach(e -> sb.append(",").append(e.getContainerID())); + String dbFile = OMStorage.getOmDbDir(conf).getPath() + "/om.db"; + cmd = new CommandLine(new OzoneDebug()).addSubcommand(new CommandLine(new KeyPathRetriever())).setOut(pstdout); + args = new String[] {"retrieve-key-fullpath", "--db", dbFile, "--containers", sb.toString()}; + } + int exitCode = cmd.execute(args); + assertEquals(0, exitCode); + String keyPrefix = volumeName + "/" + bucketName + "/dir1/"; + assertThat(stdout.toString()).contains(keyPrefix + "file1").contains(keyPrefix + "file2") + .doesNotContain(keyPrefix + "file3"); + } + private static String getSnapshotDBPath(String checkPointDir) { return OMStorage.getOmDbDir(conf) + OM_KEY_PREFIX + OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index a7793fdc9ab..784a450f692 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -381,14 +381,14 @@ public static OmMetadataManagerImpl createCheckpointMetadataManager( } /** - * Metadata constructor for checkpoints. + * Metadata constructor for checkpoints and readonly db. * * @param conf - Ozone conf. - * @param dir - Checkpoint parent directory. - * @param name - Checkpoint directory name. + * @param dir - parent directory. + * @param name - directory name. * @throws IOException */ - private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) + public OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name) throws IOException { lock = new OmReadOnlyLock(); omEpoch = 0; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/KeyPathRetriever.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/KeyPathRetriever.java new file mode 100644 index 00000000000..7288026ffb9 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/KeyPathRetriever.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.stream.Stream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.cli.DebugSubcommand; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; +import org.apache.hadoop.hdds.utils.db.LongCodec; +import org.apache.hadoop.hdds.utils.db.StringCodec; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.Table.KeyValue; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.kohsuke.MetaInfServices; +import picocli.CommandLine; + +/** + * Tool that retrieve key full path from OM db file table with pattern match. + */ +@CommandLine.Command( + name = "retrieve-key-fullpath", + description = "retrieve full key path with matching criteria") +@MetaInfServices(DebugSubcommand.class) +public class KeyPathRetriever implements Callable, DebugSubcommand { + private static final String DIRTREEDBNAME = "omdirtree.db"; + private static final DBColumnFamilyDefinition DIRTREE_COLUMN_FAMILY + = new DBColumnFamilyDefinition<>("dirTreeTable", LongCodec.get(), StringCodec.get()); + + @CommandLine.Spec + private static CommandLine.Model.CommandSpec spec; + + @CommandLine.Option(names = {"--db"}, + required = true, + description = "Database File") + private String dbFile; + + @CommandLine.Option(names = {"--containers"}, + required = false, + description = "Comma separated Container Ids") + private String strContainerIds; + + @CommandLine.Option(names = {"--container-file"}, + required = false, + description = "file with container id in new line") + private String containerFileName; + + private DBStore dirTreeDbStore = null; + private Table dirTreeTable = null; + + public void setDirTreeTable(Table table) { + this.dirTreeTable = table; + } + + @Override + public Void call() throws Exception { + // get containerIds filter + Set containerIds = new HashSet<>(); + if (!StringUtils.isEmpty(strContainerIds)) { + String[] split = strContainerIds.split(","); + for (String id : split) { + containerIds.add(Long.parseLong(id)); + } + } else if (!StringUtils.isEmpty(containerFileName)) { + try (Stream stream = Files.lines(Paths.get(containerFileName))) { + stream.forEach(e -> containerIds.add(Long.parseLong(e))); + } + } + if (containerIds.isEmpty()) { + System.err.println("No containers to be filtered"); + return null; + } + // out stream + PrintWriter writer = out(); + + // db handler + OMMetadataManager metadataManager = getOmMetadataManager(dbFile); + if (metadataManager == null) { + writer.close(); + return null; + } + + dirTreeDbStore = openDb(new File(dbFile).getParentFile()); + if (dirTreeDbStore == null) { + writer.close(); + metadataManager.stop(); + return null; + } + dirTreeTable = dirTreeDbStore.getTable(DIRTREE_COLUMN_FAMILY.getName(), Long.class, String.class); + + try { + retrieve(metadataManager, writer, containerIds); + } finally { + dirTreeDbStore.close(); + writer.close(); + metadataManager.stop(); + // delete temporary created db file + File dirTreeDbPath = new File(new File(dbFile).getParentFile(), DIRTREEDBNAME); + if (dirTreeDbPath.exists()) { + FileUtils.deleteDirectory(dirTreeDbPath); + } + } + return null; + } + + public void retrieve( + OMMetadataManager metadataManager, PrintWriter writer, Set containerIds) { + // build dir tree + Map> bucketVolMap = new HashMap<>(); + try { + prepareDirIdTree(metadataManager, bucketVolMap); + } catch (Exception e) { + System.err.println("Exception occurred reading directory Table, " + e); + return; + } + + // iterate file table and filter for container + try (TableIterator> fileItr + = metadataManager.getFileTable().iterator()) { + while (fileItr.hasNext()) { + KeyValue next = fileItr.next(); + boolean found = next.getValue().getKeyLocationVersions().stream().anyMatch( + e -> e.getLocationList().stream().anyMatch( + blk -> containerIds.contains(blk.getBlockID().getContainerID()))); + if (found) { + StringBuilder sb = new StringBuilder(next.getValue().getKeyName()); + Long prvParent = next.getValue().getParentObjectID(); + while (prvParent != null) { + // check reached for bucket volume level + if (bucketVolMap.containsKey(prvParent)) { + Pair nameParentPair = bucketVolMap.get(prvParent); + sb.insert(0, nameParentPair.getValue() + OM_KEY_PREFIX); + prvParent = nameParentPair.getKey(); + if (null == prvParent) { + // add to output as reached till volume + writer.println(sb); + break; + } + continue; + } + + // check dir tree + Pair nameParentPair = getFromDirTree(prvParent); + if (nameParentPair == null) { + break; + } + sb.insert(0, nameParentPair.getValue() + OM_KEY_PREFIX); + prvParent = nameParentPair.getKey(); + } + } + } + } catch (IOException e) { + System.err.println("Exception occurred reading file Table, " + e); + } + } + + private static OMMetadataManager getOmMetadataManager(String db) throws IOException { + if (!Files.exists(Paths.get(db))) { + System.err.println("DB with path not exist:" + db); + return null; + } + System.err.println("Db Path is:" + db); + File file = new File(db); + + OzoneConfiguration conf = new OzoneConfiguration(); + return new OmMetadataManagerImpl(conf, file.getParentFile(), file.getName()); + } + + private void prepareDirIdTree( + OMMetadataManager metadataManager, Map> bucketVolMap) throws IOException { + // add bucket volume tree + try (TableIterator> bucItr + = metadataManager.getBucketTable().iterator()) { + while (bucItr.hasNext()) { + KeyValue next = bucItr.next(); + bucketVolMap.put(next.getValue().getObjectID(), + Pair.of(metadataManager.getVolumeId(next.getValue().getVolumeName()), next.getValue().getBucketName())); + bucketVolMap.putIfAbsent(metadataManager.getVolumeId(next.getValue().getVolumeName()), + Pair.of(null, next.getValue().getVolumeName())); + } + } + // add dir tree + try (TableIterator> dirItr + = metadataManager.getDirectoryTable().iterator()) { + while (dirItr.hasNext()) { + KeyValue next = dirItr.next(); + addToDirTree(next.getValue().getObjectID(), next.getValue().getParentObjectID(), next.getValue().getName()); + } + } + } + + private DBStore openDb(File omPath) { + File dirTreeDbPath = new File(omPath, DIRTREEDBNAME); + System.err.println("Creating database of dir tree path at " + dirTreeDbPath); + try { + // Delete the DB from the last run if it exists. + if (dirTreeDbPath.exists()) { + FileUtils.deleteDirectory(dirTreeDbPath); + } + ConfigurationSource conf = new OzoneConfiguration(); + DBStoreBuilder dbStoreBuilder = DBStoreBuilder.newBuilder(conf); + dbStoreBuilder.setName(dirTreeDbPath.getName()); + dbStoreBuilder.setPath(dirTreeDbPath.getParentFile().toPath()); + dbStoreBuilder.addTable(DIRTREE_COLUMN_FAMILY.getName()); + dbStoreBuilder.addCodec(DIRTREE_COLUMN_FAMILY.getKeyType(), DIRTREE_COLUMN_FAMILY.getKeyCodec()); + dbStoreBuilder.addCodec(DIRTREE_COLUMN_FAMILY.getValueType(), DIRTREE_COLUMN_FAMILY.getValueCodec()); + return dbStoreBuilder.build(); + } catch (IOException e) { + System.err.println("Error creating omdirtree.db " + e); + return null; + } + } + + private void addToDirTree(Long objectId, Long parentId, String name) throws IOException { + if (null == parentId) { + dirTreeTable.put(objectId, name + "#"); + } else { + dirTreeTable.put(objectId, name + "#" + parentId); + } + } + + private Pair getFromDirTree(Long objectId) throws IOException { + String val = dirTreeTable.get(objectId); + if (null == val) { + return null; + } + return getDirParentNamePair(val); + } + + public static Pair getDirParentNamePair(String val) { + int hashIdx = val.lastIndexOf('#'); + String strParentId = val.substring(hashIdx + 1); + Long parentId = null; + if (!StringUtils.isEmpty(strParentId)) { + parentId = Long.parseLong(strParentId); + } + return Pair.of(parentId, val.substring(0, hashIdx)); + } + + private static PrintWriter out() { + return spec.commandLine().getOut(); + } +} diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestKeyPathRetriever.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestKeyPathRetriever.java new file mode 100644 index 00000000000..60cb6860c3c --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestKeyPathRetriever.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests the ozone key path retriever command. + */ +public class TestKeyPathRetriever { + + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private final ByteArrayOutputStream err = new ByteArrayOutputStream(); + private static final PrintStream OLD_OUT = System.out; + private static final PrintStream OLD_ERR = System.err; + private static final String DEFAULT_ENCODING = UTF_8.name(); + + private static final String OZONE_USER = "ozone"; + private static final String OLD_USER = System.getProperty("user.name"); + + @BeforeEach + public void setup() throws Exception { + System.setOut(new PrintStream(out, false, DEFAULT_ENCODING)); + System.setErr(new PrintStream(err, false, DEFAULT_ENCODING)); + System.setProperty("user.name", OZONE_USER); + } + + @AfterEach + public void reset() { + // reset stream after each unit test + out.reset(); + err.reset(); + + // restore system streams + System.setOut(OLD_OUT); + System.setErr(OLD_ERR); + System.setProperty("user.name", OLD_USER); + } + + @Test + void testRetriveFullPath() throws Exception { + KeyPathRetriever keyPathRetriever = new KeyPathRetriever(); + + Set containerSet = new HashSet<>(); + containerSet.add(500L); + containerSet.add(800L); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(out, UTF_8)); + + Map dirInfoMap = new HashMap<>(); + Table dirTreeTable = mock(Table.class); + keyPathRetriever.setDirTreeTable(dirTreeTable); + + doAnswer(invocation -> { + Long objectId = invocation.getArgument(0); + String val = invocation.getArgument(1); + dirInfoMap.put(objectId, val); + return null; + }).when(dirTreeTable).put(anyLong(), anyString()); + + when(dirTreeTable.get(anyLong())).thenAnswer(inv -> dirInfoMap.get((Long)inv.getArgument(0))); + + OMMetadataManager mock = mock(OMMetadataManager.class); + when(mock.getVolumeId(anyString())).thenReturn(1L); + + // bucket iteration mock + Table bucketTblMock = mock(Table.class); + TableIterator bucketTblItr = mock(TableIterator.class); + when(mock.getBucketTable()).thenReturn(bucketTblMock); + when(bucketTblMock.iterator()).thenReturn(bucketTblItr); + when(bucketTblItr.hasNext()).thenReturn(true, false); + when(bucketTblItr.next()).thenReturn(Table.newKeyValue("bucket1", + new OmBucketInfo.Builder().setBucketName("bucket1").setVolumeName("vol").setObjectID(2).build())); + + // dir iteration mock + Table dirTblMock = mock(Table.class); + TableIterator dirTblItr = mock(TableIterator.class); + when(mock.getDirectoryTable()).thenReturn(dirTblMock); + when(dirTblMock.iterator()).thenReturn(dirTblItr); + when(dirTblItr.hasNext()).thenReturn(true, false); + when(dirTblItr.next()).thenReturn(Table.newKeyValue("/1/2/2/dir1", + new OmDirectoryInfo.Builder().setName("dir1").setParentObjectID(2).setObjectID(3).build())); + + // file iteration mock + Table fileTblMock = mock(Table.class); + TableIterator fileTblItr = mock(TableIterator.class); + when(mock.getFileTable()).thenReturn(fileTblMock); + when(fileTblMock.iterator()).thenReturn(fileTblItr); + when(fileTblItr.hasNext()).thenReturn(true, true, true, true, false); + Table.KeyValue file1 = Table.newKeyValue("1/2/3/file1", + new OmKeyInfo.Builder().setKeyName("file1").setParentObjectID(3).setObjectID(4) + .setOmKeyLocationInfos(getLocationGrpList(500)).build()); + Table.KeyValue file2 = Table.newKeyValue("1/2/3/file2", + new OmKeyInfo.Builder().setKeyName("file2").setParentObjectID(3).setObjectID(5) + .setOmKeyLocationInfos(getLocationGrpList(500)).build()); + Table.KeyValue file3 = Table.newKeyValue("1/2/3/file3", + new OmKeyInfo.Builder().setKeyName("file3").setParentObjectID(3).setObjectID(6) + .setOmKeyLocationInfos(getLocationGrpList(600)).build()); + Table.KeyValue file4 = Table.newKeyValue("1/2/3/file4", + new OmKeyInfo.Builder().setKeyName("file4").setParentObjectID(3).setObjectID(7) + .setOmKeyLocationInfos(getLocationGrpList(800)).build()); + when(fileTblItr.next()).thenReturn(file1, file2, file3, file4); + + keyPathRetriever.retrieve(mock, writer, containerSet); + writer.close(); + assertThat(out.toString(UTF_8.name())).contains("vol/bucket1/dir1/file1").contains("vol/bucket1/dir1/file2") + .contains("vol/bucket1/dir1/file4").doesNotContain("file3"); + } + + private static List getLocationGrpList(long containerId) { + List locationGrpList = new ArrayList<>(); + List locationList = new ArrayList<>(); + locationList.add(new OmKeyLocationInfo.Builder().setBlockID(new BlockID(containerId, 500L)).build()); + locationGrpList.add(new OmKeyLocationInfoGroup(1, locationList)); + return locationGrpList; + } +}