Skip to content

HDDS-12778. Change ordering of transfer in OM bootstrap process. #8245

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft

HDDS-12778. Change ordering of transfer in OM bootstrap process. #8245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7389683
HDDS-12778.Change ordering of transfer in OM bootstrap process.
Apr 8, 2025
04b94d5
don't send empty tarball
Apr 11, 2025
d18a661
Merge branch 'master' into HDDS-12778
Apr 11, 2025
5ac00f6
fix checkstyle + findbugs
Apr 11, 2025
0d97ee9
fix test
Apr 23, 2025
92b164d
fix checkstyle + other warnings
Apr 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,13 @@ private void generateSnapshotCheckpoint(HttpServletRequest request,
}

Path tmpdir = null;
Instant start = Instant.now();
try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) {
if (LOG.isDebugEnabled()) {
Instant end = Instant.now();
long duration = Duration.between(start, end).toMillis();
LOG.debug("Time taken to acquire the BootstrapStateLock " + ": {} milliseconds", duration);
}
tmpdir = Files.createTempDirectory(bootstrapTempData.toPath(),
"bootstrap-data-");
checkpoint = getCheckpoint(tmpdir, flush);
Expand All @@ -234,7 +240,7 @@ private void generateSnapshotCheckpoint(HttpServletRequest request,
"attachment; filename=\"" +
file + ".tar\"");

Instant start = Instant.now();
start = Instant.now();
writeDbDataToStream(checkpoint, request,
response.getOutputStream(), receivedSstList, excludedSstList, tmpdir);
Instant end = Instant.now();
Expand Down Expand Up @@ -301,7 +307,7 @@ private static String[] parseFormDataParameters(HttpServletRequest request) {
sstParam.add(Streams.asString(item.openStream()));
}
} catch (Exception e) {
LOG.warn("Exception occured during form data parsing {}", e.getMessage());
LOG.error("Exception occurred during form data parsing", e);
}

return sstParam.isEmpty() ? null : sstParam.toArray(new String[0]);
Expand Down
11 changes: 8 additions & 3 deletions hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,17 +112,22 @@ public DBCheckpoint downloadDBSnapshotFromLeader(String leaderNodeID)
while (true) {
String snapshotFileName = getSnapshotFileName(leaderNodeID);
File targetFile = new File(snapshotDir, snapshotFileName);
downloadSnapshot(leaderNodeID, targetFile);
try {
downloadSnapshot(leaderNodeID, targetFile);
} catch (IOException ioe) {
LOG.error("Failed to download DB Snapshot", ioe);
throw ioe;
}
LOG.info(
"Successfully download the latest snapshot {} from leader OM: {}",
"Successfully downloaded the latest snapshot {} from leader OM: {}",
targetFile, leaderNodeID);

numDownloaded.incrementAndGet();
injectPause();

RocksDBCheckpoint checkpoint = getCheckpointFromSnapshotFile(targetFile,
candidateDir, true);
LOG.info("Successfully untar the downloaded snapshot {} at {}.",
LOG.info("Successfully un-tarred the downloaded snapshot {} at {}.",
targetFile, checkpoint.getCheckpointLocation());
if (ratisSnapshotComplete(checkpoint.getCheckpointLocation())) {
LOG.info("Ratis snapshot transfer is complete.");
Expand Down
116 changes: 17 additions & 99 deletions .../integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@
import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterEach;
Expand Down Expand Up @@ -212,6 +213,8 @@ public void write(int b) throws IOException {
ServletContext servletContextMock = mock(ServletContext.class);
when(omDbCheckpointServletMock.getServletContext())
.thenReturn(servletContextMock);
when(omDbCheckpointServletMock.getOmMetadataManager())
.thenReturn(om.getMetadataManager());

when(servletContextMock.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE))
.thenReturn(om);
Expand Down Expand Up @@ -247,7 +250,6 @@ void testWithoutACL() throws Exception {
testDoPostWithInvalidContentType();

prepSnapshotData();

testWriteDbDataWithoutOmSnapshot();
testWriteDbDataToStream();
testWriteDbDataWithToExcludeFileList();
Expand Down Expand Up @@ -446,60 +448,39 @@ private void testWriteDbDataToStream() throws Exception {
assertTrue(newDbDir.mkdirs());
FileUtil.unTar(tempFile, newDbDir);

// Move snapshot dir to correct location.
assertTrue(new File(newDbDirName, OM_SNAPSHOT_DIR)
.renameTo(new File(newDbDir.getParent(), OM_SNAPSHOT_DIR)));

// Confirm the checkpoint directories match, (after remove extras).
Path checkpointLocation = dbCheckpoint.getCheckpointLocation();
Set<String> initialCheckpointSet = getFiles(checkpointLocation,
checkpointLocation.toString().length() + 1);
Path finalCheckpointLocation = Paths.get(newDbDirName);
Set<String> finalCheckpointSet = getFiles(finalCheckpointLocation,
newDbDirLength);

assertThat(finalCheckpointSet).withFailMessage("hardlink file exists in checkpoint dir")
.contains(OM_HARDLINK_FILE);
finalCheckpointSet.remove(OM_HARDLINK_FILE);
assertEquals(initialCheckpointSet, finalCheckpointSet);

String shortSnapshotLocation =
truncateFileName(metaDirLength, Paths.get(snapshotDirName));
String shortSnapshotLocation2 =
truncateFileName(metaDirLength, Paths.get(snapshotDirName2));
String shortCompactionDirLocation =
truncateFileName(metaDirLength, compactionDirPath);

Set<String> finalFullSet =
getFiles(Paths.get(testDirName, OM_SNAPSHOT_DIR), testDirLength);

// Check each line in the hard link file.
List<String> fabricatedLinkLines = new ArrayList<>();
try (Stream<String> lines = Files.lines(Paths.get(newDbDirName,
OM_HARDLINK_FILE))) {

for (String line : lines.collect(Collectors.toList())) {
assertFalse(line.contains("CURRENT"),
"CURRENT file is not a hard link");
if (line.contains(FABRICATED_FILE_NAME)) {
fabricatedLinkLines.add(line);
} else {
checkLine(shortSnapshotLocation, shortSnapshotLocation2, line);
// add links to the final set
finalFullSet.add(line.split("\t")[0]);
}
}
}
Set<String> directories = Sets.newHashSet(
shortSnapshotLocation, shortSnapshotLocation2,
shortCompactionDirLocation);
checkFabricatedLines(directories, fabricatedLinkLines, testDirName);
OmSnapshotUtils.createHardLinks(newDbDir.toPath());
// Move snapshot dir to correct location.
assertTrue(new File(newDbDirName, OM_SNAPSHOT_DIR).renameTo(new File(newDbDir.getParent(), OM_SNAPSHOT_DIR)));

String shortSnapshotLocation =
truncateFileName(metaDirLength, Paths.get(snapshotDirName));
String shortSnapshotLocation2 =
truncateFileName(metaDirLength, Paths.get(snapshotDirName2));
String shortCompactionDirLocation =
truncateFileName(metaDirLength, compactionDirPath);

Set<String> finalFullSet =
getFiles(Paths.get(testDirName, OM_SNAPSHOT_DIR), testDirLength);

Set<String> initialFullSet =
getFiles(Paths.get(metaDir.toString(), OM_SNAPSHOT_DIR), metaDirLength);
assertThat(finalFullSet).contains(expectedLogStr);
assertThat(finalFullSet).contains(expectedSstStr);
assertEquals(initialFullSet, finalFullSet, "expected snapshot files not found");
assertTrue(initialFullSet.equals(finalFullSet), "expected snapshot files not found");
}

private static long tmpHardLinkFileCount() throws IOException {
Expand Down Expand Up @@ -764,69 +745,6 @@ private Set<String> getFiles(Path path, int truncateLength,
return fileSet;
}

/**
* Confirm fabricated link lines in hardlink file are properly
* formatted: "dir1/fabricatedFile dir2/fabricatedFile".
*
* The "fabricated" files/links are ones I've created by hand to
* fully test the code, (as opposed to the "natural" files/links
* created by the create snapshot process).
*
* @param directories Possible directories for the links to exist in.
* @param lines Text lines defining the link paths.
* @param testDirName Name of test directory.
*/
private void checkFabricatedLines(Set<String> directories, List<String> lines,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removed because the usage is removed.

String testDirName) {
// find the real file
String realDir = null;
for (String dir: directories) {
if (Paths.get(testDirName, dir, FABRICATED_FILE_NAME).toFile().exists()) {
assertNull(realDir, "Exactly one copy of the fabricated file exists in the tarball");
realDir = dir;
}
}

assertNotNull(realDir, "real directory found");
directories.remove(realDir);
Iterator<String> directoryIterator = directories.iterator();
String dir0 = directoryIterator.next();
String dir1 = directoryIterator.next();
assertNotEquals("link directories are different", dir0, dir1);

for (String line : lines) {
String[] files = line.split("\t");
assertTrue(
files[0].startsWith(dir0) || files[0].startsWith(dir1),
"fabricated entry contains valid first directory: " + line);
assertTrue(files[1].startsWith(realDir),
"fabricated entry contains correct real directory: " + line);
Path path0 = Paths.get(files[0]);
Path path1 = Paths.get(files[1]);
assertEquals(FABRICATED_FILE_NAME,
String.valueOf(path0.getFileName()),
"fabricated entries contains correct file name: " + line);
assertEquals(FABRICATED_FILE_NAME,
String.valueOf(path1.getFileName()),
"fabricated entries contains correct file name: " + line);
}
}

// Validates line in hard link file. should look something like:
// "dir1/x.sst x.sst".
private void checkLine(String shortSnapshotLocation,
String shortSnapshotLocation2,
String line) {
String[] files = line.split("\t");
assertTrue(files[0].startsWith(shortSnapshotLocation) ||
files[0].startsWith(shortSnapshotLocation2),
"hl entry starts with valid snapshot dir: " + line);

String file0 = files[0].substring(shortSnapshotLocation.length() + 1);
String file1 = files[1];
assertEquals(file0, file1, "hl filenames are the same");
}

private void testBootstrapLocking() throws Exception {
// Get the bootstrap state handlers
KeyManager keyManager = om.getKeyManager();
Expand Down
Loading