Skip to content

HDDS-12078. Improve container reconciliation CLIs #7944

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 11 commits into
base: HDDS-10239-container-reconciliation
Choose a base branch
from
Draft

HDDS-12078. Improve container reconciliation CLIs #7944

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
ef07f83
Add initial running status command
errose28 Feb 8, 2025
c430fdc
Add inital untested stdin/container list support
errose28 Feb 8, 2025
7e2b27a
Minor fixes and handle failures triggering reconcile for list
errose28 Feb 10, 2025
9f037c7
Initial test cases
errose28 Feb 12, 2025
b0d5a91
WIP test updates
errose28 Feb 19, 2025
eb9c92a
Basic test passes
errose28 Feb 20, 2025
7a0fefe
Set up tests to read from args and stdin and stub out all cases
errose28 Feb 20, 2025
fe9dd21
Checkstyle
errose28 Feb 21, 2025
9ba8efd
Fix findbugs and rat
errose28 Feb 21, 2025
d98bd25
Merge branch 'HDDS-10239-container-reconciliation' into HDDS-12078-im…
errose28 Feb 21, 2025
02b7dd4
Remove unused visible for testing
errose28 Feb 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'HDDS-10239-container-reconciliation' into HDDS-12078-im… 
…prove-cli

* HDDS-10239-container-reconciliation: (113 commits)
  HDDS-12361. Mark testGetSnapshotDiffReportJob as flaky
  HDDS-12375. Random object created and used only once (#7933)
  HDDS-12335. Fix ozone admin namespace summary to give complete output (#7908)
  HDDS-12364. Require Override annotation for overridden methods (#7923)
  HDDS-11530. Support listMultipartUploads max uploads and markers (#7817)
  HDDS-11867. Remove code paths for non-Ratis SCM. (#7911)
  HDDS-12363. Add import options to IntelliJ IDEA style settings (#7921)
  HDDS-12215. Mark testContainerStateMachineRestartWithDNChangePipeline as flaky
  HDDS-12331. BlockOutputStream.failedServers is not thread-safe (#7885)
  HDDS-12188. Move server-only upgrade classes from hdds-common to hdds-server-framework (#7903)
  HDDS-12362. Remove temporary checkstyle suppression file (#7920)
  HDDS-12343. Fix spotbugs warnings in Recon (#7902)
  HDDS-12286. Fix license headers and imports for ozone-tools (#7919)
  HDDS-12284. Fix license headers and imports for ozone-s3-secret-store (#7917)
  HDDS-12275. Fix license headers and imports for ozone-integration-test (#7904)
  HDDS-12164. Rename and deprecate DFSConfigKeysLegacy config keys (#7803)
  HDDS-12283. Fix license headers and imports for ozone-recon-codegen (#7916)
  HDDS-12330. Convert Volume, Bucket and Key count to use comma separated numbers (#7881)
  HDDS-12281. Fix license headers and imports for ozone-filesystem-hadoop3 (#7913)
  HDDS-12285. Fix license headers and imports for ozone-s3gateway (#7918)
  ...

Conflicts:
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaInfo.java
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReconcileSubcommand.java
  • Loading branch information
@errose28
errose28 committed Feb 21, 2025
commit d98bd2572c75c73741b7736827e57dab2fb60d96
4 changes: 0 additions & 4 deletions ...-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,9 @@
package org.apache.hadoop.hdds.scm.container;

import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.io.IOException;
import java.util.UUID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;

import java.util.UUID;

import org.apache.hadoop.hdds.server.JsonUtils;

/**
Expand Down
12 changes: 6 additions & 6 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,10 @@

package org.apache.hadoop.hdds.server;

import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Reader;
import java.util.List;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.MappingIterator;
import com.fasterxml.jackson.databind.ObjectMapper;
Expand All @@ -37,6 +32,11 @@
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Reader;
import java.util.List;
import org.apache.hadoop.hdds.HddsUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down
12 changes: 5 additions & 7 deletions ...dds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReconcileSubcommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,22 @@

package org.apache.hadoop.hdds.scm.cli.container;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.SequenceWriter;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Scanner;
import java.util.Iterator;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.SequenceWriter;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.cli.ScmSubcommand;
import org.apache.hadoop.hdds.scm.client.ScmClient;

import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplicaInfo;
import org.apache.hadoop.hdds.server.JsonUtils;
Expand Down
69 changes: 34 additions & 35 deletions ...tools/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestReconcileSubcommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,36 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.hdds.scm.cli.container;

import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplicaInfo;
import org.apache.hadoop.hdds.server.JsonUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import com.fasterxml.jackson.core.type.TypeReference;
import org.junit.jupiter.api.Test;
import picocli.CommandLine;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import com.fasterxml.jackson.core.type.TypeReference;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
Expand All @@ -45,18 +44,18 @@
import java.util.UUID;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.ScmClient;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplicaInfo;
import org.apache.hadoop.hdds.server.JsonUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import picocli.CommandLine;

/**
* Tests the `ozone admin container reconcile` CLI.
Expand Down
You are viewing a condensed version of this merge commit. You can view the full changes here.