Skip to content

HDDS-12928. datanode min free space configuration #8388

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
May 28, 2025
Merged

HDDS-12928. datanode min free space configuration #8388

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
b07c088
HDDS-12928. design approach for min free space configuration
sumitagrawl May 5, 2025
db7211c
HDDS-12928. min free space configuration change with default
sumitagrawl May 8, 2025
cef0da3
checkstyle and doc fix
sumitagrawl May 8, 2025
f07be0e
test case fix
sumitagrawl May 8, 2025
c268e57
checkstyle fix
sumitagrawl May 8, 2025
23f06cf
fix test case
sumitagrawl May 8, 2025
2fca212
fix test case
sumitagrawl May 8, 2025
0a215ec
fix test cases
sumitagrawl May 9, 2025
6032c71
fix review comments
sumitagrawl May 12, 2025
96a7486
check style fix
sumitagrawl May 12, 2025
914ff7f
fix review comments
sumitagrawl May 12, 2025
e3e7480
fix review comment
sumitagrawl May 13, 2025
2b74370
fix review comment for doc
sumitagrawl May 19, 2025
614acd3
Merge branch 'apache:master' into HDDS-12928
sumitagrawl May 20, 2025
c22a69c
fix test case config
sumitagrawl May 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix test case
  • Loading branch information
@sumitagrawl
sumitagrawl committed May 16, 2025
commit 2fca212ce1c1b1f84082a656e32582ea5ceae5c0
3 changes: 3 additions & 0 deletions ...e/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.hadoop.ozone.s3.awssdk.v1;

import static org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE;

import java.io.IOException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
Expand All @@ -31,6 +33,7 @@ public class TestS3SDKV1 extends AbstractS3SDKV1Tests {
@BeforeAll
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.set(HDDS_DATANODE_VOLUME_MIN_FREE_SPACE, "5GB");
conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1);
startCluster(conf);
}
Expand Down
3 changes: 3 additions & 0 deletions ...-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/TestS3SDKV1WithRatisStreaming.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.hadoop.ozone.s3.awssdk.v1;

import static org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE;

import java.io.IOException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
Expand All @@ -36,6 +38,7 @@ public static void init() throws Exception {
false);
conf.setBoolean(OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY,
true);
conf.set(HDDS_DATANODE_VOLUME_MIN_FREE_SPACE, "5GB");
conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED, true);
// Ensure that all writes use datastream
Expand Down
3 changes: 3 additions & 0 deletions ...e/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.hadoop.ozone.s3.awssdk.v2;

import static org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE;

import java.io.IOException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
Expand All @@ -31,6 +33,7 @@ public class TestS3SDKV2 extends AbstractS3SDKV2Tests {
@BeforeAll
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.set(HDDS_DATANODE_VOLUME_MIN_FREE_SPACE, "5GB");
conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, 1);
startCluster(conf);
}
Expand Down
3 changes: 3 additions & 0 deletions ...-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/TestS3SDKV2WithRatisStreaming.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.hadoop.ozone.s3.awssdk.v2;

import static org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE;

import java.io.IOException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
Expand All @@ -36,6 +38,7 @@ public static void init() throws Exception {
false);
conf.setBoolean(OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY,
true);
conf.set(HDDS_DATANODE_VOLUME_MIN_FREE_SPACE, "5GB");
conf.setBoolean(OzoneConfigKeys.HDDS_CONTAINER_RATIS_DATASTREAM_ENABLED, true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED, true);
// Ensure that all writes use datastream
Expand Down
2 changes: 2 additions & 0 deletions ...oop/ozone/container/common/statemachine/commandhandler/TestRefreshVolumeUsageHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import static org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.util.HashMap;
Expand Down Expand Up @@ -56,6 +57,7 @@ public void setup() throws Exception {
//setup a cluster (1G free space is enough for a unit test)
conf = new OzoneConfiguration();
conf.set(OZONE_SCM_CONTAINER_SIZE, "1GB");
conf.set(HDDS_DATANODE_VOLUME_MIN_FREE_SPACE, "5GB");
conf.set(HDDS_NODE_REPORT_INTERVAL, "1s");
conf.set("hdds.datanode.du.factory.classname",
"org.apache.hadoop.ozone.container.common.volume.HddsVolumeFactory");
Expand Down