Skip to content

Commit 1923096

Browse files
lfxylfxy
authored
Allow block reconstruction pending timeout to be refreshable (#4567)
Reviewed-by: Hiroyuki Adachi <hadachi@yahoo-corp.jp> Signed-off-by: Takanobu Asanuma <tasanuma@apache.org>
1 parent 21bae31 commit 1923096

File tree

5 files changed

+58
-6
lines changed

5 files changed

+58
-6
lines changed

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1067,6 +1067,26 @@ public void setBlocksReplWorkMultiplier(int newVal) {
10671067
blocksReplWorkMultiplier = newVal;
10681068
}
10691069

1070+
/**
1071+
* Updates the value used for pendingReconstruction timeout, which is set by
1072+
* {@code DFSConfigKeys.
1073+
* DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY} initially.
1074+
*
1075+
* @param newVal - Must be a positive non-zero integer.
1076+
*/
1077+
public void setReconstructionPendingTimeout(int newVal) {
1078+
ensurePositiveInt(newVal,
1079+
DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY);
1080+
pendingReconstruction.setTimeout(newVal * 1000L);
1081+
}
1082+
1083+
/** Returns the current setting for pendingReconstruction timeout, set by
1084+
* {@code DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY}.
1085+
*/
1086+
public int getReconstructionPendingTimeout() {
1087+
return (int)(pendingReconstruction.getTimeout() / 1000L);
1088+
}
1089+
10701090
public int getDefaultStorageNum(BlockInfo block) {
10711091
switch (block.getBlockType()) {
10721092
case STRIPED: return ((BlockInfoStriped) block).getRealTotalBlockNum();

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReconstructionBlocks.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,14 @@ void start() {
7676
timerThread.start();
7777
}
7878

79+
public void setTimeout(long timeoutPeriod) {
80+
this.timeout = timeoutPeriod;
81+
}
82+
83+
public long getTimeout() {
84+
return this.timeout;
85+
}
86+
7987
/**
8088
* Add a block to the list of pending reconstructions
8189
* @param block The corresponding block

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,8 @@
203203
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT;
204204
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY;
205205
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_DEFAULT;
206+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY;
207+
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT;
206208

207209
import static org.apache.hadoop.util.ExitUtil.terminate;
208210
import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
@@ -350,7 +352,8 @@ public enum OperationCategory {
350352
DFS_NAMENODE_MAX_SLOWPEER_COLLECT_NODES_KEY,
351353
DFS_BLOCK_INVALIDATE_LIMIT_KEY,
352354
DFS_DATANODE_PEER_STATS_ENABLED_KEY,
353-
DFS_DATANODE_MAX_NODES_TO_REPORT_KEY));
355+
DFS_DATANODE_MAX_NODES_TO_REPORT_KEY,
356+
DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY));
354357

355358
private static final String USAGE = "Usage: hdfs namenode ["
356359
+ StartupOption.BACKUP.getName() + "] | \n\t["
@@ -2301,7 +2304,8 @@ protected String reconfigurePropertyImpl(String property, String newVal)
23012304
} else if (property.equals(DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY)
23022305
|| property.equals(DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY)
23032306
|| property.equals(
2304-
DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION)) {
2307+
DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION)
2308+
|| property.equals(DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY)) {
23052309
return reconfReplicationParameters(newVal, property);
23062310
} else if (property.equals(DFS_BLOCK_REPLICATOR_CLASSNAME_KEY) || property
23072311
.equals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY)) {
@@ -2347,6 +2351,14 @@ private String reconfReplicationParameters(final String newVal,
23472351
DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT,
23482352
newVal));
23492353
newSetting = bm.getBlocksReplWorkMultiplier();
2354+
} else if (
2355+
property.equals(
2356+
DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY)) {
2357+
bm.setReconstructionPendingTimeout(
2358+
adjustNewVal(
2359+
DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_DEFAULT,
2360+
newVal));
2361+
newSetting = bm.getReconstructionPendingTimeout();
23502362
} else {
23512363
throw new IllegalArgumentException("Unexpected property " +
23522364
property + " in reconfReplicationParameters");

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRefreshNamenodeReplicationConfig.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ public void setup() throws IOException {
4949
config.setInt(
5050
DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
5151
12);
52+
config.setInt(
53+
DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
54+
300);
5255

5356
cluster = new MiniDFSCluster.Builder(config)
5457
.nnTopology(MiniDFSNNTopology.simpleSingleNN(0, 0))
@@ -72,6 +75,7 @@ public void testParamsCanBeReconfigured() throws ReconfigurationException {
7275
assertEquals(8, bm.getMaxReplicationStreams());
7376
assertEquals(10, bm.getReplicationStreamsHardLimit());
7477
assertEquals(12, bm.getBlocksReplWorkMultiplier());
78+
assertEquals(300, bm.getReconstructionPendingTimeout());
7579

7680
cluster.getNameNode().reconfigurePropertyImpl(
7781
DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, "20");
@@ -81,10 +85,14 @@ public void testParamsCanBeReconfigured() throws ReconfigurationException {
8185
cluster.getNameNode().reconfigurePropertyImpl(
8286
DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
8387
"24");
88+
cluster.getNameNode().reconfigurePropertyImpl(
89+
DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY,
90+
"180");
8491

8592
assertEquals(20, bm.getMaxReplicationStreams());
8693
assertEquals(22, bm.getReplicationStreamsHardLimit());
8794
assertEquals(24, bm.getBlocksReplWorkMultiplier());
95+
assertEquals(180, bm.getReconstructionPendingTimeout());
8896
}
8997

9098
/**
@@ -96,7 +104,8 @@ public void testReconfigureFailsWithInvalidValues() throws Exception {
96104
String[] keys = new String[]{
97105
DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY,
98106
DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY,
99-
DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION
107+
DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
108+
DFSConfigKeys.DFS_NAMENODE_RECONSTRUCTION_PENDING_TIMEOUT_SEC_KEY
100109
};
101110

102111
// Ensure we cannot set any of the parameters negative
@@ -112,6 +121,7 @@ public void testReconfigureFailsWithInvalidValues() throws Exception {
112121
assertEquals(8, bm.getMaxReplicationStreams());
113122
assertEquals(10, bm.getReplicationStreamsHardLimit());
114123
assertEquals(12, bm.getBlocksReplWorkMultiplier());
124+
assertEquals(300, bm.getReconstructionPendingTimeout());
115125

116126
for (String key : keys) {
117127
ReconfigurationException e =
@@ -126,6 +136,7 @@ public void testReconfigureFailsWithInvalidValues() throws Exception {
126136
assertEquals(8, bm.getMaxReplicationStreams());
127137
assertEquals(10, bm.getReplicationStreamsHardLimit());
128138
assertEquals(12, bm.getBlocksReplWorkMultiplier());
139+
assertEquals(300, bm.getReconstructionPendingTimeout());
129140

130141
// Ensure none of the parameters can be set to a string value
131142
for (String key : keys) {
@@ -139,5 +150,6 @@ public void testReconfigureFailsWithInvalidValues() throws Exception {
139150
assertEquals(8, bm.getMaxReplicationStreams());
140151
assertEquals(10, bm.getReplicationStreamsHardLimit());
141152
assertEquals(12, bm.getBlocksReplWorkMultiplier());
153+
assertEquals(300, bm.getReconstructionPendingTimeout());
142154
}
143-
}
155+
}

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ public void testNameNodeGetReconfigurableProperties() throws IOException, Interr
438438
final List<String> outs = Lists.newArrayList();
439439
final List<String> errs = Lists.newArrayList();
440440
getReconfigurableProperties("namenode", address, outs, errs);
441-
assertEquals(19, outs.size());
441+
assertEquals(20, outs.size());
442442
assertTrue(outs.get(0).contains("Reconfigurable properties:"));
443443
assertEquals(DFS_BLOCK_INVALIDATE_LIMIT_KEY, outs.get(1));
444444
assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(2));
@@ -1266,4 +1266,4 @@ public void testAllDatanodesReconfig()
12661266
outs.get(8));
12671267
}
12681268

1269-
}
1269+
}

0 commit comments

Comments
 (0)