From 28c374cf535405254f4183528864014ea5776fc8 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Mon, 25 Jul 2022 12:59:50 +0100 Subject: [PATCH 1/2] HDFS-16676. DatanodeAdminManager$Monitor reports a node as invalid continuously --- .../blockmanagement/DatanodeAdminDefaultMonitor.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java index 3ff360466c98f..a1cb7940baba7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java @@ -201,6 +201,7 @@ private void check() { iterkey).iterator(); final List toRemove = new ArrayList<>(); final List unhealthyDns = new ArrayList<>(); + boolean inValidState = false; while (it.hasNext() && !exceededNumBlocksPerCheck() && namesystem .isRunning()) { @@ -265,6 +266,7 @@ private void check() { // to track maintenance expiration. dnAdmin.setInMaintenance(dn); } else { + inValidState = true; Preconditions.checkState(false, "Node %s is in an invalid state! " + "Invalid state: %s %s blocks are on this dn.", @@ -288,7 +290,11 @@ private void check() { // an invalid state. LOG.warn("DatanodeAdminMonitor caught exception when processing node " + "{}.", dn, e); - getPendingNodes().add(dn); + if(!inValidState){ + getPendingNodes().add(dn); + } else { + LOG.warn("Ignoring the node {} which is in invalid state", dn); + } toRemove.add(dn); unhealthyDns.remove(dn); } finally { From a3c4dc90d5108fb8eb57f5efef312e295c8128e2 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Tue, 26 Jul 2022 11:05:54 +0100 Subject: [PATCH 2/2] Changing inValidState to isValidState --- .../server/blockmanagement/DatanodeAdminDefaultMonitor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java index a1cb7940baba7..e642dfba35188 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java @@ -201,7 +201,7 @@ private void check() { iterkey).iterator(); final List toRemove = new ArrayList<>(); final List unhealthyDns = new ArrayList<>(); - boolean inValidState = false; + boolean isValidState = true; while (it.hasNext() && !exceededNumBlocksPerCheck() && namesystem .isRunning()) { @@ -266,7 +266,7 @@ private void check() { // to track maintenance expiration. dnAdmin.setInMaintenance(dn); } else { - inValidState = true; + isValidState = false; Preconditions.checkState(false, "Node %s is in an invalid state! " + "Invalid state: %s %s blocks are on this dn.", @@ -290,7 +290,7 @@ private void check() { // an invalid state. LOG.warn("DatanodeAdminMonitor caught exception when processing node " + "{}.", dn, e); - if(!inValidState){ + if(isValidState){ getPendingNodes().add(dn); } else { LOG.warn("Ignoring the node {} which is in invalid state", dn);