From 84f3331ed7d67d3310a1372451f93f96d57ceaad Mon Sep 17 00:00:00 2001 From: xzn Date: Tue, 28 Sep 2021 18:09:20 +0800 Subject: [PATCH 1/6] add journalNode Id for metrics. --- .../apache/hadoop/test/MetricsAsserts.java | 25 +++++++++++++++++++ .../hdfs/qjournal/server/JournalMetrics.java | 6 +++++ .../hdfs/qjournal/server/TestJournalNode.java | 24 +++++++++++++++--- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java index eb8d938994735..53b0e8b38a2d4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java @@ -401,4 +401,29 @@ public static void assertQuantileGauges(String prefix, geq(0l)); } } + + /** + * Assert a tag of metric as expected. + * @param name of the metric tag + * @param expected value of the metric tag + * @param rb the record builder mock used to getMetrics + */ + public static void assertTag(String name, String expected, + MetricsRecordBuilder rb) { + Assert.assertEquals("Bad Tag for metric " + name, + expected, getStringTag(name, rb)); + } + + /** + * get the value tag for the metric. + * @param name of the metric tag + * @param rb value of the metric tag + * @return the value tag for the metric + */ + public static String getStringTag(String name, MetricsRecordBuilder rb) { + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + verify(rb).tag(eqName(info(name, "")), captor.capture()); + checkCaptured(captor, name); + return captor.getValue(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java index 3499da98dca8e..3032a1ed09fc5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metric.Type; import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.MetricsRegistry; @@ -99,6 +100,11 @@ String getName() { return "Journal-" + journal.getJournalId(); } + @Metric(value={"JournalId", "Current JournalId"}, type= Type.TAG) + public String getJournalId() { + return journal.getJournalId(); + } + @Metric("Current writer's epoch") public long getLastWriterEpoch() { try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java index 6e117b7687ac7..5a2fea33067bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java @@ -97,7 +97,8 @@ public void setup() throws Exception { conf.set(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY, editsDir.getAbsolutePath()); } else if (testName.getMethodName().equals( - "testJournalDefaultDirForOneNameSpace")) { + "testJournalDefaultDirForOneNameSpace") || + testName.getMethodName().equals("testJournalMetricTags")) { FileUtil.fullyDelete(new File(DFSConfigKeys .DFS_JOURNALNODE_EDITS_DIR_DEFAULT)); setFederationConf(); @@ -151,7 +152,8 @@ public void setup() throws Exception { testName.getMethodName().equals( "testJournalCommonDirAcrossNameSpace") || testName.getMethodName().equals( - "testJournalDefaultDirForOneNameSpace")) { + "testJournalDefaultDirForOneNameSpace") || + testName.getMethodName().equals("testJournalMetricTags")) { Collection nameServiceIds = DFSUtilClient.getNameServiceIds(conf); for(String nsId: nameServiceIds) { journalId = "test-journalid-" + nsId; @@ -240,6 +242,23 @@ public void testJournalDefaultDirForOneNameSpace() { File.separator + jid); assertEquals(editsDir.toString(), journalStorage.getRoot().toString()); } + + @Test(timeout=100000) + public void testJournalMetricTags() { + setupStaticHostResolution(2, "journalnode"); + String jid = "test-journalid-ns1"; + Journal nsJournal = jn.getJournal(jid); + MetricsRecordBuilder metrics = MetricsAsserts.getMetrics( + nsJournal.getMetrics().getName()); + MetricsAsserts.assertTag("JournalId", jid, metrics); + + jid = "test-journalid-ns2"; + nsJournal = jn.getJournal(jid); + metrics = MetricsAsserts.getMetrics( + nsJournal.getMetrics().getName()); + MetricsAsserts.assertTag("JournalId", jid, metrics); + } + @Test(timeout=100000) public void testJournal() throws Exception { MetricsRecordBuilder metrics = MetricsAsserts.getMetrics( @@ -281,7 +300,6 @@ public void testJournal() throws Exception { } - @Test(timeout=100000) public void testReturnsSegmentInfoAtEpochTransition() throws Exception { ch.newEpoch(1).get(); From 573de69c5307dcbb6c30b825f28216c5d68ee45e Mon Sep 17 00:00:00 2001 From: xzn Date: Wed, 29 Sep 2021 16:53:43 +0800 Subject: [PATCH 2/6] fix checkstyle 1. remove empty space --- .../org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java index 3032a1ed09fc5..574ca00350a1f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalMetrics.java @@ -100,7 +100,7 @@ String getName() { return "Journal-" + journal.getJournalId(); } - @Metric(value={"JournalId", "Current JournalId"}, type= Type.TAG) + @Metric(value={"JournalId", "Current JournalId"}, type=Type.TAG) public String getJournalId() { return journal.getJournalId(); } From f98cb220608c6bb128635a1943f2a2b6e20ef20d Mon Sep 17 00:00:00 2001 From: xzn Date: Wed, 29 Sep 2021 16:55:10 +0800 Subject: [PATCH 3/6] readd the empty line --- .../apache/hadoop/hdfs/qjournal/server/TestJournalNode.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java index 5a2fea33067bd..ac721e9d875e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java @@ -299,7 +299,8 @@ public void testJournal() throws Exception { assertTrue(lastJournalTimestamp > beginTimestamp); } - + + @Test(timeout=100000) public void testReturnsSegmentInfoAtEpochTransition() throws Exception { ch.newEpoch(1).get(); From 1d4e1b1177fad0d5f720616a7278aa61830088a1 Mon Sep 17 00:00:00 2001 From: xzn Date: Wed, 29 Sep 2021 16:58:44 +0800 Subject: [PATCH 4/6] remove the empty line --- .../org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java index ac721e9d875e1..eb23f9dcf6f2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java @@ -300,7 +300,6 @@ public void testJournal() throws Exception { } - @Test(timeout=100000) public void testReturnsSegmentInfoAtEpochTransition() throws Exception { ch.newEpoch(1).get(); From 06babcb9153fc23108f985b501ff815ef59e40e9 Mon Sep 17 00:00:00 2001 From: xzn Date: Thu, 30 Sep 2021 16:51:38 +0800 Subject: [PATCH 5/6] fix some empty line checkstyle. --- .../org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java index eb23f9dcf6f2e..ac721e9d875e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java @@ -300,6 +300,7 @@ public void testJournal() throws Exception { } + @Test(timeout=100000) public void testReturnsSegmentInfoAtEpochTransition() throws Exception { ch.newEpoch(1).get(); From fcfc7e79c84722a3ef9f31cfcaa81e07d4fdd6e5 Mon Sep 17 00:00:00 2001 From: xzn Date: Thu, 30 Sep 2021 16:55:31 +0800 Subject: [PATCH 6/6] fix some empty line checkstyle. --- .../apache/hadoop/hdfs/qjournal/server/TestJournalNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java index ac721e9d875e1..a8d4442bdff28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNode.java @@ -299,8 +299,8 @@ public void testJournal() throws Exception { assertTrue(lastJournalTimestamp > beginTimestamp); } - - + + @Test(timeout=100000) public void testReturnsSegmentInfoAtEpochTransition() throws Exception { ch.newEpoch(1).get();