From cf396e3764dc196556becc153289fccc9e533426 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 18 Aug 2022 16:10:01 +0530 Subject: [PATCH 1/9] ignornig testResilienceAsExpected --- .../lib/output/committer/manifest/TestRenameStageFailure.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java index b794f5814df36..14bcf418e3175 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java @@ -121,6 +121,7 @@ protected boolean requireRenameResilience() { @Test public void testResilienceAsExpected() throws Throwable { + Assume.assumeTrue(etagsPreserved); Assertions.assertThat(isResilientCommit()) .describedAs("resilient commit support") .isEqualTo(requireRenameResilience()); From 1060102066533c90f7dab85d20ffb31497416587 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Thu, 18 Aug 2022 18:27:19 +0530 Subject: [PATCH 2/9] HADOOP-18408. Ignoring testResilienceAsExpected for NonHNS Acc --- .../hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java index 5547d081c963f..3c817c2c6b405 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java @@ -45,7 +45,7 @@ public ITestAbfsRenameStageFailure() throws Exception { public void setup() throws Exception { binding.setup(); super.setup(); - } + } @Override protected Configuration createConfiguration() { From aa9db0257b6fa13ec8f7ad9c0695d53b28e1baf0 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Fri, 19 Aug 2022 20:11:37 +0530 Subject: [PATCH 3/9] Correcting requireRenameResilience to false for nonHNS account --- .../committer/manifest/TestRenameStageFailure.java | 1 - .../commit/ITestAbfsRenameStageFailure.java | 13 ++++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java index 14bcf418e3175..b794f5814df36 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java @@ -121,7 +121,6 @@ protected boolean requireRenameResilience() { @Test public void testResilienceAsExpected() throws Throwable { - Assume.assumeTrue(etagsPreserved); Assertions.assertThat(isResilientCommit()) .describedAs("resilient commit support") .isEqualTo(requireRenameResilience()); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java index 3c817c2c6b405..0bb683a667f75 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.azurebfs.commit; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding; import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract; import org.apache.hadoop.fs.contract.AbstractFSContract; @@ -41,6 +42,12 @@ public ITestAbfsRenameStageFailure() throws Exception { binding = new ABFSContractTestBinding(); } + protected boolean isNamespaceEnabled() { + FileSystem fs = getFileSystem(); + String namespaceEnabled = fs.getConf().get("fs.azure.test.namespace.enabled"); + return namespaceEnabled.equals("true"); + } + @Override public void setup() throws Exception { binding.setup(); @@ -59,7 +66,11 @@ protected AbstractFSContract createContract(final Configuration conf) { @Override protected boolean requireRenameResilience() { - return true; + if (isNamespaceEnabled()) { + return true; + } + + return false; } @Override From 2eab3018ce63bb2776578046cf3c7405330f9cfd Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Mon, 22 Aug 2022 12:15:58 +0530 Subject: [PATCH 4/9] Fixed extra space in setup() --- .../hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java index 0bb683a667f75..309a87b3ba2c1 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java @@ -52,7 +52,7 @@ protected boolean isNamespaceEnabled() { public void setup() throws Exception { binding.setup(); super.setup(); - } + } @Override protected Configuration createConfiguration() { From 977b4554da2184f40c3c913a6ed8218de539e792 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Tue, 23 Aug 2022 10:18:53 +0530 Subject: [PATCH 5/9] Nit changes --- .../hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java index 309a87b3ba2c1..5e01759ecb93a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java @@ -45,9 +45,8 @@ public ITestAbfsRenameStageFailure() throws Exception { protected boolean isNamespaceEnabled() { FileSystem fs = getFileSystem(); String namespaceEnabled = fs.getConf().get("fs.azure.test.namespace.enabled"); - return namespaceEnabled.equals("true"); + return ("true").equals(namespaceEnabled); } - @Override public void setup() throws Exception { binding.setup(); From d2fdfaf2e186ac240e0a5d18ce34c76b89ba26f6 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Tue, 23 Aug 2022 11:33:24 +0530 Subject: [PATCH 6/9] Removed extra line --- .../hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java index 5e01759ecb93a..5a3bc359eafef 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java @@ -68,7 +68,6 @@ protected boolean requireRenameResilience() { if (isNamespaceEnabled()) { return true; } - return false; } From aa205790a1642bd2b4cfaa4301db46e70935a354 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Tue, 23 Aug 2022 11:33:24 +0530 Subject: [PATCH 7/9] Removed extra line --- .../hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java index 5e01759ecb93a..5a3bc359eafef 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java @@ -68,7 +68,6 @@ protected boolean requireRenameResilience() { if (isNamespaceEnabled()) { return true; } - return false; } From 3e3aaf69b157485615b7e96b0f5ab0974420da2a Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Wed, 24 Aug 2022 14:27:19 +0530 Subject: [PATCH 8/9] Code style changes --- .../hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java index 5a3bc359eafef..96325e07e7958 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java @@ -47,6 +47,7 @@ protected boolean isNamespaceEnabled() { String namespaceEnabled = fs.getConf().get("fs.azure.test.namespace.enabled"); return ("true").equals(namespaceEnabled); } + @Override public void setup() throws Exception { binding.setup(); @@ -68,6 +69,7 @@ protected boolean requireRenameResilience() { if (isNamespaceEnabled()) { return true; } + return false; } From 69e8abc60010de226d4e6f4f0d54816f9db37462 Mon Sep 17 00:00:00 2001 From: sreeb-msft Date: Fri, 26 Aug 2022 16:47:39 +0530 Subject: [PATCH 9/9] Checking fs capabilities dynamically --- .../manifest/TestRenameStageFailure.java | 2 +- .../azurebfs/AbstractAbfsIntegrationTest.java | 12 +++++++++++- .../commit/ITestAbfsRenameStageFailure.java | 19 ++++++++----------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java index b794f5814df36..11d2beda5d015 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java @@ -115,7 +115,7 @@ public void setup() throws Exception { * Does this test suite require rename resilience in the store/FS? * @return true if the store operations are resilient. */ - protected boolean requireRenameResilience() { + protected boolean requireRenameResilience() throws IOException { return false; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 4a5507526c3a1..2c99267d917a4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -149,6 +149,17 @@ protected boolean getIsNamespaceEnabled(AzureBlobFileSystem fs) return fs.getIsNamespaceEnabled(getTestTracingContext(fs, false)); } + public static TracingContext getSampleTracingContext(AzureBlobFileSystem fs, + boolean needsPrimaryReqId) { + String correlationId, fsId; + TracingHeaderFormat format; + correlationId = "test-corr-id"; + fsId = "test-filesystem-id"; + format = TracingHeaderFormat.ALL_ID_FORMAT; + return new TracingContext(correlationId, fsId, + FSOperationType.TEST_OP, needsPrimaryReqId, format, null); + } + public TracingContext getTestTracingContext(AzureBlobFileSystem fs, boolean needsPrimaryReqId) { String correlationId, fsId; @@ -167,7 +178,6 @@ public TracingContext getTestTracingContext(AzureBlobFileSystem fs, FSOperationType.TEST_OP, needsPrimaryReqId, format, null); } - @Before public void setup() throws Exception { //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem. diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java index 9905ea1a828ab..6b9830e8f33fc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java @@ -19,12 +19,13 @@ package org.apache.hadoop.fs.azurebfs.commit; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding; import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestRenameStageFailure; - /** * Rename failure logic on ABFS. * This will go through the resilient rename operation. @@ -42,10 +43,9 @@ public ITestAbfsRenameStageFailure() throws Exception { binding = new ABFSContractTestBinding(); } - protected boolean isNamespaceEnabled() { - FileSystem fs = getFileSystem(); - String namespaceEnabled = fs.getConf().get("fs.azure.test.namespace.enabled"); - return ("true").equals(namespaceEnabled); + protected boolean isNamespaceEnabled() throws AzureBlobFileSystemException { + AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem(); + return fs.getAbfsStore().getIsNamespaceEnabled(AbstractAbfsIntegrationTest.getSampleTracingContext(fs, false)); } @Override @@ -65,11 +65,8 @@ protected AbstractFSContract createContract(final Configuration conf) { } @Override - protected boolean requireRenameResilience() { - if (isNamespaceEnabled()) { - return true; - } - return false; + protected boolean requireRenameResilience() throws AzureBlobFileSystemException { + return isNamespaceEnabled(); } @Override