From 7182bdbe048693a4bb1d20d23926ff5adfd4c0a2 Mon Sep 17 00:00:00 2001
From: exceptionfactory <exceptionfactory@apache.org>
Date: Tue, 28 Mar 2023 21:39:39 -0500
Subject: [PATCH] NIFI-11358 Upgraded Hadoop from 3.3.4 to 3.3.5

- Upgraded Ranger Hadoop dependencies from 3.3.3 to 3.3.5
- Aligned Iceberg Hadoop version with project Hadoop version
- Updated Atlas dependencies to align hadoop-hdfs-client version with hadoop-common
- Updated Ranger hadoop-auth version to align with other Hadoop dependencies
- Updated Spark Livy bundle to align with project Hadoop version
- Removed unnecessary dependencies from Hive Test Utilities
- Updated HBase 2 Woodstox Core from 5.3.0 to 5.4.0
- Suppressed false positive vulnerabilities for HBase client libraries
---
 nifi-dependency-check-maven/suppressions.xml  |  25 ++++
 nifi-nar-bundles/nifi-atlas-bundle/pom.xml    |  12 ++
 .../nifi-hive-test-utils/pom.xml              | 116 ++++++++++++++++++
 .../nifi-hive3-processors/pom.xml             |   6 +
 .../nifi-iceberg-processors/pom.xml           |   8 ++
 nifi-nar-bundles/nifi-iceberg-bundle/pom.xml  |  23 +++-
 .../nifi-ranger-plugin/pom.xml                |  19 ---
 nifi-nar-bundles/nifi-ranger-bundle/pom.xml   |  25 +++-
 .../nifi-livy-controller-service/pom.xml      |  10 +-
 nifi-nar-bundles/nifi-spark-bundle/pom.xml    |   6 +
 .../pom.xml                                   |  12 ++
 .../nifi-registry-ranger-plugin/pom.xml       |   2 +-
 pom.xml                                       |   4 +-
 13 files changed, 242 insertions(+), 26 deletions(-)

diff --git a/nifi-dependency-check-maven/suppressions.xml b/nifi-dependency-check-maven/suppressions.xml
index 963de089e5e0..e34867068520 100644
--- a/nifi-dependency-check-maven/suppressions.xml
+++ b/nifi-dependency-check-maven/suppressions.xml
@@ -229,4 +229,29 @@
         <packageUrl regex="true">^pkg:maven/io\.netty/.*$</packageUrl>
         <cve>CVE-2022-41881</cve>
     </suppress>
+    <suppress>
+        <notes>CVE-2021-34538 applies to Apache Hive server not the Storage API library</notes>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.hive/hive\-storage\-api@.*$</packageUrl>
+        <cve>CVE-2021-34538</cve>
+    </suppress>
+    <suppress>
+        <notes>CVE-2018-8025 applies to HBase server not the shaded libraries</notes>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.hbase\.thirdparty/hbase\-shaded\-.*$</packageUrl>
+        <cve>CVE-2018-8025</cve>
+    </suppress>
+    <suppress>
+        <notes>CVE-2018-8025 applies to HBase Server not HBase libraries</notes>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.hbase/hbase\-.*$</packageUrl>
+        <cve>CVE-2018-8025</cve>
+    </suppress>
+    <suppress>
+        <notes>CVE-2019-0212 applies to HBase Server not HBase libraries</notes>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.hbase/hbase\-.*$</packageUrl>
+        <cve>CVE-2019-0212</cve>
+    </suppress>
+    <suppress>
+        <notes>Hadoop vulnerabilities do not apply to HBase Hadoop2 compatibility library</notes>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.hbase/hbase\-hadoop2\-compat@.*$</packageUrl>
+        <cpe>cpe:/a:apache:hadoop</cpe>
+    </suppress>
 </suppressions>
diff --git a/nifi-nar-bundles/nifi-atlas-bundle/pom.xml b/nifi-nar-bundles/nifi-atlas-bundle/pom.xml
index 9559df22b28c..2785a8392c42 100644
--- a/nifi-nar-bundles/nifi-atlas-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-atlas-bundle/pom.xml
@@ -104,6 +104,18 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <!-- Override hadoop-hdfs-client 3.3.0 from Atlas 2.2.0 -->
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-hdfs-client</artifactId>
+                <version>${hadoop.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-reload4j</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 </project>
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-test-utils/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-test-utils/pom.xml
index a66a99694fec..25c3cda0d9d8 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-test-utils/pom.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-test-utils/pom.xml
@@ -24,6 +24,66 @@
     <artifactId>nifi-hive-test-utils</artifactId>
     <packaging>jar</packaging>
 
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-yarn-common</artifactId>
+                <version>${hadoop.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-yarn-server-common</artifactId>
+                <version>${hadoop.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-common</artifactId>
+                <version>${hadoop.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>log4j</groupId>
+                        <artifactId>log4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-log4j12</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-reload4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>commons-logging</groupId>
+                        <artifactId>commons-logging</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-client</artifactId>
+                <version>${hadoop.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>log4j</groupId>
+                        <artifactId>log4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-log4j12</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-reload4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>commons-logging</groupId>
+                        <artifactId>commons-logging</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
     <dependencies>
         <dependency>
             <groupId>org.apache.hive</groupId>
@@ -102,6 +162,10 @@
                     <groupId>org.apache.ivy</groupId>
                     <artifactId>ivy</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -137,8 +201,60 @@
                     <groupId>org.apache.ivy</groupId>
                     <artifactId>ivy</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-distcp</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-mapreduce-client-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-hdfs</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-archives</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-yarn-registry</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase</groupId>
+                    <artifactId>hbase-hadoop2-compat</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase</groupId>
+                    <artifactId>hbase-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase.thirdparty</groupId>
+                    <artifactId>hbase-shaded-netty</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase.thirdparty</groupId>
+                    <artifactId>hbase-shaded-protobuf</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase.thirdparty</groupId>
+                    <artifactId>hbase-shaded-miscellaneous</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-llap-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.htrace</groupId>
+                    <artifactId>htrace-core</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/pom.xml
index aa3dc1af1043..faee9bef0a59 100644
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/pom.xml
@@ -316,6 +316,12 @@
             <artifactId>groovy-all</artifactId>
             <version>2.4.21</version>
         </dependency>
+        <!-- Override Jettison 1.5.3 from Hive -->
+        <dependency>
+            <groupId>org.codehaus.jettison</groupId>
+            <artifactId>jettison</artifactId>
+            <version>1.5.4</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.nifi</groupId>
             <artifactId>nifi-hadoop-utils</artifactId>
diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml
index 69ae6ec7dcbb..61d40d03654e 100644
--- a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml
@@ -143,6 +143,14 @@
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-yarn-common</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-yarn-server-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-yarn-registry</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>org.bouncycastle</groupId>
                     <artifactId>bcprov-jdk15on</artifactId>
diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml b/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml
index 548c64bf729d..cfbf8029f08f 100644
--- a/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml
@@ -27,7 +27,6 @@
     <properties>
         <iceberg.version>1.1.0</iceberg.version>
         <hive.version>3.1.3</hive.version>
-        <hadoop.version>3.3.3</hadoop.version>
     </properties>
 
     <modules>
@@ -88,12 +87,34 @@
                 <artifactId>ant</artifactId>
                 <version>1.10.12</version>
             </dependency>
+            <!-- Override Jettison 1.5.3 from Hive -->
+            <dependency>
+                <groupId>org.codehaus.jettison</groupId>
+                <artifactId>jettison</artifactId>
+                <version>1.5.4</version>
+            </dependency>
             <!-- Override nimbus-jose-jwt 9.8.1 from hadoop-auth -->
             <dependency>
                 <groupId>com.nimbusds</groupId>
                 <artifactId>nimbus-jose-jwt</artifactId>
                 <version>9.31</version>
             </dependency>
+            <!-- Override Hadoop 3.1.0 -->
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+                <version>${hadoop.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.bouncycastle</groupId>
+                        <artifactId>bcprov-jdk15on</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.bouncycastle</groupId>
+                        <artifactId>bcpkix-jdk15on</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 </project>
diff --git a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml
index 8297402fe8ed..f5b0f72a058d 100644
--- a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml
+++ b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml
@@ -180,25 +180,6 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-auth</artifactId>
-            <version>${ranger.hadoop.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-reload4j</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-log4j12</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>log4j</groupId>
-                    <artifactId>log4j</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>commons-logging</groupId>
-                    <artifactId>commons-logging</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.nifi</groupId>
diff --git a/nifi-nar-bundles/nifi-ranger-bundle/pom.xml b/nifi-nar-bundles/nifi-ranger-bundle/pom.xml
index c78039a02a29..fd864770998d 100644
--- a/nifi-nar-bundles/nifi-ranger-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-ranger-bundle/pom.xml
@@ -32,7 +32,7 @@
     </modules>
 
     <properties>
-        <ranger.hadoop.version>3.3.3</ranger.hadoop.version>
+        <ranger.hadoop.version>3.3.5</ranger.hadoop.version>
     </properties>
 
     <dependencyManagement>
@@ -73,6 +73,29 @@
                 <artifactId>nimbus-jose-jwt</artifactId>
                 <version>9.31</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-auth</artifactId>
+                <version>${ranger.hadoop.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-reload4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-log4j12</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>log4j</groupId>
+                        <artifactId>log4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>commons-logging</groupId>
+                        <artifactId>commons-logging</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 </project>
diff --git a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-controller-service/pom.xml b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-controller-service/pom.xml
index ab109b59a1d7..8e939b1648bf 100644
--- a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-controller-service/pom.xml
+++ b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-controller-service/pom.xml
@@ -66,7 +66,7 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-auth</artifactId>
-            <version>2.7.3</version>
+            <version>${hadoop.version}</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -76,6 +76,14 @@
                     <groupId>log4j</groupId>
                     <artifactId>log4j</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-logging</groupId>
+                    <artifactId>commons-logging</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
     </dependencies>
diff --git a/nifi-nar-bundles/nifi-spark-bundle/pom.xml b/nifi-nar-bundles/nifi-spark-bundle/pom.xml
index 8b0cc2f4eddc..d4fbaca79e93 100644
--- a/nifi-nar-bundles/nifi-spark-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-spark-bundle/pom.xml
@@ -69,6 +69,12 @@
                 <artifactId>netty</artifactId>
                 <version>${netty.3.version}</version>
             </dependency>
+            <!-- Override nimbus-jose-jwt 9.8.1 from hadoop-auth -->
+            <dependency>
+                <groupId>com.nimbusds</groupId>
+                <artifactId>nimbus-jose-jwt</artifactId>
+                <version>9.31</version>
+            </dependency>
             <!-- Override zookeeper -->
             <dependency>
                 <groupId>org.apache.zookeeper</groupId>
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/pom.xml
index b615430dffe2..236904574409 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/pom.xml
@@ -79,6 +79,18 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <!-- Override nimbus-jose-jwt 9.8.1 from hadoop-auth -->
+            <dependency>
+                <groupId>com.nimbusds</groupId>
+                <artifactId>nimbus-jose-jwt</artifactId>
+                <version>9.31</version>
+            </dependency>
+            <!-- Override woodstox-core 5.3.0 from HBase -->
+            <dependency>
+                <groupId>com.fasterxml.woodstox</groupId>
+                <artifactId>woodstox-core</artifactId>
+                <version>5.4.0</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 </project>
diff --git a/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/pom.xml b/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/pom.xml
index 61885f9fab63..55f5d6eceefb 100644
--- a/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/pom.xml
+++ b/nifi-registry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/pom.xml
@@ -25,7 +25,7 @@
     <packaging>jar</packaging>
 
     <properties>
-        <ranger.hadoop.version>3.3.3</ranger.hadoop.version>
+        <ranger.hadoop.version>3.3.5</ranger.hadoop.version>
         <ranger.ozone.version>1.2.1</ranger.ozone.version>
         <ranger.gcs.version>2.1.5</ranger.gcs.version>
     </properties>
diff --git a/pom.xml b/pom.xml
index 8c20954d49fc..55f4999dc09e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -130,9 +130,7 @@
         <nifi.groovy.version>3.0.14</nifi.groovy.version>
         <groovy.eclipse.batch.version>3.0.8-01</groovy.eclipse.batch.version>
         <surefire.version>3.0.0-M8</surefire.version>
-        <!-- The Hadoop version used by nifi-hadoop-libraries-nar and any NARs that depend on it, other NARs that need
-            a specific version should override this property, or use a more specific property like abc.hadoop.version -->
-        <hadoop.version>3.3.4</hadoop.version>
+        <hadoop.version>3.3.5</hadoop.version>
         <ozone.version>1.2.1</ozone.version>
         <gcs.version>2.1.5</gcs.version>
         <aspectj.version>1.9.19</aspectj.version>