NIFI-11358 Upgraded Hadoop from 3.3.4 to 3.3.5

- Upgraded Ranger Hadoop dependencies from 3.3.3 to 3.3.5 - Aligned Iceberg Hadoop version with project Hadoop version - Updated Atlas dependencies to align hadoop-hdfs-client version with hadoop-common - Updated Ranger hadoop-auth version to align with other Hadoop dependencies - Updated Spark Livy bundle to align with project Hadoop version - Removed unnecessary dependencies from Hive Test Utilities - Updated HBase 2 Woodstox Core from 5.3.0 to 5.4.0 - Suppressed false positive vulnerabilities for HBase client libraries Signed-off-by: Pierre Villard <pierre.villard.fr@gmail.com> This closes apache#7097.
r-vandenbos · Apr 11, 2023 · 8f40be2 · 8f40be2
1 parent 98afa31
commit 8f40be2
Show file tree

Hide file tree

Showing 13 changed files with 242 additions and 26 deletions.
diff --git a/nifi-dependency-check-maven/suppressions.xml b/nifi-dependency-check-maven/suppressions.xml
@@ -229,4 +229,29 @@
         <packageUrl regex="true">^pkg:maven/io\.netty/.*$</packageUrl>
         <cve>CVE-2022-41881</cve>
     </suppress>
+    <suppress>
+        <notes>CVE-2021-34538 applies to Apache Hive server not the Storage API library</notes>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.hive/hive\-storage\-api@.*$</packageUrl>
+        <cve>CVE-2021-34538</cve>
+    </suppress>
+    <suppress>
+        <notes>CVE-2018-8025 applies to HBase server not the shaded libraries</notes>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.hbase\.thirdparty/hbase\-shaded\-.*$</packageUrl>
+        <cve>CVE-2018-8025</cve>
+    </suppress>
+    <suppress>
+        <notes>CVE-2018-8025 applies to HBase Server not HBase libraries</notes>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.hbase/hbase\-.*$</packageUrl>
+        <cve>CVE-2018-8025</cve>
+    </suppress>
+    <suppress>
+        <notes>CVE-2019-0212 applies to HBase Server not HBase libraries</notes>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.hbase/hbase\-.*$</packageUrl>
+        <cve>CVE-2019-0212</cve>
+    </suppress>
+    <suppress>
+        <notes>Hadoop vulnerabilities do not apply to HBase Hadoop2 compatibility library</notes>
+        <packageUrl regex="true">^pkg:maven/org\.apache\.hbase/hbase\-hadoop2\-compat@.*$</packageUrl>
+        <cpe>cpe:/a:apache:hadoop</cpe>
+    </suppress>
 </suppressions>
diff --git a/nifi-nar-bundles/nifi-atlas-bundle/pom.xml b/nifi-nar-bundles/nifi-atlas-bundle/pom.xml
@@ -104,6 +104,18 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <!-- Override hadoop-hdfs-client 3.3.0 from Atlas 2.2.0 -->
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-hdfs-client</artifactId>
+                <version>${hadoop.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-reload4j</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 </project>
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-test-utils/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-test-utils/pom.xml
@@ -24,6 +24,66 @@
     <artifactId>nifi-hive-test-utils</artifactId>
     <packaging>jar</packaging>
 
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-yarn-common</artifactId>
+                <version>${hadoop.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-yarn-server-common</artifactId>
+                <version>${hadoop.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-common</artifactId>
+                <version>${hadoop.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>log4j</groupId>
+                        <artifactId>log4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-log4j12</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-reload4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>commons-logging</groupId>
+                        <artifactId>commons-logging</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-client</artifactId>
+                <version>${hadoop.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>log4j</groupId>
+                        <artifactId>log4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-log4j12</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-reload4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>commons-logging</groupId>
+                        <artifactId>commons-logging</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
     <dependencies>
         <dependency>
             <groupId>org.apache.hive</groupId>
@@ -102,6 +162,10 @@
                     <groupId>org.apache.ivy</groupId>
                     <artifactId>ivy</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -137,8 +201,60 @@
                     <groupId>org.apache.ivy</groupId>
                     <artifactId>ivy</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-distcp</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-mapreduce-client-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-hdfs</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-archives</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-yarn-registry</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase</groupId>
+                    <artifactId>hbase-hadoop2-compat</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase</groupId>
+                    <artifactId>hbase-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase.thirdparty</groupId>
+                    <artifactId>hbase-shaded-netty</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase.thirdparty</groupId>
+                    <artifactId>hbase-shaded-protobuf</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase.thirdparty</groupId>
+                    <artifactId>hbase-shaded-miscellaneous</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-llap-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.htrace</groupId>
+                    <artifactId>htrace-core</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>

diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/pom.xml
@@ -316,6 +316,12 @@
             <artifactId>groovy-all</artifactId>
             <version>2.4.21</version>
         </dependency>
+        <!-- Override Jettison 1.5.3 from Hive -->
+        <dependency>
+            <groupId>org.codehaus.jettison</groupId>
+            <artifactId>jettison</artifactId>
+            <version>1.5.4</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.nifi</groupId>
             <artifactId>nifi-hadoop-utils</artifactId>

diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml b/nifi-nar-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml
@@ -143,6 +143,14 @@
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-yarn-common</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-yarn-server-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-yarn-registry</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>org.bouncycastle</groupId>
                     <artifactId>bcprov-jdk15on</artifactId>

diff --git a/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml b/nifi-nar-bundles/nifi-iceberg-bundle/pom.xml
@@ -27,7 +27,6 @@
     <properties>
         <iceberg.version>1.1.0</iceberg.version>
         <hive.version>3.1.3</hive.version>
-        <hadoop.version>3.3.3</hadoop.version>
     </properties>
 
     <modules>
@@ -88,12 +87,34 @@
                 <artifactId>ant</artifactId>
                 <version>1.10.12</version>
             </dependency>
+            <!-- Override Jettison 1.5.3 from Hive -->
+            <dependency>
+                <groupId>org.codehaus.jettison</groupId>
+                <artifactId>jettison</artifactId>
+                <version>1.5.4</version>
+            </dependency>
             <!-- Override nimbus-jose-jwt 9.8.1 from hadoop-auth -->
             <dependency>
                 <groupId>com.nimbusds</groupId>
                 <artifactId>nimbus-jose-jwt</artifactId>
                 <version>9.31</version>
             </dependency>
+            <!-- Override Hadoop 3.1.0 -->
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+                <version>${hadoop.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.bouncycastle</groupId>
+                        <artifactId>bcprov-jdk15on</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.bouncycastle</groupId>
+                        <artifactId>bcpkix-jdk15on</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 </project>
diff --git a/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml b/nifi-nar-bundles/nifi-ranger-bundle/nifi-ranger-plugin/pom.xml
@@ -180,25 +180,6 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-auth</artifactId>
-            <version>${ranger.hadoop.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-reload4j</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-log4j12</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>log4j</groupId>
-                    <artifactId>log4j</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>commons-logging</groupId>
-                    <artifactId>commons-logging</artifactId>
-                </exclusion>
-            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.nifi</groupId>

diff --git a/nifi-nar-bundles/nifi-ranger-bundle/pom.xml b/nifi-nar-bundles/nifi-ranger-bundle/pom.xml
@@ -32,7 +32,7 @@
     </modules>
 
     <properties>
-        <ranger.hadoop.version>3.3.3</ranger.hadoop.version>
+        <ranger.hadoop.version>3.3.5</ranger.hadoop.version>
     </properties>
 
     <dependencyManagement>
@@ -73,6 +73,29 @@
                 <artifactId>nimbus-jose-jwt</artifactId>
                 <version>9.31</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-auth</artifactId>
+                <version>${ranger.hadoop.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-reload4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.slf4j</groupId>
+                        <artifactId>slf4j-log4j12</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>log4j</groupId>
+                        <artifactId>log4j</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>commons-logging</groupId>
+                        <artifactId>commons-logging</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 </project>
diff --git a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-controller-service/pom.xml b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-controller-service/pom.xml
@@ -66,7 +66,7 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-auth</artifactId>
-            <version>2.7.3</version>
+            <version>${hadoop.version}</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -76,6 +76,14 @@
                     <groupId>log4j</groupId>
                     <artifactId>log4j</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-logging</groupId>
+                    <artifactId>commons-logging</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
     </dependencies>

diff --git a/nifi-nar-bundles/nifi-spark-bundle/pom.xml b/nifi-nar-bundles/nifi-spark-bundle/pom.xml
@@ -69,6 +69,12 @@
                 <artifactId>netty</artifactId>
                 <version>${netty.3.version}</version>
             </dependency>
+            <!-- Override nimbus-jose-jwt 9.8.1 from hadoop-auth -->
+            <dependency>
+                <groupId>com.nimbusds</groupId>
+                <artifactId>nimbus-jose-jwt</artifactId>
+                <version>9.31</version>
+            </dependency>
             <!-- Override zookeeper -->
             <dependency>
                 <groupId>org.apache.zookeeper</groupId>

diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-hbase_2-client-service-bundle/pom.xml
@@ -79,6 +79,18 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <!-- Override nimbus-jose-jwt 9.8.1 from hadoop-auth -->
+            <dependency>
+                <groupId>com.nimbusds</groupId>
+                <artifactId>nimbus-jose-jwt</artifactId>
+                <version>9.31</version>
+            </dependency>
+            <!-- Override woodstox-core 5.3.0 from HBase -->
+            <dependency>
+                <groupId>com.fasterxml.woodstox</groupId>
+                <artifactId>woodstox-core</artifactId>
+                <version>5.4.0</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 </project>
diff --git a/...egistry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/pom.xml b/...egistry/nifi-registry-extensions/nifi-registry-ranger/nifi-registry-ranger-plugin/pom.xml
@@ -25,7 +25,7 @@
     <packaging>jar</packaging>
 
     <properties>
-        <ranger.hadoop.version>3.3.3</ranger.hadoop.version>
+        <ranger.hadoop.version>3.3.5</ranger.hadoop.version>
         <ranger.ozone.version>1.2.1</ranger.ozone.version>
         <ranger.gcs.version>2.1.5</ranger.gcs.version>
     </properties>

diff --git a/pom.xml b/pom.xml
@@ -130,9 +130,7 @@
         <nifi.groovy.version>3.0.14</nifi.groovy.version>
         <groovy.eclipse.batch.version>3.0.8-01</groovy.eclipse.batch.version>
         <surefire.version>3.0.0-M8</surefire.version>
-        <!-- The Hadoop version used by nifi-hadoop-libraries-nar and any NARs that depend on it, other NARs that need
-            a specific version should override this property, or use a more specific property like abc.hadoop.version -->
-        <hadoop.version>3.3.4</hadoop.version>
+        <hadoop.version>3.3.5</hadoop.version>
         <ozone.version>1.2.1</ozone.version>
         <gcs.version>2.1.5</gcs.version>
         <aspectj.version>1.9.19</aspectj.version>