Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

Commit

Permalink
[NSE-382]Fix Hadoop version issue (#388)
Browse files Browse the repository at this point in the history 
* Fix Hadoop version issue

* Fix NullPointer Issue.

* Update mvn version check and enable arrow-memory unsafe by default
  • Loading branch information
@weiting-chen
weiting-chen authored Jul 5, 2021
1 parent 31af7a5 commit e39bff6
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 67 deletions.
1 change: 1 addition & 0 deletions arrow-data-source/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
Expand Down
1 change: 1 addition & 0 deletions native-sql-engine/core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
Expand Down
163 changes: 96 additions & 67 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,82 @@
</license>
</licenses>

<modules>
<module>arrow-data-source</module>
<module>native-sql-engine/core</module>
<module>shims</module>
</modules>

<profiles>
<profile>
<id>spark</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.version>${spark311.version}</spark.version>
</properties>
</profile>
<profile>
<id>spark-3.1.1</id>
<properties>
<spark.version>${spark311.version}</spark.version>
</properties>
</profile>
<profile>
<id>hadoop-2.7.4</id>
<activation>
<property>
<name>!hadoop.version</name>
</property>
</activation>
<properties>
<hadoop.version>2.7.4</hadoop.version>
</properties>
</profile>
<profile>
<id>hadoop-3.2</id>
<properties>
<hadoop.version>3.2.0</hadoop.version>
</properties>
</profile>
<profile>
<id>incremental-scala-compiler</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<scala.recompile.mode>incremental</scala.recompile.mode>
</properties>
</profile>
<profile>
<id>full-scala-compiler</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<scala.recompile.mode>all</scala.recompile.mode>
</properties>
</profile>
<profile>
<id>arrow-netty</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<id>arrow-netty</id>
<arrow-memory.artifact>arrow-memory-netty</arrow-memory.artifact>
</properties>
</profile>
</profiles>

<properties>
<scala.version>2.12.10</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<spark.version>3.1.1</spark.version>
<arrow.version>4.0.0</arrow.version>
<arrow-memory.artifact>arrow-memory-netty</arrow-memory.artifact>
<hadoop.version>2.7.4</hadoop.version>
<arrow-memory.artifact>arrow-memory-unsafe</arrow-memory.artifact>
<hadoop.version>${hadoop.version}</hadoop.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<arrow.script.dir>${project.basedir}/script</arrow.script.dir>
Expand All @@ -50,12 +119,6 @@
<spark311.version>3.1.1</spark311.version>
</properties>

<modules>
<module>arrow-data-source</module>
<module>native-sql-engine/core</module>
<module>shims</module>
</modules>

<dependencyManagement>
<dependencies>
<dependency>
Expand Down Expand Up @@ -189,63 +252,29 @@
</dependencies>
</dependencyManagement>

<profiles>
<profile>
<id>spark</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.version>${spark311.version}</spark.version>
</properties>
</profile>
<profile>
<id>spark-3.1.1</id>
<properties>
<spark.version>${spark311.version}</spark.version>
</properties>
</profile>
<profile>
<id>hadoop-3.2</id>
<properties>
<hadoop.version>3.2.0</hadoop.version>
</properties>
</profile>
<profile>
<id>incremental-scala-compiler</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<scala.recompile.mode>incremental</scala.recompile.mode>
</properties>
</profile>
<profile>
<id>full-scala-compiler</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<scala.recompile.mode>all</scala.recompile.mode>
</properties>
</profile>
<profile>
<id>arrow-netty</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<arrow-memory.artifact>arrow-memory-netty</arrow-memory.artifact>
</properties>
</profile>
<profile>
<id>arrow-unsafe</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<arrow-memory.artifact>arrow-memory-unsafe</arrow-memory.artifact>
</properties>
</profile>
</profiles>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.0.1</version>
<executions>
<execution>
<id>enforce-versions</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<requireMavenVersion>
<version>3.6.3</version>
</requireMavenVersion>
</rules>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>

0 comments on commit e39bff6

Please sign in to comment.