HADOOP-17464. Create hadoop-compression module

hadoop-common-project/hadoop-common/pom.xml

@@ -365,16 +365,6 @@
       <artifactId>wildfly-openssl-java</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>org.xerial.snappy</groupId>
-      <artifactId>snappy-java</artifactId>
-      <scope>compile</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.lz4</groupId>
-      <artifactId>lz4-java</artifactId>
-      <scope>provided</scope>
-    </dependency>
   </dependencies>
 
   <build>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java

@@ -26,7 +26,6 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.compress.zlib.ZlibDecompressor;
 import org.apache.hadoop.io.compress.zlib.ZlibFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec

@@ -13,9 +13,6 @@
 #
 org.apache.hadoop.io.compress.BZip2Codec
 org.apache.hadoop.io.compress.DefaultCodec
-org.apache.hadoop.io.compress.DeflateCodec
 org.apache.hadoop.io.compress.GzipCodec
-org.apache.hadoop.io.compress.Lz4Codec
-org.apache.hadoop.io.compress.SnappyCodec
 org.apache.hadoop.io.compress.ZStandardCodec
 

hadoop-hdfs-project/hadoop-hdfs/pom.xml

@@ -221,8 +221,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.lz4</groupId>
-      <artifactId>lz4-java</artifactId>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-compression</artifactId>
       <scope>test</scope>
     </dependency>
   </dependencies>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml

@@ -56,8 +56,8 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.lz4</groupId>
-      <artifactId>lz4-java</artifactId>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-compression</artifactId>
       <scope>test</scope>
     </dependency>
   </dependencies>

hadoop-project/pom.xml

@@ -694,6 +694,12 @@
         <type>test-jar</type>
       </dependency>
 
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-compression</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>

hadoop-tools/hadoop-compression/pom.xml

@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                      https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-project</artifactId>
+        <version>3.4.0-SNAPSHOT</version>
+        <relativePath>../../hadoop-project</relativePath>
+    </parent>
+    <artifactId>hadoop-compression</artifactId>
+    <version>3.4.0-SNAPSHOT</version>
+    <description>Apache Hadoop Compression</description>
+    <name>Apache Hadoop Compression</name>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.lz4</groupId>
+            <artifactId>lz4-java</artifactId>
+            <scope>compile</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <scope>test</scope>
+            <type>test-jar</type>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore>
+                    <forkCount>1</forkCount>
+                    <reuseForks>false</reuseForks>
+                    <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds>
+                    <argLine>-Xmx1024m</argLine>
+                    <includes>
+                        <include>**/Test*.java</include>
+                    </includes>
+                    <redirectTestOutputToFile>true</redirectTestOutputToFile>
+                    <systemProperties>
+                        <property>
+                            <name>test.build.data</name>
+                            <value>${basedir}/target/test/data</value>
+                        </property>
+                        <property>
+                            <name>hadoop.log.dir</name>
+                            <value>target/test/logs</value>
+                        </property>
+                    </systemProperties>
+                </configuration>
+            </plugin>
+            <plugin>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>deplist</id>
+                        <phase>compile</phase>
+                        <goals>
+                            <goal>list</goal>
+                        </goals>
+                        <configuration>
+                            <!-- referenced by a built-in command -->
+                            <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <mainClass>org.apache.hadoop.tools.DistCp</mainClass>
+                        </manifest>
+                    </archive>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>prepare-jar</id>
+                        <phase>prepare-package</phase>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>prepare-test-jar</id>
+                        <phase>prepare-package</phase>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <configuration>
+                    <attach>true</attach>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec

@@ -0,0 +1,16 @@
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+org.apache.hadoop.io.compress.DeflateCodec
+org.apache.hadoop.io.compress.Lz4Codec
+org.apache.hadoop.io.compress.SnappyCodec

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java → hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java

@@ -31,7 +31,6 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.compress.lz4.Lz4Compressor;
-import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
 import org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater;
 import org.apache.hadoop.io.compress.zlib.ZlibCompressor;
 import org.apache.hadoop.io.compress.zlib.ZlibFactory;
@@ -312,7 +311,7 @@ public void assertCompression(String name, Compressor compressor,
       final Joiner joiner = Joiner.on("- ");
       final ImmutableMap<Class<? extends Compressor>, Integer> emptySize = ImmutableMap
           .of(Lz4Compressor.class, 4, ZlibCompressor.class, 16,
-              SnappyCompressor.class, 4, BuiltInZlibDeflater.class, 16);
+              BuiltInZlibDeflater.class, 16);
 
       @Override
       void assertCompression(String name, Compressor compressor,
@@ -482,8 +481,6 @@ else if (compressor.getClass().isAssignableFrom(BuiltInZlibDeflater.class)
 
     else if (compressor.getClass().isAssignableFrom(ZlibCompressor.class)) {
       return ZlibFactory.isNativeZlibLoaded(new Configuration());
-    } else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class)) {
-      return true;
     }
 
     return false;      

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java → hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCodec.java

@@ -142,12 +142,6 @@ public void testBZip2NativeCodec() throws IOException {
     }
   }
 
-  @Test
-  public void testSnappyCodec() throws IOException {
-    codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec");
-    codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec");
-  }
-
   @Test
   public void testLz4Codec() throws IOException {
     conf.setBoolean(
@@ -626,48 +620,6 @@ private static void sequenceFileCodecTest(Configuration conf, int lines,
     LOG.info("SUCCESS! Completed SequenceFileCodecTest with codec \"" + codecClass + "\"");
   }
 
-  /**
-   * Regression test for HADOOP-8423: seeking in a block-compressed
-   * stream would not properly reset the block decompressor state.
-   */
-  @Test
-  public void testSnappyMapFile() throws Exception {
-    codecTestMapFile(SnappyCodec.class, CompressionType.BLOCK, 100);
-  }
-
-  private void codecTestMapFile(Class<? extends CompressionCodec> clazz,
-      CompressionType type, int records) throws Exception {
-
-    FileSystem fs = FileSystem.get(conf);
-    LOG.info("Creating MapFiles with " + records  + 
-            " records using codec " + clazz.getSimpleName());
-    Path path = new Path(GenericTestUtils.getTempPath(
-        clazz.getSimpleName() + "-" + type + "-" + records));
-
-    LOG.info("Writing " + path);
-    createMapFile(conf, fs, path, clazz.newInstance(), type, records);
-    MapFile.Reader reader = new MapFile.Reader(path, conf);
-    Text key1 = new Text("002");
-    assertNotNull(reader.get(key1, new Text()));
-    Text key2 = new Text("004");
-    assertNotNull(reader.get(key2, new Text()));
-  }
-
-  private static void createMapFile(Configuration conf, FileSystem fs, Path path, 
-      CompressionCodec codec, CompressionType type, int records) throws IOException {
-    MapFile.Writer writer = 
-        new MapFile.Writer(conf, path,
-            MapFile.Writer.keyClass(Text.class),
-            MapFile.Writer.valueClass(Text.class),
-            MapFile.Writer.compression(type, codec));
-    Text key = new Text();
-    for (int j = 0; j < records; j++) {
-        key.set(String.format("%03d", j));
-        writer.append(key, key);
-    }
-    writer.close();
-  }
-
   public static void main(String[] args) throws IOException {
     int count = 10000;
     String codecClass = "org.apache.hadoop.io.compress.DefaultCodec";

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java → hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java

@@ -207,11 +207,6 @@ public void testFinding() {
     codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName());
     checkCodec("empty factory for bzip2 codec", BZip2Codec.class, codec);
 
-    codec = factory.getCodec(new Path("/tmp/foo.snappy"));
-    checkCodec("empty factory snappy codec", SnappyCodec.class, codec);
-    codec = factory.getCodecByClassName(SnappyCodec.class.getCanonicalName());
-    checkCodec("empty factory snappy codec", SnappyCodec.class, codec);
-
     codec = factory.getCodec(new Path("/tmp/foo.lz4"));
     checkCodec("empty factory lz4 codec", Lz4Codec.class, codec);
     codec = factory.getCodecByClassName(Lz4Codec.class.getCanonicalName());

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java → hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java

@@ -21,8 +21,6 @@
 import org.apache.hadoop.io.compress.CompressDecompressTester.CompressionTestStrategy;
 import org.apache.hadoop.io.compress.lz4.Lz4Compressor;
 import org.apache.hadoop.io.compress.lz4.Lz4Decompressor;
-import org.apache.hadoop.io.compress.snappy.SnappyCompressor;
-import org.apache.hadoop.io.compress.snappy.SnappyDecompressor;
 import org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater;
 import org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -56,7 +54,6 @@ public void testCompressorDecompressor() {
     byte[] rawData = generate(SIZE);
     try {
       CompressDecompressTester.of(rawData)
-          .withCompressDecompressPair(new SnappyCompressor(), new SnappyDecompressor())
           .withCompressDecompressPair(new Lz4Compressor(), new Lz4Decompressor())
           .withCompressDecompressPair(new BuiltInZlibDeflater(), new BuiltInZlibInflater())
           .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK,
@@ -79,9 +76,6 @@ public void testCompressorDecompressorWithExceedBufferLimit() {
     byte[] rawData = generate(byteSize);
     try {
       CompressDecompressTester.of(rawData)
-          .withCompressDecompressPair(
-              new SnappyCompressor(bufferSize),
-              new SnappyDecompressor(bufferSize))
           .withCompressDecompressPair(
               new Lz4Compressor(bufferSize),
               new Lz4Decompressor(bufferSize))

hadoop-tools/pom.xml

@@ -51,6 +51,7 @@
     <module>hadoop-azure-datalake</module>
     <module>hadoop-aliyun</module>
     <module>hadoop-fs2img</module>
+    <module>hadoop-compression</module>
   </modules>
 
   <build>