From 46fd12eaa94df6c2c5a8db9c7716aec0078ccf30 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 24 Sep 2020 17:55:40 -0700 Subject: [PATCH 1/8] Move snappy code to hadoop-compression. --- hadoop-common-project/hadoop-common/pom.xml | 5 - ...apache.hadoop.io.compress.CompressionCodec | 1 - .../io/compress/CompressDecompressTester.java | 5 +- .../apache/hadoop/io/compress/TestCodec.java | 48 ----- .../hadoop/io/compress/TestCodecFactory.java | 5 - .../compress/TestCompressorDecompressor.java | 6 - hadoop-project/pom.xml | 6 + hadoop-tools/hadoop-compression/pom.xml | 177 ++++++++++++++++++ .../hadoop/io/compress/SnappyCodec.java | 0 .../snappy}/snappy/SnappyCompressor.java | 0 .../snappy}/snappy/SnappyDecompressor.java | 0 .../compress/snappy}/snappy/package-info.java | 0 ...apache.hadoop.io.compress.CompressionCodec | 15 ++ .../TestSnappyCompressorDecompressor.java | 0 hadoop-tools/pom.xml | 1 + 15 files changed, 200 insertions(+), 69 deletions(-) create mode 100644 hadoop-tools/hadoop-compression/pom.xml rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java (100%) rename {hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress => hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy}/snappy/SnappyCompressor.java (100%) rename {hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress => hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy}/snappy/SnappyDecompressor.java (100%) rename {hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress => hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy}/snappy/package-info.java (100%) create mode 100644 hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java (100%) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index f3a838e41b160..5a6bf18e944e8 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -365,11 +365,6 @@ wildfly-openssl-java provided - - org.xerial.snappy - snappy-java - compile - org.lz4 lz4-java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec index 99b6fb2a3898a..f9687b092b29a 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec +++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec @@ -16,6 +16,5 @@ org.apache.hadoop.io.compress.DefaultCodec org.apache.hadoop.io.compress.DeflateCodec org.apache.hadoop.io.compress.GzipCodec org.apache.hadoop.io.compress.Lz4Codec -org.apache.hadoop.io.compress.SnappyCodec org.apache.hadoop.io.compress.ZStandardCodec diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java index c016ff0378957..6811019888450 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java @@ -31,7 +31,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.lz4.Lz4Compressor; -import org.apache.hadoop.io.compress.snappy.SnappyCompressor; import org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater; import org.apache.hadoop.io.compress.zlib.ZlibCompressor; import org.apache.hadoop.io.compress.zlib.ZlibFactory; @@ -312,7 +311,7 @@ public void assertCompression(String name, Compressor compressor, final Joiner joiner = Joiner.on("- "); final ImmutableMap, Integer> emptySize = ImmutableMap .of(Lz4Compressor.class, 4, ZlibCompressor.class, 16, - SnappyCompressor.class, 4, BuiltInZlibDeflater.class, 16); + BuiltInZlibDeflater.class, 16); @Override void assertCompression(String name, Compressor compressor, @@ -482,8 +481,6 @@ else if (compressor.getClass().isAssignableFrom(BuiltInZlibDeflater.class) else if (compressor.getClass().isAssignableFrom(ZlibCompressor.class)) { return ZlibFactory.isNativeZlibLoaded(new Configuration()); - } else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class)) { - return true; } return false; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java index 1cc5f8b867f0b..fd7ac02a56cf3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java @@ -132,12 +132,6 @@ public void testBZip2NativeCodec() throws IOException { } } - @Test - public void testSnappyCodec() throws IOException { - codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.SnappyCodec"); - codecTest(conf, seed, count, "org.apache.hadoop.io.compress.SnappyCodec"); - } - @Test public void testLz4Codec() throws IOException { conf.setBoolean( @@ -599,48 +593,6 @@ private static void sequenceFileCodecTest(Configuration conf, int lines, LOG.info("SUCCESS! Completed SequenceFileCodecTest with codec \"" + codecClass + "\""); } - /** - * Regression test for HADOOP-8423: seeking in a block-compressed - * stream would not properly reset the block decompressor state. - */ - @Test - public void testSnappyMapFile() throws Exception { - codecTestMapFile(SnappyCodec.class, CompressionType.BLOCK, 100); - } - - private void codecTestMapFile(Class clazz, - CompressionType type, int records) throws Exception { - - FileSystem fs = FileSystem.get(conf); - LOG.info("Creating MapFiles with " + records + - " records using codec " + clazz.getSimpleName()); - Path path = new Path(GenericTestUtils.getTempPath( - clazz.getSimpleName() + "-" + type + "-" + records)); - - LOG.info("Writing " + path); - createMapFile(conf, fs, path, clazz.newInstance(), type, records); - MapFile.Reader reader = new MapFile.Reader(path, conf); - Text key1 = new Text("002"); - assertNotNull(reader.get(key1, new Text())); - Text key2 = new Text("004"); - assertNotNull(reader.get(key2, new Text())); - } - - private static void createMapFile(Configuration conf, FileSystem fs, Path path, - CompressionCodec codec, CompressionType type, int records) throws IOException { - MapFile.Writer writer = - new MapFile.Writer(conf, path, - MapFile.Writer.keyClass(Text.class), - MapFile.Writer.valueClass(Text.class), - MapFile.Writer.compression(type, codec)); - Text key = new Text(); - for (int j = 0; j < records; j++) { - key.set(String.format("%03d", j)); - writer.append(key, key); - } - writer.close(); - } - public static void main(String[] args) throws IOException { int count = 10000; String codecClass = "org.apache.hadoop.io.compress.DefaultCodec"; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java index edab634a0b877..e54a8ca97da7d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java @@ -207,11 +207,6 @@ public void testFinding() { codec = factory.getCodecByClassName(BZip2Codec.class.getCanonicalName()); checkCodec("empty factory for bzip2 codec", BZip2Codec.class, codec); - codec = factory.getCodec(new Path("/tmp/foo.snappy")); - checkCodec("empty factory snappy codec", SnappyCodec.class, codec); - codec = factory.getCodecByClassName(SnappyCodec.class.getCanonicalName()); - checkCodec("empty factory snappy codec", SnappyCodec.class, codec); - codec = factory.getCodec(new Path("/tmp/foo.lz4")); checkCodec("empty factory lz4 codec", Lz4Codec.class, codec); codec = factory.getCodecByClassName(Lz4Codec.class.getCanonicalName()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java index 43cb4df1105b2..a01bcff4cfdd2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java @@ -21,8 +21,6 @@ import org.apache.hadoop.io.compress.CompressDecompressTester.CompressionTestStrategy; import org.apache.hadoop.io.compress.lz4.Lz4Compressor; import org.apache.hadoop.io.compress.lz4.Lz4Decompressor; -import org.apache.hadoop.io.compress.snappy.SnappyCompressor; -import org.apache.hadoop.io.compress.snappy.SnappyDecompressor; import org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater; import org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater; import org.apache.hadoop.test.GenericTestUtils; @@ -56,7 +54,6 @@ public void testCompressorDecompressor() { byte[] rawData = generate(SIZE); try { CompressDecompressTester.of(rawData) - .withCompressDecompressPair(new SnappyCompressor(), new SnappyDecompressor()) .withCompressDecompressPair(new Lz4Compressor(), new Lz4Decompressor()) .withCompressDecompressPair(new BuiltInZlibDeflater(), new BuiltInZlibInflater()) .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK, @@ -79,9 +76,6 @@ public void testCompressorDecompressorWithExceedBufferLimit() { byte[] rawData = generate(byteSize); try { CompressDecompressTester.of(rawData) - .withCompressDecompressPair( - new SnappyCompressor(bufferSize), - new SnappyDecompressor(bufferSize)) .withCompressDecompressPair( new Lz4Compressor(bufferSize), new Lz4Decompressor(bufferSize)) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 6f644eb6d9c3e..a8735fdbd0133 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -694,6 +694,12 @@ test-jar + + org.apache.hadoop + hadoop-compression + ${hadoop.version} + + com.google.guava guava diff --git a/hadoop-tools/hadoop-compression/pom.xml b/hadoop-tools/hadoop-compression/pom.xml new file mode 100644 index 0000000000000..ec367cae2b42c --- /dev/null +++ b/hadoop-tools/hadoop-compression/pom.xml @@ -0,0 +1,177 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.4.0-SNAPSHOT + ../../hadoop-project + + hadoop-compression + 3.4.0-SNAPSHOT + Apache Hadoop Compression + Apache Hadoop Compression + jar + + + + org.xerial.snappy + snappy-java + provided + + + org.apache.hadoop + hadoop-common + provided + + + + org.apache.hadoop + hadoop-common + test + test-jar + + + junit + junit + test + + + org.assertj + assertj-core + test + + + + + + + src/main/resources + true + + + + + src/test/resources + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${ignoreTestFailure} + 1 + false + 600 + -Xmx1024m + + **/Test*.java + + true + + + test.build.data + ${basedir}/target/test/data + + + hadoop.log.dir + target/test/logs + + + org.apache.commons.logging.Log + org.apache.commons.logging.impl.SimpleLog + + + org.apache.commons.logging.simplelog.defaultlog + warn + + + + + + maven-dependency-plugin + + + package + + copy-dependencies + + + ${project.build.directory}/lib + + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.hadoop.tools.DistCp + + + + + + prepare-jar + prepare-package + + jar + + + + prepare-test-jar + prepare-package + + test-jar + + + + + + org.apache.maven.plugins + maven-source-plugin + + true + + + + + jar + + + + + + + diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/SnappyCompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/SnappyCompressor.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/SnappyDecompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/SnappyDecompressor.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/package-info.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/package-info.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/package-info.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/package-info.java diff --git a/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec b/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec new file mode 100644 index 0000000000000..18c73c7d21b6c --- /dev/null +++ b/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec @@ -0,0 +1,15 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.hadoop.io.compress.SnappyCodec + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/snappy/TestSnappyCompressorDecompressor.java diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index f026bc261e00b..9f6b010ca619a 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -51,6 +51,7 @@ hadoop-azure-datalake hadoop-aliyun hadoop-fs2img + hadoop-compression From 8cd023de1b4cd3d18fa15f665b0eaa4e909c3e59 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 7 Jan 2021 14:42:56 -0800 Subject: [PATCH 2/8] Move files. --- hadoop-common-project/hadoop-common/pom.xml | 5 --- .../hadoop/io/compress/DefaultCodec.java | 1 - ...apache.hadoop.io.compress.CompressionCodec | 20 ----------- hadoop-tools/hadoop-compression/pom.xml | 36 ++++++++++++++++++- .../io/compress/BlockCompressorStream.java | 0 .../io/compress/BlockDecompressorStream.java | 0 .../hadoop/io/compress/DeflateCodec.java | 0 .../apache/hadoop/io/compress/Lz4Codec.java | 0 .../hadoop/io/compress/lz4/Lz4Compressor.java | 0 .../io/compress/lz4/Lz4Decompressor.java | 0 .../hadoop/io/compress/lz4/package-info.java | 0 .../snappy/{snappy => }/SnappyCompressor.java | 0 .../{snappy => }/SnappyDecompressor.java | 0 .../snappy/{snappy => }/package-info.java | 0 ...apache.hadoop.io.compress.CompressionCodec | 7 +++- .../io/compress/CompressDecompressTester.java | 0 .../hadoop/io/compress/FakeCompressor.java | 0 .../hadoop/io/compress/FakeDecompressor.java | 0 .../compress/TestBlockDecompressorStream.java | 0 .../apache/hadoop/io/compress/TestCodec.java | 0 .../hadoop/io/compress/TestCodecFactory.java | 0 .../hadoop/io/compress/TestCodecPool.java | 0 .../compress/TestCompressionStreamReuse.java | 0 .../compress/TestCompressorDecompressor.java | 0 .../io/compress/TestCompressorStream.java | 0 .../io/compress/TestDecompressorStream.java | 0 .../hadoop/io/compress/TestGzipCodec.java | 0 .../TestBzip2CompressorDecompressor.java | 0 .../lz4/TestLz4CompressorDecompressor.java | 0 .../zlib/TestZlibCompressorDecompressor.java | 0 .../TestZStandardCompressorDecompressor.java | 0 31 files changed, 41 insertions(+), 28 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/main/java/org/apache/hadoop/io/compress/BlockCompressorStream.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/main/java/org/apache/hadoop/io/compress/BlockDecompressorStream.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/main/java/org/apache/hadoop/io/compress/DeflateCodec.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/main/java/org/apache/hadoop/io/compress/lz4/package-info.java (100%) rename hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/{snappy => }/SnappyCompressor.java (100%) rename hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/{snappy => }/SnappyDecompressor.java (100%) rename hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/{snappy => }/package-info.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/FakeCompressor.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/FakeDecompressor.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/TestBlockDecompressorStream.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/TestCodec.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/TestCompressorStream.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/TestDecompressorStream.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/TestGzipCodec.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBzip2CompressorDecompressor.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/lz4/TestLz4CompressorDecompressor.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java (100%) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 5a6bf18e944e8..d5f8446b1c9d1 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -365,11 +365,6 @@ wildfly-openssl-java provided - - org.lz4 - lz4-java - provided - diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java index d2ffb22eaafb3..b407ddb11046c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java @@ -26,7 +26,6 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.compress.zlib.ZlibDecompressor; import org.apache.hadoop.io.compress.zlib.ZlibFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec deleted file mode 100644 index f9687b092b29a..0000000000000 --- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -org.apache.hadoop.io.compress.BZip2Codec -org.apache.hadoop.io.compress.DefaultCodec -org.apache.hadoop.io.compress.DeflateCodec -org.apache.hadoop.io.compress.GzipCodec -org.apache.hadoop.io.compress.Lz4Codec -org.apache.hadoop.io.compress.ZStandardCodec - diff --git a/hadoop-tools/hadoop-compression/pom.xml b/hadoop-tools/hadoop-compression/pom.xml index ec367cae2b42c..75b7b1068453c 100644 --- a/hadoop-tools/hadoop-compression/pom.xml +++ b/hadoop-tools/hadoop-compression/pom.xml @@ -35,18 +35,52 @@ snappy-java provided + + org.lz4 + lz4-java + provided + + + + commons-logging + commons-logging + compile + + + log4j + log4j + compile + + + + org.slf4j + slf4j-api + compile + + + org.slf4j + slf4j-log4j12 + compile + + org.apache.hadoop hadoop-common provided - org.apache.hadoop hadoop-common test test-jar + + + org.apache.hadoop + hadoop-annotations + compile + + junit junit diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BlockCompressorStream.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/BlockCompressorStream.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BlockCompressorStream.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/BlockCompressorStream.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BlockDecompressorStream.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/BlockDecompressorStream.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BlockDecompressorStream.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/BlockDecompressorStream.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DeflateCodec.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/DeflateCodec.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DeflateCodec.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/DeflateCodec.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/package-info.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/lz4/package-info.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/package-info.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/lz4/package-info.java diff --git a/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/SnappyCompressor.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java similarity index 100% rename from hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/SnappyCompressor.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java diff --git a/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/SnappyDecompressor.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java similarity index 100% rename from hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/SnappyDecompressor.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java diff --git a/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/package-info.java b/hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/package-info.java similarity index 100% rename from hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/snappy/package-info.java rename to hadoop-tools/hadoop-compression/src/main/java/org/apache/hadoop/io/compress/snappy/package-info.java diff --git a/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec b/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec index 18c73c7d21b6c..568972e5bc9fb 100644 --- a/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec +++ b/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec @@ -11,5 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # +org.apache.hadoop.io.compress.BZip2Codec +org.apache.hadoop.io.compress.DefaultCodec +org.apache.hadoop.io.compress.DeflateCodec +org.apache.hadoop.io.compress.GzipCodec +org.apache.hadoop.io.compress.Lz4Codec org.apache.hadoop.io.compress.SnappyCodec - +org.apache.hadoop.io.compress.ZStandardCodec diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/CompressDecompressTester.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/FakeCompressor.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/FakeCompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/FakeCompressor.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/FakeCompressor.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/FakeDecompressor.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/FakeDecompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/FakeDecompressor.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/FakeDecompressor.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestBlockDecompressorStream.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestBlockDecompressorStream.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestBlockDecompressorStream.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestBlockDecompressorStream.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCodec.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCodec.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCompressionStreamReuse.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCompressorDecompressor.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorStream.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCompressorStream.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCompressorStream.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestCompressorStream.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestDecompressorStream.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestDecompressorStream.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestDecompressorStream.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestDecompressorStream.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestGzipCodec.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestGzipCodec.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestGzipCodec.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/TestGzipCodec.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBzip2CompressorDecompressor.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBzip2CompressorDecompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBzip2CompressorDecompressor.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/bzip2/TestBzip2CompressorDecompressor.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/lz4/TestLz4CompressorDecompressor.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/lz4/TestLz4CompressorDecompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/lz4/TestLz4CompressorDecompressor.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/lz4/TestLz4CompressorDecompressor.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/zlib/TestZlibCompressorDecompressor.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java b/hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java rename to hadoop-tools/hadoop-compression/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java From 6f072f543f26b607d5222e5ad009de922b14cdfd Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 8 Jan 2021 19:54:30 -0800 Subject: [PATCH 3/8] Remove unnecessary content. --- hadoop-tools/hadoop-compression/pom.xml | 36 ------------------------- 1 file changed, 36 deletions(-) diff --git a/hadoop-tools/hadoop-compression/pom.xml b/hadoop-tools/hadoop-compression/pom.xml index 75b7b1068453c..79f99ad4b9f52 100644 --- a/hadoop-tools/hadoop-compression/pom.xml +++ b/hadoop-tools/hadoop-compression/pom.xml @@ -41,28 +41,6 @@ provided - - commons-logging - commons-logging - compile - - - log4j - log4j - compile - - - - org.slf4j - slf4j-api - compile - - - org.slf4j - slf4j-log4j12 - compile - - org.apache.hadoop hadoop-common @@ -75,12 +53,6 @@ test-jar - - org.apache.hadoop - hadoop-annotations - compile - - junit junit @@ -129,14 +101,6 @@ hadoop.log.dir target/test/logs - - org.apache.commons.logging.Log - org.apache.commons.logging.impl.SimpleLog - - - org.apache.commons.logging.simplelog.defaultlog - warn - From 1cf778b08323701c4031577d8f4fb8bffbd1f4ac Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 8 Jan 2021 21:19:44 -0800 Subject: [PATCH 4/8] Replace lz4 dependency. --- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++-- .../hadoop-mapreduce-client-nativetask/pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 90ce0020d7caf..d56e02679dc53 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -220,8 +220,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> test - org.lz4 - lz4-java + org.apache.hadoop + hadoop-compression test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml index 4073f445de7be..ca334c38ac3dc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml @@ -72,8 +72,8 @@ test - org.lz4 - lz4-java + org.apache.hadoop + hadoop-compression test From aed38373a57c8870dde0c96e3b63a82003be2958 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 8 Jan 2021 22:30:11 -0800 Subject: [PATCH 5/8] Revert services change. --- ....apache.hadoop.io.compress.CompressionCodec | 18 ++++++++++++++++++ ....apache.hadoop.io.compress.CompressionCodec | 4 ---- 2 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec new file mode 100644 index 0000000000000..7afcfd066d6bd --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec @@ -0,0 +1,18 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.hadoop.io.compress.BZip2Codec +org.apache.hadoop.io.compress.DefaultCodec +org.apache.hadoop.io.compress.GzipCodec +org.apache.hadoop.io.compress.ZStandardCodec + diff --git a/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec b/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec index 568972e5bc9fb..6a7855978a8ab 100644 --- a/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec +++ b/hadoop-tools/hadoop-compression/src/main/resources/META-INF/services/org.apache.hadoop.io.compress.CompressionCodec @@ -11,10 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -org.apache.hadoop.io.compress.BZip2Codec -org.apache.hadoop.io.compress.DefaultCodec org.apache.hadoop.io.compress.DeflateCodec -org.apache.hadoop.io.compress.GzipCodec org.apache.hadoop.io.compress.Lz4Codec org.apache.hadoop.io.compress.SnappyCodec -org.apache.hadoop.io.compress.ZStandardCodec From 3f152f46be55cf028db2ecfdfb2094a44b3d5ebf Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Tue, 12 Jan 2021 20:39:53 -0800 Subject: [PATCH 6/8] dependencies should be compile. --- hadoop-tools/hadoop-compression/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-compression/pom.xml b/hadoop-tools/hadoop-compression/pom.xml index 79f99ad4b9f52..9ee23e7ba8581 100644 --- a/hadoop-tools/hadoop-compression/pom.xml +++ b/hadoop-tools/hadoop-compression/pom.xml @@ -33,18 +33,18 @@ org.xerial.snappy snappy-java - provided + compile org.lz4 lz4-java - provided + compile org.apache.hadoop hadoop-common - provided + compile org.apache.hadoop From 42a43c0f8480c3d146afb00dba7d597b15fa0ef3 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 13 Jan 2021 13:39:35 -0800 Subject: [PATCH 7/8] Fix TestLz4CompressorDecompressor and TestZStandardCompressorDecompressor. --- hadoop-tools/hadoop-compression/pom.xml | 6 ------ .../src/test/resources/lz4/.sequencefile.crc | Bin .../src/test/resources/lz4/sequencefile | Bin .../src/test/resources/zstd/test_file.txt | 0 .../src/test/resources/zstd/test_file.txt.zst | Bin 5 files changed, 6 deletions(-) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/resources/lz4/.sequencefile.crc (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/resources/lz4/sequencefile (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/resources/zstd/test_file.txt (100%) rename {hadoop-common-project/hadoop-common => hadoop-tools/hadoop-compression}/src/test/resources/zstd/test_file.txt.zst (100%) diff --git a/hadoop-tools/hadoop-compression/pom.xml b/hadoop-tools/hadoop-compression/pom.xml index 9ee23e7ba8581..467176a35d9b5 100644 --- a/hadoop-tools/hadoop-compression/pom.xml +++ b/hadoop-tools/hadoop-compression/pom.xml @@ -72,12 +72,6 @@ true - - - src/test/resources - true - - org.apache.maven.plugins diff --git a/hadoop-common-project/hadoop-common/src/test/resources/lz4/.sequencefile.crc b/hadoop-tools/hadoop-compression/src/test/resources/lz4/.sequencefile.crc similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/resources/lz4/.sequencefile.crc rename to hadoop-tools/hadoop-compression/src/test/resources/lz4/.sequencefile.crc diff --git a/hadoop-common-project/hadoop-common/src/test/resources/lz4/sequencefile b/hadoop-tools/hadoop-compression/src/test/resources/lz4/sequencefile similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/resources/lz4/sequencefile rename to hadoop-tools/hadoop-compression/src/test/resources/lz4/sequencefile diff --git a/hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt b/hadoop-tools/hadoop-compression/src/test/resources/zstd/test_file.txt similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt rename to hadoop-tools/hadoop-compression/src/test/resources/zstd/test_file.txt diff --git a/hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt.zst b/hadoop-tools/hadoop-compression/src/test/resources/zstd/test_file.txt.zst similarity index 100% rename from hadoop-common-project/hadoop-common/src/test/resources/zstd/test_file.txt.zst rename to hadoop-tools/hadoop-compression/src/test/resources/zstd/test_file.txt.zst From 3decd632279f77c3f77db63063b555044283ae9a Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 14 Jan 2021 11:55:21 -0800 Subject: [PATCH 8/8] Trigger Build