merge

.gitignore

@@ -9,6 +9,7 @@
 *.suo
 *.vcxproj.user
 *.patch
+*.diff
 .idea
 .svn
 .classpath
@@ -51,15 +52,10 @@ patchprocess/
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
 
-# Ignore files generated by HDDS acceptance tests.
-hadoop-ozone/acceptance-test/docker-compose.log
-hadoop-ozone/acceptance-test/junit-results.xml
-
 #robotframework outputs
 log.html
 output.xml
 report.html
 
-hadoop-hdds/docs/public
 
 .mvn

BUILDING.txt

@@ -85,7 +85,7 @@ Optional packages:
 * Linux FUSE
   $ sudo apt-get install fuse libfuse-dev
 * ZStandard compression
-    $ sudo apt-get install zstd
+  $ sudo apt-get install libzstd1-dev
 * PMDK library for storage class memory(SCM) as HDFS cache backend
   Please refer to http://pmem.io/ and https://github.com/pmem/pmdk
 
@@ -104,8 +104,6 @@ Maven main modules:
          - hadoop-hdfs-project          (Hadoop HDFS)
          - hadoop-yarn-project          (Hadoop YARN)
          - hadoop-mapreduce-project     (Hadoop MapReduce)
-         - hadoop-ozone                 (Hadoop Ozone)
-         - hadoop-hdds                  (Hadoop Distributed Data Store)
          - hadoop-tools                 (Hadoop tools like Streaming, Distcp, etc.)
          - hadoop-dist                  (Hadoop distribution assembler)
          - hadoop-client-modules        (Hadoop client modules)
@@ -419,6 +417,47 @@ Building command example:
 Note that the command above manually specified the openssl library and include
 path. This is necessary at least for Homebrewed OpenSSL.
 
+
+----------------------------------------------------------------------------------
+
+Building on CentOS 8
+
+----------------------------------------------------------------------------------
+
+
+* Install development tools such as GCC, autotools, OpenJDK and Maven.
+  $ sudo dnf group install --with-optional 'Development Tools'
+  $ sudo dnf install java-1.8.0-openjdk-devel maven
+
+* Install Protocol Buffers v3.7.1.
+  $ git clone https://github.com/protocolbuffers/protobuf
+  $ cd protobuf
+  $ git checkout v3.7.1
+  $ autoreconf -i
+  $ ./configure --prefix=/usr/local
+  $ make
+  $ sudo make install
+  $ cd ..
+
+* Install libraries provided by CentOS 8.
+  $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel
+
+* Install optional dependencies (snappy-devel).
+  $ sudo dnf --enablerepo=PowerTools snappy-devel
+
+* Install optional dependencies (libzstd-devel).
+  $ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+  $ sudo dnf --enablerepo=epel install libzstd-devel
+
+* Install optional dependencies (isa-l).
+  $ sudo dnf --enablerepo=PowerTools install nasm
+  $ git clone https://github.com/intel/isa-l
+  $ cd isa-l/
+  $ ./autogen.sh
+  $ ./configure
+  $ make
+  $ sudo make install
+
 ----------------------------------------------------------------------------------
 
 Building on Windows

Jenkinsfile

@@ -35,7 +35,7 @@ pipeline {
         DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='rel/0.10.0'
+        YETUS_VERSION='rel/0.12.0'
     }
 
     parameters {
@@ -61,7 +61,7 @@ pipeline {
             steps {
                 withCredentials(
                     [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
-                                  passwordVariable: 'GITHUB_PASSWORD',
+                                  passwordVariable: 'GITHUB_TOKEN',
                                   usernameVariable: 'GITHUB_USER'),
                     usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
                                         passwordVariable: 'JIRA_PASSWORD',
@@ -105,8 +105,7 @@ pipeline {
                         YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
 
                         # enable writing back to Github
-                        YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}")
-                        YETUS_ARGS+=(--github-user=${GITHUB_USER})
+                        YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
 
                         # enable writing back to ASF JIRA
                         YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
@@ -135,7 +134,7 @@ pipeline {
                         YETUS_ARGS+=("--plugins=all")
 
                         # use Hadoop's bundled shelldocs
-                        YETUS_ARGS+=("--shelldocs=/testptch/hadoop/dev-support/bin/shelldocs")
+                        YETUS_ARGS+=("--shelldocs=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/shelldocs")
 
                         # don't let these tests cause -1s because we aren't really paying that
                         # much attention to them
@@ -147,11 +146,14 @@ pipeline {
                         YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
 
                         # effectively treat dev-suport as a custom maven module
-                        YETUS_ARGS+=("--skip-dir=dev-support")
+                        YETUS_ARGS+=("--skip-dirs=dev-support")
 
                         # help keep the ASF boxes clean
                         YETUS_ARGS+=("--sentinel")
 
+                        # use emoji vote so it is easier to find the broken line
+                        YETUS_ARGS+=("--github-use-emoji-vote")
+
                         "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
                         '''
                 }

LICENSE-binary

@@ -254,15 +254,15 @@ commons-lang:commons-lang:2.6
 commons-logging:commons-logging:1.1.3
 commons-net:commons-net:3.6
 de.ruedigermoeller:fst:2.50
-io.grpc:grpc-context:1.15.1
-io.grpc:grpc-core:1.15.1
-io.grpc:grpc-netty:1.15.1
-io.grpc:grpc-protobuf:1.15.1
-io.grpc:grpc-protobuf-lite:1.15.1
-io.grpc:grpc-stub:1.15.1
+io.grpc:grpc-api:1.26.0
+io.grpc:grpc-context:1.26.0
+io.grpc:grpc-core:1.26.0
+io.grpc:grpc-netty:1.26.0
+io.grpc:grpc-protobuf:1.26.0
+io.grpc:grpc-protobuf-lite:1.26.0
+io.grpc:grpc-stub:1.26.0
 io.netty:netty:3.10.6.Final
-io.netty:netty-all:4.0.52.Final
-io.netty:netty-all:4.1.27.Final
+io.netty:netty-all:4.1.42.Final
 io.netty:netty-buffer:4.1.27.Final
 io.netty:netty-codec:4.1.27.Final
 io.netty:netty-codec-http:4.1.27.Final
@@ -282,7 +282,6 @@ io.swagger:swagger-annotations:1.5.4
 javax.inject:javax.inject:1
 log4j:log4j:1.2.17
 net.java.dev.jna:jna:5.2.0
-net.jpountz.lz4:lz4:1.2.0
 net.minidev:accessors-smart:1.2
 net.minidev:json-smart:2.3
 org.apache.avro:avro:1.7.7
@@ -307,7 +306,7 @@ org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.6
 org.apache.httpcomponents:httpcore:4.4.10
-org.apache.kafka:kafka-clients:0.8.2.1
+org.apache.kafka:kafka-clients:2.4.0
 org.apache.kerby:kerb-admin:1.0.1
 org.apache.kerby:kerb-client:1.0.1
 org.apache.kerby:kerb-common:1.0.1
@@ -345,7 +344,8 @@ org.eclipse.jetty:jetty-xml:9.3.27.v20190418
 org.eclipse.jetty.websocket:javax-websocket-client-impl:9.3.27.v20190418
 org.eclipse.jetty.websocket:javax-websocket-server-impl:9.3.27.v20190418
 org.ehcache:ehcache:3.3.1
-org.objenesis:objenesis:1.0:compile
+org.lz4:lz4-java:1.6.0
+org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.0.5
 org.yaml:snakeyaml:1.16:
 org.wildfly.openssl:wildfly-openssl:1.0.7.Final
@@ -364,6 +364,7 @@ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/com
 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
 
+com.github.luben:zstd-jni:1.4.3-1
 dnsjava:dnsjava:2.1.7
 org.codehaus.woodstox:stax2-api:3.1.4
 
@@ -384,7 +385,7 @@ com.google.protobuf:protobuf-java:3.6.1
 com.google.re2j:re2j:1.1
 com.jcraft:jsch:0.1.54
 com.thoughtworks.paranamer:paranamer:2.3
-javax.activation:javax.activation-api:1.2.0
+jakarta.activation:jakarta.activation-api:1.2.1
 org.fusesource.leveldbjni:leveldbjni-all:1.8
 org.jline:jline:3.9.0
 org.hamcrest:hamcrest-core:1.3

LICENSE.txt

@@ -256,3 +256,26 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/st
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL
+
+=======
+For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/cJSON.[ch]:
+
+Copyright (c) 2009-2017 Dave Gamble and cJSON contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

dev-support/bin/create-release

@@ -204,6 +204,11 @@ function set_defaults
   DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile"
   DOCKERRAN=false
 
+  CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
+  if [ "$CPU_ARCH" = "aarch64" ]; then
+    DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile_aarch64"
+  fi
+
   # Extract Java version from ${BASEDIR}/pom.xml
   # doing this outside of maven means we can do this before
   # the docker container comes up...
@@ -249,7 +254,9 @@ function startgpgagent
       eval $("${GPGAGENT}" --daemon \
         --options "${LOGDIR}/gpgagent.conf" \
         --log-file="${LOGDIR}/create-release-gpgagent.log")
-      GPGAGENTPID=$(echo "${GPG_AGENT_INFO}" | cut -f 2 -d:)
+      GPGAGENTPID=$(pgrep "${GPGAGENT}")
+      GPG_AGENT_INFO="$HOME/.gnupg/S.gpg-agent:$GPGAGENTPID:1"
+      export GPG_AGENT_INFO
     fi
 
     if [[ -n "${GPG_AGENT_INFO}" ]]; then
@@ -499,7 +506,12 @@ function dockermode
 
     # we always force build with the OpenJDK JDK
     # but with the correct version
-    echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
+    if [ "$CPU_ARCH" = "aarch64" ]; then
+      echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
+    else
+      echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
+    fi
+
     echo "USER ${user_name}"
     printf "\n\n"
   ) | docker build -t "${imgname}" -
@@ -639,10 +651,12 @@ function signartifacts
 
   big_console_header "Signing the release"
 
-  for i in ${ARTIFACTS_DIR}/*; do
+  run cd "${ARTIFACTS_DIR}"
+  for i in *; do
     ${GPG} --use-agent --armor --output "${i}.asc" --detach-sig "${i}"
     sha512sum --tag "${i}" > "${i}.sha512"
   done
+  run cd "${BASEDIR}"
 
   if [[ "${ASFRELEASE}" = true ]]; then
     echo "Fetching the Apache Hadoop KEYS file..."

dev-support/bin/dist-copynativelibs

@@ -72,7 +72,7 @@ function bundle_native_bin
   fi
 
   cd "${libdir}" || exit 1
-  ${TAR} ./*"${libpattern}"* | (cd "${TARGET_BIN_DIR}"/  || exit 1 ; ${UNTAR})
+  ${TAR} ./*"${binpattern}"* | (cd "${TARGET_BIN_DIR}"/  || exit 1 ; ${UNTAR})
   if [[ $? -ne 0 ]]; then
     echo "Bundling bin files for ${binoption} failed"
     exit 1
@@ -152,13 +152,13 @@ if [[ -d "${LIB_DIR}" ]]; then
     exit 1
   fi
 
-  bundle_native_lib "${SNAPPYLIBBUNDLE}" "snappy.lib" "snappy" "${SNAPPYLIB}"
+  bundle_native_lib "${SNAPPYLIBBUNDLE}" "snappy.lib" "libsnappy." "${SNAPPYLIB}"
 
-  bundle_native_lib "${ZSTDLIBBUNDLE}" "zstd.lib" "zstd" "${ZSTDLIB}"
+  bundle_native_lib "${ZSTDLIBBUNDLE}" "zstd.lib" "libzstd." "${ZSTDLIB}"
 
-  bundle_native_lib "${OPENSSLLIBBUNDLE}" "openssl.lib" "crypto" "${OPENSSLLIB}"
+  bundle_native_lib "${OPENSSLLIBBUNDLE}" "openssl.lib" "libcrypto." "${OPENSSLLIB}"
 
-  bundle_native_lib "${ISALBUNDLE}" "isal.lib" "isa" "${ISALLIB}"
+  bundle_native_lib "${ISALBUNDLE}" "isal.lib" "libisal." "${ISALLIB}"
 
   bundle_native_lib "${PMDKBUNDLE}" "pmdk.lib" "pmdk" "${PMDKLIB}"
 fi

dev-support/bin/dist-layout-stitching

@@ -21,9 +21,6 @@ VERSION=$1
 # project.build.directory
 BASEDIR=$2
 
-#hdds.version
-HDDS_VERSION=$3
-
 function run()
 {
   declare res

dev-support/bin/yetus-wrapper

@@ -144,7 +144,7 @@ else
   exit 1
 fi
 
-if [[ -n "${GPGBIN}" ]]; then
+if [[ -n "${GPGBIN}" && ! "${HADOOP_SKIP_YETUS_VERIFICATION}" = true ]]; then
   if ! mkdir -p .gpg; then
     yetus_error "ERROR: yetus-dl: Unable to create ${HADOOP_PATCHPROCESS}/.gpg"
     exit 1