From 777700e248d06ccf90e084734423b1f73d93ee8a Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 22 May 2021 18:10:42 +0530 Subject: [PATCH 01/20] HADOOP-17727. Modularize docker images --- dev-support/docker/Dockerfile | 136 +-------- dev-support/docker/Dockerfile_centos_8 | 5 +- dev-support/docker/Dockerfile_pkg_test | 0 .../pkg-resolver/install-common-packages.sh | 81 +++++ dev-support/docker/pkg-resolver/packages.json | 287 ++++++++++++++++++ .../docker/pkg-resolver/platforms.json | 6 + dev-support/docker/pkg-resolver/resolve.py | 41 +++ 7 files changed, 428 insertions(+), 128 deletions(-) create mode 100644 dev-support/docker/Dockerfile_pkg_test create mode 100644 dev-support/docker/pkg-resolver/install-common-packages.sh create mode 100644 dev-support/docker/pkg-resolver/packages.json create mode 100644 dev-support/docker/pkg-resolver/platforms.json create mode 100644 dev-support/docker/pkg-resolver/resolve.py diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index ef2f1562613e8..3a43610dc1d05 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -32,57 +32,11 @@ RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true -# hadolint ignore=DL3008 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends \ - ant \ - apt-utils \ - bats \ - build-essential \ - bzip2 \ - clang \ - cmake \ - curl \ - doxygen \ - fuse \ - g++ \ - gcc \ - git \ - gnupg-agent \ - hugo \ - libbcprov-java \ - libbz2-dev \ - libcurl4-openssl-dev \ - libfuse-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libsasl2-dev \ - libsnappy-dev \ - libssl-dev \ - libtool \ - libzstd-dev \ - locales \ - make \ - maven \ - nodejs \ - node-yarn \ - npm \ - openjdk-11-jdk \ - openjdk-8-jdk \ - pinentry-curses \ - pkg-config \ - python3 \ - python3-pip \ - python3-pkg-resources \ - python3-setuptools \ - python3-wheel \ - rsync \ - shellcheck \ - software-properties-common \ - sudo \ - valgrind \ - zlib1g-dev \ - && apt-get clean \ + && apt-get -q install -y --no-install-recommends python3 +COPY pkg-resolver pkg-resolver +RUN apt-get -q install -y --no-install-recommends `pkg-resolver/resolve.py ubuntu:focal` +RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* RUN locale-gen en_US.UTF-8 @@ -97,89 +51,16 @@ ENV MAVEN_HOME /usr ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ####### -# Install SpotBugs 4.2.2 +# Set env vars for SpotBugs 4.2.2 ####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Install Boost 1.72 (1.71 ships with Focal) +# Set env vars for Google Protobuf 3.7.1 ####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 - -### -# Install hadolint -#### -RUN curl -L -s -S \ - https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ - -o /bin/hadolint \ - && chmod a+rx /bin/hadolint \ - && shasum -a 512 /bin/hadolint | \ - awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' - -###### -# Intel ISA-L 2.29.0 -###### -# hadolint ignore=DL3003,DL3008 -RUN mkdir -p /opt/isa-l-src \ - && apt-get -q update \ - && apt-get install -y --no-install-recommends automake yasm \ - && apt-get clean \ - && curl -L -s -S \ - https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ - -o /opt/isa-l.tar.gz \ - && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \ - && cd /opt/isa-l-src \ - && ./autogen.sh \ - && ./configure \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/isa-l-src - ### # Avoid out of memory errors in builds ### @@ -188,6 +69,11 @@ ENV MAVEN_OPTS -Xms256m -Xmx3072m # Skip gpg verification when downloading Yetus via yetus-wrapper ENV HADOOP_SKIP_YETUS_VERIFICATION true +#### +# Install common packages +#### +RUN pkg-resolver/install-common-packages.sh + ### # Everything past this point is either not needed for testing or breaks Yetus. # So tell Yetus not to read the rest of the file: diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 7f1f6b4ac2239..92ec720f2ec21 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -33,11 +33,10 @@ RUN yum update -y \ cyrus-sasl-devel \ dnf \ fuse \ - git \ - libcurl-devel \ - fuse \ fuse-libs \ fuse-devel \ + git \ + libcurl-devel \ libtool \ libtirpc-devel \ lz4-devel \ diff --git a/dev-support/docker/Dockerfile_pkg_test b/dev-support/docker/Dockerfile_pkg_test new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/dev-support/docker/pkg-resolver/install-common-packages.sh b/dev-support/docker/pkg-resolver/install-common-packages.sh new file mode 100644 index 0000000000000..1e3d139bba488 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-common-packages.sh @@ -0,0 +1,81 @@ +####### +# Install SpotBugs 4.2.2 +####### +mkdir -p /opt/spotbugs \ + && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz \ + && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ + && chmod +x /opt/spotbugs/bin/* + +####### +# Install Boost 1.72 (1.71 ships with Focal) +####### +# hadolint ignore=DL3003 +mkdir -p /opt/boost-library \ + && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ + && mv boost_1_72_0.tar.bz2 /opt/boost-library \ + && cd /opt/boost-library \ + && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ + && cd /opt/boost-library/boost_1_72_0 \ + && ./bootstrap.sh --prefix=/usr/ \ + && ./b2 --without-python install \ + && cd /root \ + && rm -rf /opt/boost-library + +###### +# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) +###### +# hadolint ignore=DL3003 +mkdir -p /opt/protobuf-src \ + && curl -L -s -S \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz \ + && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ + && cd /opt/protobuf-src \ + && ./configure --prefix=/opt/protobuf \ + && make "-j$(nproc)" \ + && make install \ + && cd /root \ + && rm -rf /opt/protobuf-src + +###### +# Install pylint and python-dateutil +###### +pip3 install pylint==2.6.0 python-dateutil==2.8.1 + +###### +# Install bower +###### +# hadolint ignore=DL3008 +npm install -g bower@1.8.8 + +###### +# Install hadolint +###### +curl -L -s -S \ + https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ + -o /bin/hadolint \ + && chmod a+rx /bin/hadolint \ + && shasum -a 512 /bin/hadolint | \ + awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' + +###### +# Intel ISA-L 2.29.0 +###### +# hadolint ignore=DL3003,DL3008 +mkdir -p /opt/isa-l-src \ + && apt-get -q update \ + && apt-get install -y --no-install-recommends automake yasm \ + && apt-get clean \ + && curl -L -s -S \ + https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ + -o /opt/isa-l.tar.gz \ + && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \ + && cd /opt/isa-l-src \ + && ./autogen.sh \ + && ./configure \ + && make "-j$(nproc)" \ + && make install \ + && cd /root \ + && rm -rf /opt/isa-l-src + diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json new file mode 100644 index 0000000000000..cf59c63992d32 --- /dev/null +++ b/dev-support/docker/pkg-resolver/packages.json @@ -0,0 +1,287 @@ +{ + "ant": { + "ubuntu:focal": "ant", + "ubuntu:focal::arch64": "ant", + "centos:7": "ant", + "centos:8": "ant" + }, + "apt-utils": { + "ubuntu:focal": "apt-utils", + "ubuntu:focal::arch64": "apt-utils" + }, + "bats": { + "ubuntu:focal": "bats", + "ubuntu:focal::arch64": "bats" + }, + "build-essential": { + "ubuntu:focal": "build-essential", + "ubuntu:focal::arch64": "build-essential", + "centos:7": "build-essential", + "centos:8": "build-essential" + }, + "bzip2": { + "ubuntu:focal": [ + "bzip2", + "libbz2-dev" + ], + "ubuntu:focal::arch64": [ + "bzip2", + "libbz2-dev" + ], + "centos:7": [ + "bzip2", + "bzip2-devel" + ], + "centos:8": [ + "bzip2", + "bzip2-devel" + ] + }, + "clang": { + "ubuntu:focal": "clang", + "ubuntu:focal::arch64": "clang", + "centos:7": "clang", + "centos:8": "clang" + }, + "cmake": { + "ubuntu:focal": "cmake", + "ubuntu:focal::arch64": "cmake" + }, + "curl": { + "ubuntu:focal": [ + "curl", + "libcurl4-openssl-dev" + ], + "ubuntu:focal::arch64": [ + "curl", + "libcurl4-openssl-dev" + ], + "centos:7": [ + "curl", + "libcurl-devel" + ], + "centos:8": [ + "curl", + "libcurl-devel" + ] + }, + "doxygen": { + "ubuntu:focal": "doxygen", + "ubuntu:focal::arch64": "doxygen", + "centos:7": "doxygen" + }, + "dnf": { + "centos:8": "dnf" + }, + "fuse": { + "ubuntu:focal": [ + "fuse", + "libfuse-dev" + ], + "ubuntu:focal::arch64": [ + "fuse", + "libfuse-dev" + ], + "centos:7": [ + "fuse", + "fuse-libs", + "fuse-devel" + ], + "centos:8": [ + "fuse", + "fuse-libs", + "fuse-devel" + ] + }, + "gcc": { + "ubuntu:focal": [ + "gcc", + "g++" + ], + "ubuntu:focal::arch64": [ + "gcc", + "g++" + ], + "centos:7": [ + "centos-release-scl", + "devtoolset-9" + ] + }, + "git": { + "ubuntu:focal": "git", + "ubuntu:focal::arch64": "git", + "centos:7": "git", + "centos:8": "git" + }, + "gnupg-agent": { + "ubuntu:focal": "gnupg-agent", + "ubuntu:focal::arch64": "gnupg-agent" + }, + "hugo": { + "ubuntu:focal": "hugo", + "ubuntu:focal::arch64": "hugo" + }, + "libbcprov-java": { + "ubuntu:focal": "libbcprov-java", + "ubuntu:focal::arch64": "libbcprov-java" + }, + "libtool": { + "ubuntu:focal": "libtool", + "ubuntu:focal::arch64": "libtool", + "centos:7": "libtool", + "centos:8": "libtool" + }, + "openssl": { + "ubuntu:focal": "libssl-dev", + "ubuntu:focal::arch64": "libssl-dev", + "centos:7": "openssl-devel", + "centos:8": "openssl-devel" + }, + "protocol-buffers": { + "ubuntu:focal": [ + "libprotobuf-dev", + "libprotoc-dev" + ], + "ubuntu:focal::arch64": [ + "libprotobuf-dev", + "libprotoc-dev" + ] + }, + "sasl": { + "ubuntu:focal": "libsasl2-dev", + "ubuntu:focal::arch64": "libsasl2-dev", + "centos:7": "cyrus-sasl-devel", + "centos:8": "cyrus-sasl-devel" + }, + "snappy": { + "ubuntu:focal": "libsnappy-dev", + "ubuntu:focal::arch64": "libsnappy-dev", + "centos:7": "snappy-devel" + }, + "zlib": { + "ubuntu:focal": [ + "libzstd-dev", + "zlib1g-dev" + ], + "ubuntu:focal::arch64": [ + "libzstd-dev", + "zlib1g-dev" + ], + "centos:7": [ + "zlib-devel", + "lz4-devel" + ], + "centos:8": [ + "zlib-devel", + "lz4-devel" + ] + }, + "locales": { + "ubuntu:focal": "locales", + "ubuntu:focal::arch64": "locales" + }, + "libtirpc-devel": { + "centos:7": "libtirpc-devel", + "centos:8": "libtirpc-devel" + }, + "libpmem": { + "centos:7": "libpmem-devel" + }, + "make": { + "ubuntu:focal": "make", + "ubuntu:focal::arch64": "make", + "centos:7": "make", + "centos:8": "make" + }, + "maven": { + "ubuntu:focal": "maven", + "ubuntu:focal::arch64": "maven" + }, + "node.js": { + "ubuntu:focal": [ + "nodejs", + "node-yarn", + "npm" + ], + "ubuntu:focal::arch64": [ + "nodejs", + "node-yarn", + "npm" + ] + }, + "java": { + "ubuntu:focal": [ + "openjdk-8-jdk", + "openjdk-11-jdk" + ], + "ubuntu:focal::arch64": [ + "openjdk-8-jdk", + "openjdk-11-jdk" + ] + }, + "pinentry-curses": { + "ubuntu:focal": "pinentry-curses", + "ubuntu:focal::arch64": "pinentry-curses", + "centos:7": "pinentry-curses", + "centos:8": "pinentry-curses" + }, + "pkg-config": { + "ubuntu:focal": "pkg-config", + "ubuntu:focal::arch64": "pkg-config", + "centos:8": "pkg-config" + }, + "python3": { + "ubuntu:focal": [ + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], + "ubuntu:focal::arch64": [ + "python3", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + "python3-wheel" + ], + "centos:7": [ + "python3", + "python3-pip", + "python3-setuptools", + "python3-wheel" + ], + "centos:8": [ + "python3", + "python3-pip", + "python3-setuptools", + "python3-wheel" + ] + }, + "rsync": { + "ubuntu:focal": "rsync", + "ubuntu:focal::arch64": "rsync", + "centos:7": "rsync", + "centos:8": "rsync" + }, + "shellcheck": { + "ubuntu:focal": "shellcheck", + "ubuntu:focal::arch64": "shellcheck" + }, + "software-properties-common": { + "ubuntu:focal": "software-properties-common", + "ubuntu:focal::arch64": "software-properties-common" + }, + "sudo": { + "ubuntu:focal": "sudo", + "ubuntu:focal::arch64": "sudo", + "centos:7": "sudo", + "centos:8": "sudo" + }, + "valgrind": { + "ubuntu:focal": "valgrind", + "ubuntu:focal::arch64": "valgrind", + "centos:7": "valgrind", + "centos:8": "valgrind" + } +} \ No newline at end of file diff --git a/dev-support/docker/pkg-resolver/platforms.json b/dev-support/docker/pkg-resolver/platforms.json new file mode 100644 index 0000000000000..4861c5dd55c00 --- /dev/null +++ b/dev-support/docker/pkg-resolver/platforms.json @@ -0,0 +1,6 @@ +[ + "ubuntu:focal", + "ubuntu:focal::arch64", + "centos:7", + "centos:8" +] \ No newline at end of file diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py new file mode 100644 index 0000000000000..7c73bb1d96314 --- /dev/null +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import json +import sys + + +def get_packages(platform): + with open('pkg-resolver/packages.json', encoding='utf-8', mode='r') as pkg_file: + pkgs = json.loads(pkg_file.read()) + packages = [] + for platforms in pkgs.values(): + if platforms.get(platform) is None: + continue + else: + if type(platforms.get(platform)) == list: + for p in platforms.get(platform): + packages.append(p) + else: + packages.append(platforms.get(platform)) + return packages + + +def get_platforms(): + with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file: + return json.loads(platforms_file.read()) + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1), file=sys.stderr) + sys.exit(1) + + supported_platforms = get_platforms() + platform = sys.argv[1] + if platform not in supported_platforms: + print('ERROR: The given platform {} is not supported. ' + 'Please refer to platforms.json for a list of supported platforms'.format(platform), file=sys.stderr) + sys.exit(1) + + packages = get_packages(platform) + print(' '.join(packages)) From 456cc76fc65f53df3ed77289f54c5fd26a5d1419 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 22 May 2021 18:13:11 +0530 Subject: [PATCH 02/20] Add license headers --- .../pkg-resolver/install-common-packages.sh | 18 ++++++++++++++++++ dev-support/docker/pkg-resolver/resolve.py | 16 ++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/dev-support/docker/pkg-resolver/install-common-packages.sh b/dev-support/docker/pkg-resolver/install-common-packages.sh index 1e3d139bba488..00a5e8ee1cac1 100644 --- a/dev-support/docker/pkg-resolver/install-common-packages.sh +++ b/dev-support/docker/pkg-resolver/install-common-packages.sh @@ -1,3 +1,21 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ####### # Install SpotBugs 4.2.2 ####### diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py index 7c73bb1d96314..56a9b7d0fb5fb 100644 --- a/dev-support/docker/pkg-resolver/resolve.py +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -1,5 +1,21 @@ #!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import sys From e0f99d6597666156b325b994f8e9a21c467bed4f Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 22 May 2021 20:22:06 +0530 Subject: [PATCH 03/20] Add the required permissions * Added the necessary permissions to the files in pkg-resolver. --- dev-support/docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 3a43610dc1d05..37ab911640ca2 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -35,6 +35,8 @@ ENV DEBCONF_TERSE true RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends python3 COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/install-common-packages.sh pkg-resolver/resolve.py \ + && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json RUN apt-get -q install -y --no-install-recommends `pkg-resolver/resolve.py ubuntu:focal` RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* From ee688f7afe643eb9240f7a66af9262c976fed84d Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 22 May 2021 20:25:41 +0530 Subject: [PATCH 04/20] Refactor install-common-pkgs * Formatted the file and renamed it. --- dev-support/docker/Dockerfile | 4 +- .../pkg-resolver/install-common-packages.sh | 99 ------------------- .../pkg-resolver/install-common-pkgs.sh | 98 ++++++++++++++++++ 3 files changed, 100 insertions(+), 101 deletions(-) delete mode 100644 dev-support/docker/pkg-resolver/install-common-packages.sh create mode 100644 dev-support/docker/pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 37ab911640ca2..415dd2fa27bb9 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -35,7 +35,7 @@ ENV DEBCONF_TERSE true RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends python3 COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/install-common-packages.sh pkg-resolver/resolve.py \ +RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json RUN apt-get -q install -y --no-install-recommends `pkg-resolver/resolve.py ubuntu:focal` RUN apt-get clean \ @@ -74,7 +74,7 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true #### # Install common packages #### -RUN pkg-resolver/install-common-packages.sh +RUN pkg-resolver/install-common-pkgs.sh ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/dev-support/docker/pkg-resolver/install-common-packages.sh b/dev-support/docker/pkg-resolver/install-common-packages.sh deleted file mode 100644 index 00a5e8ee1cac1..0000000000000 --- a/dev-support/docker/pkg-resolver/install-common-packages.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -####### -# Install SpotBugs 4.2.2 -####### -mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* - -####### -# Install Boost 1.72 (1.71 ships with Focal) -####### -# hadolint ignore=DL3003 -mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src - -###### -# Install pylint and python-dateutil -###### -pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -###### -# Install bower -###### -# hadolint ignore=DL3008 -npm install -g bower@1.8.8 - -###### -# Install hadolint -###### -curl -L -s -S \ - https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ - -o /bin/hadolint \ - && chmod a+rx /bin/hadolint \ - && shasum -a 512 /bin/hadolint | \ - awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' - -###### -# Intel ISA-L 2.29.0 -###### -# hadolint ignore=DL3003,DL3008 -mkdir -p /opt/isa-l-src \ - && apt-get -q update \ - && apt-get install -y --no-install-recommends automake yasm \ - && apt-get clean \ - && curl -L -s -S \ - https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ - -o /opt/isa-l.tar.gz \ - && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \ - && cd /opt/isa-l-src \ - && ./autogen.sh \ - && ./configure \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/isa-l-src - diff --git a/dev-support/docker/pkg-resolver/install-common-pkgs.sh b/dev-support/docker/pkg-resolver/install-common-pkgs.sh new file mode 100644 index 0000000000000..62e8e7bd7f0e2 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-common-pkgs.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +####### +# Install SpotBugs 4.2.2 +####### +mkdir -p /opt/spotbugs && + curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz && + tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs && + chmod +x /opt/spotbugs/bin/* + +####### +# Install Boost 1.72 (1.71 ships with Focal) +####### +# hadolint ignore=DL3003 +mkdir -p /opt/boost-library && + curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 && + mv boost_1_72_0.tar.bz2 /opt/boost-library && + cd /opt/boost-library && + tar --bzip2 -xf boost_1_72_0.tar.bz2 && + cd /opt/boost-library/boost_1_72_0 && + ./bootstrap.sh --prefix=/usr/ && + ./b2 --without-python install && + cd /root && + rm -rf /opt/boost-library + +###### +# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) +###### +# hadolint ignore=DL3003 +mkdir -p /opt/protobuf-src && + curl -L -s -S \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz && + tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src && + cd /opt/protobuf-src && + ./configure --prefix=/opt/protobuf && + make "-j$(nproc)" && + make install && + cd /root && + rm -rf /opt/protobuf-src + +###### +# Install pylint and python-dateutil +###### +pip3 install pylint==2.6.0 python-dateutil==2.8.1 + +###### +# Install bower +###### +# hadolint ignore=DL3008 +npm install -g bower@1.8.8 + +###### +# Install hadolint +###### +curl -L -s -S \ + https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ + -o /bin/hadolint && + chmod a+rx /bin/hadolint && + shasum -a 512 /bin/hadolint | + awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' + +###### +# Intel ISA-L 2.29.0 +###### +# hadolint ignore=DL3003,DL3008 +mkdir -p /opt/isa-l-src && + apt-get -q update && + apt-get install -y --no-install-recommends automake yasm && + apt-get clean && + curl -L -s -S \ + https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ + -o /opt/isa-l.tar.gz && + tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src && + cd /opt/isa-l-src && + ./autogen.sh && + ./configure && + make "-j$(nproc)" && + make install && + cd /root && + rm -rf /opt/isa-l-src From a229946643d18e124d4edd7cb2267a5b875f1dfc Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sun, 23 May 2021 17:57:40 +0530 Subject: [PATCH 05/20] Fix warnings reported in CI run * Added necessary warning suppressions. * Fixed warnings reported by pylint. --- dev-support/docker/Dockerfile | 13 +++++-- dev-support/docker/Dockerfile_pkg_test | 0 dev-support/docker/pkg-resolver/resolve.py | 43 ++++++++++++++-------- 3 files changed, 37 insertions(+), 19 deletions(-) delete mode 100644 dev-support/docker/Dockerfile_pkg_test diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 415dd2fa27bb9..1de50619383b9 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -32,13 +32,18 @@ RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends python3 COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json -RUN apt-get -q install -y --no-install-recommends `pkg-resolver/resolve.py ubuntu:focal` -RUN apt-get clean \ + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008 +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends python3 \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) + && apt-get clean \ && rm -rf /var/lib/apt/lists/* RUN locale-gen en_US.UTF-8 diff --git a/dev-support/docker/Dockerfile_pkg_test b/dev-support/docker/Dockerfile_pkg_test deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py index 56a9b7d0fb5fb..f78e5727ab66b 100644 --- a/dev-support/docker/pkg-resolver/resolve.py +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -16,42 +16,55 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Platform package dependency resolver for building Apache Hadoop. +""" + import json import sys def get_packages(platform): + """ + Resolve and get the list of packages to install for the given platform. + + :param platform: The platform for which the packages needs to be resolved. + :return: A list of resolved packages to install. + """ with open('pkg-resolver/packages.json', encoding='utf-8', mode='r') as pkg_file: pkgs = json.loads(pkg_file.read()) packages = [] - for platforms in pkgs.values(): - if platforms.get(platform) is None: - continue + for platforms in filter(lambda x: x.get(platform) is not None, pkgs.values()): + if isinstance(platforms.get(platform), list): + packages.extend(platforms.get(platform)) else: - if type(platforms.get(platform)) == list: - for p in platforms.get(platform): - packages.append(p) - else: - packages.append(platforms.get(platform)) + packages.append(platforms.get(platform)) return packages def get_platforms(): + """ + :return: A list of the supported platforms managed by pkg-resolver. + """ + with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file: return json.loads(platforms_file.read()) if __name__ == '__main__': if len(sys.argv) < 2: - print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1), file=sys.stderr) + print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1), + file=sys.stderr) sys.exit(1) supported_platforms = get_platforms() - platform = sys.argv[1] - if platform not in supported_platforms: - print('ERROR: The given platform {} is not supported. ' - 'Please refer to platforms.json for a list of supported platforms'.format(platform), file=sys.stderr) + platform_arg = sys.argv[1] + if platform_arg not in supported_platforms: + print( + 'ERROR: The given platform {} is not supported. ' + 'Please refer to platforms.json for a list of supported platforms'.format( + platform_arg), file=sys.stderr) sys.exit(1) - packages = get_packages(platform) - print(' '.join(packages)) + packages_to_install = get_packages(platform_arg) + print(' '.join(packages_to_install)) From ecc6dbfabcdb3ce6f3be23ca6dd044a2102e5468 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sun, 23 May 2021 18:05:33 +0530 Subject: [PATCH 06/20] Add trailing slash * Added a trailing / for continuation of the command. --- dev-support/docker/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 1de50619383b9..7be1c31efb975 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -32,6 +32,9 @@ RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true +###### +# Platform package dependency resolver +###### COPY pkg-resolver pkg-resolver RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json @@ -42,7 +45,7 @@ RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ # hadolint ignore=DL3008 RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends python3 \ - && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From 43a794697fd19f3e5815574a131ce45cd541bce6 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sun, 23 May 2021 20:08:27 +0530 Subject: [PATCH 07/20] Add quotes to prevent word split * Added a quote as per the reported warning in CI run to prevent the splitting of word. --- dev-support/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 7be1c31efb975..9addd4b971e97 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -45,7 +45,7 @@ RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ # hadolint ignore=DL3008 RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends python3 \ - && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \ + && apt-get -q install -y --no-install-recommends "$(pkg-resolver/resolve.py ubuntu:focal)" \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From 2513a12bd3049d577c703419681d661f5cb2af69 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sun, 23 May 2021 22:21:50 +0530 Subject: [PATCH 08/20] Unquote and ignore warning * Quoting $(...) causes the command execution to fail. Thus, ignoring the warning instead. --- dev-support/docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 9addd4b971e97..19fad3aa7ab50 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -42,10 +42,10 @@ RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ ###### # Install packages from apt ###### -# hadolint ignore=DL3008 +# hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ && apt-get -q install -y --no-install-recommends python3 \ - && apt-get -q install -y --no-install-recommends "$(pkg-resolver/resolve.py ubuntu:focal)" \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From d58e29566980ce4974f20b25a5f6032e97ec2176 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 24 May 2021 20:41:10 +0530 Subject: [PATCH 09/20] Use pkg-resolver for other platforms * Using pkg-resolver for resolving the dependencies on Ubuntu Focal arch64, Centos 7 and Centos 8. --- dev-support/docker/Dockerfile_aarch64 | 64 ++--------- dev-support/docker/Dockerfile_centos_7 | 104 +++--------------- dev-support/docker/Dockerfile_centos_8 | 100 +++-------------- dev-support/docker/pkg-resolver/packages.json | 8 +- 4 files changed, 52 insertions(+), 224 deletions(-) diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 5c2613268a65b..7fd21fbdfbe0d 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -33,61 +33,19 @@ ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_TERSE true ###### -# Install common dependencies from packages. Versions here are either -# sufficient or irrelevant. +# Platform package dependency resolver ###### -# hadolint ignore=DL3008 +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ + && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json + +###### +# Install packages from apt +###### +# hadolint ignore=DL3008,SC2046 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends \ - ant \ - apt-utils \ - bats \ - build-essential \ - bzip2 \ - clang \ - cmake \ - curl \ - doxygen \ - fuse \ - g++ \ - gcc \ - git \ - gnupg-agent \ - hugo \ - libbcprov-java \ - libbz2-dev \ - libcurl4-openssl-dev \ - libfuse-dev \ - libprotobuf-dev \ - libprotoc-dev \ - libsasl2-dev \ - libsnappy-dev \ - libssl-dev \ - libtool \ - libzstd-dev \ - locales \ - make \ - maven \ - nodejs \ - node-yarn \ - npm \ - openjdk-11-jdk \ - openjdk-8-jdk \ - phantomjs \ - pinentry-curses \ - pkg-config \ - python2.7 \ - python3 \ - python3-pip \ - python3-pkg-resources \ - python3-setuptools \ - python3-wheel \ - rsync \ - shellcheck \ - software-properties-common \ - sudo \ - valgrind \ - zlib1g-dev \ + && apt-get -q install -y --no-install-recommends python3 \ + && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 319942e2f6d4a..061c29ad63e5d 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -23,39 +23,20 @@ WORKDIR /root SHELL ["/bin/bash", "-o", "pipefail", "-c"] +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ + && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json + +###### +# Install packages from yum +###### +# hadolint ignore=DL3008,SC2046 RUN yum update -y \ - && yum install -y centos-release-scl \ - && yum install -y devtoolset-9 \ - && yum install -y \ - ant \ - build-essential \ - bzip2 \ - bzip2-devel \ - clang \ - curl \ - cyrus-sasl-devel \ - doxygen \ - fuse \ - fuse-libs \ - fuse-devel \ - git \ - libcurl-devel \ - libtirpc-devel \ - libpmem-devel \ - libtool \ - lz4-devel \ - make \ - openssl-devel \ - pinentry-curses \ - python3 \ - python3-pip \ - python3-setuptools \ - python3-wheel \ - rsync \ - snappy-devel \ - sudo \ - valgrind \ - zlib-devel + && yum install -y python3 \ + && yum install -y $(pkg-resolver/resolve.py centos:7) # Set GCC 9 as the default C/C++ compiler RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc @@ -97,6 +78,11 @@ RUN locale-gen en_US.UTF-8 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' ENV PYTHONIOENCODING=utf-8 +###### +# Install common packages +###### +RUN pkg-resolver/install-common-pkgs.sh + ###### # Set env vars required to build Hadoop ###### @@ -105,49 +91,6 @@ ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 -####### -# Install SpotBugs 4.2.2 -####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* -ENV SPOTBUGS_HOME /opt/spotbugs - -####### -# Install Boost 1.72 (1.71 ships with Focal) -####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" - #### # Install Node.js #### @@ -160,14 +103,3 @@ RUN mkdir -p /tmp/node \ && make "-j$(nproc)" \ && make install \ && cd /root || exit - -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 92ec720f2ec21..5b0f8868565e8 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -23,35 +23,20 @@ WORKDIR /root SHELL ["/bin/bash", "-o", "pipefail", "-c"] +###### +# Platform package dependency resolver +###### +COPY pkg-resolver pkg-resolver +RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ + && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json + +###### +# Install packages from yum +###### +# hadolint ignore=DL3008,SC2046 RUN yum update -y \ - && yum install -y \ - ant \ - bzip2 \ - bzip2-devel \ - clang \ - curl \ - cyrus-sasl-devel \ - dnf \ - fuse \ - fuse-libs \ - fuse-devel \ - git \ - libcurl-devel \ - libtool \ - libtirpc-devel \ - lz4-devel \ - make \ - openssl-devel \ - pinentry-curses \ - pkg-config \ - python3 \ - python3-pip \ - python3-setuptools \ - python3-wheel \ - rsync \ - sudo \ - valgrind \ - zlib-devel + && yum install -y python3 \ + && yum install -y $(pkg-resolver/resolve.py centos:8) #### # Install EPEL @@ -113,6 +98,11 @@ RUN locale-gen en_US.UTF-8 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' ENV PYTHONIOENCODING=utf-8 +###### +# Install common packages +###### +RUN pkg-resolver/install-common-pkgs.sh + ###### # Set env vars required to build Hadoop ###### @@ -120,57 +110,3 @@ ENV MAVEN_HOME /opt/maven/apache-maven-3.6.3 ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 - -####### -# Install SpotBugs 4.2.2 -####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* -ENV SPOTBUGS_HOME /opt/spotbugs - -####### -# Install Boost 1.72 -####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src -ENV PROTOBUF_HOME /opt/protobuf -ENV PATH "${PATH}:/opt/protobuf/bin" - -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index cf59c63992d32..a5ce5402f3e64 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -197,7 +197,7 @@ "ubuntu:focal": "maven", "ubuntu:focal::arch64": "maven" }, - "node.js": { + "javascript": { "ubuntu:focal": [ "nodejs", "node-yarn", @@ -206,7 +206,8 @@ "ubuntu:focal::arch64": [ "nodejs", "node-yarn", - "npm" + "npm", + "phantomjs" ] }, "java": { @@ -230,7 +231,7 @@ "ubuntu:focal::arch64": "pkg-config", "centos:8": "pkg-config" }, - "python3": { + "python": { "ubuntu:focal": [ "python3", "python3-pip", @@ -239,6 +240,7 @@ "python3-wheel" ], "ubuntu:focal::arch64": [ + "python2.7", "python3", "python3-pip", "python3-pkg-resources", From 692169fc27f23b50f0556596e12e7ac579b35c64 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 24 May 2021 23:30:07 +0530 Subject: [PATCH 10/20] Install centos-release-scl as prerequisite * The centos-release-scl package must be installed as a prerequisite before installing devtoolset-9 for gcc. --- dev-support/docker/Dockerfile_centos_7 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 061c29ad63e5d..9e4f741be624f 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -35,7 +35,9 @@ RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ ###### # hadolint ignore=DL3008,SC2046 RUN yum update -y \ - && yum install -y python3 \ + && yum install -y \ + centos-release-scl \ + python3 \ && yum install -y $(pkg-resolver/resolve.py centos:7) # Set GCC 9 as the default C/C++ compiler From 50eff6995e684c6757eddc77b0b8a38205bd2566 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 29 May 2021 13:40:39 +0530 Subject: [PATCH 11/20] Fix installation of Intel ISA-L library * Intel ISA-L library has dependencies on yasm, shasum which needs to be installed prior to installing it. --- dev-support/docker/Dockerfile_centos_7 | 16 ++++++++++++---- dev-support/docker/Dockerfile_centos_8 | 9 +++++---- .../pkg-resolver/install-common-pkgs.sh | 3 --- dev-support/docker/pkg-resolver/packages.json | 19 +++++++++++++++++-- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 9e4f741be624f..44d6f7b0ac981 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -76,9 +76,17 @@ RUN mkdir -p /opt/zstd /tmp/zstd \ && make install \ && cd /root || exit -RUN locale-gen en_US.UTF-8 -ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -ENV PYTHONIOENCODING=utf-8 +# RUN locale-gen en_US.UTF-8 +# ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +# ENV PYTHONIOENCODING=utf-8 + +###### +# Install yasm +###### +RUN mkdir -p /tmp/yasm \ + && curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \ + -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm \ + && rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm ###### # Install common packages @@ -104,4 +112,4 @@ RUN mkdir -p /tmp/node \ && ./configure \ && make "-j$(nproc)" \ && make install \ - && cd /root || exit + && cd /root || exit \ No newline at end of file diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 5b0f8868565e8..8fbb2910aea8b 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -48,7 +48,8 @@ RUN mkdir -p /tmp/epel \ RUN dnf --enablerepo=powertools install -y \ doxygen \ - snappy-devel + snappy-devel \ + yasm RUN dnf install -y \ bouncycastle \ @@ -94,9 +95,9 @@ RUN mkdir -p /opt/zstd /tmp/zstd \ && make install \ && cd /root || exit -RUN locale-gen en_US.UTF-8 -ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -ENV PYTHONIOENCODING=utf-8 +# RUN locale-gen en_US.UTF-8 +# ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +# ENV PYTHONIOENCODING=utf-8 ###### # Install common packages diff --git a/dev-support/docker/pkg-resolver/install-common-pkgs.sh b/dev-support/docker/pkg-resolver/install-common-pkgs.sh index 62e8e7bd7f0e2..9e4028f4572eb 100644 --- a/dev-support/docker/pkg-resolver/install-common-pkgs.sh +++ b/dev-support/docker/pkg-resolver/install-common-pkgs.sh @@ -82,9 +82,6 @@ curl -L -s -S \ ###### # hadolint ignore=DL3003,DL3008 mkdir -p /opt/isa-l-src && - apt-get -q update && - apt-get install -y --no-install-recommends automake yasm && - apt-get clean && curl -L -s -S \ https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ -o /opt/isa-l.tar.gz && diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index a5ce5402f3e64..e6cb625f502a7 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -9,6 +9,12 @@ "ubuntu:focal": "apt-utils", "ubuntu:focal::arch64": "apt-utils" }, + "automake": { + "ubuntu:focal": "automake", + "ubuntu:focal::arch64": "automake", + "centos:7": "automake", + "centos:8": "automake" + }, "bats": { "ubuntu:focal": "bats", "ubuntu:focal::arch64": "bats" @@ -16,8 +22,7 @@ "build-essential": { "ubuntu:focal": "build-essential", "ubuntu:focal::arch64": "build-essential", - "centos:7": "build-essential", - "centos:8": "build-essential" + "centos:7": "build-essential" }, "bzip2": { "ubuntu:focal": [ @@ -270,6 +275,12 @@ "ubuntu:focal": "shellcheck", "ubuntu:focal::arch64": "shellcheck" }, + "shasum": { + "ubuntu:focal": "shasum", + "ubuntu:focal::arch64": "shasum", + "centos:7": "perl-Digest-SHA", + "centos:8": "perl-Digest-SHA" + }, "software-properties-common": { "ubuntu:focal": "software-properties-common", "ubuntu:focal::arch64": "software-properties-common" @@ -285,5 +296,9 @@ "ubuntu:focal::arch64": "valgrind", "centos:7": "valgrind", "centos:8": "valgrind" + }, + "yasm": { + "ubuntu:focal": "yasm", + "ubuntu:focal::arch64": "yasm" } } \ No newline at end of file From 701ef5ed9a0259f1ed1eea9259ff7f5442b26ca1 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 29 May 2021 16:18:04 +0530 Subject: [PATCH 12/20] Add checks for package installation * Moved the build and installation of separate packages into individual scripts. --- dev-support/docker/Dockerfile | 11 ++- .../docker/pkg-resolver/check_platform.py | 50 ++++++++++++++ .../docker/pkg-resolver/install-boost.sh | 56 ++++++++++++++++ .../pkg-resolver/install-common-pkgs.sh | 67 ------------------- .../docker/pkg-resolver/install-hadolint.sh | 35 ++++++++++ .../pkg-resolver/install-intel-isa-l.sh | 58 ++++++++++++++++ .../docker/pkg-resolver/install-protobuf.sh | 57 ++++++++++++++++ .../docker/pkg-resolver/install-spotbugs.sh | 50 ++++++++++++++ dev-support/docker/pkg-resolver/resolve.py | 13 +--- 9 files changed, 316 insertions(+), 81 deletions(-) create mode 100644 dev-support/docker/pkg-resolver/check_platform.py create mode 100644 dev-support/docker/pkg-resolver/install-boost.sh create mode 100644 dev-support/docker/pkg-resolver/install-hadolint.sh create mode 100644 dev-support/docker/pkg-resolver/install-intel-isa-l.sh create mode 100644 dev-support/docker/pkg-resolver/install-protobuf.sh create mode 100644 dev-support/docker/pkg-resolver/install-spotbugs.sh diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 19fad3aa7ab50..fac364bbd4363 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -36,8 +36,8 @@ ENV DEBCONF_TERSE true # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ - && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json ###### # Install packages from apt @@ -80,9 +80,14 @@ ENV MAVEN_OPTS -Xms256m -Xmx3072m ENV HADOOP_SKIP_YETUS_VERIFICATION true #### -# Install common packages +# Install packages #### RUN pkg-resolver/install-common-pkgs.sh +RUN pkg-resolver/install-spotbugs.sh ubuntu:focal +RUN pkg-resolver/install-boost.sh ubuntu:focal +RUN pkg-resolver/install-protobuf.sh ubuntu:focal +RUN pkg-resolver/install-hadolint.sh ubuntu:focal +RUN pkg-resolver/install-intel-isa-l.sh ubuntu:focal ### # Everything past this point is either not needed for testing or breaks Yetus. diff --git a/dev-support/docker/pkg-resolver/check_platform.py b/dev-support/docker/pkg-resolver/check_platform.py new file mode 100644 index 0000000000000..fa5529a58be20 --- /dev/null +++ b/dev-support/docker/pkg-resolver/check_platform.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks whether the given platform is supported for building Apache Hadoop +""" + +import json +import sys + + +def get_platforms(): + """ + :return: A list of the supported platforms managed by pkg-resolver. + """ + + with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file: + return json.loads(platforms_file.read()) + + +def is_supported_platform(platform): + """ + :param platform: The name of the platform + :return: Whether the platform is supported + """ + return platform in get_platforms() + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print('ERROR: Expecting 1 argument, {} were provided'.format(len(sys.argv) - 1), + file=sys.stderr) + sys.exit(1) + + sys.exit(0 if is_supported_platform(sys.argv[1]) else 1) diff --git a/dev-support/docker/pkg-resolver/install-boost.sh b/dev-support/docker/pkg-resolver/install-boost.sh new file mode 100644 index 0000000000000..eaca09effa2c0 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-boost.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="1.72.0" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "1.72.0" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "1.72.0" ]; then + # hadolint ignore=DL3003 + mkdir -p /opt/boost-library && + curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 && + mv boost_1_72_0.tar.bz2 /opt/boost-library && + cd /opt/boost-library && + tar --bzip2 -xf boost_1_72_0.tar.bz2 && + cd /opt/boost-library/boost_1_72_0 && + ./bootstrap.sh --prefix=/usr/ && + ./b2 --without-python install && + cd /root && + rm -rf /opt/boost-library +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-common-pkgs.sh b/dev-support/docker/pkg-resolver/install-common-pkgs.sh index 9e4028f4572eb..d6756eea73d79 100644 --- a/dev-support/docker/pkg-resolver/install-common-pkgs.sh +++ b/dev-support/docker/pkg-resolver/install-common-pkgs.sh @@ -16,46 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -####### -# Install SpotBugs 4.2.2 -####### -mkdir -p /opt/spotbugs && - curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz && - tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs && - chmod +x /opt/spotbugs/bin/* - -####### -# Install Boost 1.72 (1.71 ships with Focal) -####### -# hadolint ignore=DL3003 -mkdir -p /opt/boost-library && - curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 && - mv boost_1_72_0.tar.bz2 /opt/boost-library && - cd /opt/boost-library && - tar --bzip2 -xf boost_1_72_0.tar.bz2 && - cd /opt/boost-library/boost_1_72_0 && - ./bootstrap.sh --prefix=/usr/ && - ./b2 --without-python install && - cd /root && - rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -mkdir -p /opt/protobuf-src && - curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz && - tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src && - cd /opt/protobuf-src && - ./configure --prefix=/opt/protobuf && - make "-j$(nproc)" && - make install && - cd /root && - rm -rf /opt/protobuf-src - ###### # Install pylint and python-dateutil ###### @@ -66,30 +26,3 @@ pip3 install pylint==2.6.0 python-dateutil==2.8.1 ###### # hadolint ignore=DL3008 npm install -g bower@1.8.8 - -###### -# Install hadolint -###### -curl -L -s -S \ - https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ - -o /bin/hadolint && - chmod a+rx /bin/hadolint && - shasum -a 512 /bin/hadolint | - awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' - -###### -# Intel ISA-L 2.29.0 -###### -# hadolint ignore=DL3003,DL3008 -mkdir -p /opt/isa-l-src && - curl -L -s -S \ - https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ - -o /opt/isa-l.tar.gz && - tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src && - cd /opt/isa-l-src && - ./autogen.sh && - ./configure && - make "-j$(nproc)" && - make install && - cd /root && - rm -rf /opt/isa-l-src diff --git a/dev-support/docker/pkg-resolver/install-hadolint.sh b/dev-support/docker/pkg-resolver/install-hadolint.sh new file mode 100644 index 0000000000000..1e2081f38c403 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-hadolint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +curl -L -s -S \ + https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \ + -o /bin/hadolint && + chmod a+rx /bin/hadolint && + shasum -a 512 /bin/hadolint | + awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}' diff --git a/dev-support/docker/pkg-resolver/install-intel-isa-l.sh b/dev-support/docker/pkg-resolver/install-intel-isa-l.sh new file mode 100644 index 0000000000000..c6b4de782282e --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-intel-isa-l.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="2.29.0" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "2.29.0" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "2.29.0" ]; then + # hadolint ignore=DL3003,DL3008 + mkdir -p /opt/isa-l-src && + curl -L -s -S \ + https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \ + -o /opt/isa-l.tar.gz && + tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src && + cd /opt/isa-l-src && + ./autogen.sh && + ./configure && + make "-j$(nproc)" && + make install && + cd /root && + rm -rf /opt/isa-l-src +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-protobuf.sh b/dev-support/docker/pkg-resolver/install-protobuf.sh new file mode 100644 index 0000000000000..7303b4048226a --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-protobuf.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="3.7.1" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "3.7.1" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "3.7.1" ]; then + # hadolint ignore=DL3003 + mkdir -p /opt/protobuf-src && + curl -L -s -S \ + https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ + -o /opt/protobuf.tar.gz && + tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src && + cd /opt/protobuf-src && + ./configure --prefix=/opt/protobuf && + make "-j$(nproc)" && + make install && + cd /root && + rm -rf /opt/protobuf-src +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-spotbugs.sh b/dev-support/docker/pkg-resolver/install-spotbugs.sh new file mode 100644 index 0000000000000..65a8f2e692418 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-spotbugs.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="4.2.2" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "4.2.2" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "4.2.2" ]; then + mkdir -p /opt/spotbugs && + curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ + -o /opt/spotbugs.tgz && + tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs && + chmod +x /opt/spotbugs/bin/* +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py index f78e5727ab66b..3a8bb46b02747 100644 --- a/dev-support/docker/pkg-resolver/resolve.py +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -22,6 +22,7 @@ import json import sys +from check_platform import is_supported_platform def get_packages(platform): @@ -42,24 +43,14 @@ def get_packages(platform): return packages -def get_platforms(): - """ - :return: A list of the supported platforms managed by pkg-resolver. - """ - - with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file: - return json.loads(platforms_file.read()) - - if __name__ == '__main__': if len(sys.argv) < 2: print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1), file=sys.stderr) sys.exit(1) - supported_platforms = get_platforms() platform_arg = sys.argv[1] - if platform_arg not in supported_platforms: + if is_supported_platform(platform_arg): print( 'ERROR: The given platform {} is not supported. ' 'Please refer to platforms.json for a list of supported platforms'.format( From a9bf9e29375c26bb7670de0d4530ed7d125e178c Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 29 May 2021 17:12:28 +0530 Subject: [PATCH 13/20] Fix negation bug * Forgot to negate the platform check which caused a bug. --- dev-support/docker/pkg-resolver/resolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/pkg-resolver/resolve.py b/dev-support/docker/pkg-resolver/resolve.py index 3a8bb46b02747..5d62edd323e78 100644 --- a/dev-support/docker/pkg-resolver/resolve.py +++ b/dev-support/docker/pkg-resolver/resolve.py @@ -50,7 +50,7 @@ def get_packages(platform): sys.exit(1) platform_arg = sys.argv[1] - if is_supported_platform(platform_arg): + if not is_supported_platform(platform_arg): print( 'ERROR: The given platform {} is not supported. ' 'Please refer to platforms.json for a list of supported platforms'.format( From 191368f7ac635c3ea33563067527fc74f5249a6f Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 29 May 2021 17:29:23 +0530 Subject: [PATCH 14/20] Remove shasum for Ubuntu * shasum must be installed through other deps. --- dev-support/docker/pkg-resolver/packages.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index e6cb625f502a7..4ebbc900fe4de 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -276,8 +276,6 @@ "ubuntu:focal::arch64": "shellcheck" }, "shasum": { - "ubuntu:focal": "shasum", - "ubuntu:focal::arch64": "shasum", "centos:7": "perl-Digest-SHA", "centos:8": "perl-Digest-SHA" }, From e0a5ecaf73ac28ed03de5199e5b1a2201cc2d3cb Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 31 May 2021 20:21:09 +0530 Subject: [PATCH 15/20] Modularize Dockerfile for Centos 7 * Moved the packages' installation to separate scripts. --- dev-support/docker/Dockerfile_centos_7 | 75 ++++--------------- .../docker/pkg-resolver/install-cmake.sh | 52 +++++++++++++ .../docker/pkg-resolver/install-maven.sh | 49 ++++++++++++ .../docker/pkg-resolver/install-nodejs.sh | 53 +++++++++++++ .../docker/pkg-resolver/install-yasm.sh | 49 ++++++++++++ .../docker/pkg-resolver/install-zstandard.sh | 52 +++++++++++++ 6 files changed, 268 insertions(+), 62 deletions(-) create mode 100644 dev-support/docker/pkg-resolver/install-cmake.sh create mode 100644 dev-support/docker/pkg-resolver/install-maven.sh create mode 100644 dev-support/docker/pkg-resolver/install-nodejs.sh create mode 100644 dev-support/docker/pkg-resolver/install-yasm.sh create mode 100644 dev-support/docker/pkg-resolver/install-zstandard.sh diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index 44d6f7b0ac981..840de00a275c2 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -27,8 +27,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ - && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json ###### # Install packages from yum @@ -44,54 +44,7 @@ RUN yum update -y \ RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc SHELL ["/bin/bash", "--login", "-c"] -#### -# Install Maven 3.6.3 -#### -RUN mkdir -p /opt/maven /tmp/maven \ - && curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \ - -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \ - && tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven - -#### -# Install CMake 3.19 -#### -# hadolint ignore=DL3003 -RUN mkdir -p /tmp/cmake /opt/cmake \ - && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \ - && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \ - && cd /opt/cmake || exit && ./bootstrap \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -#### -# Install zstandard -#### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/zstd /tmp/zstd \ - && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \ - && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \ - && cd /opt/zstd || exit \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -# RUN locale-gen en_US.UTF-8 -# ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -# ENV PYTHONIOENCODING=utf-8 - -###### -# Install yasm -###### -RUN mkdir -p /tmp/yasm \ - && curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \ - -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm \ - && rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm - -###### -# Install common packages -###### -RUN pkg-resolver/install-common-pkgs.sh +# TODO: Set locale ###### # Set env vars required to build Hadoop @@ -101,15 +54,13 @@ ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 -#### -# Install Node.js -#### -# hadolint ignore=DL3003 -RUN mkdir -p /tmp/node \ - && curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz \ - && tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node \ - && cd /tmp/node || exit \ - && ./configure \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit \ No newline at end of file + +###### +# Install packages +###### +RUN pkg-resolver/install-maven.sh centos:7 +RUN pkg-resolver/install-cmake.sh centos:7 +RUN pkg-resolver/install-zstandard.sh centos:7 +RUN pkg-resolver/install-yasm.sh centos:7 +RUN pkg-resolver/install-nodejs.sh centos:7 +RUN pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/pkg-resolver/install-cmake.sh b/dev-support/docker/pkg-resolver/install-cmake.sh new file mode 100644 index 0000000000000..a6a2f549acc50 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-cmake.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="3.19.0" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "3.19.0" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "3.19.0" ]; then + mkdir -p /tmp/cmake /opt/cmake && + curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz && + tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake && + cd /opt/cmake || exit && ./bootstrap && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-maven.sh b/dev-support/docker/pkg-resolver/install-maven.sh new file mode 100644 index 0000000000000..f9ff961a190f9 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-maven.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="3.6.3" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "3.6.3" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "3.6.3" ]; then + mkdir -p /opt/maven /tmp/maven && + curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \ + -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz && + tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-nodejs.sh b/dev-support/docker/pkg-resolver/install-nodejs.sh new file mode 100644 index 0000000000000..7975d3e112407 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-nodejs.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="14.16.1" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "14.16.1" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "14.16.1" ]; then + mkdir -p /tmp/node && + curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz && + tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node && + cd /tmp/node || exit && + ./configure && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-yasm.sh b/dev-support/docker/pkg-resolver/install-yasm.sh new file mode 100644 index 0000000000000..a5f6162bc38d7 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-yasm.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="1.2.0-4" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "1.2.0-4" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "1.2.0-4" ]; then + mkdir -p /tmp/yasm && + curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \ + -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm && + rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-zstandard.sh b/dev-support/docker/pkg-resolver/install-zstandard.sh new file mode 100644 index 0000000000000..e4b1efa08b65f --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-zstandard.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="1.4.9" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "1.4.9" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "1.4.9" ]; then + mkdir -p /opt/zstd /tmp/zstd && + curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz && + tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd && + cd /opt/zstd || exit && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi From 8728912479d4d347cf68dceeec3b0a022ca59ab0 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 31 May 2021 22:40:47 +0530 Subject: [PATCH 16/20] Modularize Dockerfile for Centos 8 * Replaced explicit installation of packages with script invocations. --- dev-support/docker/Dockerfile_centos_8 | 58 ++++--------------- .../docker/pkg-resolver/install-cmake.sh | 1 + .../docker/pkg-resolver/install-epel.sh | 49 ++++++++++++++++ .../docker/pkg-resolver/install-nodejs.sh | 1 + .../docker/pkg-resolver/install-zstandard.sh | 1 + 5 files changed, 64 insertions(+), 46 deletions(-) create mode 100644 dev-support/docker/pkg-resolver/install-epel.sh diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index 8fbb2910aea8b..809bb2b1e8548 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -27,8 +27,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ - && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json ###### # Install packages from yum @@ -41,10 +41,7 @@ RUN yum update -y \ #### # Install EPEL #### -RUN mkdir -p /tmp/epel \ - && curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ - -o /tmp/epel/epel-release-latest-8.noarch.rpm \ - && rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm +RUN pkg-resolver/install-epel.sh centos:8 RUN dnf --enablerepo=powertools install -y \ doxygen \ @@ -63,46 +60,7 @@ RUN dnf install -y \ RUN echo "source /opt/rh/gcc-toolset-9/enable" >> /etc/bashrc SHELL ["/bin/bash", "--login", "-c"] -#### -# Install Maven 3.6.3 -#### -RUN mkdir -p /opt/maven /tmp/maven \ - && curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \ - -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \ - && tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven - -#### -# Install CMake 3.19 -#### -# hadolint ignore=DL3003 -RUN mkdir -p /tmp/cmake /opt/cmake \ - && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \ - && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \ - && cd /opt/cmake || exit && ./bootstrap \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -#### -# Install zstandard -#### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/zstd /tmp/zstd \ - && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \ - && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \ - && cd /opt/zstd || exit \ - && make "-j$(nproc)" \ - && make install \ - && cd /root || exit - -# RUN locale-gen en_US.UTF-8 -# ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -# ENV PYTHONIOENCODING=utf-8 - -###### -# Install common packages -###### -RUN pkg-resolver/install-common-pkgs.sh +# TODO: Set locale ###### # Set env vars required to build Hadoop @@ -111,3 +69,11 @@ ENV MAVEN_HOME /opt/maven/apache-maven-3.6.3 ENV PATH "${PATH}:${MAVEN_HOME}/bin" # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 + +###### +# Install packages +###### +RUN pkg-resolver/install-maven.sh centos:8 +RUN pkg-resolver/install-cmake.sh centos:8 +RUN pkg-resolver/install-zstandard.sh centos:8 +RUN pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/pkg-resolver/install-cmake.sh b/dev-support/docker/pkg-resolver/install-cmake.sh index a6a2f549acc50..29e2733e70196 100644 --- a/dev-support/docker/pkg-resolver/install-cmake.sh +++ b/dev-support/docker/pkg-resolver/install-cmake.sh @@ -39,6 +39,7 @@ if [ "$version_to_install" != "3.19.0" ]; then fi if [ "$version_to_install" == "3.19.0" ]; then + # hadolint ignore=DL3003 mkdir -p /tmp/cmake /opt/cmake && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake && diff --git a/dev-support/docker/pkg-resolver/install-epel.sh b/dev-support/docker/pkg-resolver/install-epel.sh new file mode 100644 index 0000000000000..875dce3a9ae85 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-epel.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="8" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "8" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "8" ]; then + mkdir -p /tmp/epel && + curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ + -o /tmp/epel/epel-release-latest-8.noarch.rpm && + rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-nodejs.sh b/dev-support/docker/pkg-resolver/install-nodejs.sh index 7975d3e112407..5ba1c22808640 100644 --- a/dev-support/docker/pkg-resolver/install-nodejs.sh +++ b/dev-support/docker/pkg-resolver/install-nodejs.sh @@ -39,6 +39,7 @@ if [ "$version_to_install" != "14.16.1" ]; then fi if [ "$version_to_install" == "14.16.1" ]; then + # hadolint ignore=DL3003 mkdir -p /tmp/node && curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz && tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node && diff --git a/dev-support/docker/pkg-resolver/install-zstandard.sh b/dev-support/docker/pkg-resolver/install-zstandard.sh index e4b1efa08b65f..3aafd469d2be3 100644 --- a/dev-support/docker/pkg-resolver/install-zstandard.sh +++ b/dev-support/docker/pkg-resolver/install-zstandard.sh @@ -39,6 +39,7 @@ if [ "$version_to_install" != "1.4.9" ]; then fi if [ "$version_to_install" == "1.4.9" ]; then + # hadolint ignore=DL3003 mkdir -p /opt/zstd /tmp/zstd && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd && From 325b2dd93ee217e17133c4a2d16cc2991ebbdc7a Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 31 May 2021 22:54:06 +0530 Subject: [PATCH 17/20] Modularize Dockerfile for aarch64 * Replaced explicit installations of packages with script invocations. --- dev-support/docker/Dockerfile_aarch64 | 59 ++++++--------------------- 1 file changed, 12 insertions(+), 47 deletions(-) diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 7fd21fbdfbe0d..dd0348961f464 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -36,8 +36,8 @@ ENV DEBCONF_TERSE true # Platform package dependency resolver ###### COPY pkg-resolver pkg-resolver -RUN chmod a+x pkg-resolver/install-common-pkgs.sh pkg-resolver/resolve.py \ - && chmod a+r pkg-resolver/packages.json pkg-resolver/platforms.json +RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ + && chmod a+r pkg-resolver/*.json ###### # Install packages from apt @@ -61,59 +61,16 @@ ENV MAVEN_HOME /usr ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64 ####### -# Install SpotBugs 4.2.2 +# Set env vars for SpotBugs 4.2.2 ####### -RUN mkdir -p /opt/spotbugs \ - && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \ - -o /opt/spotbugs.tgz \ - && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \ - && chmod +x /opt/spotbugs/bin/* ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Install Boost 1.72 (1.71 ships with Focal) +# Set env vars for Google Protobuf 3.7.1 ####### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/boost-library \ - && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \ - && mv boost_1_72_0.tar.bz2 /opt/boost-library \ - && cd /opt/boost-library \ - && tar --bzip2 -xf boost_1_72_0.tar.bz2 \ - && cd /opt/boost-library/boost_1_72_0 \ - && ./bootstrap.sh --prefix=/usr/ \ - && ./b2 --without-python install \ - && cd /root \ - && rm -rf /opt/boost-library - -###### -# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal) -###### -# hadolint ignore=DL3003 -RUN mkdir -p /opt/protobuf-src \ - && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \ - -o /opt/protobuf.tar.gz \ - && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \ - && cd /opt/protobuf-src \ - && ./configure --prefix=/opt/protobuf \ - && make "-j$(nproc)" \ - && make install \ - && cd /root \ - && rm -rf /opt/protobuf-src ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" -#### -# Install pylint and python-dateutil -#### -RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 - -#### -# Install bower -#### -# hadolint ignore=DL3008 -RUN npm install -g bower@1.8.8 - ### # Avoid out of memory errors in builds ### @@ -125,6 +82,14 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true # Force PhantomJS to be in 'headless' mode, do not connect to Xwindow ENV QT_QPA_PLATFORM offscreen +#### +# Install packages +#### +RUN pkg-resolver/install-common-pkgs.sh +RUN pkg-resolver/install-spotbugs.sh ubuntu:focal::arch64 +RUN pkg-resolver/install-boost.sh ubuntu:focal::arch64 +RUN pkg-resolver/install-protobuf.sh ubuntu:focal::arch64 + ### # Everything past this point is either not needed for testing or breaks Yetus. # So tell Yetus not to read the rest of the file: From d97a07aa02b0691c7ba5880e97d838015465d45a Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Mon, 31 May 2021 23:37:53 +0530 Subject: [PATCH 18/20] Add README.md * Added a readme file for describing the package management module. --- dev-support/docker/README.md | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 dev-support/docker/README.md diff --git a/dev-support/docker/README.md b/dev-support/docker/README.md new file mode 100644 index 0000000000000..4abc65fd01ac3 --- /dev/null +++ b/dev-support/docker/README.md @@ -0,0 +1,57 @@ +# Docker images for building Hadoop + +This folder contains the Dockerfiles for building Hadoop on various platforms. + +# Dependency management + +The mode of installation of the dependencies needed for building Hadoop varies from one platform to +the other. Different platforms have different toolchains. Some packages tend to be polymorphic +across platforms and most commonly, a package that's readily available in one platform's toolchain +isn't available on another. We thus, resort to building and installing the package from source, +causing duplication of code since this needs to be done for all the Dockerfiles pertaining to all +the platforms. We need a system to track a dependency - for a package - for a platform. Thus, +there's a lot of diversity that needs to be handled for managing package dependencies and +`pkg-resolver` caters to that. + +## Supported platforms + +`pkg-resolver/platforms.json` contains a list of the supported platforms for dependency management. + +## Package dependencies + +`pkg-resolver/packages.json` maps a dependency to a given platform. Here's the schema of this JSON. + +```json +{ + "dependency_1": { + "platform_1": "package_1", + "platform_2": [ + "package_1", + "package_2" + ] + }, + "dependency_2": { + "platform_1": [ + "package_1", + "package_2", + "package_3" + ] + } +} +``` + +The root JSON element contains unique _dependency_ children. This in turn contains the name of the _ +platforms_ and the list of _packages_ to be installed for that platform. Just to give an example of +how to interpret the above JSON - + +1. For `dependency_1`, `package_1` needs to be installed for `platform_1`. +2. For `dependency_2`, `package_1` and `package_2` needs to be installed for `platform_2`. +3. For `dependency_2`, `package_1`, `package_3` and `package_3` needs to be installed for + `platform_1`. + +## Standalone packages + +Most commonly, some packages are not available across the toolchains in various platforms. Thus, we +would need to build and install them. Since we need to do this across all the Dockerfiles for all +the platforms, it could lead to code duplication and managing them becomes a hassle. Thus, we put +the build steps in a `pkg-resolver/install-.sh` and invoke this in all the Dockerfiles. \ No newline at end of file From 2169f81d954346a6843ecd900b7234116b4fc596 Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Tue, 1 Jun 2021 11:42:16 +0530 Subject: [PATCH 19/20] Trigger CI --- dev-support/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/README.md b/dev-support/docker/README.md index 4abc65fd01ac3..a22a7221bd91f 100644 --- a/dev-support/docker/README.md +++ b/dev-support/docker/README.md @@ -54,4 +54,4 @@ how to interpret the above JSON - Most commonly, some packages are not available across the toolchains in various platforms. Thus, we would need to build and install them. Since we need to do this across all the Dockerfiles for all the platforms, it could lead to code duplication and managing them becomes a hassle. Thus, we put -the build steps in a `pkg-resolver/install-.sh` and invoke this in all the Dockerfiles. \ No newline at end of file +the build steps in a `pkg-resolver/install-.sh` and invoke this in all the Dockerfiles. From 49e578d730d6857ceafd65710a31b64f2a906f27 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Wed, 2 Jun 2021 11:26:02 +0530 Subject: [PATCH 20/20] Exclude JSON from RAT check * RAT check fails on JSON files since ASF license header can't be added, since there's no comment in JSON file. Thus, we're excluding JSON files from RAT checks. --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 439ae42f293c9..d74cd64376e2e 100644 --- a/pom.xml +++ b/pom.xml @@ -454,6 +454,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x **/build/** **/patchprocess/** **/*.js + **/*.json licenses/** licenses-binary/**