Add build/debug scripts of native worker docker image

Dockerfile-native

@@ -0,0 +1,52 @@
+ARG PRESTO_VERSION
+
+FROM prestocpp/prestocpp-avx-centos:root-20230613 as Builder
+
+WORKDIR  /app
+COPY . .
+RUN cd presto-native-execution && \
+    make velox-submodule && \
+    source /opt/rh/gcc-toolset-9/enable && \
+    source velox/scripts/setup-helper-functions.sh && \
+    (mkdir -p third_party && cd third_party && github_checkout aws/aws-sdk-cpp 1.9.96 --depth 1 --recurse-submodules --config advice.detachedHead=false && \
+    cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=OFF -DMINIMIZE_SIZE:BOOL=ON -DENABLE_TESTING:BOOL=OFF -DBUILD_ONLY:STRING="s3;identity-management") && \
+    EXTRA_CMAKE_FLAGS=" -DVELOX_ENABLE_INT64_BUILD_PARTITION_BOUND=ON" PRESTO_ENABLE_PARQUET=ON PRESTO_ENABLE_S3=ON PRESTO_ENABLE_TESTING=OFF MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8 NUM_THREADS=8 TREAT_WARNINGS_AS_ERRORS=0 make release \
+    || tail -n 500 _build/release/CMakeFiles/CMakeError.log
+
+RUN cd presto-native-execution && \
+    mkdir -p prestissimo && \
+    cp _build/release/presto_cpp/main/presto_server prestissimo && \
+    tar cvf prestissimo.tar entrypoint.sh velox.properties prestissimo
+
+##
+
+FROM prestocpp/prestocpp-avx-centos:root-20230613
+ENV PRESTO_HOME="/opt/presto-server"
+
+RUN dnf update -y && dnf install -y \
+        awscli \
+        gperf \
+        iproute \
+        lsof \
+        procps \
+        python3 \
+        sysstat \
+        tar \
+        vim \
+        wget \
+        which \
+        && \
+    mkdir -p $PRESTO_HOME/etc/catalog && \
+    mkdir -p /var/lib/presto/data
+
+WORKDIR  /app
+COPY --from=Builder /app/presto-native-execution/prestissimo.tar .
+RUN tar xvf prestissimo.tar && \
+    mkdir -p /opt/presto-server/etc && \
+    mv prestissimo/presto_server /usr/local/bin/ && \
+    mv velox.properties /opt/presto-server/etc/ && \
+    mv entrypoint.sh /opt/ && \
+    touch /opt/presto-native-execution-${PRESTO_VERSION}
+
+ENTRYPOINT ["/opt/entrypoint.sh"]
+

Dockerfile-native-debug

@@ -0,0 +1,52 @@
+ARG PRESTO_VERSION
+
+FROM prestocpp/prestocpp-avx-centos:root-20230613 as Builder
+
+WORKDIR  /app
+COPY . .
+RUN cd presto-native-execution && \
+    make velox-submodule && \
+    source /opt/rh/gcc-toolset-9/enable && \
+    source velox/scripts/setup-helper-functions.sh && \
+    (mkdir -p third_party && cd third_party && github_checkout aws/aws-sdk-cpp 1.9.96 --depth 1 --recurse-submodules --config advice.detachedHead=false && \
+    cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=OFF -DMINIMIZE_SIZE:BOOL=ON -DENABLE_TESTING:BOOL=OFF -DBUILD_ONLY:STRING="s3;identity-management") && \
+    EXTRA_CMAKE_FLAGS=" -DVELOX_ENABLE_INT64_BUILD_PARTITION_BOUND=ON" PRESTO_ENABLE_PARQUET=ON PRESTO_ENABLE_S3=ON PRESTO_ENABLE_TESTING=OFF MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=4 NUM_THREADS=2 TREAT_WARNINGS_AS_ERRORS=0 make debug \
+    || tail -n 500 _build/debug/CMakeFiles/CMakeError.log
+
+RUN cd presto-native-execution && \
+    mkdir -p prestissimo && \
+    cp _build/debug/presto_cpp/main/presto_server prestissimo && \
+    tar cvf prestissimo.tar entrypoint.sh velox.properties prestissimo
+
+##
+
+FROM prestocpp/prestocpp-avx-centos:root-20230613
+ENV PRESTO_HOME="/opt/presto-server"
+
+RUN dnf update -y && dnf install -y \
+        awscli \
+        gperf \
+        iproute \
+        lsof \
+        procps \
+        python3 \
+        sysstat \
+        tar \
+        vim \
+        wget \
+        which \
+        && \
+    mkdir -p $PRESTO_HOME/etc/catalog && \
+    mkdir -p /var/lib/presto/data
+
+WORKDIR  /app
+COPY --from=Builder /app/presto-native-execution/prestissimo.tar .
+RUN tar xvf prestissimo.tar && \
+    mkdir -p /opt/presto-server/etc && \
+    mv prestissimo/presto_server /usr/local/bin/ && \
+    mv velox.properties /opt/presto-server/etc/ && \
+    mv entrypoint.sh /opt/ && \
+    touch /opt/presto-native-execution-${PRESTO_VERSION}
+
+ENTRYPOINT ["/opt/entrypoint.sh"]
+

Jenkinsfile

@@ -93,6 +93,7 @@ pipeline {
                                 sh(script: "git show -s --format=%cd --date=format:'%Y%m%d%H%M%S'", returnStdout: true).trim() + "-" +
                                 env.PRESTO_COMMIT_SHA.substring(0, 7)
                             env.DOCKER_IMAGE = env.AWS_ECR + "/presto:${PRESTO_BUILD_VERSION}"
+                            env.DOCKER_NATIVE_IMAGE = env.AWS_ECR + "/presto-native:${PRESTO_BUILD_VERSION}"
                         }
                         sh 'printenv | sort'
 
@@ -168,6 +169,22 @@ pipeline {
                     }
                 }
 
+                stage('Docker Native Build') {
+                    steps {
+                        echo "Building ${DOCKER_NATIVE_IMAGE}"
+                        withCredentials([[
+                                $class:            'AmazonWebServicesCredentialsBinding',
+                                credentialsId:     "${AWS_CREDENTIAL_ID}",
+                                accessKeyVariable: 'AWS_ACCESS_KEY_ID',
+                                secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
+                            sh '''#!/bin/bash -ex
+                                docker buildx build -f Dockerfile-native --load --platform "linux/amd64" -t "${DOCKER_NATIVE_IMAGE}-amd64" \
+                                    --build-arg "PRESTO_VERSION=${PRESTO_VERSION}" .
+                            '''
+                        }
+                    }
+                }
+
                 stage('Publish Docker') {
                     when {
                         anyOf {
@@ -194,6 +211,7 @@ pipeline {
                                 docker manifest annotate "${DOCKER_IMAGE}" "${DOCKER_IMAGE}-amd64" --os linux --arch amd64
                                 docker manifest annotate "${DOCKER_IMAGE}" "${DOCKER_IMAGE}-arm64" --os linux --arch arm64
                                 docker manifest push "${DOCKER_IMAGE}"
+                                docker push "${DOCKER_NATIVE_IMAGE}-amd64"
                             '''
                         }
                     }
@@ -202,3 +220,4 @@ pipeline {
         }
     }
 }
+

jenkins/agent-dind.yaml

@@ -1,23 +1,23 @@
 apiVersion: v1
 kind: Pod
 metadata:
-    namespace: oss-agent
-    labels:
-        containers: dind
+  namespace: oss-agent
+  labels:
+    containers: dind
 spec:
-    nodeSelector:
-      eks.amazonaws.com/nodegroup: eks-oss-presto-dynamic-managed-ng
-    serviceAccountName: oss-agent
-    containers:
-    - name: dind
-      image: docker:20.10.16-dind-alpine3.15
-      securityContext:
-        privileged: true
-      tty: true
-      resources:
-        requests:
-          memory: "4Gi"
-          cpu: "2000m"
-        limits:
-          memory: "4Gi"
-          cpu: "2000m"
+  nodeSelector:
+    eks.amazonaws.com/nodegroup: eks-oss-presto-dynamic-managed-ng
+  serviceAccountName: oss-agent
+  containers:
+  - name: dind
+    image: docker:20.10.16-dind-alpine3.15
+    securityContext:
+      privileged: true
+    tty: true
+    resources:
+      requests:
+        memory: "24Gi"
+        cpu: "7000m"
+      limits:
+        memory: "24Gi"
+        cpu: "7000m"

jenkins/agent-maven.yaml

@@ -1,26 +1,27 @@
 apiVersion: v1
 kind: Pod
 metadata:
-    namespace: oss-agent
-    labels:
-        containers: maven
+  namespace: oss-agent
+  labels:
+    containers: maven
 spec:
-    nodeSelector:
-      eks.amazonaws.com/nodegroup: eks-oss-presto-dynamic-managed-ng
-    serviceAccountName: oss-agent
-    containers:
-    - name: maven
-      image: maven:3.8.6-openjdk-8-slim
-      env:
-      - name: MAVEN_OPTS
-        value: "-Xmx8000m -Xms8000m"
-      resources:
-        requests:
-          memory: "10Gi"
-          cpu: "4000m"
-        limits:
-          memory: "10Gi"
-          cpu: "4000m"
-      tty: true
-      command:
-      - cat
+  nodeSelector:
+    eks.amazonaws.com/nodegroup: eks-oss-presto-dynamic-managed-ng
+  serviceAccountName: oss-agent
+  containers:
+  - name: maven
+    image: maven:3.8.6-openjdk-8-slim
+    env:
+    - name: MAVEN_OPTS
+      value: "-Xmx8000m -Xms8000m"
+    resources:
+      requests:
+        memory: "10Gi"
+        cpu: "4000m"
+      limits:
+        memory: "10Gi"
+        cpu: "4000m"
+    tty: true
+    command:
+    - cat
+

presto-native-execution/entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "node.id=$HOSTNAME" >> /opt/presto-server/etc/node.properties
+
+# Check for the reason of setting split_preload_per_driver:
+# https://github.com/prestodb/presto/issues/20020#issuecomment-1785083459
+GLOG_logtostderr=1 presto_server \
+    --etc-dir=/opt/presto-server/etc \
+    2>&1 | tee /var/log/presto-server/console.log

presto-native-execution/velox.properties

@@ -0,0 +1,3 @@
+mutable-config=true
+expression.eval_simplified=false
+max_split_preload_per_driver=0