Merge pull request #1727 from deepfence/tomasz/arm64

[arm64] architecture aware agent build

deepfence_agent/Dockerfile

@@ -1,9 +1,13 @@
 ARG DF_IMG_TAG=latest
 ARG IMAGE_REPOSITORY=deepfenceio
+ARG VECTORSCAN_IMG_TAG=latest
+ARG VECTORSCAN_IMAGE_REPOSITORY=deepfenceio
+
 FROM $IMAGE_REPOSITORY/deepfence_secret_scanner_ce:$DF_IMG_TAG AS secret_build
 FROM $IMAGE_REPOSITORY/deepfence_package_scanner_ce:$DF_IMG_TAG AS package_build
 FROM $IMAGE_REPOSITORY/deepfence_malware_scanner_ce:$DF_IMG_TAG AS malware_build
 FROM $IMAGE_REPOSITORY/deepfence_compliance_scanner_ce:$DF_IMG_TAG AS compliance_build
+FROM $VECTORSCAN_IMAGE_REPOSITORY/deepfence_vectorscan_build:$VECTORSCAN_IMG_TAG AS vectorscan
 
 FROM debian:bullseye-slim
 
@@ -25,16 +29,33 @@ RUN export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" \
     && echo "Installing some basic stuff"
 RUN apt-get update && apt-get install -y --no-install-recommends libpcap0.8 gettext ca-certificates supervisor logrotate util-linux dnsutils net-tools cgroup-tools libcgroup1 libcap2 libaudit1 conntrack runit auditd apparmor gzip lsof file curl zip at gnupg  unzip procps cron sudo bzip2 libssl1.1 libevent-2.1-7 libevent-openssl-2.1-7 libevent-pthreads-2.1-7 libnet1 gnupg2 libfile-mimeinfo-perl libjansson4 libmagic1 wget bash python3-pip
 
-RUN apt-get -y --allow-unauthenticated install skopeo podman \
-    && echo "Installing docker" \
-    && curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKERVERSION}.tgz \
-    && tar xzvf docker-${DOCKERVERSION}.tgz --strip 1 -C /usr/local/bin docker/docker \
-    && rm docker-${DOCKERVERSION}.tgz \
-    && mkdir -p /etc/license/ /usr/local/bin /usr/local/lib \
-        /deepfenced /var/tmp/layers /usr/local/lua-waf /var/log/nginx/ \
-    && chown root:root /deepfenced && chmod 0744 /deepfenced \
-    && mkdir /usr/local/bin/compliance_check && mkdir /usr/local/discovery \
-    && apt update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt install libhyperscan5
+ARG TARGETARCH
+
+RUN <<EOF
+set -eux
+
+apt-get -y --allow-unauthenticated install skopeo podman
+if [ "$TARGETARCH" = "arm64" ]; then
+    ARCHITECTURE="aarch64"
+elif [ "$TARGETARCH" = "amd64" ]; then
+    ARCHITECTURE="x86_64"
+else
+    echo "Unsupported architecture $TARGETARCH" && exit 1;
+fi
+
+curl -fsSLO https://download.docker.com/linux/static/stable/${ARCHITECTURE}/docker-${DOCKERVERSION}.tgz
+tar xzvf docker-${DOCKERVERSION}.tgz --strip 1 -C /usr/local/bin docker/docker
+rm docker-${DOCKERVERSION}.tgz
+
+mkdir -p /etc/license/ /usr/local/bin /usr/local/lib \
+    /deepfenced /var/tmp/layers /usr/local/lua-waf /var/log/nginx/
+chown root:root /deepfenced && chmod 0744 /deepfenced
+mkdir /usr/local/bin/compliance_check && mkdir /usr/local/discovery
+
+EOF
+
+COPY --from=vectorscan /vectorscan.tar.bz2 /
+RUN tar -xjf /vectorscan.tar.bz2 -C / && rm /vectorscan.tar.bz2
 
 RUN mkdir -p /etc/td-agent-bit/
 COPY tools/apache/fluentbit/* /etc/td-agent-bit/
@@ -74,18 +95,60 @@ RUN apt-get -qq -y --no-install-recommends install libjansson4 libssl1.1 libmagi
     && cd /tmp \
     && chmod +x /usr/local/bin/start_agent
 RUN apt-get clean && apt-get -y autoremove && rm -rf /var/lib/apt/lists/*
-RUN vessel_version="0.12.0" \
-    && curl -fsSLOk https://github.com/deepfence/vessel/releases/download/v${vessel_version}/vessel_v${vessel_version}_linux_amd64.tar.gz \
-    && tar -xzf vessel_v${vessel_version}_linux_amd64.tar.gz \
-    && mv vessel /usr/local/bin/ \
-    && rm -rf vessel_v${vessel_version}_linux_amd64.tar.gz
-RUN nerdctl_version="1.6.0" \
-    && curl -fsSLOk https://github.com/containerd/nerdctl/releases/download/v${nerdctl_version}/nerdctl-${nerdctl_version}-linux-amd64.tar.gz \
-    && tar Cxzvvf /usr/local/bin nerdctl-${nerdctl_version}-linux-amd64.tar.gz \
-    && rm nerdctl-${nerdctl_version}-linux-amd64.tar.gz
-RUN crictl_version="v1.28.0" \
-    && curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/${crictl_version}/crictl-${crictl_version}-linux-amd64.tar.gz --output crictl-${crictl_version}-linux-amd64.tar.gz \
-    && tar zxvf crictl-${crictl_version}-linux-amd64.tar.gz -C /usr/local/bin \
-    && rm -f crictl-${crictl_version}-linux-amd64.tar.gz
+
+RUN <<EOF
+set -eux
+
+vessel_version="0.12.1"
+if [ "$TARGETARCH" = "arm64" ]; then
+    ARCHITECTURE="arm64"
+elif [ "$TARGETARCH" = "amd64" ]; then
+    ARCHITECTURE="amd64"
+else
+    echo "Unsupported architecture $TARGETARCH" && exit 1
+fi
+
+curl -fsSLO https://github.com/deepfence/vessel/releases/download/v${vessel_version}/vessel_v${vessel_version}_linux_${ARCHITECTURE}.tar.gz
+tar -xzf vessel_v${vessel_version}_linux_${ARCHITECTURE}.tar.gz
+mv vessel /usr/local/bin/
+rm -rf vessel_v${vessel_version}_linux_${ARCHITECTURE}.tar.gz
+
+EOF
+
+RUN <<EOF
+set -eux
+
+nerdctl_version="1.6.0"
+if [ "$TARGETARCH" = "arm64" ]; then
+    ARCHITECTURE="arm64"
+elif [ "$TARGETARCH" = "amd64" ]; then
+    ARCHITECTURE="amd64"
+else
+    echo "Unsupported architecture $TARGETARCH" && exit 1
+fi
+
+curl -fsSLO https://github.com/containerd/nerdctl/releases/download/v${nerdctl_version}/nerdctl-${nerdctl_version}-linux-${ARCHITECTURE}.tar.gz
+tar Cxzvvf /usr/local/bin nerdctl-${nerdctl_version}-linux-${ARCHITECTURE}.tar.gz
+rm nerdctl-${nerdctl_version}-linux-${ARCHITECTURE}.tar.gz
+
+EOF
+
+RUN <<EOF
+set -eux
+
+crictl_version="v1.28.0"
+if [ "$TARGETARCH" = "arm64" ]; then
+    ARCHITECTURE="arm64"
+elif [ "$TARGETARCH" = "amd64" ]; then
+    ARCHITECTURE="amd64"
+else
+    echo "Unsupported architecture $TARGETARCH" && exit 1
+fi
+
+curl -fsSLO https://github.com/kubernetes-sigs/cri-tools/releases/download/${crictl_version}/crictl-${crictl_version}-linux-${ARCHITECTURE}.tar.gz
+tar zxvf crictl-${crictl_version}-linux-${ARCHITECTURE}.tar.gz -C /usr/local/bin
+rm -f crictl-${crictl_version}-linux-${ARCHITECTURE}.tar.gz
+
+EOF
 
 ENTRYPOINT ["/usr/local/bin/start_agent"]

deepfence_worker/Dockerfile

@@ -16,7 +16,6 @@ ADD deepfence_utils/postgresql/migrate /usr/local/postgresql-migrate
 RUN apk add --no-cache curl kafkacat docker-cli openrc bash skopeo jansson-dev \
         libmagic libstdc++ libx11 libxrender libxext libssl1.1 ca-certificates \
         fontconfig freetype ttf-droid ttf-freefont ttf-liberation postgresql15-client
-RUN apk add hyperscan --repository=https://dl-cdn.alpinelinux.org/alpine/v3.13/community
 
 RUN curl -fsSL https://raw.githubusercontent.com/pressly/goose/master/install.sh | sh
 

deepfence_worker/Makefile

@@ -15,7 +15,7 @@ vendor: go.mod $(shell find ../deepfence_utils -path ../deepfence_utils/vendor -
 	go mod vendor
 
 deepfence_worker: vendor $(shell find . -path ./vendor -prune -o -name '*.go')
-	go build -buildvcs=false -ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}"
+	go build -buildvcs=false -buildmode=pie -ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}"
 
 clean:
 	-rm deepfence_worker

docker_builders/Dockerfile-alpine

@@ -1,5 +1,8 @@
-FROM golang:1.21-alpine3.18
+ARG VECTORSCAN_IMG_TAG=latest
+ARG VECTORSCAN_IMAGE_REPOSITORY=deepfenceio
+FROM $VECTORSCAN_IMAGE_REPOSITORY/deepfence_vectorscan_build:$VECTORSCAN_IMG_TAG AS vectorscan
 
+FROM golang:1.21-alpine3.18
 
 RUN apk add --no-cache \
     git \
@@ -47,7 +50,9 @@ RUN apk update && apk add --no-cache --upgrade \
     curl tar libstdc++ libgcc python3 \
     py3-pip bash gcc musl-dev pkgconfig \
     g++ git protoc jansson-dev libc-dev openssl-dev
-RUN apk add hyperscan-dev --repository=https://dl-cdn.alpinelinux.org/alpine/v3.13/community
+
+COPY --from=vectorscan /vectorscan.tar.bz2 /
+RUN tar -xjf /vectorscan.tar.bz2 -C / && rm /vectorscan.tar.bz2
 
 ENV PKG_CONFIG_PATH=/usr/local/include/hs/:$(PKG_CONFIG_PATH) \
     CGO_CFLAGS="-I/usr/local/include/hyperscan/src" \

docker_builders/Dockerfile-debian

@@ -1,3 +1,7 @@
+ARG VECTORSCAN_IMG_TAG=latest
+ARG VECTORSCAN_IMAGE_REPOSITORY=deepfenceio
+FROM $VECTORSCAN_IMAGE_REPOSITORY/deepfence_vectorscan_build:$VECTORSCAN_IMG_TAG AS vectorscan
+
 ARG DF_IMG_TAG=latest
 ARG IMAGE_REPOSITORY=deepfenceio
 
@@ -9,7 +13,7 @@ RUN apt-get install -y
 RUN apt-get -qq -y --no-install-recommends install \
     build-essential automake libtool make gcc pkg-config libssl-dev git protoc-gen-go \
     bash make git gcc libc-dev lsb-release software-properties-common libz-dev apt-utils\
-    protobuf-compiler libhyperscan-dev ca-certificates libpcap-dev time file shellcheck curl \
+    protobuf-compiler ca-certificates libpcap-dev time file shellcheck curl \
     libjansson-dev libmagic-dev \
     cmake flex bison libyaml-dev
 
@@ -24,6 +28,9 @@ RUN cd /root  \
     && cd /usr/local/ \
     && tar -czf yara.tar.gz yara
 
+COPY --from=vectorscan /vectorscan.tar.bz2 /
+RUN tar -xjf /vectorscan.tar.bz2 -C / && rm /vectorscan.tar.bz2
+
 RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30.0
 RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0