From 7c59ff5f2cf65ff79a5efaa318ef17d74e0a30fe Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Thu, 9 May 2024 08:46:52 +0000 Subject: [PATCH] user libc images for server and worker --- Makefile | 4 +-- deepfence_server/Dockerfile | 17 +++++++++-- deepfence_server/Makefile | 2 +- deepfence_worker/Dockerfile | 38 ++++++++++++++++++++---- deepfence_worker/Makefile | 4 +-- docker_builders/Dockerfile-debian | 49 +++++++++++++++++++++++-------- 6 files changed, 89 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 93b24ce673..3996688140 100644 --- a/Makefile +++ b/Makefile @@ -103,11 +103,11 @@ file-server: docker build -t $(IMAGE_REPOSITORY)/deepfence_file_server_ce:$(DF_IMG_TAG) $(DEEPFENCE_FILE_SERVER_DIR) .PHONY: server -server: alpine_builder +server: debian_builder (cd ./deepfence_server && VERSION=$(VERSION) make image) .PHONY: worker -worker: alpine_builder agent-binary-tar +worker: debian_builder agent-binary-tar (cd ./deepfence_worker && VERSION=$(VERSION) AGENT_BINARY_DIST_RELATIVE=$(AGENT_BINARY_DIST_RELATIVE) make image) .PHONY: jaeger diff --git a/deepfence_server/Dockerfile b/deepfence_server/Dockerfile index db10b528bf..36097d5058 100644 --- a/deepfence_server/Dockerfile +++ b/deepfence_server/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.18 +FROM debian:12-slim MAINTAINER Deepfence Inc LABEL deepfence.role=system @@ -9,7 +9,20 @@ ADD deepfence_server/auth /auth ADD deepfence_server/cloud_controls /cloud_controls COPY deepfence_server/entrypoint.sh /entrypoint.sh -RUN apk add --no-cache --update bash curl libpcap tar kafkacat postgresql15-client +RUN apt update && \ + apt install -y \ + bash \ + curl \ + wget \ + netcat \ + libpcap-dev \ + tar \ + kafkacat \ + postgresql-client && \ + apt clean && \ + apt autoclean && \ + apt auto-remove -y && \ + rm -rf /var/lib/{apt,dpkg,cache,log}/ RUN chmod +x /entrypoint.sh diff --git a/deepfence_server/Makefile b/deepfence_server/Makefile index cefa1d07a2..11a298284f 100644 --- a/deepfence_server/Makefile +++ b/deepfence_server/Makefile @@ -7,7 +7,7 @@ all: deepfence_server local: deepfence_server image: - docker run --rm -i -e VERSION=${VERSION} -e GIT_COMMIT=${GIT_COMMIT} -e BUILD_TIME=${BUILD_TIME} -v $(ROOT_MAKEFILE_DIR):/src:rw -v /tmp/go:/go:rw $(IMAGE_REPOSITORY)/deepfence_builder_ce:$(DF_IMG_TAG) bash -c 'cd /src/deepfence_server && make deepfence_server' + docker run --rm -i -e VERSION=${VERSION} -e GIT_COMMIT=${GIT_COMMIT} -e BUILD_TIME=${BUILD_TIME} -v $(ROOT_MAKEFILE_DIR):/src:rw -v /tmp/go:/go:rw $(IMAGE_REPOSITORY)/deepfence_glibc_builder_ce:$(DF_IMG_TAG) bash -c 'cd /src/deepfence_server && make deepfence_server' docker build -f ./Dockerfile -t $(IMAGE_REPOSITORY)/deepfence_server_ce:$(DF_IMG_TAG) .. vendor: go.mod $(shell find ../deepfence_utils -path ../deepfence_utils/vendor -prune -o -name '*.go') diff --git a/deepfence_worker/Dockerfile b/deepfence_worker/Dockerfile index f44c226e02..b8be246f1d 100644 --- a/deepfence_worker/Dockerfile +++ b/deepfence_worker/Dockerfile @@ -4,9 +4,9 @@ ARG IMAGE_REPOSITORY=deepfenceio FROM $IMAGE_REPOSITORY/deepfence_package_scanner_ce:$DF_IMG_TAG AS packagescanner FROM $IMAGE_REPOSITORY/deepfence_secret_scanner_ce:$DF_IMG_TAG AS secretscanner FROM $IMAGE_REPOSITORY/deepfence_malware_scanner_ce:$DF_IMG_TAG AS yarahunter -FROM $IMAGE_REPOSITORY/deepfence_builder_ce:$DF_IMG_TAG AS builder-yara +FROM $IMAGE_REPOSITORY/deepfence_glibc_builder_ce:$DF_IMG_TAG AS builder-yara -FROM alpine:3.18 AS final +FROM debian:12-slim AS final ARG AGENT_BINARY_DIST_RELATIVE @@ -15,9 +15,35 @@ LABEL deepfence.role=system ADD deepfence_utils/postgresql/migrate /usr/local/postgresql-migrate -RUN apk add --no-cache curl kafkacat docker-cli openrc bash skopeo jansson-dev \ - libmagic libstdc++ libx11 libxrender libxext libssl1.1 ca-certificates \ - fontconfig freetype ttf-droid ttf-freefont ttf-liberation postgresql15-client +RUN apt-get update && apt install -y curl && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \ + chmod a+r /etc/apt/keyrings/docker.asc && \ + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null && \ + apt-get update && \ + apt-get install docker-ce-cli -y + +RUN apt install -y \ + netcat \ + kafkacat \ + bash \ + skopeo \ + libjansson-dev \ + libmagic-dev \ + libstdc++6 \ + libssl3 \ + ca-certificates \ + postgresql-client \ + libvectorscan5 \ + gdb \ + strace && \ + apt clean && \ + apt autoclean && \ + apt auto-remove -y && \ + rm -rf /var/lib/{apt,dpkg,cache,log}/ RUN curl -fsSL https://raw.githubusercontent.com/pressly/goose/master/install.sh | sh @@ -62,4 +88,4 @@ COPY ./${AGENT_BINARY_DIST_RELATIVE}/* /opt/deepfence COPY --from=builder-yara /go/bin/asynq /usr/local/bin/asynq ENTRYPOINT ["/entrypoint.sh"] -CMD ["/usr/local/bin/deepfence_worker"] +CMD ["/usr/local/bin/deepfence_worker"] \ No newline at end of file diff --git a/deepfence_worker/Makefile b/deepfence_worker/Makefile index 3b813a34d5..cf4d37b1a1 100644 --- a/deepfence_worker/Makefile +++ b/deepfence_worker/Makefile @@ -7,7 +7,7 @@ all: deepfence_worker local: deepfence_worker image: - docker run --rm -i -e VERSION=${VERSION} -e GIT_COMMIT=${GIT_COMMIT} -e BUILD_TIME=${BUILD_TIME} -v $(ROOT_MAKEFILE_DIR):/src:rw -v /tmp/go:/go:rw $(IMAGE_REPOSITORY)/deepfence_builder_ce:$(DF_IMG_TAG) bash -c 'cd /src/deepfence_worker && make deepfence_worker' + docker run --rm -i -e VERSION=${VERSION} -e GIT_COMMIT=${GIT_COMMIT} -e BUILD_TIME=${BUILD_TIME} -v $(ROOT_MAKEFILE_DIR):/src:rw -v /tmp/go:/go:rw $(IMAGE_REPOSITORY)/deepfence_glibc_builder_ce:$(DF_IMG_TAG) bash -c 'cd /src/deepfence_worker && make deepfence_worker' docker build -f ./Dockerfile --build-arg IMAGE_REPOSITORY=$(IMAGE_REPOSITORY) --build-arg DF_IMG_TAG=$(DF_IMG_TAG) --build-arg AGENT_BINARY_DIST_RELATIVE=$(AGENT_BINARY_DIST_RELATIVE) -t $(IMAGE_REPOSITORY)/deepfence_worker_ce:$(DF_IMG_TAG) .. vendor: go.mod $(shell find ../deepfence_utils -path ../deepfence_utils/vendor -prune -o -name '*.go') @@ -15,7 +15,7 @@ vendor: go.mod $(shell find ../deepfence_utils -path ../deepfence_utils/vendor - go mod vendor deepfence_worker: vendor $(shell find . -path ./vendor -prune -o -name '*.go') - CGO_LDFLAGS="-ljansson -lcrypto -lmagic" go build -buildvcs=false -ldflags="-s -w -X github.com/deepfence/ThreatMapper/deepfence_worker/utils.Version=${VERSION} -X github.com/deepfence/ThreatMapper/deepfence_worker/utils.Commit=${GIT_COMMIT} -X github.com/deepfence/ThreatMapper/deepfence_worker/utils.BuildTime=${BUILD_TIME} -extldflags='-static'" + CGO_LDFLAGS="-ljansson -lcrypto -lmagic" go build -buildvcs=false -ldflags="-s -w -X github.com/deepfence/ThreatMapper/deepfence_worker/utils.Version=${VERSION} -X github.com/deepfence/ThreatMapper/deepfence_worker/utils.Commit=${GIT_COMMIT} -X github.com/deepfence/ThreatMapper/deepfence_worker/utils.BuildTime=${BUILD_TIME}" clean: -rm deepfence_worker diff --git a/docker_builders/Dockerfile-debian b/docker_builders/Dockerfile-debian index 66c5bf426e..07c16024ca 100644 --- a/docker_builders/Dockerfile-debian +++ b/docker_builders/Dockerfile-debian @@ -1,7 +1,3 @@ -ARG VECTORSCAN_IMG_TAG=latest -ARG VECTORSCAN_IMAGE_REPOSITORY=deepfenceio -FROM $VECTORSCAN_IMAGE_REPOSITORY/deepfence_vectorscan_build:$VECTORSCAN_IMG_TAG AS vectorscan - ARG DF_IMG_TAG=latest ARG IMAGE_REPOSITORY=deepfenceio @@ -11,11 +7,37 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update RUN apt-get install -y RUN apt-get -qq -y --no-install-recommends install \ - build-essential automake libtool make gcc pkg-config libssl-dev git protoc-gen-go \ - bash make git gcc libc-dev lsb-release software-properties-common libz-dev apt-utils\ - protobuf-compiler ca-certificates libpcap-dev time file shellcheck curl \ - libjansson-dev libmagic-dev \ - cmake flex bison libyaml-dev + build-essential \ + automake \ + libtool \ + make \ + gcc \ + pkg-config \ + libssl-dev \ + git \ + protoc-gen-go \ + bash \ + make \ + gcc \ + libc-dev \ + lsb-release \ + software-properties-common \ + libz-dev \ + apt-utils \ + protobuf-compiler \ + ca-certificates \ + libpcap-dev \ + time \ + file \ + shellcheck \ + curl \ + libjansson-dev \ + libmagic-dev \ + cmake \ + flex \ + bison \ + libyaml-dev \ + libvectorscan-dev RUN cd /root \ && wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.3.2.tar.gz \ @@ -28,12 +50,13 @@ RUN cd /root \ && cd /usr/local/ \ && tar -czf yara.tar.gz yara -COPY --from=vectorscan /vectorscan.tar.bz2 / -RUN tar -xjf /vectorscan.tar.bz2 -C / && rm /vectorscan.tar.bz2 - RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.32.0 RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/include/hs/lib:$LD_LIBRARY_PATH \ + PKG_CONFIG_PATH=/usr/local/yara/lib/pkgconfig:$(PKG_CONFIG_PATH) \ + GOWORK=off + RUN mkdir /home/deepfence COPY deepfence_agent/build_scripts/*.sh /home/deepfence/ @@ -41,3 +64,5 @@ ARG DF_AGENT_SRC=/go/src/github.com/deepfence/deepfence_agent WORKDIR $DF_AGENT_SRC ENV GOWORK=off + +RUN go install github.com/hibiken/asynq/tools/asynq@latest