Make all plugins statically built

.gitmodules

@@ -26,3 +26,7 @@
 	path = deepfence_agent/plugins/yara-rules
 	url = https://github.com/deepfence/yara-rules
 	branch = main
+[submodule "deepfence_agent/plugins/fluent-bit"]
+	path = deepfence_agent/plugins/fluent-bit
+	url = https://github.com/deepfence/fluent-bit
+    branch = master

Makefile

@@ -33,8 +33,8 @@ alpine_builder:
 	docker build --tag=$(IMAGE_REPOSITORY)/deepfence_builder_ce:$(DF_IMG_TAG) -f docker_builders/Dockerfile-alpine .
 
 .PHONY: debian_builder
-debian_builder: alpine_builder
-	docker build --build-arg DF_IMG_TAG=${DF_IMG_TAG} --build-arg IMAGE_REPOSITORY=${IMAGE_REPOSITORY} --tag=$(IMAGE_REPOSITORY)/deepfence_agent_builder_ce:$(DF_IMG_TAG) -f docker_builders/Dockerfile-debian .
+debian_builder:
+	docker build --build-arg DF_IMG_TAG=${DF_IMG_TAG} --build-arg IMAGE_REPOSITORY=${IMAGE_REPOSITORY} --tag=$(IMAGE_REPOSITORY)/deepfence_glibc_builder_ce:$(DF_IMG_TAG) -f docker_builders/Dockerfile-debian .
 
 .PHONY: bootstrap-agent-plugins
 bootstrap-agent-plugins:
@@ -89,7 +89,7 @@ graphdb:
 ui:
 	git log --format="%h" -n 1 > $(DEEPFENCE_FRONTEND_DIR)/console_version.txt && \
 	echo $(VERSION) > $(DEEPFENCE_FRONTEND_DIR)/product_version.txt && \
-	docker run -i --rm --entrypoint=bash -v $(DEEPFENCE_FRONTEND_DIR):/app node:18-bullseye-slim -c "cd /app && corepack enable && corepack prepare pnpm@7.17.1 --activate && PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=true pnpm install --frozen-lockfile --prefer-offline && pnpm run build" && \
+	docker run --rm --entrypoint=bash -v $(DEEPFENCE_FRONTEND_DIR):/app node:18-bullseye-slim -c "cd /app && corepack enable && corepack prepare pnpm@7.17.1 --activate && PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=true pnpm install --frozen-lockfile --prefer-offline && pnpm run build" && \
 	docker build -f $(DEEPFENCE_FRONTEND_DIR)/Dockerfile -t $(IMAGE_REPOSITORY)/deepfence_ui_ce:$(DF_IMG_TAG) $(DEEPFENCE_FRONTEND_DIR) && \
 	rm -rf $(DEEPFENCE_FRONTEND_DIR)/console_version.txt $(DEEPFENCE_FRONTEND_DIR)/product_version.txt
 
@@ -111,7 +111,7 @@ compliancescanner:
 
 .PHONY: openapi
 openapi: server
-	docker run --rm -i \
+	docker run --rm \
 	--entrypoint=/usr/local/bin/deepfence_server \
 	-v $(PWD):/app $(IMAGE_REPOSITORY)/deepfence_server_ce:$(DF_IMG_TAG) \
 	--export-api-docs-path /app/openapi.yaml
@@ -198,9 +198,7 @@ publish-graphdb:
 
 .PHONY: clean
 clean:
-	(cd $(DEEPFENCE_AGENT_DIR)/plugins && make clean)
-	(cd $(DEEPFENCE_AGENT_DIR) && make clean)
-	(cd $(ROOT_MAKEFILE_DIR)/deepfence_server && make clean)
-	(cd $(ROOT_MAKEFILE_DIR)/deepfence_worker && make clean)
-	(cd $(DEEPFENCED) && make clean && rm $(DEEPFENCE_AGENT_DIR)/deepfenced)
-	(cd $(DEEPFENCE_AGENT_DIR)/tools/apache/fluentbit/out_deepfence && make clean)
+	-(cd $(DEEPFENCE_AGENT_DIR) && make clean)
+	-(cd $(ROOT_MAKEFILE_DIR)/deepfence_server && make clean)
+	-(cd $(ROOT_MAKEFILE_DIR)/deepfence_worker && make clean)
+	-(cd $(DEEPFENCED) && make clean && rm $(DEEPFENCE_AGENT_DIR)/deepfenced)

deepfence_agent/Dockerfile

@@ -37,16 +37,11 @@ RUN apt-get -y --allow-unauthenticated install skopeo podman \
     && mkdir /usr/local/bin/compliance_check && mkdir /usr/local/discovery \
     && apt update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt install libhyperscan5
 
-# Installing FluentBit
-RUN wget -qO - https://packages.fluentbit.io/fluentbit.key | apt-key add - \
-    && echo "deb https://packages.fluentbit.io/debian/buster buster main " >> /etc/apt/sources.list
-RUN apt-get update --allow-insecure-repositories
-RUN apt-get install -y td-agent-bit=1.6.10 && mkdir -p /etc/td-agent-bit/
-
+RUN mkdir -p /etc/td-agent-bit/
 COPY tools/apache/fluentbit/* /etc/td-agent-bit/
 
 # Copy fluentbit plugin
-COPY tools/apache/fluentbit/out_deepfence/out_deepfence.so /opt/td-agent-bit/bin/
+COPY plugins/fluent-bit/build/bin/fluent-bit /opt/td-agent-bit/bin/
 COPY tools/apache/deepfence/df-utils/get_cloud_instance_id/getCloudInstanceId /usr/local/bin/getCloudInstanceId
 COPY etc/fenced_logrotate.conf /etc/logrotate.d/fenced_logrotate.conf
 COPY etc/certs/* /etc/filebeat/

deepfence_agent/Makefile

@@ -32,14 +32,16 @@ gocode:
 
 install:
 
-agent:
+tools/apache/scope/vendor: tools/apache/scope/go.mod $(shell find ../deepfence_utils -name '*.go')
+	($(CD) tools/apache/scope && go mod tidy -v)
+	($(CD) tools/apache/scope && go mod vendor)
+
+agent: tools/apache/scope/vendor
 	($(CD) tools/apache/scope && \
-		go mod tidy -v && \
-		go mod vendor && \
 		env GOGC=off \
 		CGO_ENABLED=1 \
 		go build -buildvcs=false \
-		-ldflags "-X main.version=$GIT_COMMIT -X github.com/weaveworks/scope/probe/host.agentCommitID=$GIT_COMMIT -X github.com/weaveworks/scope/probe/host.agentBuildTime=$(shell date +"%s%d%m%y") -s -w"\
+		-ldflags "-X main.version=$GIT_COMMIT -X main.version=$(shell git rev-parse --short HEAD)  -X github.com/weaveworks/scope/probe/host.agentCommitID=$(shell git rev-parse --short HEAD) -X github.com/weaveworks/scope/probe/host.agentBuildTime=$(shell date +"%s%d%m%y") -s -w -extldflags=-static"\
 		-tags 'netgo osusergo unsafe' \
 		-o docker/deepfence_exe \
 		./prog)

deepfence_agent/build.sh

@@ -7,16 +7,52 @@ GIT_COMMIT=$(git rev-parse --short HEAD)
 building_image(){
 
     echo "Building GetCloudInstanceId"
-    docker run --rm -i -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_agent_builder_ce:$DF_IMG_TAG bash -x /home/deepfence/gocode-build.sh
+    docker run --rm --workdir /go/src/github.com/deepfence/deepfence_agent -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_builder_ce:$DF_IMG_TAG bash -x /home/deepfence/gocode-build.sh
     build_result=$?
     if [ $build_result -ne 0 ]
     then
         echo "Deepfence code compilation failed, bailing out"
         exit 1
     fi
 
+    echo "Prepare Fluentbit"
+    docker run --rm --workdir /go/src/github.com/deepfence/deepfence_agent  -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_glibc_builder_ce:$DF_IMG_TAG bash -c "\
+        cd plugins/fluent-bit/build && \
+        cmake \
+        -DFLB_WASM=No \
+        -DFLB_LUAJIT=No \
+        -DFLB_DEBUG=No \
+        -DFLB_RELEASE=Yes \
+        -DFLB_SHARED_LIB=No \
+        -DCMAKE_FIND_LIBRARY_SUFFIXES='.a' \
+        -DFLB_CONFIG_YAML=Off \
+        -DBUILD_SHARED_LIBS=OFF \
+        -DCMAKE_EXE_LINKER_FLAGS='-static' \
+        -DOPENSSL_USE_STATIC_LIBS=Yes \
+        -DZLIB_USE_STATIC_LIBS=Yes \
+        -DCMAKE_C_FLAGS='-fcommon' \
+        .."
+    build_result=$?
+    if [ $build_result -ne 0 ]
+    then
+        echo "Fluentbit deepfence out plugins build failed, bailing out"
+        exit 1
+    fi
+
     echo "Building Fluentbit deepfence output plugin"
-    docker run --rm -i -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_agent_builder_ce:$DF_IMG_TAG bash -x /home/deepfence/deepfence-out-plugin-build.sh
+    docker run --rm --workdir /go/src/github.com/deepfence/deepfence_agent  -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_glibc_builder_ce:$DF_IMG_TAG bash -c "cd plugins/fluent-bit/plugins/out_deepfence && make out_deepfence.a"
+    build_result=$?
+    if [ $build_result -ne 0 ]
+    then
+        echo "Fluentbit deepfence out plugins build failed, bailing out"
+        exit 1
+    fi
+
+    echo "Building Fluentbit"
+    docker run --rm --workdir /go/src/github.com/deepfence/deepfence_agent  -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_glibc_builder_ce:$DF_IMG_TAG bash -c "cd plugins/fluent-bit/build \
+        && make flb-plugin-out_deepfence\
+        && cp ../plugins/out_deepfence/out_deepfence.a ./library/libflb-plugin-out_deepfence.a\
+        && make"
     build_result=$?
     if [ $build_result -ne 0 ]
     then
@@ -25,7 +61,7 @@ building_image(){
     fi
 
     echo "Building Agent Executable"
-    docker run --rm -i -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host -e GIT_COMMIT="$GIT_COMMIT" $IMAGE_REPOSITORY/deepfence_agent_builder_ce:$DF_IMG_TAG bash -x /home/deepfence/agent-build.sh
+    docker run --rm --workdir /go/src/github.com/deepfence/deepfence_agent -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host -e GIT_COMMIT="$GIT_COMMIT" $IMAGE_REPOSITORY/deepfence_builder_ce:$DF_IMG_TAG bash -x /home/deepfence/agent-build.sh
     build_result=$?
     if [ $build_result -ne 0 ]
     then
@@ -34,7 +70,7 @@ building_image(){
     fi
 
     echo "Building Cluster Agent Image"
-    docker build --network host --rm=true --tag=$IMAGE_REPOSITORY/deepfence_cluster_agent_ce:$DF_IMG_TAG -f tools/apache/scope/docker/Dockerfile.cluster-agent tools/apache
+    docker build --network host --rm=true --tag=$IMAGE_REPOSITORY/deepfence_cluster_agent_ce:$DF_IMG_TAG -f tools/apache/scope/docker/Dockerfile.cluster-agent .
     build_result=$?
     if [ $build_result -ne 0 ]
     then

deepfence_agent/plugins/Makefile

@@ -1,37 +1,14 @@
-all: localinit proto bin/SecretScanner bin/package-scanner bin/YaraHunter
-# bin/open-tracer
+all: localinit
 
 localinit:
 
-bin/SecretScanner: ./SecretScanner/**/*.go
-	(cd SecretScanner && make)
-	-mkdir $(PWD)/bin/secret-scanner/
-	cp $(PWD)/SecretScanner/SecretScanner $(PWD)/bin/secret-scanner/
-	cp $(PWD)/SecretScanner/config.yaml $(PWD)/bin/secret-scanner/
-
-bin/YaraHunter: ./YaraHunter/**/*.go
-	(cd YaraHunter && make)
-	-mkdir $(PWD)/bin/yara-hunter/
-	cp $(PWD)/YaraHunter/YaraHunter $(PWD)/bin/yara-hunter/
-	cp $(PWD)/YaraHunter/config.yaml $(PWD)/bin/yara-hunter/
-
-bin/package-scanner:
-	(cd package-scanner && make)
-	cp $(PWD)/package-scanner/package-scanner $(PWD)/bin
-
-bin/compliance:
-	(cd compliance && make)
-	cp $(PWD)/compliance/compliance $(PWD)/bin
-
-proto: ./agent-plugins-grpc/proto/*.proto
-	(cd agent-plugins-grpc && make go)
-	cp agent-plugins-grpc/proto/*.go $(PWD)/proto
-
 clean:
 	-rm -rf $(PWD)/bin/secret-scanner/
 	-rm -rf $(PWD)/bin/yara-hunter/
 	-rm $(PWD)/bin/package-scanner
 	-rm $(PWD)/bin/compliance
-	(cd agent-plugins-grpc && make clean)
+	-(cd agent-plugins-grpc && make clean)
+	-(rm -rf $(PWD)/fluent-bit/build && mkdir $(PWD)/fluent-bit/build)
+	-(cd $(PWD)/fluent-bit/plugins/out_deepfence && make clean)
 
-.PHONY: clean localinit
+.PHONY: clean localinit all

deepfence_agent/tools/apache/deepfence/supervisord_cluster_agent.conf

@@ -19,7 +19,7 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 ; the name of your supervisord program
 [program:fluentbit]
 environment=SCOPE_HOSTNAME=${SCOPE_HOSTNAME}
-command=/opt/td-agent-bit/bin/td-agent-bit -c /etc/td-agent-bit/td-agent-bit.conf
+command=/opt/td-agent-bit/bin/fluent-bit -c /etc/td-agent-bit/td-agent-bit.conf
 numprocs=1
 autostart=true
 autorestart=true
@@ -37,4 +37,4 @@ autorestart=true
 startsecs=0
 stopwaitsecs=30
 killasgroup=true
-priority=10
+priority=10

deepfence_agent/tools/apache/fluentbit/fluentbit-plugins.conf

@@ -1,2 +1 @@
 [PLUGINS]
-  Path /opt/td-agent-bit/bin/out_deepfence.so