diff --git a/.gitmodules b/.gitmodules index ec280d8b88..f177b91155 100644 --- a/.gitmodules +++ b/.gitmodules @@ -26,3 +26,7 @@ path = deepfence_agent/plugins/yara-rules url = https://github.com/deepfence/yara-rules branch = main +[submodule "deepfence_agent/plugins/fluent-bit"] + path = deepfence_agent/plugins/fluent-bit + url = https://github.com/deepfence/fluent-bit + branch = master diff --git a/Makefile b/Makefile index 706eb11875..cfd5822ace 100644 --- a/Makefile +++ b/Makefile @@ -33,8 +33,8 @@ alpine_builder: docker build --tag=$(IMAGE_REPOSITORY)/deepfence_builder_ce:$(DF_IMG_TAG) -f docker_builders/Dockerfile-alpine . .PHONY: debian_builder -debian_builder: alpine_builder - docker build --build-arg DF_IMG_TAG=${DF_IMG_TAG} --build-arg IMAGE_REPOSITORY=${IMAGE_REPOSITORY} --tag=$(IMAGE_REPOSITORY)/deepfence_agent_builder_ce:$(DF_IMG_TAG) -f docker_builders/Dockerfile-debian . +debian_builder: + docker build --build-arg DF_IMG_TAG=${DF_IMG_TAG} --build-arg IMAGE_REPOSITORY=${IMAGE_REPOSITORY} --tag=$(IMAGE_REPOSITORY)/deepfence_glibc_builder_ce:$(DF_IMG_TAG) -f docker_builders/Dockerfile-debian . .PHONY: bootstrap-agent-plugins bootstrap-agent-plugins: @@ -89,7 +89,7 @@ graphdb: ui: git log --format="%h" -n 1 > $(DEEPFENCE_FRONTEND_DIR)/console_version.txt && \ echo $(VERSION) > $(DEEPFENCE_FRONTEND_DIR)/product_version.txt && \ - docker run -i --rm --entrypoint=bash -v $(DEEPFENCE_FRONTEND_DIR):/app node:18-bullseye-slim -c "cd /app && corepack enable && corepack prepare pnpm@7.17.1 --activate && PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=true pnpm install --frozen-lockfile --prefer-offline && pnpm run build" && \ + docker run --rm --entrypoint=bash -v $(DEEPFENCE_FRONTEND_DIR):/app node:18-bullseye-slim -c "cd /app && corepack enable && corepack prepare pnpm@7.17.1 --activate && PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=true pnpm install --frozen-lockfile --prefer-offline && pnpm run build" && \ docker build -f $(DEEPFENCE_FRONTEND_DIR)/Dockerfile -t $(IMAGE_REPOSITORY)/deepfence_ui_ce:$(DF_IMG_TAG) $(DEEPFENCE_FRONTEND_DIR) && \ rm -rf $(DEEPFENCE_FRONTEND_DIR)/console_version.txt $(DEEPFENCE_FRONTEND_DIR)/product_version.txt @@ -111,7 +111,7 @@ compliancescanner: .PHONY: openapi openapi: server - docker run --rm -i \ + docker run --rm \ --entrypoint=/usr/local/bin/deepfence_server \ -v $(PWD):/app $(IMAGE_REPOSITORY)/deepfence_server_ce:$(DF_IMG_TAG) \ --export-api-docs-path /app/openapi.yaml @@ -198,9 +198,7 @@ publish-graphdb: .PHONY: clean clean: - (cd $(DEEPFENCE_AGENT_DIR)/plugins && make clean) - (cd $(DEEPFENCE_AGENT_DIR) && make clean) - (cd $(ROOT_MAKEFILE_DIR)/deepfence_server && make clean) - (cd $(ROOT_MAKEFILE_DIR)/deepfence_worker && make clean) - (cd $(DEEPFENCED) && make clean && rm $(DEEPFENCE_AGENT_DIR)/deepfenced) - (cd $(DEEPFENCE_AGENT_DIR)/tools/apache/fluentbit/out_deepfence && make clean) + -(cd $(DEEPFENCE_AGENT_DIR) && make clean) + -(cd $(ROOT_MAKEFILE_DIR)/deepfence_server && make clean) + -(cd $(ROOT_MAKEFILE_DIR)/deepfence_worker && make clean) + -(cd $(DEEPFENCED) && make clean && rm $(DEEPFENCE_AGENT_DIR)/deepfenced) diff --git a/deepfence_agent/Dockerfile b/deepfence_agent/Dockerfile index 119e3a8c0e..b65992cf73 100644 --- a/deepfence_agent/Dockerfile +++ b/deepfence_agent/Dockerfile @@ -37,16 +37,11 @@ RUN apt-get -y --allow-unauthenticated install skopeo podman \ && mkdir /usr/local/bin/compliance_check && mkdir /usr/local/discovery \ && apt update --allow-insecure-repositories && DEBIAN_FRONTEND=noninteractive apt install libhyperscan5 -# Installing FluentBit -RUN wget -qO - https://packages.fluentbit.io/fluentbit.key | apt-key add - \ - && echo "deb https://packages.fluentbit.io/debian/buster buster main " >> /etc/apt/sources.list -RUN apt-get update --allow-insecure-repositories -RUN apt-get install -y td-agent-bit=1.6.10 && mkdir -p /etc/td-agent-bit/ - +RUN mkdir -p /etc/td-agent-bit/ COPY tools/apache/fluentbit/* /etc/td-agent-bit/ # Copy fluentbit plugin -COPY tools/apache/fluentbit/out_deepfence/out_deepfence.so /opt/td-agent-bit/bin/ +COPY plugins/fluent-bit/build/bin/fluent-bit /opt/td-agent-bit/bin/ COPY tools/apache/deepfence/df-utils/get_cloud_instance_id/getCloudInstanceId /usr/local/bin/getCloudInstanceId COPY etc/fenced_logrotate.conf /etc/logrotate.d/fenced_logrotate.conf COPY etc/certs/* /etc/filebeat/ diff --git a/deepfence_agent/Makefile b/deepfence_agent/Makefile index 25e8be0090..8c175d37e0 100644 --- a/deepfence_agent/Makefile +++ b/deepfence_agent/Makefile @@ -32,14 +32,16 @@ gocode: install: -agent: +tools/apache/scope/vendor: tools/apache/scope/go.mod $(shell find ../deepfence_utils -name '*.go') + ($(CD) tools/apache/scope && go mod tidy -v) + ($(CD) tools/apache/scope && go mod vendor) + +agent: tools/apache/scope/vendor ($(CD) tools/apache/scope && \ - go mod tidy -v && \ - go mod vendor && \ env GOGC=off \ CGO_ENABLED=1 \ go build -buildvcs=false \ - -ldflags "-X main.version=$GIT_COMMIT -X github.com/weaveworks/scope/probe/host.agentCommitID=$GIT_COMMIT -X github.com/weaveworks/scope/probe/host.agentBuildTime=$(shell date +"%s%d%m%y") -s -w"\ + -ldflags "-X main.version=$GIT_COMMIT -X main.version=$(shell git rev-parse --short HEAD) -X github.com/weaveworks/scope/probe/host.agentCommitID=$(shell git rev-parse --short HEAD) -X github.com/weaveworks/scope/probe/host.agentBuildTime=$(shell date +"%s%d%m%y") -s -w -extldflags=-static"\ -tags 'netgo osusergo unsafe' \ -o docker/deepfence_exe \ ./prog) diff --git a/deepfence_agent/build.sh b/deepfence_agent/build.sh index 17059c7e2c..0e5161307e 100755 --- a/deepfence_agent/build.sh +++ b/deepfence_agent/build.sh @@ -7,7 +7,7 @@ GIT_COMMIT=$(git rev-parse --short HEAD) building_image(){ echo "Building GetCloudInstanceId" - docker run --rm -i -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_agent_builder_ce:$DF_IMG_TAG bash -x /home/deepfence/gocode-build.sh + docker run --rm --workdir /go/src/github.com/deepfence/deepfence_agent -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_builder_ce:$DF_IMG_TAG bash -x /home/deepfence/gocode-build.sh build_result=$? if [ $build_result -ne 0 ] then @@ -15,8 +15,44 @@ building_image(){ exit 1 fi + echo "Prepare Fluentbit" + docker run --rm --workdir /go/src/github.com/deepfence/deepfence_agent -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_glibc_builder_ce:$DF_IMG_TAG bash -c "\ + cd plugins/fluent-bit/build && \ + cmake \ + -DFLB_WASM=No \ + -DFLB_LUAJIT=No \ + -DFLB_DEBUG=No \ + -DFLB_RELEASE=Yes \ + -DFLB_SHARED_LIB=No \ + -DCMAKE_FIND_LIBRARY_SUFFIXES='.a' \ + -DFLB_CONFIG_YAML=Off \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_EXE_LINKER_FLAGS='-static' \ + -DOPENSSL_USE_STATIC_LIBS=Yes \ + -DZLIB_USE_STATIC_LIBS=Yes \ + -DCMAKE_C_FLAGS='-fcommon' \ + .." + build_result=$? + if [ $build_result -ne 0 ] + then + echo "Fluentbit deepfence out plugins build failed, bailing out" + exit 1 + fi + echo "Building Fluentbit deepfence output plugin" - docker run --rm -i -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_agent_builder_ce:$DF_IMG_TAG bash -x /home/deepfence/deepfence-out-plugin-build.sh + docker run --rm --workdir /go/src/github.com/deepfence/deepfence_agent -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_glibc_builder_ce:$DF_IMG_TAG bash -c "cd plugins/fluent-bit/plugins/out_deepfence && make out_deepfence.a" + build_result=$? + if [ $build_result -ne 0 ] + then + echo "Fluentbit deepfence out plugins build failed, bailing out" + exit 1 + fi + + echo "Building Fluentbit" + docker run --rm --workdir /go/src/github.com/deepfence/deepfence_agent -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host $IMAGE_REPOSITORY/deepfence_glibc_builder_ce:$DF_IMG_TAG bash -c "cd plugins/fluent-bit/build \ + && make flb-plugin-out_deepfence\ + && cp ../plugins/out_deepfence/out_deepfence.a ./library/libflb-plugin-out_deepfence.a\ + && make" build_result=$? if [ $build_result -ne 0 ] then @@ -25,7 +61,7 @@ building_image(){ fi echo "Building Agent Executable" - docker run --rm -i -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host -e GIT_COMMIT="$GIT_COMMIT" $IMAGE_REPOSITORY/deepfence_agent_builder_ce:$DF_IMG_TAG bash -x /home/deepfence/agent-build.sh + docker run --rm --workdir /go/src/github.com/deepfence/deepfence_agent -v $(pwd)/../golang_deepfence_sdk:/go/src/github.com/deepfence/golang_deepfence_sdk -v $(pwd)/../deepfence_utils:/go/src/github.com/deepfence/deepfence_utils -v $(pwd):/go/src/github.com/deepfence/deepfence_agent:rw --net=host -e GIT_COMMIT="$GIT_COMMIT" $IMAGE_REPOSITORY/deepfence_builder_ce:$DF_IMG_TAG bash -x /home/deepfence/agent-build.sh build_result=$? if [ $build_result -ne 0 ] then @@ -34,7 +70,7 @@ building_image(){ fi echo "Building Cluster Agent Image" - docker build --network host --rm=true --tag=$IMAGE_REPOSITORY/deepfence_cluster_agent_ce:$DF_IMG_TAG -f tools/apache/scope/docker/Dockerfile.cluster-agent tools/apache + docker build --network host --rm=true --tag=$IMAGE_REPOSITORY/deepfence_cluster_agent_ce:$DF_IMG_TAG -f tools/apache/scope/docker/Dockerfile.cluster-agent . build_result=$? if [ $build_result -ne 0 ] then diff --git a/deepfence_agent/plugins/Makefile b/deepfence_agent/plugins/Makefile index a2c497a7b0..b04cf93ac9 100644 --- a/deepfence_agent/plugins/Makefile +++ b/deepfence_agent/plugins/Makefile @@ -1,37 +1,14 @@ -all: localinit proto bin/SecretScanner bin/package-scanner bin/YaraHunter -# bin/open-tracer +all: localinit localinit: -bin/SecretScanner: ./SecretScanner/**/*.go - (cd SecretScanner && make) - -mkdir $(PWD)/bin/secret-scanner/ - cp $(PWD)/SecretScanner/SecretScanner $(PWD)/bin/secret-scanner/ - cp $(PWD)/SecretScanner/config.yaml $(PWD)/bin/secret-scanner/ - -bin/YaraHunter: ./YaraHunter/**/*.go - (cd YaraHunter && make) - -mkdir $(PWD)/bin/yara-hunter/ - cp $(PWD)/YaraHunter/YaraHunter $(PWD)/bin/yara-hunter/ - cp $(PWD)/YaraHunter/config.yaml $(PWD)/bin/yara-hunter/ - -bin/package-scanner: - (cd package-scanner && make) - cp $(PWD)/package-scanner/package-scanner $(PWD)/bin - -bin/compliance: - (cd compliance && make) - cp $(PWD)/compliance/compliance $(PWD)/bin - -proto: ./agent-plugins-grpc/proto/*.proto - (cd agent-plugins-grpc && make go) - cp agent-plugins-grpc/proto/*.go $(PWD)/proto - clean: -rm -rf $(PWD)/bin/secret-scanner/ -rm -rf $(PWD)/bin/yara-hunter/ -rm $(PWD)/bin/package-scanner -rm $(PWD)/bin/compliance - (cd agent-plugins-grpc && make clean) + -(cd agent-plugins-grpc && make clean) + -(rm -rf $(PWD)/fluent-bit/build && mkdir $(PWD)/fluent-bit/build) + -(cd $(PWD)/fluent-bit/plugins/out_deepfence && make clean) -.PHONY: clean localinit +.PHONY: clean localinit all diff --git a/deepfence_agent/plugins/fluent-bit b/deepfence_agent/plugins/fluent-bit new file mode 160000 index 0000000000..b1fe5f98b8 --- /dev/null +++ b/deepfence_agent/plugins/fluent-bit @@ -0,0 +1 @@ +Subproject commit b1fe5f98b8ab6cfcde25790808f7d70e702bc4d2 diff --git a/deepfence_agent/tools/apache/deepfence/supervisord_cluster_agent.conf b/deepfence_agent/tools/apache/deepfence/supervisord_cluster_agent.conf index 1f2ed58586..8a041c0616 100644 --- a/deepfence_agent/tools/apache/deepfence/supervisord_cluster_agent.conf +++ b/deepfence_agent/tools/apache/deepfence/supervisord_cluster_agent.conf @@ -19,7 +19,7 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface ; the name of your supervisord program [program:fluentbit] environment=SCOPE_HOSTNAME=${SCOPE_HOSTNAME} -command=/opt/td-agent-bit/bin/td-agent-bit -c /etc/td-agent-bit/td-agent-bit.conf +command=/opt/td-agent-bit/bin/fluent-bit -c /etc/td-agent-bit/td-agent-bit.conf numprocs=1 autostart=true autorestart=true @@ -37,4 +37,4 @@ autorestart=true startsecs=0 stopwaitsecs=30 killasgroup=true -priority=10 \ No newline at end of file +priority=10 diff --git a/deepfence_agent/tools/apache/fluentbit/fluentbit-plugins.conf b/deepfence_agent/tools/apache/fluentbit/fluentbit-plugins.conf index 9911449401..1829da5d7e 100644 --- a/deepfence_agent/tools/apache/fluentbit/fluentbit-plugins.conf +++ b/deepfence_agent/tools/apache/fluentbit/fluentbit-plugins.conf @@ -1,2 +1 @@ [PLUGINS] - Path /opt/td-agent-bit/bin/out_deepfence.so diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/Dockerfile b/deepfence_agent/tools/apache/fluentbit/out_deepfence/Dockerfile deleted file mode 100644 index 198744699f..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM golang:1.20 as gobuilder - -WORKDIR /root - -ENV GOOS=linux\ - GOARCH=amd64 - -COPY / /root/ - -# RUN go mod edit -replace github.com/fluent/fluent-bit-go=github.com/fluent/fluent-bit-go@master -RUN go mod download -RUN make all - -FROM fluent/fluent-bit:1.6 - -COPY --from=gobuilder /root/out_deepfence.so /fluent-bit/bin/ -COPY --from=gobuilder /root/fluent-bit.conf /fluent-bit/etc/ -COPY --from=gobuilder /root/plugins.conf /fluent-bit/etc/ - -EXPOSE 2020 - -# CMD ["/fluent-bit/bin/fluent-bit", "--plugin", "/fluent-bit/bin/out_deepfence.so", "--config", "/fluent-bit/etc/fluent-bit.conf"] -CMD ["/fluent-bit/bin/fluent-bit", "--config", "/fluent-bit/etc/fluent-bit.conf"] diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/Makefile b/deepfence_agent/tools/apache/fluentbit/out_deepfence/Makefile deleted file mode 100644 index ce42ae0d22..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -FLUENT_BIT?=/opt/fluent-bit/bin/fluent-bit - -all: out_deepfence.so - -out_deepfence.so: ./*.go - go mod tidy -v - go build -buildmode=c-shared -o out_deepfence.so - -clean: - rm -rf ./*.so ./*.h - -docker: - docker build -t fluentbit:deepfence . - -run: all - $(FLUENT_BIT) -c local-fluent-bit.conf diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/README.md b/deepfence_agent/tools/apache/fluentbit/out_deepfence/README.md deleted file mode 100644 index 6a77033217..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# out_deepfence - -out_deepfence is a fluentbit output plugin \ No newline at end of file diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/cluster_agent_plugin.conf b/deepfence_agent/tools/apache/fluentbit/out_deepfence/cluster_agent_plugin.conf deleted file mode 100644 index d2a7917fb2..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/cluster_agent_plugin.conf +++ /dev/null @@ -1,2 +0,0 @@ -[PLUGINS] - Path /etc/td-agent-bit/out_deepfence.so diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/fluent-bit.conf b/deepfence_agent/tools/apache/fluentbit/out_deepfence/fluent-bit.conf deleted file mode 100644 index 47ab4aef25..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/fluent-bit.conf +++ /dev/null @@ -1,28 +0,0 @@ -[SERVICE] - Flush 5 - Daemon Off - Log_Level info - Parsers_File parsers.conf - Plugins_File plugins.conf - HTTP_Server Off - HTTP_Listen 0.0.0.0 - HTTP_Port 2020 - -[INPUT] - Name cpu - Tag cpu.local - Interval_Sec 1 - -[INPUT] - Name dummy - Tag dummy.local - -[OUTPUT] - Name deepfence - Match cpu* - Id cpu_metrics - -[OUTPUT] - Name deepfence - Match dummy* - Id dummy_metrics diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/go.mod b/deepfence_agent/tools/apache/fluentbit/out_deepfence/go.mod deleted file mode 100644 index 358d6b08a7..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/go.mod +++ /dev/null @@ -1,42 +0,0 @@ -module github.com/deepfence/ThreatMapper/deepfence_agent/tools/apache/fluentbit/out_deepfence - -go 1.20 - -replace github.com/deepfence/golang_deepfence_sdk/client => ../../../../../golang_deepfence_sdk/client - -replace github.com/deepfence/golang_deepfence_sdk/utils => ../../../../../golang_deepfence_sdk/utils - -replace github.com/deepfence/ThreatMapper/deepfence_utils => ../../../../../deepfence_utils - -require ( - github.com/deepfence/ThreatMapper/deepfence_utils v0.0.0-00010101000000-000000000000 - github.com/deepfence/golang_deepfence_sdk/client v0.0.0-00010101000000-000000000000 - github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-00010101000000-000000000000 - github.com/fluent/fluent-bit-go v0.0.0-20230515084116-b93d969da46d - github.com/hashicorp/go-retryablehttp v0.7.4 -) - -require ( - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/klauspost/compress v1.16.5 // indirect - github.com/lestrrat-go/blackmagic v1.0.1 // indirect - github.com/lestrrat-go/httpcc v1.0.1 // indirect - github.com/lestrrat-go/httprc v1.0.4 // indirect - github.com/lestrrat-go/iter v1.0.2 // indirect - github.com/lestrrat-go/jwx/v2 v2.0.11 // indirect - github.com/lestrrat-go/option v1.0.1 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect - github.com/pierrec/lz4/v4 v4.1.17 // indirect - github.com/rs/zerolog v1.29.1 // indirect - github.com/segmentio/asm v1.2.0 // indirect - github.com/twmb/franz-go v1.13.5 // indirect - github.com/twmb/franz-go/pkg/kadm v1.8.1 // indirect - github.com/twmb/franz-go/pkg/kmsg v1.4.0 // indirect - github.com/ugorji/go/codec v1.1.7 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/sys v0.8.0 // indirect -) diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/go.sum b/deepfence_agent/tools/apache/fluentbit/out_deepfence/go.sum deleted file mode 100644 index be188a09d1..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/go.sum +++ /dev/null @@ -1,111 +0,0 @@ -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/fluent/fluent-bit-go v0.0.0-20230515084116-b93d969da46d h1:b4a4JIzP5VT7l64NHnXF1nP95zRRN7wVMsD5D1jJiY0= -github.com/fluent/fluent-bit-go v0.0.0-20230515084116-b93d969da46d/go.mod h1:L92h+dgwElEyUuShEwjbiHjseW410WIcNz+Bjutc8YQ= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= -github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80= -github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= -github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= -github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= -github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8= -github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= -github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= -github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.0.11 h1:ViHMnaMeaO0qV16RZWBHM7GTrAnX2aFLVKofc7FuKLQ= -github.com/lestrrat-go/jwx/v2 v2.0.11/go.mod h1:ZtPtMFlrfDrH2Y0iwfa3dRFn8VzwBrB+cyrm3IBWdDg= -github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= -github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= -github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= -github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= -github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= -github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/twmb/franz-go v1.13.5 h1:7Hk47eZ7XRb4yWXQZk1GZU4BthkrKuZUfKOuP9Sgp24= -github.com/twmb/franz-go v1.13.5/go.mod h1:jm/FtYxmhxDTN0gNSb26XaJY0irdSVcsckLiR5tQNMk= -github.com/twmb/franz-go/pkg/kadm v1.8.1 h1:SrzL855I7gQTGdMtOYGTHhebs7TPgPN29FPtjusqwlE= -github.com/twmb/franz-go/pkg/kadm v1.8.1/go.mod h1:qUSM7pxoMCU1UNu5H4USE64ODcVmeG9LS96mysv1nu8= -github.com/twmb/franz-go/pkg/kmsg v1.4.0 h1:tbp9hxU6m8qZhQTlpGiaIJOm4BXix5lsuEZ7K00dF0s= -github.com/twmb/franz-go/pkg/kmsg v1.4.0/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/local-fluent-bit.conf b/deepfence_agent/tools/apache/fluentbit/out_deepfence/local-fluent-bit.conf deleted file mode 100644 index e5c34e6202..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/local-fluent-bit.conf +++ /dev/null @@ -1,39 +0,0 @@ -[SERVICE] - Flush 5 - Daemon Off - Log_Level info - # Parsers_File parsers.conf - Plugins_File /root/ThreatMapper/deepfence_agent/tools/apache/fluentbit/out_deepfence/local-plugins.conf - HTTP_Server Off - HTTP_Listen 0.0.0.0 - HTTP_Port 2020 - -[INPUT] - Name cpu - Tag dummy.cve - # Interval_Sec 1 - -[INPUT] - Name dummy - Tag dummy.local - # Interval_Sec 3 - -[OUTPUT] - Name deepfence - Match dummy.cve - Id cve - Schema https - Console_host 159.89.174.123 - Console_port 443 - Path /deepfence/ingest/vulnerabilities - Token c369d073-1858-46c1-b59b-391852108411 - -[OUTPUT] - Name deepfence - Match dummy.local - Id local - Schema https - Console_host 159.89.174.123 - Console_port 443 - Path /deepfence/ingest/vulnerabilities - Token c369d073-1858-46c1-b59b-391852108411 \ No newline at end of file diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/local-plugins.conf b/deepfence_agent/tools/apache/fluentbit/out_deepfence/local-plugins.conf deleted file mode 100644 index d62edaedb0..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/local-plugins.conf +++ /dev/null @@ -1,2 +0,0 @@ -[PLUGINS] - Path /root/ThreatMapper/deepfence_agent/tools/apache/fluentbit/out_deepfence/out_deepfence.so diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/out_deepfence.go b/deepfence_agent/tools/apache/fluentbit/out_deepfence/out_deepfence.go deleted file mode 100644 index b18c042bb9..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/out_deepfence.go +++ /dev/null @@ -1,395 +0,0 @@ -package main - -import ( - "bytes" - "context" - "crypto/tls" - "crypto/x509" - "encoding/json" - "errors" - "io" - "log" - "net" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "time" - "unsafe" - - "C" - "github.com/fluent/fluent-bit-go/output" - - deepfenceUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" - dsc "github.com/deepfence/golang_deepfence_sdk/client" - dschttp "github.com/deepfence/golang_deepfence_sdk/utils/http" - rhttp "github.com/hashicorp/go-retryablehttp" -) - -var ( - cfg map[string]Config - hc *http.Client - instance int = 0 -) - -type Config struct { - ConsoleURL string - URL string - Key string - AccessToken string - RefreshToken string -} - -func getURLWithPath(schema, host, port, path string) string { - u := &url.URL{ - Scheme: schema, - Host: net.JoinHostPort(host, port), - Path: path, - } - return u.String() -} - -func getURL(schema, host, port string) string { - u := &url.URL{ - Scheme: schema, - Host: net.JoinHostPort(host, port), - } - return u.String() -} - -func toMapStringInterface(inputRecord map[interface{}]interface{}) map[string]interface{} { - return parseValue(inputRecord).(map[string]interface{}) -} - -func parseValue(value interface{}) interface{} { - switch value := value.(type) { - case []byte: - return string(value) - case map[interface{}]interface{}: - remapped := make(map[string]interface{}) - for k, v := range value { - remapped[k.(string)] = parseValue(v) - } - return remapped - case []interface{}: - remapped := make([]interface{}, len(value)) - for i, v := range value { - remapped[i] = parseValue(v) - } - return remapped - default: - return value - } -} - -// // data needs to be in this format -// // {"records":[{"value":},{"value":record2}]} -// func toKafkaRestFormat(data []map[string]interface{}) *bytes.Buffer { -// values := make([]string, len(data)) -// for i, u := range data { -// encoded, err := json.Marshal(u) -// if err != nil { -// log.Printf("error marshal doc %s\ndoc:%s", err, u) -// continue -// } -// values[i] = "{\"value\":" + string(encoded) + "}" -// } -// result := strings.Join(values, ",") -// return bytes.NewBuffer([]byte("{\"records\":[" + result + "]}")) -// } - -func Authenticate(url string, apiToken string) (string, string, error) { - var ( - accessToken string - refreshToken string - ) - cfg := dsc.NewConfiguration() - cfg.HTTPClient = hc - cfg.Servers = dsc.ServerConfigurations{ - {URL: url, Description: "deepfence_server"}, - } - - apiClient := dsc.NewAPIClient(cfg) - - req := apiClient.AuthenticationAPI.AuthToken(context.Background()). - ModelApiAuthRequest( - dsc.ModelApiAuthRequest{ApiToken: apiToken}, - ) - - resp, _, err := apiClient.AuthenticationAPI.AuthTokenExecute(req) - if err != nil { - return "", "", err - } - - accessToken = resp.GetAccessToken() - refreshToken = resp.GetRefreshToken() - if accessToken == "" || refreshToken == "" { - return "", "", errors.New("auth tokens are nil: failed to authenticate") - } - - log.Print("authenticated with console successfully") - - return accessToken, refreshToken, nil -} - -func RefreshToken(url string, apiToken string) (string, string, error) { - var ( - accessToken string - refreshToken string - ) - cfg := dsc.NewConfiguration() - cfg.HTTPClient = hc - cfg.Servers = dsc.ServerConfigurations{ - {URL: url, Description: "deepfence_server"}, - } - - cfg.AddDefaultHeader("Authorization", "Bearer "+apiToken) - - apiClient := dsc.NewAPIClient(cfg) - - req := apiClient.AuthenticationAPI.AuthTokenRefresh(context.Background()) - - resp, _, err := apiClient.AuthenticationAPI.AuthTokenRefreshExecute(req) - if err != nil { - return "", "", err - } - - accessToken = resp.GetAccessToken() - refreshToken = resp.GetRefreshToken() - if accessToken == "" || refreshToken == "" { - return "", "", errors.New("auth tokens are nil: failed to authenticate") - } - - log.Print("refreshed tokens from console successfully") - - return accessToken, refreshToken, nil -} - -func validateTokens(cfg Config) (Config, bool, error) { - if !deepfenceUtils.IsJWTExpired(cfg.AccessToken) { - return cfg, false, nil - } else { - var ( - access string - refresh string - err error - ) - access, refresh, err = RefreshToken(cfg.ConsoleURL, cfg.RefreshToken) - if err != nil { - access, refresh, err = Authenticate(cfg.ConsoleURL, cfg.Key) - if err != nil { - return cfg, false, err - } - } - cfg.AccessToken = access - cfg.RefreshToken = refresh - return cfg, true, nil - } -} - -//export FLBPluginRegister -func FLBPluginRegister(def unsafe.Pointer) int { - log.SetFlags(log.LstdFlags | log.Lshortfile) - return output.FLBPluginRegister(def, "deepfence", "deepfence output plugin") -} - -//export FLBPluginInit -func FLBPluginInit(plugin unsafe.Pointer) int { - if cfg == nil { - cfg = make(map[string]Config) - } - - id := output.FLBPluginConfigKey(plugin, "id") - host := output.FLBPluginConfigKey(plugin, "console_host") - port := output.FLBPluginConfigKey(plugin, "console_port") - path := output.FLBPluginConfigKey(plugin, "path") - schema := output.FLBPluginConfigKey(plugin, "schema") - apiToken := output.FLBPluginConfigKey(plugin, "token") - certPath := output.FLBPluginConfigKey(plugin, "cert_file") - certKey := output.FLBPluginConfigKey(plugin, "key_file") - log.Printf("id=%s schema=%s host=%s port=%s path=%s", - id, schema, host, port, path) - - // setup http client - tlsConfig := &tls.Config{RootCAs: x509.NewCertPool(), InsecureSkipVerify: true} - rhc := rhttp.NewClient() - rhc.HTTPClient.Timeout = 10 * time.Second - rhc.RetryMax = 3 - rhc.RetryWaitMin = 1 * time.Second - rhc.RetryWaitMax = 10 * time.Second - rhc.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { - if err != nil || resp == nil { - return false, err - } - if resp.StatusCode == http.StatusServiceUnavailable { - return false, err - } - return rhttp.DefaultRetryPolicy(ctx, resp, err) - } - rhc.Logger = log.New(os.Stderr, "", log.LstdFlags|log.Lshortfile) - if schema == "https" { - if len(certPath) > 0 && len(certKey) > 0 { - cer, err := tls.LoadX509KeyPair(certPath, certKey) - if err != nil { - log.Printf("error loading certs %s", err) - return output.FLB_ERROR - } - tlsConfig.Certificates = []tls.Certificate{cer} - } - tr := &http.Transport{ - TLSClientConfig: tlsConfig, - DisableKeepAlives: false, - } - rhc.HTTPClient = &http.Client{Transport: tr} - } - - hc = rhc.StandardClient() - - if dschttp.IsConsoleAgent(host) && strings.Trim(apiToken, "\"") == "" { - internalURL := os.Getenv("MGMT_CONSOLE_URL_INTERNAL") - internalPort := os.Getenv("MGMT_CONSOLE_PORT_INTERNAL") - var err error - if apiToken, err = dschttp.GetConsoleApiToken(internalURL, internalPort); err != nil { - log.Panic(err) - } - } - - access, refresh, err := Authenticate(getURL(schema, host, port), apiToken) - if err != nil { - log.Printf("failed to authenticate %s", err) - } - - if len(id) == 0 { - id = "deepfence." + strconv.Itoa(instance) - instance = instance + 1 - } - - cfg[id] = Config{ - ConsoleURL: getURL(schema, host, port), - URL: getURLWithPath(schema, host, port, path), - Key: apiToken, - AccessToken: access, - RefreshToken: refresh, - } - - log.Printf("api token set %t for id %s", apiToken != "", id) - log.Printf("push to url %s", cfg[id].URL) - - output.FLBPluginSetContext(plugin, id) - - return output.FLB_OK -} - -//export FLBPluginFlush -func FLBPluginFlush(data unsafe.Pointer, length C.int, tag *C.char) int { - log.Printf("flush called on unknown instance") - return output.FLB_OK -} - -//export FLBPluginFlushCtx -func FLBPluginFlushCtx(ctx, data unsafe.Pointer, length C.int, tag *C.char) int { - id := output.FLBPluginGetContext(ctx).(string) - idCfg, ok := cfg[id] - if !ok { - log.Printf("push to unknown id topic %s", id) - return output.FLB_ERROR - } - - newConfig, changed, err := validateTokens(idCfg) - if err != nil { - log.Print(err.Error()) - return output.FLB_ERROR - } - if changed { - idCfg = newConfig - cfg[id] = newConfig - } - - // fluent-bit decoder - dec := output.NewDecoder(data, int(length)) - - records := make([]map[string]interface{}, 0) - - for { - ret, _, record := output.GetRecord(dec) - if ret != 0 { - break - } - records = append(records, toMapStringInterface(record)) - } - - rawRecords, err := json.Marshal(records) - if err != nil { - log.Printf("error marshaling records: %s", err) - return output.FLB_ERROR - } - - req, err := http.NewRequest(http.MethodPost, idCfg.URL, bytes.NewReader(rawRecords)) - if err != nil { - log.Printf("error creating request %s", err) - return output.FLB_ERROR - } - - req.Header.Add("Authorization", "Bearer "+idCfg.AccessToken) - req.Header.Add("Content-Type", "application/json") - - resp, err := hc.Do(req) - if err != nil { - if os.IsTimeout(err) { - // timeout error - log.Printf(" retry request timeout error: %s", err) - return output.FLB_RETRY - } - log.Printf(" error making request %s", err) - return output.FLB_ERROR - } - - defer resp.Body.Close() - - if resp.StatusCode == http.StatusBadGateway || - resp.StatusCode == http.StatusServiceUnavailable || - resp.StatusCode == http.StatusGatewayTimeout || - resp.StatusCode == http.StatusTooManyRequests || - resp.StatusCode == http.StatusUnauthorized { - log.Printf("retry response code %s", resp.Status) - return output.FLB_RETRY - } else if resp.StatusCode != http.StatusOK { - log.Printf("error response code %s", resp.Status) - return output.FLB_ERROR - } - - _, err = io.ReadAll(resp.Body) - if err != nil { - log.Printf("error reading response %s", err) - return output.FLB_ERROR - } - - return output.FLB_OK -} - -//export FLBPluginExit -func FLBPluginExit() int { - log.Printf("exit called on unknown instance") - return output.FLB_OK -} - -//export FLBPluginExitCtx -func FLBPluginExitCtx(ctx unsafe.Pointer) int { - id := output.FLBPluginGetContext(ctx).(string) - _, ok := cfg[id] - if !ok { - log.Printf("exit called on unknown id topic %s", id) - return output.FLB_ERROR - } - log.Printf("exit called on id topic %s", id) - return output.FLB_OK -} - -//export FLBPluginUnregister -func FLBPluginUnregister(ctx unsafe.Pointer) { - log.Print("unregister called") - output.FLBPluginUnregister(ctx) -} - -func main() {} diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/out_deepfence.h b/deepfence_agent/tools/apache/fluentbit/out_deepfence/out_deepfence.h deleted file mode 100644 index f29c68b76a..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/out_deepfence.h +++ /dev/null @@ -1,87 +0,0 @@ -/* Code generated by cmd/cgo; DO NOT EDIT. */ - -/* package github.com/deepfence/ThreatMapper/deepfence_agent/tools/apache/fluentbit/out_deepfence */ - - -#line 1 "cgo-builtin-export-prolog" - -#include - -#ifndef GO_CGO_EXPORT_PROLOGUE_H -#define GO_CGO_EXPORT_PROLOGUE_H - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef struct { const char *p; ptrdiff_t n; } _GoString_; -#endif - -#endif - -/* Start of preamble from import "C" comments. */ - - - - -/* End of preamble from import "C" comments. */ - - -/* Start of boilerplate cgo prologue. */ -#line 1 "cgo-gcc-export-header-prolog" - -#ifndef GO_CGO_PROLOGUE_H -#define GO_CGO_PROLOGUE_H - -typedef signed char GoInt8; -typedef unsigned char GoUint8; -typedef short GoInt16; -typedef unsigned short GoUint16; -typedef int GoInt32; -typedef unsigned int GoUint32; -typedef long long GoInt64; -typedef unsigned long long GoUint64; -typedef GoInt64 GoInt; -typedef GoUint64 GoUint; -typedef size_t GoUintptr; -typedef float GoFloat32; -typedef double GoFloat64; -#ifdef _MSC_VER -#include -typedef _Fcomplex GoComplex64; -typedef _Dcomplex GoComplex128; -#else -typedef float _Complex GoComplex64; -typedef double _Complex GoComplex128; -#endif - -/* - static assertion to make sure the file is being used on architecture - at least with matching size of GoInt. -*/ -typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef _GoString_ GoString; -#endif -typedef void *GoMap; -typedef void *GoChan; -typedef struct { void *t; void *v; } GoInterface; -typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; - -#endif - -/* End of boilerplate cgo prologue. */ - -#ifdef __cplusplus -extern "C" { -#endif - -extern GoInt FLBPluginRegister(void* def); -extern GoInt FLBPluginInit(void* plugin); -extern GoInt FLBPluginFlush(void* data, int length, char* tag); -extern GoInt FLBPluginFlushCtx(void* ctx, void* data, int length, char* tag); -extern GoInt FLBPluginExit(); -extern GoInt FLBPluginExitCtx(void* ctx); -extern void FLBPluginUnregister(void* ctx); - -#ifdef __cplusplus -} -#endif diff --git a/deepfence_agent/tools/apache/fluentbit/out_deepfence/plugins.conf b/deepfence_agent/tools/apache/fluentbit/out_deepfence/plugins.conf deleted file mode 100644 index 97c431024b..0000000000 --- a/deepfence_agent/tools/apache/fluentbit/out_deepfence/plugins.conf +++ /dev/null @@ -1,2 +0,0 @@ -[PLUGINS] - Path /fluent-bit/bin/out_deepfence.so diff --git a/deepfence_agent/tools/apache/scope/docker/Dockerfile.cluster-agent b/deepfence_agent/tools/apache/scope/docker/Dockerfile.cluster-agent index d27f54f88b..428b389b9e 100644 --- a/deepfence_agent/tools/apache/scope/docker/Dockerfile.cluster-agent +++ b/deepfence_agent/tools/apache/scope/docker/Dockerfile.cluster-agent @@ -28,29 +28,26 @@ RUN /bin/sh -c "$(curl -fsSL https://raw.githubusercontent.com/turbot/steampipe/ RUN useradd -rm -d /home/deepfence -s /bin/bash -g root -G sudo -u 1001 deepfence # Installing FluentBit -RUN wget -qO - https://packages.fluentbit.io/fluentbit.key | apt-key add - \ - && echo "deb https://packages.fluentbit.io/debian/buster buster main " >> /etc/apt/sources.list -RUN apt-get update --allow-insecure-repositories -RUN apt-get install -y td-agent-bit=1.6.10 && mkdir -p /etc/td-agent-bit/ +RUN mkdir -p /etc/td-agent-bit/ -COPY fluentbit/* /etc/td-agent-bit/ -COPY scope/docker/fluent-bit.conf /etc/td-agent-bit/td-agent-bit.conf -COPY fluentbit/out_deepfence/out_deepfence.so /opt/td-agent-bit/bin/ +COPY tools/apache/scope/docker/fluent-bit.conf /etc/td-agent-bit/td-agent-bit.conf +COPY plugins/fluent-bit/build/bin/fluent-bit /opt/td-agent-bit/bin/ WORKDIR /opt/steampipe -COPY scope/probe/kubernetes/kubeconfig /home/deepfence/.kube/config -COPY scope/probe/kubernetes/token.sh /home/deepfence/token.sh +COPY tools/apache/scope/probe/kubernetes/kubeconfig /home/deepfence/.kube/config +COPY tools/apache/scope/probe/kubernetes/token.sh /home/deepfence/token.sh RUN chown -R deepfence /opt/ /home/deepfence /var/log /var/run/ /etc/td-agent-bit/ USER deepfence -COPY deepfence/supervisord_cluster_agent.conf /home/deepfence/supervisord-temp.conf -COPY scope/docker/run_cluster_agent.sh /home/deepfence/ -COPY scope/docker/entrypoint.sh /home/deepfence/ +COPY tools/apache/deepfence/supervisord_cluster_agent.conf /home/deepfence/supervisord-temp.conf +COPY tools/apache/scope/docker/run_cluster_agent.sh /home/deepfence/ +COPY tools/apache/scope/docker/entrypoint.sh /home/deepfence/ RUN steampipe plugin install steampipe \ && steampipe plugin install kubernetes \ && git clone https://github.com/turbot/steampipe-mod-kubernetes-compliance.git -COPY scope/docker/deepfence_exe /home/deepfence/ +COPY tools/apache/scope/docker/deepfence_exe /home/deepfence/ +COPY deepfenced /bin/deepfenced USER root diff --git a/deepfence_bootstrapper/Makefile b/deepfence_bootstrapper/Makefile index 0268846003..f1149ecd60 100644 --- a/deepfence_bootstrapper/Makefile +++ b/deepfence_bootstrapper/Makefile @@ -7,10 +7,11 @@ local: deepfence_bootstrapper prepare: docker run --rm -i -v $(ROOT_MAKEFILE_DIR):/src:rw -v /tmp/go:/go:rw deepfenceio/deepfence_builder_ce:$(DF_IMG_TAG) bash -c 'cd /src/deepfence_bootstrapper && make deepfence_bootstrapper' -tidy: +vendor: go.mod $(shell find ../deepfence_utils -name '*.go') go mod tidy -v + go mod vendor -deepfence_bootstrapper: tidy $(shell find . -name '*.go') +deepfence_bootstrapper: vendor $(shell find . -name '*.go') CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w -X main.Version=${VERSION} -extldflags=-static" -buildvcs=false -upx ./deepfence_bootstrapper diff --git a/deepfence_bootstrapper/assets/config.ini b/deepfence_bootstrapper/assets/config.ini index ffcf923a4b..05e3013175 100644 --- a/deepfence_bootstrapper/assets/config.ini +++ b/deepfence_bootstrapper/assets/config.ini @@ -3,8 +3,8 @@ maxcpu = 10 maxmem = 100000 [process:fluentbit] -path=$DF_INSTALL_DIR/opt/td-agent-bit/bin/td-agent-bit -command=$DF_INSTALL_DIR/opt/td-agent-bit/bin/td-agent-bit -c /etc/td-agent-bit/td-agent-bit.conf +command=/bin/bash -c "$DF_INSTALL_DIR/opt/td-agent-bit/bin/fluent-bit -c /etc/td-agent-bit/td-agent-bit.conf" +path=$DF_INSTALL_DIR/opt/td-agent-bit/bin/fluent-bit autostart=true autorestart=true diff --git a/deepfence_bootstrapper/supervisor/process.go b/deepfence_bootstrapper/supervisor/process.go index 4ee262d415..8ce9843903 100644 --- a/deepfence_bootstrapper/supervisor/process.go +++ b/deepfence_bootstrapper/supervisor/process.go @@ -91,10 +91,10 @@ func startLogging(name string, cmd *exec.Cmd) { log.Error().Msgf("Cannot start logging: %v", err) return } - scanner := bufio.NewScanner(cmdReader) go func() { defer f.Close() for { + scanner := bufio.NewScanner(cmdReader) for scanner.Scan() { m := scanner.Bytes() _, err := f.Write(m) diff --git a/deepfence_server/Makefile b/deepfence_server/Makefile index fb9ab6232e..8bc225d557 100644 --- a/deepfence_server/Makefile +++ b/deepfence_server/Makefile @@ -6,10 +6,11 @@ image: docker run --rm -i -v $(ROOT_MAKEFILE_DIR):/src:rw -v /tmp/go:/go:rw deepfenceio/deepfence_builder_ce:$(DF_IMG_TAG) bash -c 'cd /src/deepfence_server && make deepfence_server' docker build -f ./Dockerfile -t $(IMAGE_REPOSITORY)/deepfence_server_ce:$(DF_IMG_TAG) .. -tidy: +vendor: go.mod $(shell find ../deepfence_utils -name '*.go') go mod tidy -v + go mod vendor -deepfence_server: tidy $(shell find . -name '*.go') +deepfence_server: vendor $(shell find . -name '*.go') go build -buildvcs=false clean: diff --git a/deepfence_worker/Makefile b/deepfence_worker/Makefile index 75335dc2ba..8cc24d476f 100644 --- a/deepfence_worker/Makefile +++ b/deepfence_worker/Makefile @@ -6,10 +6,11 @@ image: docker run --rm -i -v $(ROOT_MAKEFILE_DIR):/src:rw -v /tmp/go:/go:rw deepfenceio/deepfence_builder_ce:$(DF_IMG_TAG) bash -c 'cd /src/deepfence_worker && make deepfence_worker' docker build -f ./Dockerfile --build-arg IMAGE_REPOSITORY=$(IMAGE_REPOSITORY) --build-arg DF_IMG_TAG=$(DF_IMG_TAG) -t $(IMAGE_REPOSITORY)/deepfence_worker_ce:$(DF_IMG_TAG) .. -tidy: +vendor: go.mod $(shell find ../deepfence_utils -name '*.go') go mod tidy -v + go mod vendor -deepfence_worker: tidy $(shell find . -name '*.go') +deepfence_worker: vendor $(shell find . -name '*.go') go build -buildvcs=false clean: diff --git a/docker_builders/Dockerfile-alpine b/docker_builders/Dockerfile-alpine index c28391d921..a12fae6089 100644 --- a/docker_builders/Dockerfile-alpine +++ b/docker_builders/Dockerfile-alpine @@ -55,5 +55,7 @@ ENV PKG_CONFIG_PATH=/usr/local/include/hs/:$(PKG_CONFIG_PATH) \ PKG_CONFIG_PATH=/usr/local/yara/lib/pkgconfig:$(PKG_CONFIG_PATH) \ GOWORK=off -RUN git clone https://github.com/the-tcpdump-group/libpcap -RUN cd libpcap && ./autogen.sh && ./configure && make && mv ./libpcap.a /lib +RUN mkdir /home/deepfence +COPY deepfence_agent/build_scripts/*.sh /home/deepfence/ + +RUN apk add cmake fts-dev openssl-dev openssl-libs-static diff --git a/docker_builders/Dockerfile-debian b/docker_builders/Dockerfile-debian index 7964f3b711..ac4c2f4bb0 100644 --- a/docker_builders/Dockerfile-debian +++ b/docker_builders/Dockerfile-debian @@ -1,16 +1,17 @@ ARG DF_IMG_TAG=latest ARG IMAGE_REPOSITORY=deepfenceio -FROM $IMAGE_REPOSITORY/deepfence_builder_ce:$DF_IMG_TAG AS alpine_builder -FROM golang:1.20-bullseye +FROM golang:1.20-bookworm +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update RUN apt-get install -y RUN apt-get -qq -y --no-install-recommends install \ build-essential automake libtool make gcc pkg-config libssl-dev git protoc-gen-go \ bash make git gcc libc-dev lsb-release software-properties-common libz-dev apt-utils\ protobuf-compiler libhyperscan-dev ca-certificates libpcap-dev time file shellcheck curl \ - libjansson-dev libmagic-dev + libjansson-dev libmagic-dev \ + cmake flex bison libyaml-dev RUN cd /root \ && wget https://github.com/VirusTotal/yara/archive/refs/tags/v4.3.2.tar.gz \ @@ -32,6 +33,4 @@ COPY deepfence_agent/build_scripts/*.sh /home/deepfence/ ARG DF_AGENT_SRC=/go/src/github.com/deepfence/deepfence_agent WORKDIR $DF_AGENT_SRC -COPY --from=alpine_builder /lib/libpcap.a /lib - ENV GOWORK=off