WIP: add vulnerability scanner (#733)

deepfence · Dec 22, 2022 · 9d948de · 9d948de
1 parent a9424ec
commit 9d948de
Show file tree

Hide file tree

Showing 12 changed files with 433 additions and 99 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,11 +1,11 @@
 [submodule "deepfence_agent/plugins/agent-plugins-grpc"]
 	path = deepfence_agent/plugins/agent-plugins-grpc
 	url = https://github.com/deepfence/agent-plugins-grpc
-	branch = master
+	branch = package-scanner
 [submodule "deepfence_agent/plugins/package-scanner"]
 	path = deepfence_agent/plugins/package-scanner
 	url = https://github.com/deepfence/package-scanner
-	branch = kafka-rest
+	branch = merge-vulnerability-mapper
 [submodule "deepfence_agent/plugins/SecretScanner"]
 	path = deepfence_agent/plugins/SecretScanner
 	url = https://github.com/deepfence/SecretScanner

diff --git a/deepfence_agent/tools/apache/fluentbit/td-agent-bit.conf b/deepfence_agent/tools/apache/fluentbit/td-agent-bit.conf
@@ -81,6 +81,30 @@
     #
     storage.backlog.mem_limit 5M
 
+[INPUT]
+    Name tail
+    Path ${DF_INSTALL_DIR}/var/log/fenced/vulnerability-scan/*.log
+    Tag vulnerability-scan
+    storage.type filesystem
+    Buffer_Chunk_Size 4K
+    Mem_Buf_Limit 5MB
+    Refresh_Interval 10
+    Skip_Long_Lines On
+    DB ${DF_INSTALL_DIR}/home/deepfence/fluentbit/vulnerability-scan.db
+    Parser json
+
+[INPUT]
+    Name tail
+    Path ${DF_INSTALL_DIR}/var/log/fenced/vulnerability-scan-log/*.log
+    Tag vulnerability-scan-log
+    storage.type filesystem
+    Buffer_Chunk_Size 4K
+    Mem_Buf_Limit 5MB
+    Refresh_Interval 10
+    Skip_Long_Lines On
+    DB ${DF_INSTALL_DIR}/home/deepfence/fluentbit/vulnerability-scan-scan.db
+    Parser json
+
 [INPUT]
     Name tail
     Path ${DF_INSTALL_DIR}/var/log/fenced/secret-scan/*.log
@@ -153,6 +177,30 @@
     DB ${DF_INSTALL_DIR}/home/deepfence/fluentbit/malware-scan-log.db
     Parser json
 
+[OUTPUT]
+    Name  deepfence
+    Match vulnerability-scan
+    Id vulnerability-scan
+    Schema https
+    Console_host ${MGMT_CONSOLE_URL}
+    Console_port ${MGMT_CONSOLE_PORT}
+    Path /deepfence/ingest/vulnerabilities
+    Token ${DEEPFENCE_KEY}
+    #cert_file ${DF_INSTALL_DIR}/etc/td-agent-bit/fluentbit-client.crt
+    #key_file ${DF_INSTALL_DIR}/etc/td-agent-bit/fluentbit-client.key
+
+[OUTPUT]
+    Name  deepfence
+    Match vulnerability-scan-log
+    Id vulnerability-scan-log
+    Schema https
+    Console_host ${MGMT_CONSOLE_URL}
+    Console_port ${MGMT_CONSOLE_PORT}
+    Path /deepfence/ingest/vulnerabilities-scan-log
+    Token ${DEEPFENCE_KEY}
+    #cert_file ${DF_INSTALL_DIR}/etc/td-agent-bit/fluentbit-client.crt
+    #key_file ${DF_INSTALL_DIR}/etc/td-agent-bit/fluentbit-client.key
+
 [OUTPUT]
     Name  deepfence
     Match secret-scan

diff --git a/deepfence_agent/tools/apache/scope/probe/host/controls.go b/deepfence_agent/tools/apache/scope/probe/host/controls.go
@@ -4,13 +4,14 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
-	dfUtils "github.com/deepfence/df-utils"
-	log "github.com/sirupsen/logrus"
-	"github.com/weaveworks/scope/common/xfer"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
+
+	dfUtils "github.com/deepfence/df-utils"
+	log "github.com/sirupsen/logrus"
+	"github.com/weaveworks/scope/common/xfer"
 )
 
 // Control IDs used by the host integration.
@@ -21,11 +22,11 @@ const (
 	AddUserDefinedTags    = "host_add_user_defined_tags"
 	DeleteUserDefinedTags = "host_delete_user_defined_tags"
 	//StartSecretsScan      = "secret_scan_start"
-	secretScanSocket      = "/tmp/secretScanner.sock"
-	StartMalwareScan      = "malware_scan_start"
-	malwareScanSocket      = "/tmp/yaraHunter.sock"
-	unixProtocol          = "unix"
-	tcpProtocol           = "tcp"
+	secretScanSocket  = "/tmp/secretScanner.sock"
+	StartMalwareScan  = "malware_scan_start"
+	malwareScanSocket = "/tmp/yaraHunter.sock"
+	unixProtocol      = "unix"
+	tcpProtocol       = "tcp"
 )
 
 var (
@@ -40,7 +41,7 @@ func init() {
 func (r *Reporter) registerControls() {
 	r.handlerRegistry.Register(StartComplianceScan, r.startComplianceScan)
 	r.handlerRegistry.Register(GetLogsFromAgent, r.getLogsFromAgent)
-	r.handlerRegistry.Register(GenerateSBOM, r.handleGenerateSBOM)
+	// r.handlerRegistry.Register(GenerateSBOM, r.handleGenerateSBOM)
 	r.handlerRegistry.Register(AddUserDefinedTags, r.addUserDefinedTags)
 	r.handlerRegistry.Register(DeleteUserDefinedTags, r.deleteUserDefinedTags)
 	//r.handlerRegistry.Register(StartSecretsScan, r.startSecretsScan)

diff --git a/deepfence_agent/tools/apache/scope/probe/host/controls_linux.go b/deepfence_agent/tools/apache/scope/probe/host/controls_linux.go
@@ -7,7 +7,6 @@ import (
 	"syscall"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/weaveworks/scope/common/xfer"
 
 	"github.com/willdonnelly/passwd"
 )
@@ -89,46 +88,46 @@ func isProbeContainerized() bool {
 	return selfMountNamespaceID != statT.Ino
 }
 
-func (r *Reporter) handleGenerateSBOM(req xfer.Request) xfer.Response {
-	var imageName = "host"
-	var imageId = ""
-	var scanId = ""
-	var kubernetesClusterName = ""
-	var containerName = ""
-	var containerId = ""
-
-	if imageNameArg, ok := req.ControlArgs["image_name"]; ok {
-		imageName = imageNameArg
-	}
-	if containerNameArg, ok := req.ControlArgs["container_name"]; ok {
-		containerName = containerNameArg
-	}
-	if kubernetesClusterNameArg, ok := req.ControlArgs["kubernetes_cluster_name"]; ok {
-		kubernetesClusterName = kubernetesClusterNameArg
-	}
-	if imageIdArg, ok := req.ControlArgs["image_id"]; ok {
-		imageId = imageIdArg
-	}
-	if containerIdArg, ok := req.ControlArgs["container_id"]; ok {
-		containerId = containerIdArg
-	}
-	if imageName != "host" && imageId == "" {
-		return xfer.ResponseErrorf("image_id is required for container/image vulnerability scan")
-	}
-	scanType := "all"
-	if scanTypeArg, ok := req.ControlArgs["scan_type"]; ok {
-		scanType = scanTypeArg
-	}
-	if scanIdArg, ok := req.ControlArgs["scan_id"]; ok {
-		scanId = scanIdArg
-	}
-	log.Infof("uploading %s tar to console...", imageName)
-	// call package scanner plugin
-	go func() {
-		err := GenerateSbomForVulnerabilityScan(imageName, imageId, scanId,containerId, kubernetesClusterName, containerName, scanType)
-		if err != nil {
-			log.Error(err.Error())
-		}
-	}()
-	return xfer.Response{CVEInfo: "Image upload started"}
-}
+// func (r *Reporter) handleGenerateSBOM(req xfer.Request) xfer.Response {
+// 	var imageName = "host"
+// 	var imageId = ""
+// 	var scanId = ""
+// 	var kubernetesClusterName = ""
+// 	var containerName = ""
+// 	var containerId = ""
+
+// 	if imageNameArg, ok := req.ControlArgs["image_name"]; ok {
+// 		imageName = imageNameArg
+// 	}
+// 	if containerNameArg, ok := req.ControlArgs["container_name"]; ok {
+// 		containerName = containerNameArg
+// 	}
+// 	if kubernetesClusterNameArg, ok := req.ControlArgs["kubernetes_cluster_name"]; ok {
+// 		kubernetesClusterName = kubernetesClusterNameArg
+// 	}
+// 	if imageIdArg, ok := req.ControlArgs["image_id"]; ok {
+// 		imageId = imageIdArg
+// 	}
+// 	if containerIdArg, ok := req.ControlArgs["container_id"]; ok {
+// 		containerId = containerIdArg
+// 	}
+// 	if imageName != "host" && imageId == "" {
+// 		return xfer.ResponseErrorf("image_id is required for container/image vulnerability scan")
+// 	}
+// 	scanType := "all"
+// 	if scanTypeArg, ok := req.ControlArgs["scan_type"]; ok {
+// 		scanType = scanTypeArg
+// 	}
+// 	if scanIdArg, ok := req.ControlArgs["scan_id"]; ok {
+// 		scanId = scanIdArg
+// 	}
+// 	log.Infof("uploading %s tar to console...", imageName)
+// 	// call package scanner plugin
+// 	go func() {
+// 		err := GenerateSbomForVulnerabilityScan(imageName, imageId, scanId, containerId, kubernetesClusterName, containerName, scanType)
+// 		if err != nil {
+// 			log.Error(err.Error())
+// 		}
+// 	}()
+// 	return xfer.Response{CVEInfo: "Image upload started"}
+// }