diff --git a/.gitmodules b/.gitmodules index 7de2b5bc02..f88239e4be 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,11 @@ [submodule "deepfence_agent/plugins/agent-plugins-grpc"] path = deepfence_agent/plugins/agent-plugins-grpc url = https://github.com/deepfence/agent-plugins-grpc - branch = master + branch = package-scanner [submodule "deepfence_agent/plugins/package-scanner"] path = deepfence_agent/plugins/package-scanner url = https://github.com/deepfence/package-scanner - branch = kafka-rest + branch = merge-vulnerability-mapper [submodule "deepfence_agent/plugins/SecretScanner"] path = deepfence_agent/plugins/SecretScanner url = https://github.com/deepfence/SecretScanner diff --git a/deepfence_agent/tools/apache/fluentbit/td-agent-bit.conf b/deepfence_agent/tools/apache/fluentbit/td-agent-bit.conf index e1667ad516..1d3ed75bce 100644 --- a/deepfence_agent/tools/apache/fluentbit/td-agent-bit.conf +++ b/deepfence_agent/tools/apache/fluentbit/td-agent-bit.conf @@ -81,6 +81,30 @@ # storage.backlog.mem_limit 5M +[INPUT] + Name tail + Path ${DF_INSTALL_DIR}/var/log/fenced/vulnerability-scan/*.log + Tag vulnerability-scan + storage.type filesystem + Buffer_Chunk_Size 4K + Mem_Buf_Limit 5MB + Refresh_Interval 10 + Skip_Long_Lines On + DB ${DF_INSTALL_DIR}/home/deepfence/fluentbit/vulnerability-scan.db + Parser json + +[INPUT] + Name tail + Path ${DF_INSTALL_DIR}/var/log/fenced/vulnerability-scan-log/*.log + Tag vulnerability-scan-log + storage.type filesystem + Buffer_Chunk_Size 4K + Mem_Buf_Limit 5MB + Refresh_Interval 10 + Skip_Long_Lines On + DB ${DF_INSTALL_DIR}/home/deepfence/fluentbit/vulnerability-scan-scan.db + Parser json + [INPUT] Name tail Path ${DF_INSTALL_DIR}/var/log/fenced/secret-scan/*.log @@ -153,6 +177,30 @@ DB ${DF_INSTALL_DIR}/home/deepfence/fluentbit/malware-scan-log.db Parser json +[OUTPUT] + Name deepfence + Match vulnerability-scan + Id vulnerability-scan + Schema https + Console_host ${MGMT_CONSOLE_URL} + Console_port ${MGMT_CONSOLE_PORT} + Path /deepfence/ingest/vulnerabilities + Token ${DEEPFENCE_KEY} + #cert_file ${DF_INSTALL_DIR}/etc/td-agent-bit/fluentbit-client.crt + #key_file ${DF_INSTALL_DIR}/etc/td-agent-bit/fluentbit-client.key + +[OUTPUT] + Name deepfence + Match vulnerability-scan-log + Id vulnerability-scan-log + Schema https + Console_host ${MGMT_CONSOLE_URL} + Console_port ${MGMT_CONSOLE_PORT} + Path /deepfence/ingest/vulnerabilities-scan-log + Token ${DEEPFENCE_KEY} + #cert_file ${DF_INSTALL_DIR}/etc/td-agent-bit/fluentbit-client.crt + #key_file ${DF_INSTALL_DIR}/etc/td-agent-bit/fluentbit-client.key + [OUTPUT] Name deepfence Match secret-scan diff --git a/deepfence_agent/tools/apache/scope/probe/host/controls.go b/deepfence_agent/tools/apache/scope/probe/host/controls.go index 8c554e82ac..14ee41b1d3 100644 --- a/deepfence_agent/tools/apache/scope/probe/host/controls.go +++ b/deepfence_agent/tools/apache/scope/probe/host/controls.go @@ -4,13 +4,14 @@ import ( "bufio" "encoding/json" "fmt" - dfUtils "github.com/deepfence/df-utils" - log "github.com/sirupsen/logrus" - "github.com/weaveworks/scope/common/xfer" "io/ioutil" "os" "path/filepath" "strings" + + dfUtils "github.com/deepfence/df-utils" + log "github.com/sirupsen/logrus" + "github.com/weaveworks/scope/common/xfer" ) // Control IDs used by the host integration. @@ -21,11 +22,11 @@ const ( AddUserDefinedTags = "host_add_user_defined_tags" DeleteUserDefinedTags = "host_delete_user_defined_tags" //StartSecretsScan = "secret_scan_start" - secretScanSocket = "/tmp/secretScanner.sock" - StartMalwareScan = "malware_scan_start" - malwareScanSocket = "/tmp/yaraHunter.sock" - unixProtocol = "unix" - tcpProtocol = "tcp" + secretScanSocket = "/tmp/secretScanner.sock" + StartMalwareScan = "malware_scan_start" + malwareScanSocket = "/tmp/yaraHunter.sock" + unixProtocol = "unix" + tcpProtocol = "tcp" ) var ( @@ -40,7 +41,7 @@ func init() { func (r *Reporter) registerControls() { r.handlerRegistry.Register(StartComplianceScan, r.startComplianceScan) r.handlerRegistry.Register(GetLogsFromAgent, r.getLogsFromAgent) - r.handlerRegistry.Register(GenerateSBOM, r.handleGenerateSBOM) + // r.handlerRegistry.Register(GenerateSBOM, r.handleGenerateSBOM) r.handlerRegistry.Register(AddUserDefinedTags, r.addUserDefinedTags) r.handlerRegistry.Register(DeleteUserDefinedTags, r.deleteUserDefinedTags) //r.handlerRegistry.Register(StartSecretsScan, r.startSecretsScan) diff --git a/deepfence_agent/tools/apache/scope/probe/host/controls_linux.go b/deepfence_agent/tools/apache/scope/probe/host/controls_linux.go index a81c33c3a2..0e709e6fd0 100644 --- a/deepfence_agent/tools/apache/scope/probe/host/controls_linux.go +++ b/deepfence_agent/tools/apache/scope/probe/host/controls_linux.go @@ -7,7 +7,6 @@ import ( "syscall" log "github.com/sirupsen/logrus" - "github.com/weaveworks/scope/common/xfer" "github.com/willdonnelly/passwd" ) @@ -89,46 +88,46 @@ func isProbeContainerized() bool { return selfMountNamespaceID != statT.Ino } -func (r *Reporter) handleGenerateSBOM(req xfer.Request) xfer.Response { - var imageName = "host" - var imageId = "" - var scanId = "" - var kubernetesClusterName = "" - var containerName = "" - var containerId = "" - - if imageNameArg, ok := req.ControlArgs["image_name"]; ok { - imageName = imageNameArg - } - if containerNameArg, ok := req.ControlArgs["container_name"]; ok { - containerName = containerNameArg - } - if kubernetesClusterNameArg, ok := req.ControlArgs["kubernetes_cluster_name"]; ok { - kubernetesClusterName = kubernetesClusterNameArg - } - if imageIdArg, ok := req.ControlArgs["image_id"]; ok { - imageId = imageIdArg - } - if containerIdArg, ok := req.ControlArgs["container_id"]; ok { - containerId = containerIdArg - } - if imageName != "host" && imageId == "" { - return xfer.ResponseErrorf("image_id is required for container/image vulnerability scan") - } - scanType := "all" - if scanTypeArg, ok := req.ControlArgs["scan_type"]; ok { - scanType = scanTypeArg - } - if scanIdArg, ok := req.ControlArgs["scan_id"]; ok { - scanId = scanIdArg - } - log.Infof("uploading %s tar to console...", imageName) - // call package scanner plugin - go func() { - err := GenerateSbomForVulnerabilityScan(imageName, imageId, scanId,containerId, kubernetesClusterName, containerName, scanType) - if err != nil { - log.Error(err.Error()) - } - }() - return xfer.Response{CVEInfo: "Image upload started"} -} +// func (r *Reporter) handleGenerateSBOM(req xfer.Request) xfer.Response { +// var imageName = "host" +// var imageId = "" +// var scanId = "" +// var kubernetesClusterName = "" +// var containerName = "" +// var containerId = "" + +// if imageNameArg, ok := req.ControlArgs["image_name"]; ok { +// imageName = imageNameArg +// } +// if containerNameArg, ok := req.ControlArgs["container_name"]; ok { +// containerName = containerNameArg +// } +// if kubernetesClusterNameArg, ok := req.ControlArgs["kubernetes_cluster_name"]; ok { +// kubernetesClusterName = kubernetesClusterNameArg +// } +// if imageIdArg, ok := req.ControlArgs["image_id"]; ok { +// imageId = imageIdArg +// } +// if containerIdArg, ok := req.ControlArgs["container_id"]; ok { +// containerId = containerIdArg +// } +// if imageName != "host" && imageId == "" { +// return xfer.ResponseErrorf("image_id is required for container/image vulnerability scan") +// } +// scanType := "all" +// if scanTypeArg, ok := req.ControlArgs["scan_type"]; ok { +// scanType = scanTypeArg +// } +// if scanIdArg, ok := req.ControlArgs["scan_id"]; ok { +// scanId = scanIdArg +// } +// log.Infof("uploading %s tar to console...", imageName) +// // call package scanner plugin +// go func() { +// err := GenerateSbomForVulnerabilityScan(imageName, imageId, scanId, containerId, kubernetesClusterName, containerName, scanType) +// if err != nil { +// log.Error(err.Error()) +// } +// }() +// return xfer.Response{CVEInfo: "Image upload started"} +// } diff --git a/deepfence_agent/tools/apache/scope/probe/host/generate_sbom.go b/deepfence_agent/tools/apache/scope/probe/host/generate_sbom.go index 1a00a0d60a..82b0db4bfe 100644 --- a/deepfence_agent/tools/apache/scope/probe/host/generate_sbom.go +++ b/deepfence_agent/tools/apache/scope/probe/host/generate_sbom.go @@ -2,30 +2,68 @@ package host import ( "context" + "encoding/json" + "errors" + "fmt" "os" + "path/filepath" + "strconv" + "strings" + "github.com/Jeffail/tunny" + ctl "github.com/deepfence/ThreatMapper/deepfence_utils/controls" + log "github.com/sirupsen/logrus" scopeHostname "github.com/weaveworks/scope/common/hostname" pb "github.com/weaveworks/scope/proto" "google.golang.org/grpc" ) const ( - packageScannerSocket = "/tmp/package-scanner.sock" + packageScannerSocket = "/tmp/package-scanner.sock" + defaultVulnerabilityScanConcurrency = 5 ) var ( - scanPath = "dir:/fenced/mnt/host/" + scanPath = "dir:/fenced/mnt/host/" + grpcVulnScanWorkerPool *tunny.Pool + vulnerabilityScanFile = getDfInstallDir() + "/var/log/fenced/vulnerability-scan/vulnerability_scan.log" + vulnerabilityScanStatusFile = getDfInstallDir() + "/var/log/fenced/vulnerability-scan-log/vulnerability_scan_log.log" ) +type vulnScanParameters struct { + client pb.PackageScannerClient + req *pb.SBOMRequest + controlArgs map[string]string + hostName string +} + func init() { + os.MkdirAll(filepath.Dir(vulnerabilityScanFile), 0755) + var err error + scanConcurrency, err = strconv.Atoi(os.Getenv("VULNERABILITY_SCAN_CONCURRENCY")) + if err != nil { + scanConcurrency = defaultVulnerabilityScanConcurrency + } + grpcVulnScanWorkerPool = tunny.NewFunc(scanConcurrency, + getAndPublishVulnerabilityScanResultsWrapper) + mgmtConsoleUrl = os.Getenv("MGMT_CONSOLE_URL") + consolePort := os.Getenv("MGMT_CONSOLE_PORT") + if consolePort != "" && consolePort != "443" { + mgmtConsoleUrl += ":" + consolePort + } + deepfenceKey = os.Getenv("DEEPFENCE_KEY") if os.Getenv("DF_SERVERLESS") == "true" { - scanPath = "dir:/" + certPath = "/deepfence/etc/filebeat/filebeat.crt" + scanDir = "/" + } else { + scanDir = HostMountDir } } func createPackageScannerClient() (pb.PackageScannerClient, error) { maxMsgSize := 1024 * 1024 * 1 // 1 mb - conn, err := grpc.Dial("unix://"+packageScannerSocket, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)), + conn, err := grpc.Dial("unix://"+packageScannerSocket, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)), grpc.WithAuthority("dummy"), grpc.WithInsecure()) if err != nil { return nil, err @@ -33,8 +71,39 @@ func createPackageScannerClient() (pb.PackageScannerClient, error) { return pb.NewPackageScannerClient(conn), nil } -func GenerateSbomForVulnerabilityScan(imageName, imageId, scanId, containerId, kubernetesClusterName, containerName, scanType string) error { - ctx := context.Background() +func generateSbomRequest(req ctl.StartVulnerabilityScanRequest) (*pb.SBOMRequest, error) { + var imageName = "host" + var imageId = "" + var scanId = "" + var kubernetesClusterName = "" + var containerName = "" + var containerId = "" + + if imageNameArg, ok := req.BinArgs["image_name"]; ok { + imageName = imageNameArg + } + if containerNameArg, ok := req.BinArgs["container_name"]; ok { + containerName = containerNameArg + } + if kubernetesClusterNameArg, ok := req.BinArgs["kubernetes_cluster_name"]; ok { + kubernetesClusterName = kubernetesClusterNameArg + } + if imageIdArg, ok := req.BinArgs["image_id"]; ok { + imageId = imageIdArg + } + if containerIdArg, ok := req.BinArgs["container_id"]; ok { + containerId = containerIdArg + } + if imageName != "host" && imageId == "" { + return nil, errors.New("image_id is required for container/image vulnerability scan") + } + scanType := "all" + if scanTypeArg, ok := req.BinArgs["scan_type"]; ok { + scanType = scanTypeArg + } + if scanIdArg, ok := req.BinArgs["scan_id"]; ok { + scanId = scanIdArg + } hostName := scopeHostname.Get() var nodeType string @@ -45,16 +114,14 @@ func GenerateSbomForVulnerabilityScan(imageName, imageId, scanId, containerId, k } else { nodeType = "container_image" } - packageScannerClient, err := createPackageScannerClient() - if err != nil { - return err - } + var source string if imageName == "host" { source = scanPath } else { source = imageName } + sbomRequest := &pb.SBOMRequest{ Source: source, ScanType: scanType, @@ -67,7 +134,173 @@ func GenerateSbomForVulnerabilityScan(imageName, imageId, scanId, containerId, k RegistryId: "", ContainerId: containerId, } - _, err = packageScannerClient.GenerateSBOM(ctx, sbomRequest) + return sbomRequest, nil +} + +func StartVulnerabilityScan(req ctl.StartVulnerabilityScanRequest) error { + + sbomRequest, err := generateSbomRequest(req) + if err != nil { + return err + } + + packageScannerClient, err := createPackageScannerClient() + if err != nil { + return err + } + go grpcScanWorkerPool.Process(vulnScanParameters{ + client: packageScannerClient, + req: sbomRequest, + controlArgs: req.BinArgs, + hostName: req.Hostname, + }) + return nil +} + +func getAndPublishVulnerabilityScanResultsWrapper(scanParametersInterface interface{}) interface{} { + scanParameters, ok := scanParametersInterface.(vulnScanParameters) + if !ok { + fmt.Println("Error reading input from grpc API") + return nil + } + getAndPublishVulnerabilityScanResults(scanParameters.client, scanParameters.req, + scanParameters.controlArgs, scanParameters.hostName) + return nil +} + +func getAndPublishVulnerabilityScanResults(client pb.PackageScannerClient, req *pb.SBOMRequest, + controlArgs map[string]string, hostName string) { + var scanLog = make(map[string]interface{}) + scanLog["node_id"] = controlArgs["node_id"] + scanLog["node_type"] = controlArgs["node_type"] + scanLog["node_name"] = hostName + scanLog["container_name"] = controlArgs["container_name"] + scanLog["kubernetes_cluster_name"] = controlArgs["kubernetes_cluster_name"] + scanLog["host_name"] = hostName + scanLog["scan_id"] = controlArgs["scan_id"] + scanLog["masked"] = "false" + scanLog["scan_status"] = "IN_PROGRESS" + scanLog["time_stamp"] = getTimestamp() + scanLog["@timestamp"] = getCurrentTime() + + byteJson, err := json.Marshal(scanLog) + if err != nil { + log.Errorf("err marshalling json: %s", err) + return + } + + err = writeVulnScanDataToFile(string(byteJson), vulnerabilityScanStatusFile) + if err != nil { + log.Errorf("error in sending data to mark in progress: %s" + err.Error()) + } + + res, err := client.GenerateSBOM(context.Background(), req) + if err != nil { + scanLog["scan_status"] = "ERROR" + scanLog["scan_message"] = err.Error() + scanLog["time_stamp"] = getTimestamp() + scanLog["@timestamp"] = getCurrentTime() + byteJson, err = json.Marshal(scanLog) + if err != nil { + log.Errorf("error marshalling json: %s", err) + return + } + writeVulnScanDataToFile(string(byteJson), vulnerabilityScanStatusFile) + return + } + + sbom, err := os.ReadFile(res.GetSbomPath()) + if err != nil { + scanLog["scan_status"] = "ERROR" + scanLog["scan_message"] = err.Error() + scanLog["time_stamp"] = getTimestamp() + scanLog["@timestamp"] = getCurrentTime() + byteJson, err = json.Marshal(scanLog) + if err != nil { + log.Errorf("error marshalling json: %s", err) + return + } + writeVulnScanDataToFile(string(byteJson), vulnerabilityScanStatusFile) + return + } + defer os.Remove(res.GetSbomPath()) + + sbomData := make(map[string]interface{}) + if err := json.Unmarshal(sbom, &sbomData); err != nil { + scanLog["scan_status"] = "ERROR" + scanLog["scan_message"] = err.Error() + scanLog["time_stamp"] = getTimestamp() + scanLog["@timestamp"] = getCurrentTime() + byteJson, err = json.Marshal(scanLog) + if err != nil { + log.Errorf("error marshalling json: %s", err) + return + } + writeScanDataToFile(string(byteJson), vulnerabilityScanStatusFile) + return + } + + var scanDoc = make(map[string]interface{}) + scanDoc["node_id"] = controlArgs["node_id"] + scanDoc["node_type"] = controlArgs["node_type"] + scanDoc["node_name"] = hostName + scanDoc["host_name"] = hostName + scanDoc["scan_id"] = controlArgs["scan_id"] + scanDoc["container_name"] = controlArgs["container_name"] + scanDoc["kubernetes_cluster_name"] = controlArgs["kubernetes_cluster_name"] + scanDoc["sbom"] = sbomData + byteJson, err = json.Marshal(scanDoc) + if err != nil { + scanLog["scan_status"] = "ERROR" + scanLog["scan_message"] = err.Error() + scanLog["time_stamp"] = getTimestamp() + scanLog["@timestamp"] = getCurrentTime() + byteJson, err = json.Marshal(scanLog) + if err != nil { + log.Errorf("error marshalling json: %s", err) + return + } + writeVulnScanDataToFile(string(byteJson), vulnerabilityScanStatusFile) + return + } + err = writeVulnScanDataToFile(string(byteJson), vulnerabilityScanFile) + if err != nil { + scanLog["scan_status"] = "ERROR" + scanLog["scan_message"] = err.Error() + scanLog["time_stamp"] = getTimestamp() + scanLog["@timestamp"] = getCurrentTime() + byteJson, err = json.Marshal(scanLog) + if err != nil { + log.Errorf("error marshalling json: %s", err) + return + } + writeVulnScanDataToFile(string(byteJson), vulnerabilityScanStatusFile) + return + } + + scanLog["scan_status"] = "COMPLETE" + scanLog["time_stamp"] = getTimestamp() + scanLog["@timestamp"] = getCurrentTime() + byteJson, err = json.Marshal(scanLog) + if err != nil { + log.Errorf("error marshalling json: %s", err) + return + } + err = writeVulnScanDataToFile(string(byteJson), vulnerabilityScanStatusFile) + if err != nil { + log.Errorf("error in sending data %s", err.Error()) + } + +} + +func writeVulnScanDataToFile(msg string, filename string) error { + out, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return err + } + defer out.Close() + + _, err = out.WriteString(strings.Replace(msg, "\n", " ", -1) + "\n") if err != nil { return err } diff --git a/deepfence_agent/tools/apache/scope/probe/host/malware_scanner.go b/deepfence_agent/tools/apache/scope/probe/host/malware_scanner.go index 1c4132be54..acca94c77a 100644 --- a/deepfence_agent/tools/apache/scope/probe/host/malware_scanner.go +++ b/deepfence_agent/tools/apache/scope/probe/host/malware_scanner.go @@ -19,9 +19,9 @@ import ( "time" "github.com/Jeffail/tunny" + log "github.com/sirupsen/logrus" "github.com/weaveworks/scope/common/xfer" pb "github.com/weaveworks/scope/proto" - log "github.com/sirupsen/logrus" "google.golang.org/grpc" ) @@ -57,7 +57,8 @@ func init() { if err != nil { MalwareScanConcurrency = defaultMalwareScanConcurrency } - MalwareGrpcScanWorkerPool = tunny.NewFunc(MalwareScanConcurrency, getAndPublishMalwareScanResultsWrapper) + MalwareGrpcScanWorkerPool = tunny.NewFunc(MalwareScanConcurrency, + getAndPublishMalwareScanResultsWrapper) MalwareMgmtConsoleUrl = os.Getenv("MGMT_CONSOLE_URL") consolePort := os.Getenv("MGMT_CONSOLE_PORT") if consolePort != "" && consolePort != "443" { @@ -95,7 +96,7 @@ func (r *Reporter) startMalwareScan(req xfer.Request) xfer.Response { Image: &pb.MalwareDockerImage{Id: imageId, Name: imageName}, }} } else if nodeType == nodeTypeHost { - log.Info("malware scan started final " ,nodeType,MalwareScanDir,HostMountDir,"hello") + log.Info("malware scan started final ", nodeType, MalwareScanDir, HostMountDir, "hello") greq = pb.MalwareRequest{Input: &pb.MalwareRequest_Path{Path: MalwareScanDir}} } ssClient, err := newMalwareScannerClient() @@ -172,9 +173,9 @@ func getAndPublishMalwareScanResults(client pb.MalwareScannerClient, req pb.Malw if err != nil { fmt.Println("Error in sending data to malwareScanLogsIndex to mark in progress:" + err.Error()) } - log.Info("started conrext background",context.Background(), req) + log.Info("started conrext background", context.Background(), req) res, err := client.FindMalwareInfo(context.Background(), &req) - log.Error("the error is",err) + log.Error("the error is", err) if req.GetPath() != "" && err == nil && res != nil { if MalwareScanDir == HostMountDir { for _, malware := range res.Malwares { @@ -220,11 +221,11 @@ func getAndPublishMalwareScanResults(client pb.MalwareScannerClient, req pb.Malw log.Error("Error in marshalling malware result object to json:" + err.Error()) return } - // byteJson := formatToKafka(malwareScanDoc) - err = writeMalwareScanDataToFile(string(byteJson), malwareScanIndexName) - if err != nil { - fmt.Println("Error in sending data to malwareScanIndex:" + err.Error()) - } + // byteJson := formatToKafka(malwareScanDoc) + err = writeMalwareScanDataToFile(string(byteJson), malwareScanIndexName) + if err != nil { + fmt.Println("Error in sending data to malwareScanIndex:" + err.Error()) + } } if err == nil { malwareScanLogDoc["scan_status"] = "COMPLETE" diff --git a/deepfence_agent/tools/apache/scope/probe/host/secret_scanner.go b/deepfence_agent/tools/apache/scope/probe/host/secret_scanner.go index e910923dc3..5eb146c157 100644 --- a/deepfence_agent/tools/apache/scope/probe/host/secret_scanner.go +++ b/deepfence_agent/tools/apache/scope/probe/host/secret_scanner.go @@ -56,7 +56,8 @@ func init() { if err != nil { scanConcurrency = defaultScanConcurrency } - grpcScanWorkerPool = tunny.NewFunc(scanConcurrency, getAndPublishSecretScanResultsWrapper) + grpcScanWorkerPool = tunny.NewFunc(scanConcurrency, + getAndPublishSecretScanResultsWrapper) mgmtConsoleUrl = os.Getenv("MGMT_CONSOLE_URL") consolePort := os.Getenv("MGMT_CONSOLE_PORT") if consolePort != "" && consolePort != "443" { @@ -110,8 +111,8 @@ func getAndPublishSecretScanResultsWrapper(scanParametersInterface interface{}) fmt.Println("Error reading input from grpc API") return nil } - getAndPublishSecretScanResults(scanParameters.client, scanParameters.req, scanParameters.controlArgs, - scanParameters.hostName) + getAndPublishSecretScanResults(scanParameters.client, scanParameters.req, + scanParameters.controlArgs, scanParameters.hostName) return nil } @@ -251,7 +252,8 @@ func writeScanDataToFile(secretScanMsg string, index string) error { } func newSecretScannerClient() (pb.SecretScannerClient, error) { - conn, err := grpc.Dial("unix://"+ebpfSocketPath, grpc.WithAuthority("dummy"), grpc.WithInsecure()) + conn, err := grpc.Dial("unix://"+ebpfSocketPath, grpc.WithAuthority("dummy"), + grpc.WithInsecure()) if err != nil { fmt.Printf("error in creating secret scanner client: %s\n", err.Error()) return nil, err diff --git a/deepfence_agent/tools/apache/scope/prog/probe.go b/deepfence_agent/tools/apache/scope/prog/probe.go index 8b3b963627..6dd90cbee4 100644 --- a/deepfence_agent/tools/apache/scope/prog/probe.go +++ b/deepfence_agent/tools/apache/scope/prog/probe.go @@ -110,30 +110,32 @@ func checkFlagsRequiringRoot(flags probeFlags) { } func setControls() { - err := controls.RegisterControl(ctl.StartVulnerabilityScan, func(req ctl.StartVulnerabilityScanRequest) error { - log.Info("Start Vulnerability Scan") - //TODO - return nil - }) + err := controls.RegisterControl(ctl.StartVulnerabilityScan, + func(req ctl.StartVulnerabilityScanRequest) error { + return host.StartVulnerabilityScan(req) + }) if err != nil { log.Errorf("set controls: %v", err) } - err = controls.RegisterControl(ctl.StartSecretScan, func(req ctl.StartSecretScanRequest) error { - return host.StartSecretsScan(req) - }) + err = controls.RegisterControl(ctl.StartSecretScan, + func(req ctl.StartSecretScanRequest) error { + return host.StartSecretsScan(req) + }) if err != nil { log.Errorf("set controls: %v", err) } - err = controls.RegisterControl(ctl.StartComplianceScan, func(req ctl.StartComplianceScanRequest) error { - log.Info("Start Compliance Scan") - //TODO - return nil - }) - err = controls.RegisterControl(ctl.StartMalwareScan, func(req ctl.StartMalwareScanRequest) error { - log.Info("Start Malware Scan") - //TODO - return nil - }) + err = controls.RegisterControl(ctl.StartComplianceScan, + func(req ctl.StartComplianceScanRequest) error { + log.Info("Start Compliance Scan") + //TODO + return nil + }) + err = controls.RegisterControl(ctl.StartMalwareScan, + func(req ctl.StartMalwareScanRequest) error { + log.Info("Start Malware Scan") + //TODO + return nil + }) if err != nil { log.Errorf("set controls: %v", err) } diff --git a/deepfence_server/handler/scan_reports.go b/deepfence_server/handler/scan_reports.go index 8e44088fd6..80af468b69 100644 --- a/deepfence_server/handler/scan_reports.go +++ b/deepfence_server/handler/scan_reports.go @@ -189,6 +189,11 @@ func (h *Handler) IngestVulnerabilityReportHandler(w http.ResponseWriter, r *htt ingest_scan_report_kafka(w, r, ingester, h.IngestChan) } +func (h *Handler) IngestVulnerabilityScanStatusHandler(w http.ResponseWriter, r *http.Request) { + ingester := ingesters.NewVulnerabilityStatusIngester() + ingest_scan_report_kafka(w, r, ingester, h.IngestChan) +} + func (h *Handler) IngestSecretReportHandler(w http.ResponseWriter, r *http.Request) { ingester := ingesters.NewSecretIngester() ingest_scan_report_kafka(w, r, ingester, h.IngestChan) diff --git a/deepfence_server/ingesters/vulnerability_ingester.go b/deepfence_server/ingesters/vulnerability_ingester.go index 0fbd9b972f..d5afb475cc 100644 --- a/deepfence_server/ingesters/vulnerability_ingester.go +++ b/deepfence_server/ingesters/vulnerability_ingester.go @@ -47,3 +47,39 @@ func (tc *VulnerabilityIngester) Ingest( return nil } + +type VulnerabilityStatusIngester struct{} + +func NewVulnerabilityStatusIngester() KafkaIngester[[]ingesters.VulnerabilityScanStatus] { + return &VulnerabilityStatusIngester{} +} + +func (tc *VulnerabilityStatusIngester) Ingest( + ctx context.Context, + statuses []ingesters.VulnerabilityScanStatus, + ingestC chan *kgo.Record, +) error { + tenantID, err := directory.ExtractNamespace(ctx) + if err != nil { + return err + } + + rh := []kgo.RecordHeader{ + {Key: "tenant_id", Value: []byte(tenantID)}, + } + + for _, c := range statuses { + cb, err := json.Marshal(c) + if err != nil { + log.Error().Msg(err.Error()) + } else { + ingestC <- &kgo.Record{ + Topic: utils.VULNERABILITY_SCAN_STATUS, + Value: cb, + Headers: rh, + } + } + } + + return nil +} diff --git a/deepfence_server/router/router.go b/deepfence_server/router/router.go index d13115dae7..d8ee99dd31 100644 --- a/deepfence_server/router/router.go +++ b/deepfence_server/router/router.go @@ -138,6 +138,7 @@ func SetupRoutes(r *chi.Mux, serverPort string, jwtSecret []byte, serveOpenapiDo r.Post("/cloud-resources", dfHandler.AuthHandler(ResourceCloudReport, PermissionIngest, dfHandler.IngestCloudResourcesReportHandler)) // below api's write to kafka r.Post("/vulnerabilities", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestVulnerabilityReportHandler)) + r.Post("/vulnerabilities-scan-logs", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestSecretScanStatusHandler)) r.Post("/secrets", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestSecretReportHandler)) r.Post("/secret-scan-logs", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestSecretScanStatusHandler)) r.Post("/compliance", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestComplianceReportHandler)) diff --git a/deepfence_utils/controls/agent.go b/deepfence_utils/controls/agent.go index 014990edfe..ee33e2e095 100644 --- a/deepfence_utils/controls/agent.go +++ b/deepfence_utils/controls/agent.go @@ -45,7 +45,13 @@ func StringToResourceType(s string) ScanResource { return -1 } -type StartVulnerabilityScanRequest struct{} +type StartVulnerabilityScanRequest struct { + ResourceId string `json:"resource_id" required:"true"` + ResourceType ScanResource `json:"resource_type" required:"true"` + BinArgs map[string]string `json:"bin_args" required:"true"` + Hostname string `json:"hostname" required:"true"` +} + type StartSecretScanRequest struct { ResourceId string `json:"resource_id" required:"true"` ResourceType ScanResource `json:"resource_type" required:"true"`