diff --git a/deepfence_server/handler/scan_reports.go b/deepfence_server/handler/scan_reports.go index 4df02b7c65..4a17e67c4e 100644 --- a/deepfence_server/handler/scan_reports.go +++ b/deepfence_server/handler/scan_reports.go @@ -1508,7 +1508,21 @@ func (h *Handler) scanResultActionHandler(w http.ResponseWriter, r *http.Request } h.AuditUserActivity(r, req.ScanType, ACTION_DELETE, req, true) case "notify": - err = reporters_scan.NotifyScanResult(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, req.ResultIDs) + if req.NotifyIndividual { + for _, resultID := range req.ResultIDs { + err = reporters_scan.NotifyScanResult(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, []string{resultID}) + if err != nil { + h.respondError(err, w) + return + } + } + } else { + err = reporters_scan.NotifyScanResult(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, req.ResultIDs) + if err != nil { + h.respondError(err, w) + return + } + } h.AuditUserActivity(r, req.ScanType, ACTION_NOTIFY, req, true) } if err != nil { @@ -1668,16 +1682,27 @@ func (h *Handler) BulkDeleteScans(w http.ResponseWriter, r *http.Request) { log.Info().Msgf("bulk delete %s scans filters %+v", req.ScanType, req.Filters) - scanType := utils.DetectedNodeScanType[req.ScanType] - scansList, err := reporters_scan.GetScansList(r.Context(), scanType, nil, req.Filters, model.FetchWindow{}) + err = h.bulkDeleteScanResults(r.Context(), req) if err != nil { - h.respondError(&ValidatorError{err: err}, w) + h.respondError(err, w) return } + h.AuditUserActivity(r, ACTION_BULK, ACTION_DELETE, req, true) + + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) bulkDeleteScanResults(ctx context.Context, req model.BulkDeleteScansRequest) error { + scanType := utils.DetectedNodeScanType[req.ScanType] + scansList, err := reporters_scan.GetScansList(ctx, scanType, nil, req.Filters, model.FetchWindow{}) + if err != nil { + return err + } + for _, s := range scansList.ScansInfo { log.Info().Msgf("delete scan %s %s", req.ScanType, s.ScanId) - err = reporters_scan.DeleteScan(r.Context(), scanType, s.ScanId, []string{}) + err = reporters_scan.DeleteScan(ctx, scanType, s.ScanId, []string{}) if err != nil { log.Error().Err(err).Msgf("failed to delete scan id %s", s.ScanId) continue @@ -1685,16 +1710,12 @@ func (h *Handler) BulkDeleteScans(w http.ResponseWriter, r *http.Request) { } if len(scansList.ScansInfo) > 0 && (scanType == utils.NEO4J_COMPLIANCE_SCAN || scanType == utils.NEO4J_CLOUD_COMPLIANCE_SCAN) { - err = h.CachePostureProviders(r.Context()) + err = h.CachePostureProviders(ctx) if err != nil { - h.respondError(err, w) - return + return err } } - - h.AuditUserActivity(r, ACTION_BULK, ACTION_DELETE, req, true) - - w.WriteHeader(http.StatusNoContent) + return nil } func (h *Handler) GetAllNodesInScanResultBulkHandler(w http.ResponseWriter, r *http.Request) { @@ -2134,6 +2155,7 @@ func StartMultiScan(ctx context.Context, func StartMultiCloudComplianceScan(ctx context.Context, reqs []model.NodeIdentifier, benchmarkTypes []string) ([]string, string, error) { driver, err := directory.Neo4jClient(ctx) + if err != nil { return nil, "", err } diff --git a/deepfence_server/model/cloud_node.go b/deepfence_server/model/cloud_node.go index 53156c9be6..cf894a544e 100644 --- a/deepfence_server/model/cloud_node.go +++ b/deepfence_server/model/cloud_node.go @@ -367,7 +367,6 @@ func GetCloudComplianceNodesList(ctx context.Context, cloudProvider string, fw F isOrgListing := false neo4jNodeType := "CloudNode" passStatus := []string{"ok", "info", "skip"} - scanType := utils.NEO4J_CLOUD_COMPLIANCE_SCAN if cloudProvider == PostureProviderAWSOrg { cloudProvider = PostureProviderAWS isOrgListing = true @@ -381,85 +380,38 @@ func GetCloudComplianceNodesList(ctx context.Context, cloudProvider string, fw F passStatus = []string{"warn", "pass"} } var res neo4j.Result - if isOrgListing { - res, err = tx.Run(fmt.Sprintf(` - MATCH (m:%s{cloud_provider:$cloud_provider+'_org'}) -[:IS_CHILD]-> (n:%s{cloud_provider: $cloud_provider}) - WITH DISTINCT m.node_id AS node_id, m.node_name AS node_name, m.updated_at AS updated_at, m.active AS active - UNWIND node_id AS x - OPTIONAL MATCH (m:%s{cloud_provider:$cloud_provider+'_org', node_id: x}) -[:IS_CHILD]-> (n:%s{cloud_provider: $cloud_provider})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c:CloudCompliance) - WITH x, node_name, updated_at, COUNT(c) AS total_compliance_count, active - OPTIONAL MATCH (m:%s{cloud_provider:$cloud_provider+'_org', node_id: x}) -[:IS_CHILD]-> (n:%s{cloud_provider: $cloud_provider})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c1:CloudCompliance) - WHERE c1.status IN $pass_status - WITH x, node_name, $cloud_provider+'_org' AS cloud_provider, CASE WHEN total_compliance_count = 0 THEN 0.0 ELSE COUNT(c1.status)*100.0/total_compliance_count END AS compliance_percentage, updated_at, active - CALL { - WITH x - OPTIONAL MATCH (m:%s{cloud_provider:$cloud_provider+'_org', node_id: x}) -[:IS_CHILD]-> (n:%s{cloud_provider: $cloud_provider})<-[:SCANNED]-(s1:%s) - RETURN s1.node_id AS last_scan_id, s1.status AS last_scan_status - ORDER BY s1.updated_at DESC LIMIT 1 - } - RETURN x, node_name, cloud_provider, compliance_percentage, active, updated_at, COALESCE(last_scan_id, ''), COALESCE(last_scan_status, '') - ORDER BY updated_at`, neo4jNodeType, neo4jNodeType, neo4jNodeType, neo4jNodeType, scanType, neo4jNodeType, - neo4jNodeType, scanType, neo4jNodeType, scanType, utils.NodeTypeCloudNode)+fw.FetchWindow2CypherQuery(), - map[string]interface{}{"cloud_provider": cloudProvider, "pass_status": passStatus}) - if err != nil { - return CloudNodeAccountsListResp{Total: 0}, err - } - } else if cloudProvider == PostureProviderKubernetes || cloudProvider == PostureProviderLinux { - scanType = utils.NEO4J_COMPLIANCE_SCAN + var query string + if cloudProvider == PostureProviderKubernetes || cloudProvider == PostureProviderLinux { nonKubeFilter := "" if cloudProvider == PostureProviderLinux { nonKubeFilter = "{kubernetes_cluster_id:''}" } - res, err = tx.Run(fmt.Sprintf(` - MATCH (n:%s%s) + query = ` + MATCH (n:` + string(neo4jNodeType) + nonKubeFilter + `) WHERE n.pseudo=false - WITH n.node_id AS node_id, n.node_name AS node_name, n.updated_at AS updated_at, n.active AS active - UNWIND node_id AS x - OPTIONAL MATCH (n:%s{node_id: x})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c:Compliance) - WITH x, node_name, updated_at, COUNT(c) AS total_compliance_count, active - OPTIONAL MATCH (n:%s{node_id: x})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c1:Compliance) - WHERE c1.status IN $pass_status - WITH x, node_name, CASE WHEN total_compliance_count = 0 THEN 0.0 ELSE COUNT(c1.status)*100.0/total_compliance_count END AS compliance_percentage, updated_at, active - CALL { - WITH x - OPTIONAL MATCH (n:%s{node_id: x})<-[:SCANNED]-(s1:%s) - RETURN s1.node_id AS last_scan_id, s1.status AS last_scan_status - ORDER BY s1.updated_at DESC LIMIT 1 - } - RETURN x, node_name, $cloud_provider, compliance_percentage, active, updated_at, COALESCE(last_scan_id, ''), COALESCE(last_scan_status, '') - ORDER BY updated_at`, neo4jNodeType, nonKubeFilter, neo4jNodeType, scanType, neo4jNodeType, scanType, neo4jNodeType, scanType)+fw.FetchWindow2CypherQuery(), - map[string]interface{}{ - "cloud_provider": cloudProvider, - "pass_status": passStatus, - }) - if err != nil { - return CloudNodeAccountsListResp{Total: 0}, err - } + RETURN n.node_id, n.node_name, $cloud_provider, n.active, n.updated_at, COALESCE(n.compliance_latest_scan_id, ''), COALESCE(n.compliance_latest_scan_status, '') + ORDER BY n.updated_at` + fw.FetchWindow2CypherQuery() + } else if isOrgListing { + query = ` + MATCH (m:` + string(neo4jNodeType) + `{cloud_provider:$cloud_provider+'_org'}) -[:IS_CHILD]-> (n:` + string(neo4jNodeType) + `) + RETURN n.node_id, n.node_name, $cloud_provider, n.active, n.updated_at, COALESCE(n.cloud_compliance_latest_scan_id, ''), COALESCE(n.cloud_compliance_latest_scan_status, '') + ORDER BY n.updated_at` + fw.FetchWindow2CypherQuery() } else { - res, err = tx.Run(fmt.Sprintf(` - MATCH (n:%s{cloud_provider: $cloud_provider}) - WITH n.node_id AS node_id, n.node_name AS node_name, n.cloud_provider AS cloud_provider, n.updated_at AS updated_at, n.active AS active - UNWIND node_id AS x - OPTIONAL MATCH (n:%s{cloud_provider: $cloud_provider, node_id: x})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c:CloudCompliance) - WITH x, node_name, cloud_provider, updated_at, COUNT(c) AS total_compliance_count, active - OPTIONAL MATCH (n:%s{cloud_provider: $cloud_provider, node_id: x})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c1:CloudCompliance) - WHERE c1.status IN $pass_status - WITH x, node_name, cloud_provider, CASE WHEN total_compliance_count = 0 THEN 0.0 ELSE COUNT(c1.status)*100.0/total_compliance_count END AS compliance_percentage, updated_at, active - CALL { - WITH x - OPTIONAL MATCH (n:%s{cloud_provider: $cloud_provider, node_id: x})<-[:SCANNED]-(s1:%s) - RETURN s1.node_id AS last_scan_id, s1.status AS last_scan_status - ORDER BY s1.updated_at DESC LIMIT 1 - } - RETURN x, node_name, cloud_provider, compliance_percentage, active, updated_at, COALESCE(last_scan_id, ''), COALESCE(last_scan_status, '') - ORDER BY updated_at`, neo4jNodeType, neo4jNodeType, scanType, neo4jNodeType, scanType, neo4jNodeType, scanType)+fw.FetchWindow2CypherQuery(), - map[string]interface{}{ - "cloud_provider": cloudProvider, - "pass_status": passStatus, - }) - if err != nil { - return CloudNodeAccountsListResp{Total: 0}, err - } + query = ` + MATCH (n:` + string(neo4jNodeType) + `{cloud_provider: $cloud_provider}) + RETURN n.node_id, n.node_name, $cloud_provider, n.active, n.updated_at, COALESCE(n.cloud_compliance_latest_scan_id, ''), COALESCE(n.cloud_compliance_latest_scan_status, '') + ORDER BY n.updated_at` + fw.FetchWindow2CypherQuery() + } + + log.Debug().Msgf("posture query: %v", query) + res, err = tx.Run( + query, + map[string]interface{}{ + "cloud_provider": cloudProvider, + "pass_status": passStatus}, + ) + if err != nil { + return CloudNodeAccountsListResp{Total: 0}, err } recs, err := res.Collect() @@ -473,10 +425,10 @@ func GetCloudComplianceNodesList(ctx context.Context, cloudProvider string, fw F NodeId: rec.Values[0].(string), NodeName: rec.Values[1].(string), CloudProvider: rec.Values[2].(string), - CompliancePercentage: rec.Values[3].(float64), - Active: rec.Values[4].(bool), - LastScanId: rec.Values[6].(string), - LastScanStatus: rec.Values[7].(string), + CompliancePercentage: 0, + Active: rec.Values[3].(bool), + LastScanId: rec.Values[5].(string), + LastScanStatus: rec.Values[6].(string), } cloud_node_accounts_info = append(cloud_node_accounts_info, tmp) } @@ -527,7 +479,7 @@ func GetActiveCloudControls(ctx context.Context, complianceTypes []string, cloud var res neo4j.Result res, err = tx.Run(` MATCH (n:CloudComplianceBenchmark) -[:INCLUDES]-> (m:CloudComplianceControl) - WHERE m.active = true + WHERE m.active = true AND m.compliance_type IN $compliance_types AND n.cloud_provider = $cloud_provider RETURN n.benchmark_id, n.compliance_type, collect(m.control_id) diff --git a/deepfence_server/model/scans.go b/deepfence_server/model/scans.go index 75d9f823c3..c24682eaf3 100644 --- a/deepfence_server/model/scans.go +++ b/deepfence_server/model/scans.go @@ -136,10 +136,10 @@ type ScanResultsMaskRequest struct { } type ScanResultsActionRequest struct { - ScanID string `json:"scan_id" validate:"required" required:"true"` - ResultIDs []string `json:"result_ids" validate:"required,gt=0,dive,min=1" required:"true"` - ScanType string `json:"scan_type" validate:"required,oneof=SecretScan VulnerabilityScan MalwareScan ComplianceScan CloudComplianceScan" required:"true" enum:"SecretScan,VulnerabilityScan,MalwareScan,ComplianceScan,CloudComplianceScan"` - //utils.Neo4jScanType + ScanID string `json:"scan_id" validate:"required" required:"true"` + ResultIDs []string `json:"result_ids" validate:"required,gt=0,dive,min=1" required:"true"` + ScanType string `json:"scan_type" validate:"required,oneof=SecretScan VulnerabilityScan MalwareScan ComplianceScan CloudComplianceScan" required:"true" enum:"SecretScan,VulnerabilityScan,MalwareScan,ComplianceScan,CloudComplianceScan"` + NotifyIndividual bool `json:"notify_individual"` } type DownloadReportResponse struct { diff --git a/deepfence_server/reporters/scan/scan_result_actions.go b/deepfence_server/reporters/scan/scan_result_actions.go index 3795b808e6..403f1010af 100644 --- a/deepfence_server/reporters/scan/scan_result_actions.go +++ b/deepfence_server/reporters/scan/scan_result_actions.go @@ -278,10 +278,10 @@ func StopScan(ctx context.Context, scanType, scanId string) error { if _, err = tx.Run(fmt.Sprintf(query, scanType), map[string]interface{}{ - "scan_id": scanId, - "starting": utils.SCAN_STATUS_STARTING, - "in_progress": utils.SCAN_STATUS_INPROGRESS, - "cancel_pending": utils.SCAN_STATUS_CANCEL_PENDING, + "scan_id": scanId, + "starting": utils.SCAN_STATUS_STARTING, + "in_progress": utils.SCAN_STATUS_INPROGRESS, + "cancelling": utils.SCAN_STATUS_CANCEL_PENDING, }); err != nil { log.Error().Msgf("StopScan: Error in setting the state in neo4j: %v", err) return err @@ -291,5 +291,105 @@ func StopScan(ctx context.Context, scanType, scanId string) error { } func NotifyScanResult(ctx context.Context, scanType utils.Neo4jScanType, scanId string, scanIDs []string) error { + switch scanType { + case utils.NEO4J_VULNERABILITY_SCAN: + res, common, err := GetSelectedScanResults[model.Vulnerability](ctx, scanType, scanId, scanIDs) + if err != nil { + return err + } + if err := Notify[model.Vulnerability](ctx, res, common, string(scanType)); err != nil { + return err + } + case utils.NEO4J_SECRET_SCAN: + res, common, err := GetSelectedScanResults[model.Secret](ctx, scanType, scanId, scanIDs) + if err != nil { + return err + } + if err := Notify[model.Secret](ctx, res, common, string(scanType)); err != nil { + return err + } + case utils.NEO4J_MALWARE_SCAN: + res, common, err := GetSelectedScanResults[model.Malware](ctx, scanType, scanId, scanIDs) + if err != nil { + return err + } + if err := Notify[model.Malware](ctx, res, common, string(scanType)); err != nil { + return err + } + case utils.NEO4J_COMPLIANCE_SCAN: + res, common, err := GetSelectedScanResults[model.Compliance](ctx, scanType, scanId, scanIDs) + if err != nil { + return err + } + if err := Notify[model.Compliance](ctx, res, common, string(scanType)); err != nil { + return err + } + case utils.NEO4J_CLOUD_COMPLIANCE_SCAN: + res, common, err := GetSelectedScanResults[model.CloudCompliance](ctx, scanType, scanId, scanIDs) + if err != nil { + return err + } + if err := Notify[model.CloudCompliance](ctx, res, common, string(scanType)); err != nil { + return err + } + } + return nil } + +func GetSelectedScanResults[T any](ctx context.Context, scanType utils.Neo4jScanType, scanId string, scanIDs []string) ([]T, model.ScanResultsCommon, error) { + res := []T{} + common := model.ScanResultsCommon{} + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return res, common, err + } + session := driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close() + + tx, err := session.BeginTransaction(neo4j.WithTxTimeout(15 * time.Second)) + if err != nil { + return res, common, err + } + defer tx.Close() + + query := `MATCH (n:%s) -[:DETECTED]-> (m) + WHERE m.node_id IN $scan_ids + AND n.node_id = $scan_id + RETURN m{.*}` + + result, err := tx.Run(fmt.Sprintf(query, scanType), map[string]interface{}{"scan_ids": scanIDs, "scan_id": scanId}) + if err != nil { + log.Error().Msgf("NotifyScanResult: Error in getting the scan result nodes from neo4j: %v", err) + return res, common, err + } + + recs, err := result.Collect() + if err != nil { + log.Error().Msgf("NotifyScanResult: Error in collecting the scan result nodes from neo4j: %v", err) + return res, common, err + } + + for _, rec := range recs { + var tmp T + utils.FromMap(rec.Values[0].(map[string]interface{}), &tmp) + res = append(res, tmp) + } + + ncommonres, err := tx.Run(` + MATCH (m:`+string(scanType)+`{node_id: $scan_id}) -[:SCANNED]-> (n) + RETURN n{.*, scan_id: m.node_id, updated_at:m.updated_at, created_at:m.created_at}`, + map[string]interface{}{"scan_id": scanId}) + if err != nil { + return res, common, err + } + + rec, err := ncommonres.Single() + if err != nil { + return res, common, err + } + + utils.FromMap(rec.Values[0].(map[string]interface{}), &common) + + return res, common, err +} diff --git a/deepfence_utils/utils/constants.go b/deepfence_utils/utils/constants.go index d62070cb3a..99c0c6898d 100644 --- a/deepfence_utils/utils/constants.go +++ b/deepfence_utils/utils/constants.go @@ -25,7 +25,6 @@ const ( // task names const ( - SetUpGraphDBTask = "set_up_graph_db" CleanUpGraphDBTask = "clean_up_graph_db" CleanUpPostgresqlTask = "clean_up_postgresql" CleanupDiagnosisLogs = "clean_up_diagnosis_logs" @@ -178,7 +177,6 @@ var Topics = []string{ // list of task names to create topics var Tasks = []string{ - SetUpGraphDBTask, CleanUpGraphDBTask, CleanUpPostgresqlTask, CleanupDiagnosisLogs, diff --git a/deepfence_worker/tasks/sbom/generate_sbom.go b/deepfence_worker/tasks/sbom/generate_sbom.go index 839df087ac..8b63193727 100644 --- a/deepfence_worker/tasks/sbom/generate_sbom.go +++ b/deepfence_worker/tasks/sbom/generate_sbom.go @@ -120,6 +120,9 @@ func (s SbomGenerator) GenerateSbom(msg *message.Message) ([]*message.Message, e defer func() { log.Info().Msgf("remove auth directory %s", authFile) + if authFile == "" { + return + } if err := os.RemoveAll(authFile); err != nil { log.Error().Msg(err.Error()) } @@ -166,7 +169,6 @@ func (s SbomGenerator) GenerateSbom(msg *message.Message) ([]*message.Message, e log.Error().Msg(err.Error()) statusChan <- NewSbomScanStatus(params, utils.SCAN_STATUS_FAILED, err.Error(), nil) } - return nil, nil }