diff --git a/deepfence_server/handler/scan_reports.go b/deepfence_server/handler/scan_reports.go index ac622da1ed..b17ae60365 100644 --- a/deepfence_server/handler/scan_reports.go +++ b/deepfence_server/handler/scan_reports.go @@ -1525,7 +1525,7 @@ func (h *Handler) scanResultActionHandler(w http.ResponseWriter, r *http.Request } switch action { case "delete": - err = reportersScan.DeleteScan(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, req.ResultIDs) + err = reportersScan.DeleteScanResults(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, req.ResultIDs) if req.ScanType == string(utils.NEO4JCloudComplianceScan) { err := h.CachePostureProviders(r.Context()) if err != nil { @@ -1672,7 +1672,7 @@ func (h *Handler) scanIDActionHandler(w http.ResponseWriter, r *http.Request, ac h.AuditUserActivity(r, req.ScanType, ActionDownload, req, true) case "delete": - err = reportersScan.DeleteScan(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, []string{}) + err = reportersScan.DeleteScan(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID) if err != nil { h.respondError(err, w) return diff --git a/deepfence_server/reporters/scan/scan_reporters.go b/deepfence_server/reporters/scan/scan_reporters.go index 32ec38df3c..ece1cb6a55 100644 --- a/deepfence_server/reporters/scan/scan_reporters.go +++ b/deepfence_server/reporters/scan/scan_reporters.go @@ -472,12 +472,20 @@ func nodeType2Neo4jType(nodeType string) string { return "Container" case "image": return "ContainerImage" + case "container_image": + return "ContainerImage" case "host": return "Node" case "cluster": return "KubernetesCluster" case "cloud_account": return "CloudNode" + case "aws": + return "CloudNode" + case "gcp": + return "CloudNode" + case "azure": + return "CloudNode" } return "unknown" } diff --git a/deepfence_server/reporters/scan/scan_result_actions.go b/deepfence_server/reporters/scan/scan_result_actions.go index 6c94bb64f5..7dab7be2b5 100644 --- a/deepfence_server/reporters/scan/scan_result_actions.go +++ b/deepfence_server/reporters/scan/scan_result_actions.go @@ -7,7 +7,6 @@ import ( "time" "github.com/deepfence/ThreatMapper/deepfence_server/model" - "github.com/deepfence/ThreatMapper/deepfence_server/reporters" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" @@ -171,7 +170,8 @@ func UpdateScanResultMasked(ctx context.Context, req *model.ScanResultsMaskReque return tx.Commit(ctx) } -func DeleteScan(ctx context.Context, scanType utils.Neo4jScanType, scanID string, docIds []string) error { +// DeleteScanResults Delete selected scan results (cves, secrets, etc) +func DeleteScanResults(ctx context.Context, scanType utils.Neo4jScanType, scanID string, nodeIDs []string) error { driver, err := directory.Neo4jClient(ctx) if err != nil { return err @@ -186,32 +186,25 @@ func DeleteScan(ctx context.Context, scanType utils.Neo4jScanType, scanID string } defer tx.Close(ctx) - if len(docIds) > 0 { - _, err = tx.Run(ctx, ` + _, err = tx.Run(ctx, ` MATCH (m:`+string(scanType)+`) -[r:DETECTED]-> (n) WHERE n.node_id IN $node_ids AND m.node_id = $scan_id - DELETE r`, map[string]interface{}{"node_ids": docIds, "scan_id": scanID}) - if err != nil { - return err - } - } else { - _, err = tx.Run(ctx, ` - MATCH (m:`+string(scanType)+`{node_id: $scan_id}) - OPTIONAL MATCH (m)-[r:DETECTED]-> (n:`+utils.ScanTypeDetectedNode[scanType]+`) - DETACH DELETE m,r`, map[string]interface{}{"scan_id": scanID}) - if err != nil { - return err - } + DELETE r`, map[string]interface{}{"node_ids": nodeIDs, "scan_id": scanID}) + if err != nil { + return err } + err = tx.Commit(ctx) if err != nil { return err } + tx2, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) if err != nil { return err } defer tx2.Close(ctx) + // Delete results which are not part of any scans now _, err = tx2.Run(ctx, `MATCH (n:`+utils.ScanTypeDetectedNode[scanType]+`) @@ -224,108 +217,133 @@ func DeleteScan(ctx context.Context, scanType utils.Neo4jScanType, scanID string if err != nil { return err } - if scanType == utils.NEO4JVulnerabilityScan { - tx3, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) - if err != nil { - return err - } - defer tx3.Close(ctx) - _, err = tx3.Run(ctx, - `MATCH (n:`+reporters.ScanResultMaskNode[scanType]+`) - WHERE not (n)<-[:IS]-(:`+utils.ScanTypeDetectedNode[scanType]+`) - DETACH DELETE (n)`, map[string]interface{}{}) - if err != nil { - return err - } - err = tx3.Commit(ctx) - if err != nil { - return err - } - removeSBOM := false - if len(docIds) > 0 { - // This means we are deleting some of scan results - removeSBOM, err = checkForSBMORemoval(ctx, scanID) - if err != nil { - log.Error().Msgf(err.Error()) - return err - } - } else { - // This means we are deleting entire scan - removeSBOM = true - } + return nil +} - // remove sbom - if removeSBOM { - mc, err := directory.FileServerClient(ctx) - if err != nil { - log.Error().Err(err).Msg("failed to get minio client") - return err - } - sbomFile := path.Join("/sbom", utils.ScanIDReplacer.Replace(scanID)+".json.gz") - err = mc.DeleteFile(ctx, sbomFile, true, minio.RemoveObjectOptions{ForceDelete: true}) - if err != nil { - log.Error().Err(err).Msgf("failed to delete sbom for scan id %s", scanID) - return err - } - runtimeSbomFile := path.Join("/sbom", "runtime-"+utils.ScanIDReplacer.Replace(scanID)+".json") - err = mc.DeleteFile(ctx, runtimeSbomFile, true, minio.RemoveObjectOptions{ForceDelete: true}) - if err != nil { - log.Error().Err(err).Msgf("failed to delete runtime sbom for scan id %s", scanID) - return err - } - } +func getScanNodeID(ctx context.Context, res neo4j.ResultWithContext) (nodeID string, nodeType string, err error) { + rec, err := res.Single(ctx) + if err != nil { + return "", "", err + } + if rec.Values[0] != nil && rec.Values[1] != nil { + return rec.Values[0].(string), rec.Values[1].(string), nil } + return "", "", nil +} - // update nodes scan result - query := "" - switch scanType { - case utils.NEO4JVulnerabilityScan: - query = `MATCH (n) - WHERE (n:Node OR n:Container or n:ContainerImage) - AND n.vulnerability_latest_scan_id="%s" - SET n.vulnerability_latest_scan_id="", n.vulnerabilities_count=0, n.vulnerability_scan_status=""` - case utils.NEO4JSecretScan: - query = `MATCH (n) - WHERE (n:Node OR n:Container or n:ContainerImage) - AND n.secret_latest_scan_id="%s" - SET n.secret_latest_scan_id="", n.secrets_count=0, n.secret_scan_status=""` - case utils.NEO4JMalwareScan: - query = `MATCH (n) - WHERE (n:Node OR n:Container or n:ContainerImage) - AND n.malware_latest_scan_id="%s" - SET n.malware_latest_scan_id="", n.malwares_count=0, n.malware_scan_status=""` - case utils.NEO4JComplianceScan: - query = `MATCH (n) - WHERE (n:Node OR n:KubernetesCluster) - AND n.compliance_latest_scan_id="%s" - SET n.compliance_latest_scan_id="", n.compliances_count=0, n.compliance_scan_status=""` - case utils.NEO4JCloudComplianceScan: - query = `MATCH (n) - WHERE (n:CloudResource) - AND n.cloud_compliance_latest_scan_id="%s" - SET n.cloud_compliance_latest_scan_id="", n.cloud_compliances_count=0, n.cloud_compliance_scan_status=""` +// DeleteScan Delete entire scan +func DeleteScan(ctx context.Context, scanType utils.Neo4jScanType, scanID string) error { + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return err + } + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return err } + defer tx.Close(ctx) - if len(query) < 1 { - return nil + res, err := tx.Run(ctx, ` + MATCH (m:`+string(scanType)+`{node_id: $scan_id})-[:SCANNED]-> (n) + RETURN n.node_id, n.node_type`, map[string]interface{}{"scan_id": scanID}) + if err != nil { + return err + } + nodeID, nodeType, err := getScanNodeID(ctx, res) + if err != nil { + // This error can be ignored + log.Warn().Msg(err.Error()) + } + + _, err = tx.Run(ctx, ` + MATCH (m:`+string(scanType)+`{node_id: $scan_id}) + OPTIONAL MATCH (m)-[r:DETECTED]-> (n:`+utils.ScanTypeDetectedNode[scanType]+`) + DETACH DELETE m,r`, map[string]interface{}{"scan_id": scanID}) + if err != nil { + return err + } + err = tx.Commit(ctx) + if err != nil { + return err } - tx4, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + tx2, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) if err != nil { return err } - defer tx4.Close(ctx) + defer tx2.Close(ctx) - _, err = tx4.Run(ctx, fmt.Sprintf(query, scanID), map[string]interface{}{}) + // Delete results which are not part of any scans now + _, err = tx2.Run(ctx, + `MATCH (n:`+utils.ScanTypeDetectedNode[scanType]+`) + WHERE not (n)<-[:DETECTED]-(:`+string(scanType)+`) + DETACH DELETE (n)`, map[string]interface{}{}) if err != nil { return err } - err = tx4.Commit(ctx) + err = tx2.Commit(ctx) if err != nil { return err } + // Reset node's latest_scan_id to the previous scan id, if any + if nodeID != "" && nodeType != "" { + latestScanIDField := ingestersUtil.LatestScanIDField[scanType] + scanStatusField := ingestersUtil.ScanStatusField[scanType] + scanCountField := ingestersUtil.ScanCountField[scanType] + + query := `MATCH (m:` + nodeType2Neo4jType(nodeType) + `{node_id:"` + nodeID + `"}) + SET m.` + latestScanIDField + `="", m.` + scanCountField + `=0, m.` + scanStatusField + `="" + WITH m + OPTIONAL MATCH (s:` + string(scanType) + `) - [:SCANNED] -> (m) + WITH max(s.updated_at) as most_recent + MATCH (m) <-[:SCANNED]- (s:` + string(scanType) + `{updated_at: most_recent})-[:DETECTED]->(c:` + utils.ScanTypeDetectedNode[scanType] + `) + WITH s, m, count(distinct c) as scan_count + SET m.` + latestScanIDField + `=s.node_id, m.` + scanCountField + `=scan_count, m.` + scanStatusField + `=s.status` + + log.Debug().Msgf("Query to reset scan status: %v", query) + + tx4, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return err + } + defer tx4.Close(ctx) + + _, err = tx4.Run(ctx, query, map[string]interface{}{}) + if err != nil { + return err + } + err = tx4.Commit(ctx) + if err != nil { + return err + } + } + + if scanType == utils.NEO4JVulnerabilityScan { + mc, err := directory.FileServerClient(ctx) + if err != nil { + log.Error().Err(err).Msg("failed to get minio client") + return err + } + sbomFile := path.Join("/sbom", utils.ScanIDReplacer.Replace(scanID)+".json.gz") + err = mc.DeleteFile(ctx, sbomFile, true, minio.RemoveObjectOptions{ForceDelete: true}) + if err != nil { + log.Error().Err(err).Msgf("failed to delete sbom for scan id %s", scanID) + return err + } + runtimeSbomFile := path.Join("/sbom", "runtime-"+utils.ScanIDReplacer.Replace(scanID)+".json") + err = mc.DeleteFile(ctx, runtimeSbomFile, true, minio.RemoveObjectOptions{ForceDelete: true}) + if err != nil { + log.Error().Err(err).Msgf("failed to delete runtime sbom for scan id %s", scanID) + return err + } + } + return nil } @@ -542,38 +560,3 @@ func GetSelectedScanResults[T any](ctx context.Context, scanType utils.Neo4jScan return res, common, err } - -func checkForSBMORemoval(ctx context.Context, scanID string) (bool, error) { - driver, err := directory.Neo4jClient(ctx) - if err != nil { - return false, err - } - - session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) - defer session.Close(ctx) - - txCount, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) - if err != nil { - log.Error().Msgf(err.Error()) - return false, err - } - defer txCount.Close(ctx) - - countRes, err := txCount.Run(ctx, - `MATCH (n:VulnerabilityScan{node_id: $scan_id}) -[:DETECTED]-> (v:Vulnerability) - RETURN COUNT(v) <> 0 AS Exists`, - map[string]interface{}{"scan_id": scanID}) - if err != nil { - log.Error().Msgf(err.Error()) - return false, err - } - - countRec, err := countRes.Single(ctx) - if err != nil { - log.Error().Msgf(err.Error()) - return false, err - } - exists := countRec.Values[0].(bool) - removeSBOM := !exists - return removeSBOM, err -} diff --git a/deepfence_worker/tasks/scans/bulk_delete.go b/deepfence_worker/tasks/scans/bulk_delete.go index a7bc9be33c..d95aeaf600 100644 --- a/deepfence_worker/tasks/scans/bulk_delete.go +++ b/deepfence_worker/tasks/scans/bulk_delete.go @@ -35,7 +35,7 @@ func BulkDeleteScans(ctx context.Context, task *asynq.Task) error { for _, s := range scansList.ScansInfo { log.Info().Msgf("delete scan %s %s", req.ScanType, s.ScanID) - err = reporters_scan.DeleteScan(ctx, scanType, s.ScanID, []string{}) + err = reporters_scan.DeleteScan(ctx, scanType, s.ScanID) if err != nil { log.Error().Err(err).Msgf("failed to delete scan id %s", s.ScanID) continue diff --git a/deepfence_worker/tasks/scans/delete_cloud_accounts.go b/deepfence_worker/tasks/scans/delete_cloud_accounts.go index 158db591b7..4b2b45fe6d 100644 --- a/deepfence_worker/tasks/scans/delete_cloud_accounts.go +++ b/deepfence_worker/tasks/scans/delete_cloud_accounts.go @@ -106,7 +106,7 @@ func deleteScans(ctx context.Context, accountID string) error { defer log.Info().Msgf("deleted %d scans for account %s", len(scans.ScansInfo), accountID) for _, s := range scans.ScansInfo { - err := reportersScan.DeleteScan(ctx, utils.NEO4JCloudComplianceScan, s.ScanID, []string{}) + err := reportersScan.DeleteScan(ctx, utils.NEO4JCloudComplianceScan, s.ScanID) if err != nil { log.Error().Err(err).Msgf("failed to delete scan id %s", s.ScanID) }