On scan delete, set previous scan id on the nodes (#2099)

deepfence_server/handler/scan_reports.go

@@ -1525,7 +1525,7 @@ func (h *Handler) scanResultActionHandler(w http.ResponseWriter, r *http.Request
 	}
 	switch action {
 	case "delete":
-		err = reportersScan.DeleteScan(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, req.ResultIDs)
+		err = reportersScan.DeleteScanResults(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, req.ResultIDs)
 		if req.ScanType == string(utils.NEO4JCloudComplianceScan) {
 			err := h.CachePostureProviders(r.Context())
 			if err != nil {
@@ -1672,7 +1672,7 @@ func (h *Handler) scanIDActionHandler(w http.ResponseWriter, r *http.Request, ac
 		h.AuditUserActivity(r, req.ScanType, ActionDownload, req, true)
 
 	case "delete":
-		err = reportersScan.DeleteScan(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, []string{})
+		err = reportersScan.DeleteScan(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID)
 		if err != nil {
 			h.respondError(err, w)
 			return

deepfence_server/reporters/scan/scan_reporters.go

@@ -472,12 +472,20 @@ func nodeType2Neo4jType(nodeType string) string {
 		return "Container"
 	case "image":
 		return "ContainerImage"
+	case "container_image":
+		return "ContainerImage"
 	case "host":
 		return "Node"
 	case "cluster":
 		return "KubernetesCluster"
 	case "cloud_account":
 		return "CloudNode"
+	case "aws":
+		return "CloudNode"
+	case "gcp":
+		return "CloudNode"
+	case "azure":
+		return "CloudNode"
 	}
 	return "unknown"
 }

deepfence_server/reporters/scan/scan_result_actions.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/deepfence/ThreatMapper/deepfence_server/model"
-	"github.com/deepfence/ThreatMapper/deepfence_server/reporters"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/directory"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/utils"
@@ -171,7 +170,8 @@ func UpdateScanResultMasked(ctx context.Context, req *model.ScanResultsMaskReque
 	return tx.Commit(ctx)
 }
 
-func DeleteScan(ctx context.Context, scanType utils.Neo4jScanType, scanID string, docIds []string) error {
+// DeleteScanResults Delete selected scan results (cves, secrets, etc)
+func DeleteScanResults(ctx context.Context, scanType utils.Neo4jScanType, scanID string, nodeIDs []string) error {
 	driver, err := directory.Neo4jClient(ctx)
 	if err != nil {
 		return err
@@ -186,32 +186,25 @@ func DeleteScan(ctx context.Context, scanType utils.Neo4jScanType, scanID string
 	}
 	defer tx.Close(ctx)
 
-	if len(docIds) > 0 {
-		_, err = tx.Run(ctx, `
+	_, err = tx.Run(ctx, `
 		MATCH (m:`+string(scanType)+`) -[r:DETECTED]-> (n)
 		WHERE n.node_id IN $node_ids AND m.node_id = $scan_id
-		DELETE r`, map[string]interface{}{"node_ids": docIds, "scan_id": scanID})
-		if err != nil {
-			return err
-		}
-	} else {
-		_, err = tx.Run(ctx, `
-		MATCH (m:`+string(scanType)+`{node_id: $scan_id})
-		OPTIONAL MATCH (m)-[r:DETECTED]-> (n:`+utils.ScanTypeDetectedNode[scanType]+`)
-		DETACH DELETE m,r`, map[string]interface{}{"scan_id": scanID})
-		if err != nil {
-			return err
-		}
+		DELETE r`, map[string]interface{}{"node_ids": nodeIDs, "scan_id": scanID})
+	if err != nil {
+		return err
 	}
+
 	err = tx.Commit(ctx)
 	if err != nil {
 		return err
 	}
+
 	tx2, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second))
 	if err != nil {
 		return err
 	}
 	defer tx2.Close(ctx)
+
 	// Delete results which are not part of any scans now
 	_, err = tx2.Run(ctx,
 		`MATCH (n:`+utils.ScanTypeDetectedNode[scanType]+`)
@@ -224,108 +217,133 @@ func DeleteScan(ctx context.Context, scanType utils.Neo4jScanType, scanID string
 	if err != nil {
 		return err
 	}
-	if scanType == utils.NEO4JVulnerabilityScan {
-		tx3, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second))
-		if err != nil {
-			return err
-		}
-		defer tx3.Close(ctx)
-		_, err = tx3.Run(ctx,
-			`MATCH (n:`+reporters.ScanResultMaskNode[scanType]+`)
-			WHERE not (n)<-[:IS]-(:`+utils.ScanTypeDetectedNode[scanType]+`)
-			DETACH DELETE (n)`, map[string]interface{}{})
-		if err != nil {
-			return err
-		}
-		err = tx3.Commit(ctx)
-		if err != nil {
-			return err
-		}
 
-		removeSBOM := false
-		if len(docIds) > 0 {
-			// This means we are deleting some of scan results
-			removeSBOM, err = checkForSBMORemoval(ctx, scanID)
-			if err != nil {
-				log.Error().Msgf(err.Error())
-				return err
-			}
-		} else {
-			// This means we are deleting entire scan
-			removeSBOM = true
-		}
+	return nil
+}
 
-		// remove sbom
-		if removeSBOM {
-			mc, err := directory.FileServerClient(ctx)
-			if err != nil {
-				log.Error().Err(err).Msg("failed to get minio client")
-				return err
-			}
-			sbomFile := path.Join("/sbom", utils.ScanIDReplacer.Replace(scanID)+".json.gz")
-			err = mc.DeleteFile(ctx, sbomFile, true, minio.RemoveObjectOptions{ForceDelete: true})
-			if err != nil {
-				log.Error().Err(err).Msgf("failed to delete sbom for scan id %s", scanID)
-				return err
-			}
-			runtimeSbomFile := path.Join("/sbom", "runtime-"+utils.ScanIDReplacer.Replace(scanID)+".json")
-			err = mc.DeleteFile(ctx, runtimeSbomFile, true, minio.RemoveObjectOptions{ForceDelete: true})
-			if err != nil {
-				log.Error().Err(err).Msgf("failed to delete runtime sbom for scan id %s", scanID)
-				return err
-			}
-		}
+func getScanNodeID(ctx context.Context, res neo4j.ResultWithContext) (nodeID string, nodeType string, err error) {
+	rec, err := res.Single(ctx)
+	if err != nil {
+		return "", "", err
+	}
+	if rec.Values[0] != nil && rec.Values[1] != nil {
+		return rec.Values[0].(string), rec.Values[1].(string), nil
 	}
+	return "", "", nil
+}
 
-	// update nodes scan result
-	query := ""
-	switch scanType {
-	case utils.NEO4JVulnerabilityScan:
-		query = `MATCH (n)
-		WHERE (n:Node OR n:Container or n:ContainerImage)
-		AND n.vulnerability_latest_scan_id="%s"
-		SET n.vulnerability_latest_scan_id="", n.vulnerabilities_count=0, n.vulnerability_scan_status=""`
-	case utils.NEO4JSecretScan:
-		query = `MATCH (n)
-		WHERE (n:Node OR n:Container or n:ContainerImage)
-		AND n.secret_latest_scan_id="%s"
-		SET n.secret_latest_scan_id="", n.secrets_count=0, n.secret_scan_status=""`
-	case utils.NEO4JMalwareScan:
-		query = `MATCH (n)
-		WHERE (n:Node OR n:Container or n:ContainerImage)
-		AND n.malware_latest_scan_id="%s"
-		SET n.malware_latest_scan_id="", n.malwares_count=0, n.malware_scan_status=""`
-	case utils.NEO4JComplianceScan:
-		query = `MATCH (n)
-		WHERE (n:Node OR n:KubernetesCluster)
-		AND n.compliance_latest_scan_id="%s"
-		SET n.compliance_latest_scan_id="", n.compliances_count=0, n.compliance_scan_status=""`
-	case utils.NEO4JCloudComplianceScan:
-		query = `MATCH (n)
-		WHERE (n:CloudResource)
-		AND n.cloud_compliance_latest_scan_id="%s"
-		SET n.cloud_compliance_latest_scan_id="", n.cloud_compliances_count=0, n.cloud_compliance_scan_status=""`
+// DeleteScan Delete entire scan
+func DeleteScan(ctx context.Context, scanType utils.Neo4jScanType, scanID string) error {
+	driver, err := directory.Neo4jClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
+	defer session.Close(ctx)
+
+	tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second))
+	if err != nil {
+		return err
 	}
+	defer tx.Close(ctx)
 
-	if len(query) < 1 {
-		return nil
+	res, err := tx.Run(ctx, `
+		MATCH (m:`+string(scanType)+`{node_id: $scan_id})-[:SCANNED]-> (n)
+		RETURN n.node_id, n.node_type`, map[string]interface{}{"scan_id": scanID})
+	if err != nil {
+		return err
+	}
+	nodeID, nodeType, err := getScanNodeID(ctx, res)
+	if err != nil {
+		// This error can be ignored
+		log.Warn().Msg(err.Error())
+	}
+
+	_, err = tx.Run(ctx, `
+		MATCH (m:`+string(scanType)+`{node_id: $scan_id})
+		OPTIONAL MATCH (m)-[r:DETECTED]-> (n:`+utils.ScanTypeDetectedNode[scanType]+`)
+		DETACH DELETE m,r`, map[string]interface{}{"scan_id": scanID})
+	if err != nil {
+		return err
+	}
+	err = tx.Commit(ctx)
+	if err != nil {
+		return err
 	}
 
-	tx4, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second))
+	tx2, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second))
 	if err != nil {
 		return err
 	}
-	defer tx4.Close(ctx)
+	defer tx2.Close(ctx)
 
-	_, err = tx4.Run(ctx, fmt.Sprintf(query, scanID), map[string]interface{}{})
+	// Delete results which are not part of any scans now
+	_, err = tx2.Run(ctx,
+		`MATCH (n:`+utils.ScanTypeDetectedNode[scanType]+`)
+		WHERE not (n)<-[:DETECTED]-(:`+string(scanType)+`)
+		DETACH DELETE (n)`, map[string]interface{}{})
 	if err != nil {
 		return err
 	}
-	err = tx4.Commit(ctx)
+	err = tx2.Commit(ctx)
 	if err != nil {
 		return err
 	}
 
+	// Reset node's latest_scan_id to the previous scan id, if any
+	if nodeID != "" && nodeType != "" {
+		latestScanIDField := ingestersUtil.LatestScanIDField[scanType]
+		scanStatusField := ingestersUtil.ScanStatusField[scanType]
+		scanCountField := ingestersUtil.ScanCountField[scanType]
+
+		query := `MATCH (m:` + nodeType2Neo4jType(nodeType) + `{node_id:"` + nodeID + `"})
+		SET m.` + latestScanIDField + `="", m.` + scanCountField + `=0, m.` + scanStatusField + `=""
+		WITH m
+		OPTIONAL MATCH (s:` + string(scanType) + `) - [:SCANNED] -> (m)
+		WITH max(s.updated_at) as most_recent
+		MATCH (m) <-[:SCANNED]- (s:` + string(scanType) + `{updated_at: most_recent})-[:DETECTED]->(c:` + utils.ScanTypeDetectedNode[scanType] + `)
+		WITH s, m, count(distinct c) as scan_count
+		SET m.` + latestScanIDField + `=s.node_id, m.` + scanCountField + `=scan_count, m.` + scanStatusField + `=s.status`
+
+		log.Debug().Msgf("Query to reset scan status: %v", query)
+
+		tx4, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second))
+		if err != nil {
+			return err
+		}
+		defer tx4.Close(ctx)
+
+		_, err = tx4.Run(ctx, query, map[string]interface{}{})
+		if err != nil {
+			return err
+		}
+		err = tx4.Commit(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	if scanType == utils.NEO4JVulnerabilityScan {
+		mc, err := directory.FileServerClient(ctx)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to get minio client")
+			return err
+		}
+		sbomFile := path.Join("/sbom", utils.ScanIDReplacer.Replace(scanID)+".json.gz")
+		err = mc.DeleteFile(ctx, sbomFile, true, minio.RemoveObjectOptions{ForceDelete: true})
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to delete sbom for scan id %s", scanID)
+			return err
+		}
+		runtimeSbomFile := path.Join("/sbom", "runtime-"+utils.ScanIDReplacer.Replace(scanID)+".json")
+		err = mc.DeleteFile(ctx, runtimeSbomFile, true, minio.RemoveObjectOptions{ForceDelete: true})
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to delete runtime sbom for scan id %s", scanID)
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -542,38 +560,3 @@ func GetSelectedScanResults[T any](ctx context.Context, scanType utils.Neo4jScan
 
 	return res, common, err
 }
-
-func checkForSBMORemoval(ctx context.Context, scanID string) (bool, error) {
-	driver, err := directory.Neo4jClient(ctx)
-	if err != nil {
-		return false, err
-	}
-
-	session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
-	defer session.Close(ctx)
-
-	txCount, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second))
-	if err != nil {
-		log.Error().Msgf(err.Error())
-		return false, err
-	}
-	defer txCount.Close(ctx)
-
-	countRes, err := txCount.Run(ctx,
-		`MATCH (n:VulnerabilityScan{node_id: $scan_id}) -[:DETECTED]-> (v:Vulnerability)
-                RETURN COUNT(v) <> 0 AS Exists`,
-		map[string]interface{}{"scan_id": scanID})
-	if err != nil {
-		log.Error().Msgf(err.Error())
-		return false, err
-	}
-
-	countRec, err := countRes.Single(ctx)
-	if err != nil {
-		log.Error().Msgf(err.Error())
-		return false, err
-	}
-	exists := countRec.Values[0].(bool)
-	removeSBOM := !exists
-	return removeSBOM, err
-}

deepfence_worker/tasks/scans/bulk_delete.go

@@ -35,7 +35,7 @@ func BulkDeleteScans(ctx context.Context, task *asynq.Task) error {
 
 	for _, s := range scansList.ScansInfo {
 		log.Info().Msgf("delete scan %s %s", req.ScanType, s.ScanID)
-		err = reporters_scan.DeleteScan(ctx, scanType, s.ScanID, []string{})
+		err = reporters_scan.DeleteScan(ctx, scanType, s.ScanID)
 		if err != nil {
 			log.Error().Err(err).Msgf("failed to delete scan id %s", s.ScanID)
 			continue

deepfence_worker/tasks/scans/delete_cloud_accounts.go

@@ -106,7 +106,7 @@ func deleteScans(ctx context.Context, accountID string) error {
 	defer log.Info().Msgf("deleted %d scans for account %s", len(scans.ScansInfo), accountID)
 
 	for _, s := range scans.ScansInfo {
-		err := reportersScan.DeleteScan(ctx, utils.NEO4JCloudComplianceScan, s.ScanID, []string{})
+		err := reportersScan.DeleteScan(ctx, utils.NEO4JCloudComplianceScan, s.ScanID)
 		if err != nil {
 			log.Error().Err(err).Msgf("failed to delete scan id %s", s.ScanID)
 		}