Add list & results Scan API #733

deepfence_server/apiDocs/operation.go

@@ -1,9 +1,10 @@
 package apiDocs
 
 import (
-	"github.com/deepfence/ThreatMapper/deepfence_server/diagnosis"
 	"net/http"
 
+	"github.com/deepfence/ThreatMapper/deepfence_server/diagnosis"
+
 	ingester "github.com/deepfence/ThreatMapper/deepfence_ingester/ingesters"
 	"github.com/deepfence/ThreatMapper/deepfence_server/ingesters"
 	"github.com/deepfence/ThreatMapper/deepfence_server/model"
@@ -138,6 +139,34 @@ func (d *OpenApiDocs) AddScansOperations() {
 	d.AddOperation("statusMalwareScan", http.MethodGet, "/deepfence/scan/status/malware",
 		"Get Malware Scan Status", "Get Malware Scan status on agent or registry",
 		http.StatusOK, []string{tagMalwareScan}, bearerToken, new(model.ScanStatusReq), new(model.ScanStatusResp))
+
+	// List scans
+	d.AddOperation("listVulnerabilityScans", http.MethodGet, "/deepfence/scan/list/vulnerability",
+		"Get Vulnerability Scans List", "Get Vulnerability Scan list on agent or registry",
+		http.StatusOK, []string{tagVulnerability}, bearerToken, new(model.ScanListReq), new(model.ScanListResp))
+	d.AddOperation("listSecretScan", http.MethodGet, "/deepfence/scan/list/secret",
+		"Get Secret Scans List", "Get Secret Scans list on agent or registry",
+		http.StatusOK, []string{tagSecretScan}, bearerToken, new(model.ScanListReq), new(model.ScanListResp))
+	d.AddOperation("listComplianceScan", http.MethodGet, "/deepfence/scan/list/compliance",
+		"Get Compliance Scans List", "Get Compliance Scans list on agent or registry",
+		http.StatusOK, []string{tagCompliance}, bearerToken, new(model.ScanListReq), new(model.ScanListResp))
+	d.AddOperation("listMalwareScan", http.MethodGet, "/deepfence/scan/list/malware",
+		"Get Malware Scans List", "Get Malware Scans list on agent or registry",
+		http.StatusOK, []string{tagMalwareScan}, bearerToken, new(model.ScanListReq), new(model.ScanListResp))
+
+	// Scans' Results
+	d.AddOperation("resultsVulnerabilityScans", http.MethodGet, "/deepfence/scan/results/vulnerability",
+		"Get Vulnerability Scans Results", "Get Vulnerability Scan results on agent or registry",
+		http.StatusOK, []string{tagVulnerability}, bearerToken, new(model.ScanResultsReq), new(model.ScanResultsResp))
+	d.AddOperation("resultsSecretScan", http.MethodGet, "/deepfence/scan/results/secret",
+		"Get Secret Scans Results", "Get Secret Scans results on agent or registry",
+		http.StatusOK, []string{tagSecretScan}, bearerToken, new(model.ScanResultsReq), new(model.ScanResultsResp))
+	d.AddOperation("resultsComplianceScan", http.MethodGet, "/deepfence/scan/results/compliance",
+		"Get Compliance Scans Results", "Get Compliance Scans results on agent or registry",
+		http.StatusOK, []string{tagCompliance}, bearerToken, new(model.ScanResultsReq), new(model.ScanResultsResp))
+	d.AddOperation("resultsMalwareScan", http.MethodGet, "/deepfence/scan/results/malware",
+		"Get Malware Scans Results", "Get Malware Scans results on agent or registry",
+		http.StatusOK, []string{tagMalwareScan}, bearerToken, new(model.ScanResultsReq), new(model.ScanResultsResp))
 }
 
 func (d *OpenApiDocs) AddDiagnosisOperations() {

deepfence_server/handler/scan_reports.go

@@ -351,10 +351,22 @@ func extractScanTrigger(w http.ResponseWriter, r *http.Request) (model.ScanTrigg
 }
 
 func (h *Handler) StatusVulnerabilityScanHandler(w http.ResponseWriter, r *http.Request) {
-	//	Get status of scan
+	statusScanHandler(w, r, utils.NEO4J_VULNERABILITY_SCAN)
 }
 
 func (h *Handler) StatusSecretScanHandler(w http.ResponseWriter, r *http.Request) {
+	statusScanHandler(w, r, utils.NEO4J_SECRET_SCAN)
+}
+
+func (h *Handler) StatusComplianceScanHandler(w http.ResponseWriter, r *http.Request) {
+	statusScanHandler(w, r, utils.NEO4J_COMPLIANCE_SCAN)
+}
+
+func (h *Handler) StatusMalwareScanHandler(w http.ResponseWriter, r *http.Request) {
+	statusScanHandler(w, r, utils.NEO4J_MALWARE_SCAN)
+}
+
+func statusScanHandler(w http.ResponseWriter, r *http.Request, scan_type utils.Neo4jScanType) {
 	defer r.Body.Close()
 	var req model.ScanStatusReq
 	err := httpext.DecodeQueryParams(r, &req)
@@ -364,7 +376,7 @@ func (h *Handler) StatusSecretScanHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	status, err := reporters.GetSecretScanStatus(r.Context(), req.ScanId)
+	status, err := reporters.GetScanStatus(r.Context(), scan_type, req.ScanId)
 	if err != nil {
 		log.Error().Msgf("%v", err)
 		httpext.JSON(w, http.StatusInternalServerError, model.Response{Success: false})
@@ -374,10 +386,74 @@ func (h *Handler) StatusSecretScanHandler(w http.ResponseWriter, r *http.Request
 	httpext.JSON(w, http.StatusOK, model.Response{Success: true, Data: status})
 }
 
-func (h *Handler) StatusComplianceScanHandler(w http.ResponseWriter, r *http.Request) {
-	//	Get status of scan
+func (h *Handler) ListVulnerabilityScansHandler(w http.ResponseWriter, r *http.Request) {
+	listScansHandler(w, r, utils.NEO4J_VULNERABILITY_SCAN)
 }
 
-func (h *Handler) StatusMalwareScanHandler(w http.ResponseWriter, r *http.Request) {
-	//	Get status of scan
+func (h *Handler) ListSecretScansHandler(w http.ResponseWriter, r *http.Request) {
+	listScansHandler(w, r, utils.NEO4J_SECRET_SCAN)
+}
+
+func (h *Handler) ListComplianceScansHandler(w http.ResponseWriter, r *http.Request) {
+	listScansHandler(w, r, utils.NEO4J_COMPLIANCE_SCAN)
+}
+
+func (h *Handler) ListMalwareScansHandler(w http.ResponseWriter, r *http.Request) {
+	listScansHandler(w, r, utils.NEO4J_MALWARE_SCAN)
+}
+
+func listScansHandler(w http.ResponseWriter, r *http.Request, scan_type utils.Neo4jScanType) {
+	defer r.Body.Close()
+	var req model.ScanListReq
+	err := httpext.DecodeQueryParams(r, &req)
+	if err != nil {
+		log.Error().Msgf("%v", err)
+		httpext.JSON(w, http.StatusBadRequest, model.Response{Success: false})
+		return
+	}
+
+	infos, err := reporters.GetScansList(r.Context(), scan_type, req.NodeId, req.Window)
+	if err != nil {
+		log.Error().Msgf("%v", err)
+		httpext.JSON(w, http.StatusInternalServerError, model.Response{Success: false})
+		return
+	}
+
+	httpext.JSON(w, http.StatusOK, model.Response{Success: true, Data: infos})
+}
+
+func (h *Handler) ListVulnerabilityScanResultsHandler(w http.ResponseWriter, r *http.Request) {
+	listScanResultsHandler(w, r, utils.NEO4J_VULNERABILITY_SCAN)
+}
+
+func (h *Handler) ListSecretScanResultsHandler(w http.ResponseWriter, r *http.Request) {
+	listScanResultsHandler(w, r, utils.NEO4J_SECRET_SCAN)
+}
+
+func (h *Handler) ListComplianceScanResultsHandler(w http.ResponseWriter, r *http.Request) {
+	listScanResultsHandler(w, r, utils.NEO4J_COMPLIANCE_SCAN)
+}
+
+func (h *Handler) ListMalwareScanResultsHandler(w http.ResponseWriter, r *http.Request) {
+	listScanResultsHandler(w, r, utils.NEO4J_MALWARE_SCAN)
+}
+
+func listScanResultsHandler(w http.ResponseWriter, r *http.Request, scan_type utils.Neo4jScanType) {
+	defer r.Body.Close()
+	var req model.ScanResultsReq
+	err := httpext.DecodeQueryParams(r, &req)
+	if err != nil {
+		log.Error().Msgf("%v", err)
+		httpext.JSON(w, http.StatusBadRequest, model.Response{Success: false})
+		return
+	}
+
+	results, err := reporters.GetScanResults(r.Context(), scan_type, req.ScanId, req.Window)
+	if err != nil {
+		log.Error().Msgf("%v", err)
+		httpext.JSON(w, http.StatusInternalServerError, model.Response{Success: false})
+		return
+	}
+
+	httpext.JSON(w, http.StatusOK, model.Response{Success: true, Data: results})
 }

deepfence_server/model/common.go

@@ -15,3 +15,8 @@ type ResponseAccessToken struct {
 	AccessToken  string `json:"access_token"`
 	RefreshToken string `json:"refresh_token"`
 }
+
+type FetchWindow struct {
+	Offset int `query:"offset" json:"offset" required:"true"`
+	Size   int `query:"size" json:"size" required:"true"`
+}

deepfence_server/model/scans.go

@@ -12,6 +12,12 @@ type ScanTriggerReq struct {
 
 type ScanStatus string
 
+type ScanInfo struct {
+	ScanId    string `json:"scan_id" required:"true"`
+	Status    string `json:"status" required:"true"`
+	UpdatedAt int64  `json:"updated_at" required:"true"`
+}
+
 const (
 	SCAN_STATUS_SUCCESS    = utils.SCAN_STATUS_SUCCESS
 	SCAN_STATUS_STARTING   = utils.SCAN_STATUS_STARTING
@@ -29,3 +35,21 @@ type ScanStatusReq struct {
 type ScanStatusResp struct {
 	Status ScanStatus `json:"status" required:"true"`
 }
+
+type ScanListReq struct {
+	NodeId string      `query:"node_id" required:"true" json:"node_id"`
+	Window FetchWindow `json:"window" query:"window" required:"true"`
+}
+
+type ScanListResp struct {
+	ScansInfo []ScanInfo `json:"scans_info" required:"true"`
+}
+
+type ScanResultsReq struct {
+	ScanId string      `query:"scan_id" required:"true" json:"scan_id"`
+	Window FetchWindow `json:"window" query:"window" required:"true"`
+}
+
+type ScanResultsResp struct {
+	Results []map[string]interface{} `json:"results" required:"true"`
+}

deepfence_server/reporters/scan_reporters.go

@@ -5,15 +5,11 @@ import (
 
 	"github.com/deepfence/ThreatMapper/deepfence_server/model"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/directory"
+	"github.com/deepfence/ThreatMapper/deepfence_utils/utils"
 	"github.com/neo4j/neo4j-go-driver/v4/neo4j"
 )
 
-const (
-	STATUS_PROGRESS = "progress"
-	STATUS_COMPLETE = "complete"
-)
-
-func GetSecretScanStatus(ctx context.Context, scan_id string) (model.ScanStatusResp, error) {
+func GetScanStatus(ctx context.Context, scan_type utils.Neo4jScanType, scan_id string) (model.ScanStatusResp, error) {
 	driver, err := directory.Neo4jClient(ctx)
 	if err != nil {
 		return model.ScanStatusResp{}, err
@@ -31,7 +27,7 @@ func GetSecretScanStatus(ctx context.Context, scan_id string) (model.ScanStatusR
 	}
 	defer tx.Close()
 
-	res, err := tx.Run("MATCH (m:SecretScan{node_id: $scan_id}) RETURN m.status",
+	res, err := tx.Run(`MATCH (m:`+string(scan_type)+`{node_id: $scan_id}) RETURN m.status`,
 		map[string]interface{}{"scan_id": scan_id})
 	if err != nil {
 		return model.ScanStatusResp{}, err
@@ -44,3 +40,86 @@ func GetSecretScanStatus(ctx context.Context, scan_id string) (model.ScanStatusR
 
 	return model.ScanStatusResp{Status: model.ScanStatus(rec.Values[0].(string))}, nil
 }
+
+func GetScansList(ctx context.Context, scan_type utils.Neo4jScanType, node_id string, fw model.FetchWindow) (model.ScanListResp, error) {
+	driver, err := directory.Neo4jClient(ctx)
+	if err != nil {
+		return model.ScanListResp{}, err
+	}
+
+	session := driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead})
+	if err != nil {
+		return model.ScanListResp{}, err
+	}
+	defer session.Close()
+
+	tx, err := session.BeginTransaction()
+	if err != nil {
+		return model.ScanListResp{}, err
+	}
+	defer tx.Close()
+
+	res, err := tx.Run(`MATCH (m:`+string(scan_type)+`) -[:SCANNED]-> (:Node{node_id: $node_id}) RETURN m.node_id, m.status, m.updated_at ORDER BY m.updated_at SKIP $skip LIMIT $limit`,
+		map[string]interface{}{"node_id": node_id, "skip": fw.Offset, "limit": fw.Size})
+	if err != nil {
+		return model.ScanListResp{}, err
+	}
+
+	recs, err := res.Collect()
+	if err != nil {
+		return model.ScanListResp{}, err
+	}
+
+	scans_info := []model.ScanInfo{}
+	for _, rec := range recs {
+		tmp := model.ScanInfo{
+			ScanId:    rec.Values[0].(string),
+			Status:    rec.Values[1].(string),
+			UpdatedAt: rec.Values[2].(int64),
+		}
+		scans_info = append(scans_info, tmp)
+	}
+
+	return model.ScanListResp{ScansInfo: scans_info}, nil
+}
+
+func GetScanResults(ctx context.Context, scan_type utils.Neo4jScanType, scan_id string, fw model.FetchWindow) (model.ScanResultsResp, error) {
+	driver, err := directory.Neo4jClient(ctx)
+	if err != nil {
+		return model.ScanResultsResp{}, err
+	}
+
+	session := driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead})
+	if err != nil {
+		return model.ScanResultsResp{}, err
+	}
+	defer session.Close()
+
+	tx, err := session.BeginTransaction()
+	if err != nil {
+		return model.ScanResultsResp{}, err
+	}
+	defer tx.Close()
+
+	res, err := tx.Run(`MATCH (m:`+string(scan_type)+`{node_id: $scan_id}) -[:DETECTED]-> (d) RETURN d SKIP $skip LIMIT $limit`,
+		map[string]interface{}{"scan_id": scan_id, "skip": fw.Offset, "limit": fw.Size})
+	if err != nil {
+		return model.ScanResultsResp{}, err
+	}
+
+	recs, err := res.Collect()
+	if err != nil {
+		return model.ScanResultsResp{}, err
+	}
+
+	scan_result := []map[string]interface{}{}
+	for _, rec := range recs {
+		tmp := map[string]interface{}{}
+		for i, key := range rec.Keys {
+			tmp[key] = rec.Values[i]
+		}
+		scan_result = append(scan_result, tmp)
+	}
+
+	return model.ScanResultsResp{Results: scan_result}, nil
+}

deepfence_server/router/router.go

@@ -167,6 +167,18 @@ func SetupRoutes(r *chi.Mux, serverPort string, jwtSecret []byte, serveOpenapiDo
 				r.Get("/compliance", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.StatusComplianceScanHandler))
 				r.Get("/malware", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.StatusMalwareScanHandler))
 			})
+			r.Route("/scan/list", func(r chi.Router) {
+				r.Get("/vulnerability", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListVulnerabilityScansHandler))
+				r.Get("/secret", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListSecretScansHandler))
+				r.Get("/compliance", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListComplianceScansHandler))
+				r.Get("/malware", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListMalwareScansHandler))
+			})
+			r.Route("/scan/results", func(r chi.Router) {
+				r.Get("/vulnerability", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListVulnerabilityScanResultsHandler))
+				r.Get("/secret", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListSecretScanResultsHandler))
+				r.Get("/compliance", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListComplianceScanResultsHandler))
+				r.Get("/malware", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListMalwareScanResultsHandler))
+			})
 
 			openApiDocs.AddDiagnosisOperations()
 			r.Route("/diagnosis", func(r chi.Router) {

deepfence_server_client/.openapi-generator/FILES

@@ -7,6 +7,7 @@ api_cloud_compliance.go
 api_cloud_resources.go
 api_compliance.go
 api_controls.go
+api_diagnosis.go
 api_malware_scan.go
 api_secret_scan.go
 api_threat.go
@@ -24,6 +25,8 @@ docs/ComplianceApi.md
 docs/ControlsAction.md
 docs/ControlsAgentControls.md
 docs/ControlsApi.md
+docs/DiagnosisApi.md
+docs/DiagnosisDiagnosticNotification.md
 docs/IngestersCloudCompliance.md
 docs/IngestersCloudResource.md
 docs/IngestersCompliance.md
@@ -40,6 +43,8 @@ docs/ModelLoginRequest.md
 docs/ModelRawReport.md
 docs/ModelResponse.md
 docs/ModelResponseAccessToken.md
+docs/ModelScanInfo.md
+docs/ModelScanListResp.md
 docs/ModelScanStatusResp.md
 docs/ModelScanTriggerReq.md
 docs/ModelScanTriggerResp.md
@@ -63,6 +68,7 @@ model_api_docs_bad_request_response.go
 model_api_docs_failure_response.go
 model_controls_action.go
 model_controls_agent_controls.go
+model_diagnosis_diagnostic_notification.go
 model_ingesters_cloud_compliance.go
 model_ingesters_cloud_resource.go
 model_ingesters_compliance.go
@@ -78,6 +84,8 @@ model_model_login_request.go
 model_model_raw_report.go
 model_model_response.go
 model_model_response_access_token.go
+model_model_scan_info.go
+model_model_scan_list_resp.go
 model_model_scan_status_resp.go
 model_model_scan_trigger_req.go
 model_model_scan_trigger_resp.go
@@ -90,4 +98,5 @@ model_reporters_rendered_graph.go
 model_reporters_threat_node_info.go
 model_reporters_topology_filters.go
 response.go
+test/api_diagnosis_test.go
 utils.go