From 25efe1e68d63fe83473aaf1fec170dcbcc07db97 Mon Sep 17 00:00:00 2001 From: Jatin Baweja Date: Wed, 8 Feb 2023 09:57:13 +0530 Subject: [PATCH] Add Bulk Cloud Compliance Status API deepfence/ThreatMapper#838 (#873) * Add ingest scan status and scan status API for cloud compliance scans * Merge cloud compliance and compliance start scan APIs and make status response for cloud compliance and compliance consistent * Fix Malware scan request and add linux to node_types for compliance start scan API * Replace hard coded strings and removing start cloud compliance API from docs --- deepfence_server/apiDocs/operation.go | 12 +- deepfence_server/handler/cloud_node.go | 25 +++- deepfence_server/handler/scan_reports.go | 72 ++++++++++- .../ingesters/cloud_compliance_ingester.go | 36 ++++++ deepfence_server/model/cloud_node.go | 7 +- deepfence_server/model/scans.go | 30 ++++- deepfence_server/reporters/scan_reporters.go | 120 ++++++++++++++++-- deepfence_server/router/router.go | 5 +- .../ingesters/cloud_compliance.go | 86 ++++++++++--- 9 files changed, 338 insertions(+), 55 deletions(-) diff --git a/deepfence_server/apiDocs/operation.go b/deepfence_server/apiDocs/operation.go index a821ac0de6..1893fea554 100644 --- a/deepfence_server/apiDocs/operation.go +++ b/deepfence_server/apiDocs/operation.go @@ -190,6 +190,10 @@ func (d *OpenApiDocs) AddIngestersOperations() { "Ingest Cloud Compliances", "Ingest Cloud compliances found while scanning cloud provider", http.StatusOK, []string{tagCloudScanner}, bearerToken, new([]ingester.CloudCompliance), nil) + d.AddOperation("ingestCloudComplianceScanStatus", http.MethodPost, "/deepfence/ingest/cloud-compliance-scan-status", + "Ingest Cloud Compliances", "Ingest Cloud compliances found while scanning cloud provider", + http.StatusOK, []string{tagCloudScanner}, bearerToken, new([]ingester.CloudCompliance), nil) + d.AddOperation("ingestMalwareScanStatus", http.MethodPost, "/deepfence/ingest/malware-scan-logs", "Ingest Malware Scan Status", "Ingest malware scan status from the agent", http.StatusOK, []string{tagMalwareScan}, bearerToken, new([]ingester.MalwareScanStatus), nil) @@ -217,9 +221,6 @@ func (d *OpenApiDocs) AddScansOperations() { d.AddOperation("startMalwareScan", http.MethodPost, "/deepfence/scan/start/malware", "Start Malware Scan", "Start Malware Scan on agent or registry", http.StatusAccepted, []string{tagMalwareScan}, bearerToken, new(model.MalwareScanTriggerReq), new(model.ScanTriggerResp)) - d.AddOperation("startCloudComplianceScans", http.MethodPost, "/deepfence/scan/start/cloud-compliance", - "Start Cloud Compliance Scans", "Start Cloud Compliance Scans on cloud nodes", http.StatusAccepted, - []string{tagCloudScanner}, bearerToken, new(model.CloudComplianceScanTriggerReq), new(model.ScanTriggerResp)) // Stop scan d.AddOperation("stopVulnerabilityScan", http.MethodPost, "/deepfence/scan/stop/vulnerability", @@ -244,10 +245,13 @@ func (d *OpenApiDocs) AddScansOperations() { http.StatusOK, []string{tagSecretScan}, bearerToken, new(model.ScanStatusReq), new(model.ScanStatusResp)) d.AddOperation("statusComplianceScan", http.MethodGet, "/deepfence/scan/status/compliance", "Get Compliance Scan Status", "Get Compliance Scan Status on agent or registry", - http.StatusOK, []string{tagCompliance}, bearerToken, new(model.ScanStatusReq), new(model.ScanStatusResp)) + http.StatusOK, []string{tagCompliance}, bearerToken, new(model.ScanStatusReq), new(model.ComplianceScanStatusResp)) d.AddOperation("statusMalwareScan", http.MethodGet, "/deepfence/scan/status/malware", "Get Malware Scan Status", "Get Malware Scan status on agent or registry", http.StatusOK, []string{tagMalwareScan}, bearerToken, new(model.ScanStatusReq), new(model.ScanStatusResp)) + d.AddOperation("statusCloudComplianceScan", http.MethodGet, "/deepfence/scan/status/cloud-compliance", + "Get Cloud Compliance Scan Status", "Get Cloud Compliance Scan Status on cloud node", + http.StatusOK, []string{tagCloudScanner}, bearerToken, new(model.ScanStatusReq), new(model.ComplianceScanStatusResp)) // List scans d.AddOperation("listVulnerabilityScans", http.MethodPost, "/deepfence/scan/list/vulnerability", diff --git a/deepfence_server/handler/cloud_node.go b/deepfence_server/handler/cloud_node.go index 8fe3555b29..9edff02b79 100644 --- a/deepfence_server/handler/cloud_node.go +++ b/deepfence_server/handler/cloud_node.go @@ -14,6 +14,17 @@ import ( "github.com/sirupsen/logrus" ) +var ( + AWS_DEFAULT_CONTROLS = map[string][]string{ + "cis": []string{"control.cis_v140_4_1", "control.cis_v140_4_2", "control.cis_v140_4_3", "control.cis_v140_4_4", "control.cis_v140_4_5", "control.cis_v140_4_6", "control.cis_v140_4_7", "control.cis_v140_4_8", "control.cis_v140_4_9", "control.cis_v140_4_10", "control.cis_v140_4_11", "control.cis_v140_4_12", "control.cis_v140_4_13", "control.cis_v140_4_14", "control.cis_v140_4_15", "control.cis_v140_5_1", "control.cis_v140_5_2", "control.cis_v140_5_3", "control.cis_v140_5_4", "control.cis_v140_3_1", "control.cis_v140_3_2", "control.cis_v140_3_3", "control.cis_v140_3_4", "control.cis_v140_3_5", "control.cis_v140_3_6", "control.cis_v140_3_7", "control.cis_v140_3_8", "control.cis_v140_3_9", "control.cis_v140_3_10", "control.cis_v140_3_11", "control.cis_v140_1_1", "control.cis_v140_1_2", "control.cis_v140_1_3", "control.cis_v140_1_4", "control.cis_v140_1_5", "control.cis_v140_1_6", "control.cis_v140_1_7", "control.cis_v140_1_8", "control.cis_v140_1_9", "control.cis_v140_1_10", "control.cis_v140_1_11", "control.cis_v140_1_12", "control.cis_v140_1_13", "control.cis_v140_1_14", "control.cis_v140_1_15", "control.cis_v140_1_16", "control.cis_v140_1_17", "control.cis_v140_1_18", "control.cis_v140_1_19", "control.cis_v140_1_20", "control.cis_v140_1_21", "control.cis_v140_2_1_1", "control.cis_v140_2_1_2", "control.cis_v140_2_1_3", "control.cis_v140_2_1_4", "control.cis_v140_2_1_5", "control.cis_v140_2_2_1", "control.cis_v140_2_3_1"}, + "gdpr": []string{"benchmark.article_30", "benchmark.article_32", "benchmark.article_25"}, + "hipaa": []string{"benchmark.hipaa_164_308_a_8", "benchmark.hipaa_164_308_a_4_ii_b", "benchmark.hipaa_164_308_a_4_ii_c", "benchmark.hipaa_164_308_a_1_ii_a", "benchmark.hipaa_164_308_a_7_i", "benchmark.hipaa_164_308_a_1_ii_b", "benchmark.hipaa_164_308_a_7_ii_b", "benchmark.hipaa_164_308_a_5_ii_b", "benchmark.hipaa_164_308_a_3_ii_b", "benchmark.hipaa_164_308_a_6_ii", "benchmark.hipaa_164_308_a_7_ii_c", "benchmark.hipaa_164_308_a_3_ii_c", "benchmark.hipaa_164_308_a_4_i", "benchmark.hipaa_164_308_a_5_ii_d", "benchmark.hipaa_164_308_a_5_ii_c", "benchmark.hipaa_164_308_a_4_ii_a", "benchmark.hipaa_164_308_a_1_ii_d", "benchmark.hipaa_164_308_a_3_ii_a", "benchmark.hipaa_164_308_a_3_i", "benchmark.hipaa_164_308_a_6_i", "benchmark.hipaa_164_308_a_7_ii_a", "benchmark.hipaa_164_312_d", "benchmark.hipaa_164_312_a_2_ii", "benchmark.hipaa_164_312_e_2_i", "benchmark.hipaa_164_312_e_1", "benchmark.hipaa_164_312_c_1", "benchmark.hipaa_164_312_a_1", "benchmark.hipaa_164_312_a_2_i", "benchmark.hipaa_164_312_a_2_iv", "benchmark.hipaa_164_312_b", "benchmark.hipaa_164_312_c_2", "benchmark.hipaa_164_312_e_2_ii"}, + "nist": []string{"benchmark.nist_800_53_rev_4_sc_2", "benchmark.nist_800_53_rev_4_sc_4", "benchmark.nist_800_53_rev_4_sc_5", "benchmark.nist_800_53_rev_4_sc_7", "benchmark.nist_800_53_rev_4_sc_8", "benchmark.nist_800_53_rev_4_sc_12", "benchmark.nist_800_53_rev_4_sc_13", "benchmark.nist_800_53_rev_4_sc_23", "benchmark.nist_800_53_rev_4_sc_28", "benchmark.nist_800_53_rev_4_cp_9", "benchmark.nist_800_53_rev_4_cp_10", "benchmark.nist_800_53_rev_4_ca_7", "benchmark.nist_800_53_rev_4_sa_3", "benchmark.nist_800_53_rev_4_sa_10", "benchmark.nist_800_53_rev_4_si_2_2", "benchmark.nist_800_53_rev_4_si_4", "benchmark.nist_800_53_rev_4_si_7", "benchmark.nist_800_53_rev_4_si_12", "benchmark.nist_800_53_rev_4_ac_2", "benchmark.nist_800_53_rev_4_ac_3", "benchmark.nist_800_53_rev_4_ac_4", "benchmark.nist_800_53_rev_4_ac_5", "benchmark.nist_800_53_rev_4_ac_6", "benchmark.nist_800_53_rev_4_ac_17_1", "benchmark.nist_800_53_rev_4_ac_17_2", "benchmark.nist_800_53_rev_4_ac_17_3", "benchmark.nist_800_53_rev_4_ac_21", "benchmark.nist_800_53_rev_4_ir_4_1", "benchmark.nist_800_53_rev_4_ir_6_1", "benchmark.nist_800_53_rev_4_ir_7_1", "benchmark.nist_800_53_rev_4_ra_5", "benchmark.nist_800_53_rev_4_cm_2", "benchmark.nist_800_53_rev_4_cm_7", "benchmark.nist_800_53_rev_4_cm_8_1", "benchmark.nist_800_53_rev_4_cm_8_3", "benchmark.nist_800_53_rev_4_au_2", "benchmark.nist_800_53_rev_4_au_3", "benchmark.nist_800_53_rev_4_au_6_1", "benchmark.nist_800_53_rev_4_au_6_3", "benchmark.nist_800_53_rev_4_au_7_1", "benchmark.nist_800_53_rev_4_au_9", "benchmark.nist_800_53_rev_4_au_11", "benchmark.nist_800_53_rev_4_au_12", "benchmark.nist_800_53_rev_4_ia_2", "benchmark.nist_800_53_rev_4_ia_5_1", "benchmark.nist_800_53_rev_4_ia_5_4", "benchmark.nist_800_53_rev_4_ia_5_7"}, + "pci": []string{"control.pci_v321_config_1", "control.pci_v321_dms_1", "control.pci_v321_sagemaker_1", "control.pci_v321_rds_1", "control.pci_v321_rds_2", "control.pci_v321_lambda_1", "control.pci_v321_lambda_2", "control.pci_v321_es_1", "control.pci_v321_es_2", "control.pci_v321_redshift_1", "control.pci_v321_cw_1", "control.pci_v321_elbv2_1", "control.pci_v321_kms_1", "control.pci_v321_ssm_1", "control.pci_v321_ssm_2", "control.pci_v321_ssm_3", "control.pci_v321_codebuild_1", "control.pci_v321_codebuild_2", "control.pci_v321_opensearch_1", "control.pci_v321_opensearch_2", "control.pci_v321_s3_1", "control.pci_v321_s3_2", "control.pci_v321_s3_3", "control.pci_v321_s3_4", "control.pci_v321_s3_5", "control.pci_v321_s3_6", "control.pci_v321_autoscaling_1", "control.pci_v321_cloudtrail_1", "control.pci_v321_cloudtrail_2", "control.pci_v321_cloudtrail_3", "control.pci_v321_cloudtrail_4", "control.pci_v321_guardduty_1", "control.pci_v321_iam_1", "control.pci_v321_iam_2", "control.pci_v321_iam_3", "control.pci_v321_iam_4", "control.pci_v321_iam_5", "control.pci_v321_iam_6", "control.pci_v321_iam_7", "control.pci_v321_iam_8", "control.pci_v321_ec2_1", "control.pci_v321_ec2_2", "control.pci_v321_ec2_3", "control.pci_v321_ec2_4", "control.pci_v321_ec2_5", "control.pci_v321_ec2_6"}, + "soc2": []string{"benchmark.soc_2_cc_5_1", "benchmark.soc_2_cc_5_2", "benchmark.soc_2_cc_5_3", "benchmark.soc_2_p_2_1", "benchmark.soc_2_p_6_1", "benchmark.soc_2_p_6_2", "benchmark.soc_2_p_6_3", "benchmark.soc_2_p_6_4", "benchmark.soc_2_p_6_5", "benchmark.soc_2_p_6_6", "benchmark.soc_2_p_6_7", "benchmark.soc_2_cc_4_1", "benchmark.soc_2_cc_4_2", "benchmark.soc_2_p_4_1", "benchmark.soc_2_p_4_2", "benchmark.soc_2_p_4_3", "benchmark.soc_2_cc_8_1", "benchmark.soc_2_p_8_1", "benchmark.soc_2_p_3_1", "benchmark.soc_2_p_3_2", "benchmark.soc_2_cc_1_1", "benchmark.soc_2_cc_1_2", "benchmark.soc_2_cc_1_3", "benchmark.soc_2_cc_1_4", "benchmark.soc_2_cc_1_5", "benchmark.soc_2_cc_a_1_1", "benchmark.soc_2_cc_a_1_2", "benchmark.soc_2_cc_a_1_3", "benchmark.soc_2_p_7_1", "benchmark.soc_2_p_1_1", "benchmark.soc_2_cc_9_1", "benchmark.soc_2_cc_9_2", "benchmark.soc_2_cc_2_1", "benchmark.soc_2_cc_2_2", "benchmark.soc_2_cc_2_3", "benchmark.soc_2_cc_6_1", "benchmark.soc_2_cc_6_2", "benchmark.soc_2_cc_6_3", "benchmark.soc_2_cc_6_4", "benchmark.soc_2_cc_6_5", "benchmark.soc_2_cc_6_6", "benchmark.soc_2_cc_6_7", "benchmark.soc_2_cc_6_8", "benchmark.soc_2_p_5_1", "benchmark.soc_2_p_5_2", "benchmark.soc_2_cc_3_1", "benchmark.soc_2_cc_3_2", "benchmark.soc_2_cc_3_3", "benchmark.soc_2_cc_3_4", "benchmark.soc_2_cc_c_1_1", "benchmark.soc_2_cc_c_1_2", "benchmark.soc_2_cc_7_1", "benchmark.soc_2_cc_7_2", "benchmark.soc_2_cc_7_3", "benchmark.soc_2_cc_7_4", "benchmark.soc_2_cc_7_5"}, + } +) + func (h *Handler) RegisterCloudNodeAccountHandler(w http.ResponseWriter, r *http.Request) { req, err := extractCloudNodeDetails(w, r) if err != nil { @@ -60,15 +71,17 @@ func (h *Handler) RegisterCloudNodeAccountHandler(w http.ResponseWriter, r *http if err != nil { complianceError(w, err.Error()) } - pendingScansList, err := reporters.GetPendingScansList(ctx, utils.CLOUD_COMPLIANCE_SCAN, monitoredNodeId) + pendingScansList, err := reporters.GetCloudCompliancePendingScansList(ctx, utils.NEO4J_CLOUD_COMPLIANCE_SCAN, monitoredNodeId) if err != nil { continue } for _, scan := range pendingScansList.ScansInfo { + controls, _ := AWS_DEFAULT_CONTROLS[scan.BenchmarkType] scanDetail := model.CloudComplianceScanDetails{ ScanId: scan.ScanId, - ScanType: "cis", + ScanType: scan.BenchmarkType, AccountId: monitoredNodeId, + Controls: controls, } scanList[scan.ScanId] = scanDetail } @@ -86,7 +99,7 @@ func (h *Handler) RegisterCloudNodeAccountHandler(w http.ResponseWriter, r *http logrus.Infof("Error while upserting node: %+v", err) complianceError(w, err.Error()) } - pendingScansList, err := reporters.GetPendingScansList(ctx, utils.CLOUD_COMPLIANCE_SCAN, nodeId) + pendingScansList, err := reporters.GetCloudCompliancePendingScansList(ctx, utils.NEO4J_CLOUD_COMPLIANCE_SCAN, nodeId) if err != nil || len(pendingScansList.ScansInfo) == 0 { logrus.Debugf("No pending scans found for node id: %s", nodeId) httpext.JSON(w, http.StatusOK, @@ -95,10 +108,12 @@ func (h *Handler) RegisterCloudNodeAccountHandler(w http.ResponseWriter, r *http return } for _, scan := range pendingScansList.ScansInfo { + controls, _ := AWS_DEFAULT_CONTROLS[scan.BenchmarkType] scanDetail := model.CloudComplianceScanDetails{ ScanId: scan.ScanId, - ScanType: utils.CLOUD_COMPLIANCE_SCAN, - AccountId: nodeId, + ScanType: scan.BenchmarkType, + AccountId: req.CloudAccount, + Controls: controls, } scanList[scan.ScanId] = scanDetail } diff --git a/deepfence_server/handler/scan_reports.go b/deepfence_server/handler/scan_reports.go index 5b6160e64e..71b6f627fb 100644 --- a/deepfence_server/handler/scan_reports.go +++ b/deepfence_server/handler/scan_reports.go @@ -208,8 +208,8 @@ func (h *Handler) StartSecretScanHandler(w http.ResponseWriter, r *http.Request) } func (h *Handler) StartComplianceScanHandler(w http.ResponseWriter, r *http.Request) { - var reqq model.ComplianceScanTriggerReq - err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &reqq) + var req model.ComplianceScanTriggerReq + err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req) if err != nil { log.Error().Msgf("%v", err) respondError(&BadDecoding{err}, w) @@ -232,7 +232,7 @@ func (h *Handler) StartComplianceScanHandler(w http.ResponseWriter, r *http.Requ } func (h *Handler) StartMalwareScanHandler(w http.ResponseWriter, r *http.Request) { - var reqs model.ComplianceScanTriggerReq + var reqs model.MalwareScanTriggerReq err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &reqs) if err != nil { log.Error().Msgf("%v", err) @@ -290,7 +290,7 @@ func (h *Handler) StartMalwareScanHandler(w http.ResponseWriter, r *http.Request } func (h *Handler) StartCloudComplianceScanHandler(w http.ResponseWriter, r *http.Request) { - var reqs model.CloudComplianceScanTriggerReq + var reqs model.ComplianceScanTriggerReq err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &reqs) if err != nil { log.Error().Msgf("%v", err) @@ -298,7 +298,19 @@ func (h *Handler) StartCloudComplianceScanHandler(w http.ResponseWriter, r *http return } - scanIds, bulkId, err := startMultiCloudComplianceScan(r.Context(), reqs.ScanTriggers) + var scanTrigger model.ComplianceScanTrigger + if len(reqs.ScanTriggers) > 0 { + scanTrigger = reqs.ScanTriggers[0] + } + + var scanIds []string + var bulkId string + if scanTrigger.NodeType == reporters.CLOUD_AWS || scanTrigger.NodeType == reporters.CLOUD_GCP || scanTrigger.NodeType == reporters.CLOUD_AZURE { + scanIds, bulkId, err = startMultiCloudComplianceScan(r.Context(), reqs.ScanTriggers) + } else { + scanIds, bulkId, err = startMultiComplianceScan(r.Context(), reqs.ScanTriggers) + } + if err != nil { log.Error().Msgf("%v", err) respondError(err, w) @@ -470,6 +482,11 @@ func (h *Handler) IngestMalwareScanStatusReportHandler(w http.ResponseWriter, r ingest_scan_report_kafka(w, r, ingester, h.IngestChan) } +func (h *Handler) IngestCloudComplianceScanStatusReportHandler(w http.ResponseWriter, r *http.Request) { + ingester := ingesters.NewCloudComplianceScanStatusIngester() + ingest_scan_report_kafka(w, r, ingester, h.IngestChan) +} + func ingest_scan_report_kafka[T any]( respWrite http.ResponseWriter, req *http.Request, @@ -519,13 +536,17 @@ func (h *Handler) StatusSecretScanHandler(w http.ResponseWriter, r *http.Request } func (h *Handler) StatusComplianceScanHandler(w http.ResponseWriter, r *http.Request) { - statusScanHandler(w, r, utils.NEO4J_COMPLIANCE_SCAN) + complianceStatusScanHandler(w, r, utils.NEO4J_COMPLIANCE_SCAN) } func (h *Handler) StatusMalwareScanHandler(w http.ResponseWriter, r *http.Request) { statusScanHandler(w, r, utils.NEO4J_MALWARE_SCAN) } +func (h *Handler) StatusCloudComplianceScanHandler(w http.ResponseWriter, r *http.Request) { + complianceStatusScanHandler(w, r, utils.NEO4J_CLOUD_COMPLIANCE_SCAN) +} + func statusScanHandler(w http.ResponseWriter, r *http.Request, scan_type utils.Neo4jScanType) { defer r.Body.Close() var req model.ScanStatusReq @@ -556,6 +577,32 @@ func statusScanHandler(w http.ResponseWriter, r *http.Request, scan_type utils.N httpext.JSON(w, http.StatusOK, statuses) } +func complianceStatusScanHandler(w http.ResponseWriter, r *http.Request, scan_type utils.Neo4jScanType) { + defer r.Body.Close() + var req model.ScanStatusReq + err := httpext.DecodeQueryParams(r, &req) + if err != nil { + log.Error().Msgf("%v", err) + respondError(&BadDecoding{err}, w) + return + } + + var statuses model.ComplianceScanStatusResp + if req.BulkScanId != "" { + statuses, err = reporters.GetComplianceBulkScans(r.Context(), scan_type, req.BulkScanId) + } else { + statuses, err = reporters.GetComplianceScanStatus(r.Context(), scan_type, req.ScanIds) + } + + if err != nil { + log.Error().Msgf("%v, req=%v", err, req) + respondError(err, w) + return + } + + httpext.JSON(w, http.StatusOK, statuses) +} + func (h *Handler) ListVulnerabilityScansHandler(w http.ResponseWriter, r *http.Request) { listScansHandler(w, r, utils.NEO4J_VULNERABILITY_SCAN) } @@ -811,7 +858,7 @@ func startMultiScan(ctx context.Context, gen_bulk_id bool, scan_type utils.Neo4j return scanIds, bulkId, tx.Commit() } -func startMultiCloudComplianceScan(ctx context.Context, reqs []model.CloudComplianceScanTrigger) ([]string, string, error) { +func startMultiCloudComplianceScan(ctx context.Context, reqs []model.ComplianceScanTrigger) ([]string, string, error) { driver, err := directory.Neo4jClient(ctx) if err != nil { @@ -864,3 +911,14 @@ func startMultiCloudComplianceScan(ctx context.Context, reqs []model.CloudCompli return scanIds, bulkId, tx.Commit() } + +func startMultiComplianceScan(ctx context.Context, reqs []model.ComplianceScanTrigger) ([]string, string, error) { + scanIds := []string{} + bulkId := bulkScanId() + for _, req := range reqs { + for _, benchmarkType := range req.BenchmarkTypes { + scanIds = append(scanIds, cloudComplianceScanId(req.NodeId, benchmarkType)) + } + } + return scanIds, bulkId, nil +} diff --git a/deepfence_server/ingesters/cloud_compliance_ingester.go b/deepfence_server/ingesters/cloud_compliance_ingester.go index 79c21c20c1..a57827c718 100644 --- a/deepfence_server/ingesters/cloud_compliance_ingester.go +++ b/deepfence_server/ingesters/cloud_compliance_ingester.go @@ -47,3 +47,39 @@ func (tc *CloudComplianceIngester) Ingest( return nil } + +type CloudComplianceScanStatusIngester struct{} + +func NewCloudComplianceScanStatusIngester() KafkaIngester[ingesters.CloudComplianceScanStatus] { + return &CloudComplianceScanStatusIngester{} +} + +func (tc *CloudComplianceScanStatusIngester) Ingest( + ctx context.Context, + cs ingesters.CloudComplianceScanStatus, + ingestC chan *kgo.Record, +) error { + + tenantID, err := directory.ExtractNamespace(ctx) + if err != nil { + return err + } + + rh := []kgo.RecordHeader{ + {Key: "tenant_id", Value: []byte(tenantID)}, + } + + cb, err := json.Marshal(cs) + if err != nil { + log.Error().Msg(err.Error()) + } else { + ingestC <- &kgo.Record{ + Topic: utils.CLOUD_COMPLIANCE_SCAN_STATUS, + Value: cb, + Headers: rh, + } + } + + return nil + +} diff --git a/deepfence_server/model/cloud_node.go b/deepfence_server/model/cloud_node.go index 5f6292d8dc..19159cdba9 100644 --- a/deepfence_server/model/cloud_node.go +++ b/deepfence_server/model/cloud_node.go @@ -50,9 +50,10 @@ type CloudNodeAccountInfo struct { } type CloudComplianceScanDetails struct { - ScanId string `json:"scan_id"` - ScanType string `json:"scan_type"` - AccountId string `json:"account_id"` + ScanId string `json:"scan_id"` + ScanType string `json:"scan_type"` + AccountId string `json:"account_id"` + Controls []string `json:"controls"` } type CloudNodeCloudtrailTrail struct { diff --git a/deepfence_server/model/scans.go b/deepfence_server/model/scans.go index 864202036c..1d5a679d55 100644 --- a/deepfence_server/model/scans.go +++ b/deepfence_server/model/scans.go @@ -22,11 +22,7 @@ type MalwareScanTriggerReq struct { } type ComplianceScanTriggerReq struct { - ScanTriggerCommon -} - -type CloudComplianceScanTriggerReq struct { - ScanTriggers []CloudComplianceScanTrigger `json:"scan_triggers" required:"true"` + ScanTriggers []ComplianceScanTrigger `json:"scan_triggers" required:"true"` } type KeyValue struct { @@ -54,8 +50,9 @@ type NodeIdentifier struct { NodeType string `json:"node_type" required:"true" enum:"image,host,container"` } -type CloudComplianceScanTrigger struct { +type ComplianceScanTrigger struct { NodeId string `json:"node_id" required:"true"` + NodeType string `json:"node_type" required:"true" enum:"aws,gcp,azure,linux,kubernetes_cluster"` BenchmarkTypes []string `json:"benchmark_types" required:"true"` } @@ -69,6 +66,13 @@ type ScanInfo struct { NodeType string `json:"node_type" required:"true"` } +type CloudComplianceScanInfo struct { + ScanId string `json:"scan_id" required:"true"` + BenchmarkType string `json:"benchmark_type" required:"true"` + Status string `json:"status" required:"true"` + UpdatedAt int64 `json:"updated_at" required:"true" format:"int64"` +} + const ( SCAN_STATUS_SUCCESS = utils.SCAN_STATUS_SUCCESS SCAN_STATUS_STARTING = utils.SCAN_STATUS_STARTING @@ -89,6 +93,16 @@ type ScanStatusResp struct { Statuses map[string]ScanInfo `json:"statuses" required:"true"` } +type ComplianceScanStatusResp struct { + Statuses []ComplianceScanStatus `json:"statuses" required:"true"` +} + +type ComplianceScanStatus struct { + ScanId string `json:"scan_id"` + BenchmarkType string `json:"benchmark_type"` + Status string `json:"status"` +} + type ScanListReq struct { NodeId string `json:"node_id" required:"true"` NodeType string `json:"node_type" required:"true" enum:"image,host,container"` @@ -99,6 +113,10 @@ type ScanListResp struct { ScansInfo []ScanInfo `json:"scans_info" required:"true"` } +type CloudComplianceScanListResp struct { + ScansInfo []CloudComplianceScanInfo `json:"scans_info" required:"true"` +} + type ScanResultsReq struct { ScanId string `json:"scan_id" required:"true"` Window FetchWindow `json:"window" required:"true"` diff --git a/deepfence_server/reporters/scan_reporters.go b/deepfence_server/reporters/scan_reporters.go index 17269eba09..4a182007b8 100644 --- a/deepfence_server/reporters/scan_reporters.go +++ b/deepfence_server/reporters/scan_reporters.go @@ -90,6 +90,53 @@ func GetScanStatus(ctx context.Context, scan_type utils.Neo4jScanType, scan_ids return model.ScanStatusResp{Statuses: statuses}, nil } +func GetComplianceScanStatus(ctx context.Context, scanType utils.Neo4jScanType, scanIds []string) (model.ComplianceScanStatusResp, error) { + scanResponse := model.ComplianceScanStatusResp{ + Statuses: []model.ComplianceScanStatus{}, + } + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return scanResponse, err + } + + session := driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + if err != nil { + return scanResponse, err + } + defer session.Close() + + tx, err := session.BeginTransaction() + if err != nil { + return scanResponse, err + } + defer tx.Close() + + res, err := tx.Run(fmt.Sprintf(` + MATCH (m:%s) + WHERE m.node_id IN $scan_ids + RETURN m.node_id, m.benchmark_type, m.status`, scanType), + map[string]interface{}{"scan_ids": scanIds}) + if err != nil { + return scanResponse, err + } + + recs, err := res.Collect() + if err != nil { + return scanResponse, err + } + + for _, rec := range recs { + tmp := model.ComplianceScanStatus{ + ScanId: rec.Values[0].(string), + BenchmarkType: rec.Values[1].(string), + Status: rec.Values[2].(string), + } + scanResponse.Statuses = append(scanResponse.Statuses, tmp) + } + + return scanResponse, nil +} + func GetScansList(ctx context.Context, scan_type utils.Neo4jScanType, node_id string, @@ -141,49 +188,50 @@ func GetScansList(ctx context.Context, return model.ScanListResp{ScansInfo: scans_info}, nil } -func GetPendingScansList(ctx context.Context, scan_type utils.Neo4jScanType, node_id string) (model.ScanListResp, error) { +func GetCloudCompliancePendingScansList(ctx context.Context, scan_type utils.Neo4jScanType, node_id string) (model.CloudComplianceScanListResp, error) { driver, err := directory.Neo4jClient(ctx) if err != nil { - return model.ScanListResp{}, err + return model.CloudComplianceScanListResp{}, err } session := driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) if err != nil { - return model.ScanListResp{}, err + return model.CloudComplianceScanListResp{}, err } defer session.Close() tx, err := session.BeginTransaction() if err != nil { - return model.ScanListResp{}, err + return model.CloudComplianceScanListResp{}, err } defer tx.Close() res, err := tx.Run(` MATCH (m:`+string(scan_type)+`) -[:SCANNED]-> (:Node{node_id: $node_id}) WHERE NOT m.status = $complete AND NOT m.status = $failed AND NOT m.status = $in_progress - RETURN m.node_id, m.status, m.updated_at ORDER BY m.updated_at`, + RETURN m.node_id, m.benchmark_type, m.status, m.updated_at ORDER BY m.updated_at`, map[string]interface{}{"node_id": node_id, "complete": utils.SCAN_STATUS_SUCCESS, "failed": utils.SCAN_STATUS_FAILED, "in_progress": utils.SCAN_STATUS_INPROGRESS}) if err != nil { - return model.ScanListResp{}, err + return model.CloudComplianceScanListResp{}, err } recs, err := res.Collect() if err != nil { - return model.ScanListResp{}, err + return model.CloudComplianceScanListResp{}, err } - scans_info := []model.ScanInfo{} + scans_info := []model.CloudComplianceScanInfo{} for _, rec := range recs { - tmp := model.ScanInfo{ - ScanId: rec.Values[0].(string), - Status: rec.Values[1].(string), - UpdatedAt: rec.Values[2].(int64), + tmp := model.CloudComplianceScanInfo{ + ScanId: rec.Values[0].(string), + BenchmarkType: rec.Values[1].(string), + Status: rec.Values[2].(string), + UpdatedAt: rec.Values[3].(int64), } scans_info = append(scans_info, tmp) } - return model.ScanListResp{ScansInfo: scans_info}, nil + return model.CloudComplianceScanListResp{ScansInfo: scans_info}, nil } func GetScanResults[T any](ctx context.Context, scan_type utils.Neo4jScanType, scan_id string, fw model.FetchWindow) ([]T, model.ScanResultsCommon, error) { @@ -387,3 +435,49 @@ func GetBulkScans(ctx context.Context, scan_type utils.Neo4jScanType, scan_id st return scan_ids, nil } + +func GetComplianceBulkScans(ctx context.Context, scanType utils.Neo4jScanType, scanId string) (model.ComplianceScanStatusResp, error) { + scanIds := model.ComplianceScanStatusResp{ + Statuses: []model.ComplianceScanStatus{}, + } + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return scanIds, err + } + + session := driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + if err != nil { + return scanIds, err + } + defer session.Close() + + tx, err := session.BeginTransaction() + if err != nil { + return scanIds, err + } + defer tx.Close() + + neo_res, err := tx.Run(` + MATCH (m:Bulk`+string(scanType)+`{node_id:$scan_id}) -[:BATCH]-> (d:`+string(scanType)+`) + RETURN d.node_id, d.benchmark_type, d.status`, + map[string]interface{}{"scan_id": scanId}) + if err != nil { + return scanIds, err + } + + recs, err := neo_res.Collect() + if err != nil { + return scanIds, err + } + + for _, rec := range recs { + tmp := model.ComplianceScanStatus{ + ScanId: rec.Values[0].(string), + BenchmarkType: rec.Values[1].(string), + Status: rec.Values[2].(string), + } + scanIds.Statuses = append(scanIds.Statuses, tmp) + } + + return scanIds, nil +} diff --git a/deepfence_server/router/router.go b/deepfence_server/router/router.go index 29e53979f8..7e530c4c2d 100644 --- a/deepfence_server/router/router.go +++ b/deepfence_server/router/router.go @@ -168,9 +168,10 @@ func SetupRoutes(r *chi.Mux, serverPort string, jwtSecret []byte, serveOpenapiDo r.Post("/secrets", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestSecretReportHandler)) r.Post("/secret-scan-logs", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestSecretScanStatusHandler)) r.Post("/compliance", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestComplianceReportHandler)) - r.Post("/cloud-compliance", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestCloudComplianceReportHandler)) r.Post("/malware", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestMalwareReportHandler)) r.Post("/malware-scan-logs", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestMalwareScanStatusHandler)) + r.Post("/cloud-compliance", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestCloudComplianceReportHandler)) + r.Post("/cloud-compliance-status", dfHandler.AuthHandler(ResourceScanReport, PermissionIngest, dfHandler.IngestCloudComplianceScanStatusReportHandler)) }) r.Route("/cloud-node", func(r chi.Router) { @@ -184,7 +185,6 @@ func SetupRoutes(r *chi.Mux, serverPort string, jwtSecret []byte, serveOpenapiDo r.Post("/secret", dfHandler.AuthHandler(ResourceScan, PermissionStart, dfHandler.StartSecretScanHandler)) r.Post("/compliance", dfHandler.AuthHandler(ResourceScan, PermissionStart, dfHandler.StartComplianceScanHandler)) r.Post("/malware", dfHandler.AuthHandler(ResourceScan, PermissionStart, dfHandler.StartMalwareScanHandler)) - r.Post("/cloud-compliance", dfHandler.AuthHandler(ResourceScan, PermissionStart, dfHandler.StartCloudComplianceScanHandler)) }) r.Route("/scan/stop", func(r chi.Router) { r.Post("/vulnerability", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.StopVulnerabilityScanHandler)) @@ -197,6 +197,7 @@ func SetupRoutes(r *chi.Mux, serverPort string, jwtSecret []byte, serveOpenapiDo r.Get("/secret", dfHandler.AuthHandler(ResourceScan, PermissionRead, dfHandler.StatusSecretScanHandler)) r.Get("/compliance", dfHandler.AuthHandler(ResourceScan, PermissionRead, dfHandler.StatusComplianceScanHandler)) r.Get("/malware", dfHandler.AuthHandler(ResourceScan, PermissionRead, dfHandler.StatusMalwareScanHandler)) + r.Get("/cloud-compliance", dfHandler.AuthHandler(ResourceScan, PermissionRead, dfHandler.StatusCloudComplianceScanHandler)) }) r.Route("/scan/list", func(r chi.Router) { r.Post("/vulnerability", dfHandler.AuthHandler(ResourceScanReport, PermissionRead, dfHandler.ListVulnerabilityScansHandler)) diff --git a/deepfence_worker/ingesters/cloud_compliance.go b/deepfence_worker/ingesters/cloud_compliance.go index e8bf949b6d..b20e844f85 100644 --- a/deepfence_worker/ingesters/cloud_compliance.go +++ b/deepfence_worker/ingesters/cloud_compliance.go @@ -2,25 +2,34 @@ package ingesters import ( "encoding/json" + "fmt" "time" "github.com/deepfence/golang_deepfence_sdk/utils/directory" - "github.com/deepfence/golang_deepfence_sdk/utils/log" "github.com/deepfence/golang_deepfence_sdk/utils/utils" "github.com/neo4j/neo4j-go-driver/v4/neo4j" ) +type ComplianceStats struct { + Alarm int `json:"alarm"` + Ok int `json:"ok"` + Info int `json:"info"` + Skip int `json:"skip"` + Error int `json:"error"` + CompliancePercentage float64 `json:"compliance_percentage"` +} + type CloudComplianceScanStatus struct { - Timestamp time.Time `json:"@timestamp"` - ContainerName string `json:"container_name"` - HostName string `json:"host_name"` - KubernetesClusterName string `json:"kubernetes_cluster_name"` - Masked string `json:"masked"` - NodeID string `json:"node_id"` - NodeName string `json:"node_name"` - NodeType string `json:"node_type"` - ScanID string `json:"scan_id"` - ScanStatus string `json:"scan_status"` + Timestamp time.Time `json:"@timestamp"` + ComplianceCheckType string `json:"compliance_check_type"` + Masked string `json:"masked"` + NodeID string `json:"node_id"` + Result ComplianceStats `json:"result" nested_json:"true"` + ScanID string `json:"scan_id"` + ScanMessage string `json:"scan_message"` + Status string `json:"status"` + Type string `json:"type"` + TotalChecks int `json:"total_checks"` } type CloudCompliance struct { @@ -66,12 +75,21 @@ func CommitFuncCloudCompliance(ns string, data []CloudCompliance) error { } defer tx.Close() - if _, err = tx.Run("UNWIND $batch as row MERGE (n:CloudCompliance{resource:row.resource, reason: row.reason}) MERGE (m:CloudResource{node_id:row.resource}) MERGE (n) -[:SCANNED]-> (m) SET n+= row", + if _, err = tx.Run(` + UNWIND $batch as row + MERGE (n:CloudComplianceResult{resource:row.resource, scan_id: row.scan_id, control_id: row.control_id}) + MERGE (m:CloudResource{node_id: row.resource}) + MERGE (n) -[:SCANNED]-> (m) + SET n+= row`, map[string]interface{}{"batch": CloudCompliancesToMaps(data)}); err != nil { return err } - if _, err = tx.Run("MATCH (n:CloudCompliance) MERGE (m:CloudComplianceScan{node_id: n.scan_id, time_stamp: timestamp()}) MERGE (m) -[:DETECTED]-> (n)", + if _, err = tx.Run(fmt.Sprintf(` + MATCH (n:CloudComplianceResult) + MERGE (m:%s{node_id: n.scan_id}) + SET m.time_stamp = timestamp() + MERGE (m) -[:DETECTED]-> (n)`, utils.NEO4J_CLOUD_COMPLIANCE_SCAN), map[string]interface{}{}); err != nil { return err } @@ -98,8 +116,16 @@ func CommitFuncCloudComplianceScanStatus(ns string, data []CloudComplianceScanSt } defer tx.Close() - // TODO: add query to commit for scan status - log.Error().Msg("Not implemented") + ccScanMaps := CloudComplianceScansToMaps(data) + if _, err = tx.Run(fmt.Sprintf(` + UNWIND $batch as row + MATCH (n:%s{node_id: row.scan_id}) + MATCH (m:Node{node_id: row.connected_node_id}) + MATCH (n) -[:SCANNED]-> (m) + SET n+= row`, utils.NEO4J_CLOUD_COMPLIANCE_SCAN), + map[string]interface{}{"batch": ccScanMaps}); err != nil { + return err + } return tx.Commit() } @@ -112,6 +138,15 @@ func CloudCompliancesToMaps(ms []CloudCompliance) []map[string]interface{} { return res } +func CloudComplianceScansToMaps(ms []CloudComplianceScanStatus) []map[string]interface{} { + var res []map[string]interface{} + for _, v := range ms { + out := v.ToMap() + res = append(res, out) + } + return res +} + func (c CloudCompliance) ToMap() map[string]interface{} { out, err := json.Marshal(c) if err != nil { @@ -121,3 +156,24 @@ func (c CloudCompliance) ToMap() map[string]interface{} { _ = json.Unmarshal(out, &bb) return bb } + +func (c CloudComplianceScanStatus) ToMap() map[string]interface{} { + out, err := json.Marshal(c) + if err != nil { + return nil + } + bb := map[string]interface{}{} + _ = json.Unmarshal(out, &bb) + if results, ok := bb["result"]; ok { + resultsByte, err := json.Marshal(results) + if err != nil { + resultsByte = []byte{} + } + bb["result"] = string(resultsByte) + } + if nodeId, ok := bb["node_id"]; ok { + bb["connected_node_id"] = nodeId + delete(bb, "node_id") + } + return bb +}