From b02133287b65e730092a0e62d95b30a7ee5a30d8 Mon Sep 17 00:00:00 2001 From: Ramanan Ravikumar Date: Thu, 6 Apr 2023 17:46:22 +0530 Subject: [PATCH 1/2] Add scans data as part of node metadata api #725 --- deepfence_server/model/lookup.go | 127 +++++++++++------- .../reporters/graph/threat_graph.go | 48 +++---- deepfence_server/reporters/lookup/lookup.go | 32 ----- deepfence_worker/cronjobs/threat.go | 60 ++++----- deepfence_worker/ingesters/common.go | 23 +--- tests/loading/neo4j-setup/filler/main.go | 5 +- 6 files changed, 135 insertions(+), 160 deletions(-) diff --git a/deepfence_server/model/lookup.go b/deepfence_server/model/lookup.go index 787efc8b7f..dc83dd31d8 100644 --- a/deepfence_server/model/lookup.go +++ b/deepfence_server/model/lookup.go @@ -41,12 +41,12 @@ func (kc KubernetesCluster) id() string { } type RegularScanStatus struct { - VulnerabilitiesCount int64 `json:"vulnerabilities_count" required:"true"` + VulnerabilitiesCount int64 `json:"vulnerabilities_count" required:"true"` VulnerabilityScanStatus string `json:"vulnerability_scan_status" required:"true"` VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id" required:"true"` SecretsCount int64 `json:"secrets_count" required:"true"` SecretScanStatus string `json:"secret_scan_status" required:"true"` - SecretLatestScanId string `json:"secret_latest_scan" required:"true"` + SecretLatestScanId string `json:"secret_latest_scan_id" required:"true"` MalwaresCount int64 `json:"malwares_count" required:"true"` MalwareScanStatus string `json:"malware_scan_status" required:"true"` MalwareLatestScanId string `json:"malware_latest_scan_id" required:"true"` @@ -55,7 +55,7 @@ type RegularScanStatus struct { ComplianceLatestScanId string `json:"compliance_latest_scan_id" required:"true"` CloudCompliancesCount int64 `json:"cloud_compliances_count" required:"true"` CloudComplianceScanStatus string `json:"cloud_compliance_scan_status" required:"true"` - CloudComplianceLatestScanId string `json:"cloud_compliance_latest scan_id" required:"true"` + CloudComplianceLatestScanId string `json:"cloud_compliance_latest_scan_id" required:"true"` } type BasicNode struct { @@ -66,37 +66,48 @@ type BasicNode struct { } type Host struct { - ID string `json:"node_id" required:"true"` - HostName string `json:"host_name" required:"true"` - NodeName string `json:"node_name" required:"true"` - Containers []Container `json:"containers" required:"true"` - Processes []Process `json:"processes" required:"true"` - Pods []Pod `json:"pods" required:"true"` - ContainerImages []ContainerImage `json:"container_images" required:"true"` - InterfaceNames []string `json:"interface_names" required:"true"` - InterfaceIps []string `json:"interface_ips" required:"true"` - KernelVersion string `json:"kernel_version" required:"true"` - Uptime int `json:"uptime" required:"true"` - Version string `json:"version" required:"true"` - AgentRunning bool `json:"agent_running" required:"true"` - IsConsoleVm bool `json:"is_console_vm" required:"true"` - LocalCIDRs []string `json:"local_cidr" required:"true"` - Os string `json:"os" required:"true"` - LocalNetworks []string `json:"local_networks,omitempty"` - InstanceID string `json:"instance_id,omitempty"` - CloudProvider string `json:"cloud_provider" required:"true"` - InstanceType string `json:"instance_type,omitempty"` - PublicIP []string `json:"public_ip,omitempty"` - PrivateIP []string `json:"private_ip,omitempty"` - AvailabilityZone string `json:"availability_zone,omitempty"` - KernelId string `json:"kernel_id,omitempty"` - CloudRegion string `json:"cloud_region" required:"true"` - ResourceGroup string `json:"resource_group,omitempty"` - CpuMax float64 `json:"cpu_max,omitempty"` - CpuUsage float64 `json:"cpu_usage,omitempty"` - MemoryMax int64 `json:"memory_max,omitempty"` - MemoryUsage int64 `json:"memory_usage,omitempty"` - RegularScanStatus + ID string `json:"node_id" required:"true"` + HostName string `json:"host_name" required:"true"` + NodeName string `json:"node_name" required:"true"` + Containers []Container `json:"containers" required:"true"` + Processes []Process `json:"processes" required:"true"` + Pods []Pod `json:"pods" required:"true"` + ContainerImages []ContainerImage `json:"container_images" required:"true"` + InterfaceNames []string `json:"interface_names" required:"true"` + InterfaceIps []string `json:"interface_ips" required:"true"` + KernelVersion string `json:"kernel_version" required:"true"` + Uptime int `json:"uptime" required:"true"` + Version string `json:"version" required:"true"` + AgentRunning bool `json:"agent_running" required:"true"` + IsConsoleVm bool `json:"is_console_vm" required:"true"` + LocalCIDRs []string `json:"local_cidr" required:"true"` + Os string `json:"os" required:"true"` + LocalNetworks []string `json:"local_networks,omitempty"` + InstanceID string `json:"instance_id,omitempty"` + CloudProvider string `json:"cloud_provider" required:"true"` + InstanceType string `json:"instance_type,omitempty"` + PublicIP []string `json:"public_ip,omitempty"` + PrivateIP []string `json:"private_ip,omitempty"` + AvailabilityZone string `json:"availability_zone,omitempty"` + KernelId string `json:"kernel_id,omitempty"` + CloudRegion string `json:"cloud_region" required:"true"` + ResourceGroup string `json:"resource_group,omitempty"` + CpuMax float64 `json:"cpu_max,omitempty"` + CpuUsage float64 `json:"cpu_usage,omitempty"` + MemoryMax int64 `json:"memory_max,omitempty"` + MemoryUsage int64 `json:"memory_usage,omitempty"` + VulnerabilitiesCount int64 `json:"vulnerabilities_count"` + VulnerabilityScanStatus string `json:"vulnerability_scan_status"` + VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id"` + SecretsCount int64 `json:"secrets_count"` + SecretScanStatus string `json:"secret_scan_status"` + SecretLatestScanId string `json:"secret_latest_scan_id"` + MalwaresCount int64 `json:"malwares_count"` + MalwareScanStatus string `json:"malware_scan_status"` + MalwareLatestScanId string `json:"malware_latest_scan_id"` + CompliancesCount int64 `json:"compliances_count"` + ComplianceScanStatus string `json:"compliance_scan_status"` + ComplianceLatestScanId string `json:"compliance_latest_scan_id"` } func (Host) NodeType() string { @@ -211,7 +222,15 @@ type Container struct { CpuUsage float64 `json:"cpu_usage,omitempty"` MemoryMax int64 `json:"memory_max,omitempty"` MemoryUsage int64 `json:"memory_usage,omitempty"` - RegularScanStatus + VulnerabilitiesCount int64 `json:"vulnerabilities_count"` + VulnerabilityScanStatus string `json:"vulnerability_scan_status"` + VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id"` + SecretsCount int64 `json:"secrets_count"` + SecretScanStatus string `json:"secret_scan_status"` + SecretLatestScanId string `json:"secret_latest_scan_id"` + MalwaresCount int64 `json:"malwares_count"` + MalwareScanStatus string `json:"malware_scan_status"` + MalwareLatestScanId string `json:"malware_latest_scan_id"` } func (Container) NodeType() string { @@ -269,16 +288,24 @@ func (p Process) id() string { } type ContainerImage struct { - ID string `json:"node_id" required:"true"` - NodeName string `json:"node_name" required:"true"` - Name string `json:"docker_image_name" required:"true"` - Tag string `json:"docker_image_tag" required:"true"` - Size string `json:"docker_image_size" required:"true"` - DockerImageCreatedAt string `json:"docker_image_created_at,omitempty"` - DockerImageVirtualSize string `json:"docker_image_virtual_size,omitempty"` - DockerImageID string `json:"docker_image_id,omitempty"` - Metadata Metadata `json:"metadata" required:"true" nested_json:"true"` - RegularScanStatus + ID string `json:"node_id" required:"true"` + NodeName string `json:"node_name" required:"true"` + Name string `json:"docker_image_name" required:"true"` + Tag string `json:"docker_image_tag" required:"true"` + Size string `json:"docker_image_size" required:"true"` + DockerImageCreatedAt string `json:"docker_image_created_at,omitempty"` + DockerImageVirtualSize string `json:"docker_image_virtual_size,omitempty"` + DockerImageID string `json:"docker_image_id,omitempty"` + Metadata Metadata `json:"metadata" required:"true" nested_json:"true"` + VulnerabilitiesCount int64 `json:"vulnerabilities_count"` + VulnerabilityScanStatus string `json:"vulnerability_scan_status"` + VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id"` + SecretsCount int64 `json:"secrets_count"` + SecretScanStatus string `json:"secret_scan_status"` + SecretLatestScanId string `json:"secret_latest_scan_id"` + MalwaresCount int64 `json:"malwares_count"` + MalwareScanStatus string `json:"malware_scan_status"` + MalwareLatestScanId string `json:"malware_latest_scan_id"` } func (ContainerImage) NodeType() string { @@ -302,10 +329,12 @@ func (ci ContainerImage) id() string { } type CloudResource struct { - ID string `json:"node_id" required:"true"` - Name string `json:"node_name" required:"true"` - Type string `json:"node_type" required:"true"` - RegularScanStatus + ID string `json:"node_id" required:"true"` + Name string `json:"node_name" required:"true"` + Type string `json:"node_type" required:"true"` + CloudCompliancesCount int64 `json:"cloud_compliances_count" required:"true"` + CloudComplianceScanStatus string `json:"cloud_compliance_scan_status" required:"true"` + CloudComplianceLatestScanId string `json:"cloud_compliance_latest_scan_id" required:"true"` } func (CloudResource) NodeType() string { diff --git a/deepfence_server/reporters/graph/threat_graph.go b/deepfence_server/reporters/graph/threat_graph.go index 163ff264a3..ac6761d871 100644 --- a/deepfence_server/reporters/graph/threat_graph.go +++ b/deepfence_server/reporters/graph/threat_graph.go @@ -151,15 +151,15 @@ func (tc *ThreatGraphReporter) GetRawThreatGraph(filters ThreatFilters) (map[str MATCH (n:CloudResource) WHERE n.depth IS NOT NULL AND ( - CASE WHEN $type = 'vulnerability' or $type = 'all' THEN n.num_cve > 0 ELSE false END + CASE WHEN $type = 'vulnerability' or $type = 'all' THEN n.vulnerabilities_count > 0 ELSE false END OR - CASE WHEN $type = 'secret' or $type = 'all' THEN n.num_secrets > 0 ELSE false END + CASE WHEN $type = 'secret' or $type = 'all' THEN n.secrets_count > 0 ELSE false END OR - CASE WHEN $type = 'malware' or $type = 'all' THEN n.num_malware > 0 ELSE false END + CASE WHEN $type = 'malware' or $type = 'all' THEN n.malwares_count > 0 ELSE false END OR - CASE WHEN $type = 'compliance' or $type = 'all' THEN n.num_compliance > 0 ELSE false END + CASE WHEN $type = 'compliance' or $type = 'all' THEN n.compliances_count > 0 ELSE false END OR - CASE WHEN $type = 'cloud_compliance' or $type = 'all' THEN n.num_cloud_compliance > 0 ELSE false END + CASE WHEN $type = 'cloud_compliance' or $type = 'all' THEN n.cloud_compliances_count > 0 ELSE false END ) WITH n, n.cloud_provider as provider WHERE CASE WHEN size($aws_ids) = 0 OR provider <> 'aws' then true ELSE n.account_id IN $aws_ids END @@ -181,15 +181,15 @@ func (tc *ThreatGraphReporter) GetRawThreatGraph(filters ThreatFilters) (map[str MATCH (n:Node) WHERE n.depth IS NOT NULL AND ( - CASE WHEN $type = 'vulnerability' or $type = 'all' THEN n.num_cve > 0 ELSE false END + CASE WHEN $type = 'vulnerability' or $type = 'all' THEN n.vulnerabilities_count > 0 ELSE false END OR - CASE WHEN $type = 'secret' or $type = 'all' THEN n.num_secrets > 0 ELSE false END + CASE WHEN $type = 'secret' or $type = 'all' THEN n.secrets_count > 0 ELSE false END OR - CASE WHEN $type = 'malware' or $type = 'all' THEN n.num_malware > 0 ELSE false END + CASE WHEN $type = 'malware' or $type = 'all' THEN n.malwares_count > 0 ELSE false END OR - CASE WHEN $type = 'compliance' or $type = 'all' THEN n.num_compliance > 0 ELSE false END + CASE WHEN $type = 'compliance' or $type = 'all' THEN n.compliances_count > 0 ELSE false END OR - CASE WHEN $type = 'cloud_compliance' or $type = 'all' THEN n.num_cloud_compliance > 0 ELSE false END + CASE WHEN $type = 'cloud_compliance' or $type = 'all' THEN n.cloud_compliances_count > 0 ELSE false END ) SET n:ThreatNode `, map[string]interface{}{ @@ -207,7 +207,7 @@ func (tc *ThreatGraphReporter) GetRawThreatGraph(filters ThreatFilters) (map[str if res, err = tx.Run(` CALL apoc.nodes.group(['ThreatCloudResource','ThreatNode'], ['node_type', 'depth', 'cloud_provider'], [{`+"`*`"+`: 'count', sum_cve: 'sum', sum_secrets: 'sum', sum_compliance: 'sum', sum_cloud_compliance: 'sum', - node_id:'collect', num_cve: 'collect', num_secrets:'collect', num_compliance:'collect', num_cloud_compliance: 'collect'}, + node_id:'collect', vulnerabilities_count: 'collect', secrets_count:'collect', compliances_count:'collect', cloud_compliances_count: 'collect'}, {`+"`*`"+`: 'count'}], {selfRels: false}) YIELD node, relationships WHERE apoc.any.property(node, 'cloud_provider') = '`+cloud_provider+`' @@ -218,7 +218,7 @@ func (tc *ThreatGraphReporter) GetRawThreatGraph(filters ThreatFilters) (map[str if res, err = tx.Run(` CALL apoc.nodes.group(['ThreatNode'], ['node_type', 'depth', 'cloud_provider'], [{`+"`*`"+`: 'count', sum_cve: 'sum', sum_secrets: 'sum', sum_compliance: 'sum', sum_cloud_compliance: 'sum', - node_id:'collect', num_cve: 'collect', num_secrets:'collect', num_compliance:'collect', num_cloud_compliance:'collect'}, + node_id:'collect', vulnerabilities_count: 'collect', secrets_count:'collect', compliances_count:'collect', cloud_compliances_count:'collect'}, {`+"`*`"+`: 'count'}], {selfRels: false}) YIELD node, relationships WHERE NOT apoc.any.property(node, 'cloud_provider') IN ['aws', 'gcp', 'azure'] @@ -416,12 +416,12 @@ func (ap AttackPaths) getNodeInfos() map[int64]ThreatNodeInfo { cloud_compliance_count = v.collect_num_cloud_compliance[i] } Nodes[Node_id] = NodeInfo{ - Node_id: Node_id, - Name: Node_id, - Vulnerability_count: vuln_count, - Secrets_count: secrets_count, - Compliance_count: compliance_count, - CloudCompliance_count: cloud_compliance_count, + Node_id: Node_id, + Name: Node_id, + Vulnerability_count: vuln_count, + Secrets_count: secrets_count, + Compliance_count: compliance_count, + CloudCompliance_count: cloud_compliance_count, } } res[v.identity] = ThreatNodeInfo{ @@ -467,10 +467,10 @@ type ThreatNodeInfo struct { } type NodeInfo struct { - Node_id string `json:"node_id" required:"true"` - Name string `json:"name" required:"true"` - Vulnerability_count int64 `json:"vulnerability_count" required:"true"` - Secrets_count int64 `json:"secrets_count" required:"true"` - Compliance_count int64 `json:"compliance_count" required:"true"` - CloudCompliance_count int64 `json:"cloud_compliance_count" required:"true"` + Node_id string `json:"node_id" required:"true"` + Name string `json:"name" required:"true"` + Vulnerability_count int64 `json:"vulnerability_count" required:"true"` + Secrets_count int64 `json:"secrets_count" required:"true"` + Compliance_count int64 `json:"compliance_count" required:"true"` + CloudCompliance_count int64 `json:"cloud_compliance_count" required:"true"` } diff --git a/deepfence_server/reporters/lookup/lookup.go b/deepfence_server/reporters/lookup/lookup.go index bed12fc5d9..d0f0c1e02f 100644 --- a/deepfence_server/reporters/lookup/lookup.go +++ b/deepfence_server/reporters/lookup/lookup.go @@ -5,7 +5,6 @@ import ( "github.com/deepfence/ThreatMapper/deepfence_server/model" "github.com/deepfence/ThreatMapper/deepfence_server/reporters" - reporters_scan "github.com/deepfence/ThreatMapper/deepfence_server/reporters/scan" "github.com/deepfence/golang_deepfence_sdk/utils/directory" "github.com/deepfence/golang_deepfence_sdk/utils/utils" "github.com/neo4j/neo4j-go-driver/v4/neo4j" @@ -33,11 +32,6 @@ func GetHostsReport(ctx context.Context, filter LookupFilter) ([]model.Host, err host_ids = model.ExtractNodeIDs(hosts) } - statuses, err := reporters_scan.GetScanStatuses[model.Host](ctx, host_ids) - if err != nil { - return nil, err - } - for i := range hosts { processes, err := getHostProcesses(ctx, hosts[i]) if err != nil { @@ -56,7 +50,6 @@ func GetHostsReport(ctx context.Context, filter LookupFilter) ([]model.Host, err return nil, err } hosts[i].ContainerImages = container_images - hosts[i].RegularScanStatus = statuses[i] } return hosts, nil } @@ -91,15 +84,6 @@ func GetContainersReport(ctx context.Context, filter LookupFilter) ([]model.Cont container_ids = model.ExtractNodeIDs(containers) } - statuses, err := reporters_scan.GetScanStatuses[model.Container](ctx, container_ids) - if err != nil { - return nil, err - } - - for i := range containers { - containers[i].RegularScanStatus = statuses[i] - } - return containers, nil } @@ -130,14 +114,6 @@ func GetContainerImagesReport(ctx context.Context, filter LookupFilter) ([]model images_ids = model.ExtractNodeIDs(images) } - statuses, err := reporters_scan.GetScanStatuses[model.ContainerImage](ctx, images_ids) - if err != nil { - return nil, err - } - - for i := range images { - images[i].RegularScanStatus = statuses[i] - } return images, nil } @@ -160,14 +136,6 @@ func GetCloudResourcesReport(ctx context.Context, filter LookupFilter) ([]model. entries_ids = model.ExtractNodeIDs(entries) } - statuses, err := reporters_scan.GetScanStatuses[model.CloudResource](ctx, entries_ids) - if err != nil { - return nil, err - } - - for i := range entries { - entries[i].RegularScanStatus = statuses[i] - } return entries, nil } diff --git a/deepfence_worker/cronjobs/threat.go b/deepfence_worker/cronjobs/threat.go index efa7b9c7e2..bf122eaecf 100644 --- a/deepfence_worker/cronjobs/threat.go +++ b/deepfence_worker/cronjobs/threat.go @@ -90,8 +90,8 @@ func computeThreatGraph(session neo4j.Session) error { MATCH (s:VulnerabilityScan) -[:SCANNED]-> (m) WITH distinct m, max(s.updated_at) as most_recent MATCH (m) <-[:SCANNED]- (s:VulnerabilityScan{updated_at: most_recent})-[:DETECTED]->(c:Vulnerability) - WITH m, count(distinct c) as num_cve - SET m.num_cve = num_cve`, map[string]interface{}{}); err != nil { + WITH s, m, count(distinct c) as vulnerabilities_count + SET m.vulnerabilities_count = vulnerabilities_count, m.vulnerability_scan_status = s.status, m.vulnerability_latest_scan_id = s.node_id`, map[string]interface{}{}); err != nil { return err } @@ -99,8 +99,8 @@ func computeThreatGraph(session neo4j.Session) error { MATCH (s:SecretScan) -[:SCANNED]-> (m) WITH distinct m, max(s.updated_at) as most_recent MATCH (m) <-[:SCANNED]- (s:SecretScan{updated_at: most_recent})-[:DETECTED]->(c:Secret) - WITH m, count(distinct c) as num_secrets - SET m.num_secrets = num_secrets`, map[string]interface{}{}); err != nil { + WITH s, m, count(distinct c) as secrets_count + SET m.secrets_count = secrets_count, m.secret_scan_status = s.status, m.secret_latest_scan_id = s.node_id`, map[string]interface{}{}); err != nil { return err } @@ -108,8 +108,8 @@ func computeThreatGraph(session neo4j.Session) error { MATCH (s:MalwareScan) -[:SCANNED]-> (m) WITH distinct m, max(s.updated_at) as most_recent MATCH (m) <-[:SCANNED]- (s:MalwareScan{updated_at: most_recent})-[:DETECTED]->(c:Malware) - WITH m, count(distinct c) as num_malware - SET m.num_malware = num_malware`, map[string]interface{}{}); err != nil { + WITH s, m, count(distinct c) as malwares_count + SET m.malwares_count = malwares_count, m.malware_scan_status = s.status, m.malware_latest_scan_id = s.node_id`, map[string]interface{}{}); err != nil { return err } @@ -117,8 +117,8 @@ func computeThreatGraph(session neo4j.Session) error { MATCH (s:ComplianceScan) -[:SCANNED]-> (m) WITH distinct m, max(s.updated_at) as most_recent MATCH (m) <-[:SCANNED]- (s:ComplianceScan{updated_at: most_recent})-[:DETECTED]->(c:Compliance) - WITH m, count(distinct c) as num_compliance - SET m.num_compliance = num_compliance`, map[string]interface{}{}); err != nil { + WITH s, m, count(distinct c) as compliances_count + SET m.compliances_count = compliances_count, m.compliance_scan_status = s.status, m.compliance_latest_scan_id = s.node_id`, map[string]interface{}{}); err != nil { return err } @@ -126,41 +126,41 @@ func computeThreatGraph(session neo4j.Session) error { MATCH (s:CloudComplianceScan) -[:SCANNED]-> (m) WITH distinct m, max(s.updated_at) as most_recent MATCH (m) <-[:SCANNED]- (s:ComplianceScan{updated_at: most_recent})-[:DETECTED]->(c:CloudComplianceResult) - WITH m, count(distinct c) as num_cloud_compliance - SET m.num_cloud_compliance = num_cloud_compliance`, map[string]interface{}{}); err != nil { + WITH s, m, count(distinct c) as cloud_compliances_count + SET m.cloud_compliances_count = cloud_compliances_count, m.cloud_compliance_scan_status = s.status, m.cloud_compliance_latest_scan_id = s.node_id`, map[string]interface{}{}); err != nil { return err } if _, err = tx.Run(` MATCH (n) WHERE n:Node OR n:CloudResource - SET n.num_cve = COALESCE(n.num_cve, 0), - n.num_secrets = COALESCE(n.num_secrets, 0), - n.num_malware = COALESCE(n.num_malware, 0), - n.num_compliance = COALESCE(n.num_compliance, 0), - n.num_cloud_compliance = COALESCE(n.num_cloud_compliance, 0)`, map[string]interface{}{}); err != nil { + SET n.vulnerabilities_count = COALESCE(n.vulnerabilities_count, 0), + n.secrets_count = COALESCE(n.secrets_count, 0), + n.malwares_count = COALESCE(n.malwares_count, 0), + n.compliances_count = COALESCE(n.compliances_count, 0), + n.cloud_compliances_count = COALESCE(n.cloud_compliances_count, 0)`, map[string]interface{}{}); err != nil { return err } if _, err = tx.Run(` MATCH (n) WHERE n:Node OR n:CloudResource - SET n.sum_cve = COALESCE(n.num_cve, 0), - n.sum_secrets = COALESCE(n.num_secrets, 0), - n.sum_malware = COALESCE(n.num_malware, 0), - n.sum_compliance = COALESCE(n.num_compliance, 0), - n.sum_cloud_compliance = COALESCE(n.num_cloud_compliance, 0)`, map[string]interface{}{}); err != nil { + SET n.sum_cve = COALESCE(n.vulnerabilities_count, 0), + n.sum_secrets = COALESCE(n.secrets_count, 0), + n.sum_malware = COALESCE(n.malwares_count, 0), + n.sum_compliance = COALESCE(n.compliances_count, 0), + n.sum_cloud_compliance = COALESCE(n.cloud_compliances_count, 0)`, map[string]interface{}{}); err != nil { return err } if _, err = tx.Run(` MATCH (n) -[:HOSTS]-> (m) WHERE n:Node OR n:CloudResource - SET n.sum_cve = n.sum_cve + COALESCE(m.num_cve, 0), - n.sum_secrets = n.sum_secrets + COALESCE(m.num_secrets, 0), - n.sum_malware = n.sum_malware + COALESCE(m.num_malware, 0), - n.sum_compliance = n.sum_compliance + COALESCE(m.num_compliance, 0), - n.sum_cloud_compliance = n.sum_cloud_compliance + COALESCE(m.num_cloud_compliance, 0)`, map[string]interface{}{}); err != nil { + SET n.sum_cve = n.sum_cve + COALESCE(m.vulnerabilities_count, 0), + n.sum_secrets = n.sum_secrets + COALESCE(m.secrets_count, 0), + n.sum_malware = n.sum_malware + COALESCE(m.malwares_count, 0), + n.sum_compliance = n.sum_compliance + COALESCE(m.compliances_count, 0), + n.sum_cloud_compliance = n.sum_cloud_compliance + COALESCE(m.cloud_compliances_count, 0)`, map[string]interface{}{}); err != nil { return err } @@ -183,11 +183,11 @@ func computeThreatGraph(session neo4j.Session) error { MATCH (n:Node) -[:CONNECTS]->(m:Node) WHERE n.depth = m.depth - 1 WITH n, m - SET n.sum_cve = COALESCE(n.sum_cve, 0) + COALESCE(m.sum_cve, m.num_cve, 0), - n.sum_malware = COALESCE(n.sum_malware, 0) + COALESCE(m.sum_malware, m.num_malware, 0) , - n.sum_secrets = COALESCE(n.sum_secrets, 0) + COALESCE(m.sum_secrets, m.num_secrets, 0), - n.sum_compliance = COALESCE(n.sum_compliance, 0) + COALESCE(m.sum_compliance, m.num_compliance, 0), - n.sum_cloud_compliance = COALESCE(n.sum_cloud_compliance, 0) + COALESCE(m.sum_cloud_compliance, m.num_cloud_compliance, 0)`, map[string]interface{}{}); err != nil { + SET n.sum_cve = COALESCE(n.sum_cve, 0) + COALESCE(m.sum_cve, m.vulnerabilities_count, 0), + n.sum_malware = COALESCE(n.sum_malware, 0) + COALESCE(m.sum_malware, m.malwares_count, 0) , + n.sum_secrets = COALESCE(n.sum_secrets, 0) + COALESCE(m.sum_secrets, m.secrets_count, 0), + n.sum_compliance = COALESCE(n.sum_compliance, 0) + COALESCE(m.sum_compliance, m.compliances_count, 0), + n.sum_cloud_compliance = COALESCE(n.sum_cloud_compliance, 0) + COALESCE(m.sum_cloud_compliance, m.cloud_compliances_count, 0)`, map[string]interface{}{}); err != nil { return err } diff --git a/deepfence_worker/ingesters/common.go b/deepfence_worker/ingesters/common.go index 38e7ab2419..0453be4022 100644 --- a/deepfence_worker/ingesters/common.go +++ b/deepfence_worker/ingesters/common.go @@ -9,22 +9,6 @@ import ( "github.com/neo4j/neo4j-go-driver/v4/neo4j" ) -func status_count_field(ts utils.Neo4jScanType) string { - switch ts { - case utils.NEO4J_SECRET_SCAN: - return "secrets_count" - case utils.NEO4J_VULNERABILITY_SCAN: - return "vulnerabilities_count" - case utils.NEO4J_MALWARE_SCAN: - return "malwares_count" - case utils.NEO4J_COMPLIANCE_SCAN: - return "compliances_count" - case utils.NEO4J_CLOUD_COMPLIANCE_SCAN: - return "cloud_compliances_count" - } - return "unknown" -} - func CommitFuncStatus[Status any](ts utils.Neo4jScanType) func(ns string, data []Status) error { return func(ns string, data []Status) error { ctx := directory.NewContextWithNameSpace(directory.NamespaceID(ns)) @@ -53,12 +37,7 @@ func CommitFuncStatus[Status any](ts utils.Neo4jScanType) func(ns string, data [ if _, err = tx.Run(` UNWIND $batch as row MERGE (n:`+string(ts)+`{node_id: row.scan_id}) - SET n.status = row.scan_status, n.scan_message = row.scan_message, n.updated_at = TIMESTAMP() - WITH n - MATCH (n) -[:DETECTED]- (m) - WITH n, count(m) as count - MATCH (n) -[:SCANNED]- (r) - SET r.`+status_count_field(ts)+` = count`, + SET n.status = row.scan_status, n.scan_message = row.scan_message, n.updated_at = TIMESTAMP()`, map[string]interface{}{"batch": statusesToMaps(data)}); err != nil { log.Error().Msgf("Error while updating scan status: %+v", err) return err diff --git a/tests/loading/neo4j-setup/filler/main.go b/tests/loading/neo4j-setup/filler/main.go index 8cabec4522..c4ec1b721a 100644 --- a/tests/loading/neo4j-setup/filler/main.go +++ b/tests/loading/neo4j-setup/filler/main.go @@ -88,7 +88,6 @@ func apply(round int) { "host_node_id": "flac-desktop;", "node_type": "host", "updated_at": time.Now().Unix(), - "num_cve": 80, "sum_cve": 170, "interface_ips": "{\"172.17.0.1\":\"255.255.0.0\",\"192.168.1.4\":\"255.255.255.0\",\"192.168.122.1\":\"255.255.255.0\"}", "sum_secrets": 0, @@ -96,7 +95,7 @@ func apply(round int) { "os": "linux", "kubernetes_cluster_id": "", "probeId": "17919ef2ef95d01d", - "num_compliance": 0, + "compliances_count": 0, "cloud_provider": "Serverless", "is_ui_vm": "false", "interfaceNames": "lo;enp3s0;virbr0;docker0", @@ -105,7 +104,7 @@ func apply(round int) { "uptime": "119863", "depth": 1, "kernel_version": "6.1.0-5-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.12-1 (2023-02-15)", - "num_secrets": 0, + "secrets_count": 0, "sum_compliance": 0, "user_defined_tags": "", "host_name": "flac-desktop", From a3dde7a48c12a35e673b8963b8aadadc678643d3 Mon Sep 17 00:00:00 2001 From: Ramanan Ravikumar Date: Thu, 6 Apr 2023 18:56:52 +0530 Subject: [PATCH 2/2] Send all metadata fields in api response #725 --- deepfence_server/model/lookup.go | 148 ++++++++---------- .../reporters/scan/scan_results.go | 125 --------------- 2 files changed, 65 insertions(+), 208 deletions(-) delete mode 100644 deepfence_server/reporters/scan/scan_results.go diff --git a/deepfence_server/model/lookup.go b/deepfence_server/model/lookup.go index dc83dd31d8..398959fa51 100644 --- a/deepfence_server/model/lookup.go +++ b/deepfence_server/model/lookup.go @@ -40,24 +40,6 @@ func (kc KubernetesCluster) id() string { return kc.ID } -type RegularScanStatus struct { - VulnerabilitiesCount int64 `json:"vulnerabilities_count" required:"true"` - VulnerabilityScanStatus string `json:"vulnerability_scan_status" required:"true"` - VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id" required:"true"` - SecretsCount int64 `json:"secrets_count" required:"true"` - SecretScanStatus string `json:"secret_scan_status" required:"true"` - SecretLatestScanId string `json:"secret_latest_scan_id" required:"true"` - MalwaresCount int64 `json:"malwares_count" required:"true"` - MalwareScanStatus string `json:"malware_scan_status" required:"true"` - MalwareLatestScanId string `json:"malware_latest_scan_id" required:"true"` - CompliancesCount int64 `json:"compliances_count" required:"true"` - ComplianceScanStatus string `json:"compliance_scan_status" required:"true"` - ComplianceLatestScanId string `json:"compliance_latest_scan_id" required:"true"` - CloudCompliancesCount int64 `json:"cloud_compliances_count" required:"true"` - CloudComplianceScanStatus string `json:"cloud_compliance_scan_status" required:"true"` - CloudComplianceLatestScanId string `json:"cloud_compliance_latest_scan_id" required:"true"` -} - type BasicNode struct { NodeId string `json:"node_id" required:"true"` Name string `json:"name" required:"true"` @@ -82,32 +64,32 @@ type Host struct { IsConsoleVm bool `json:"is_console_vm" required:"true"` LocalCIDRs []string `json:"local_cidr" required:"true"` Os string `json:"os" required:"true"` - LocalNetworks []string `json:"local_networks,omitempty"` - InstanceID string `json:"instance_id,omitempty"` + LocalNetworks []string `json:"local_networks" required:"true"` + InstanceID string `json:"instance_id" required:"true"` CloudProvider string `json:"cloud_provider" required:"true"` - InstanceType string `json:"instance_type,omitempty"` - PublicIP []string `json:"public_ip,omitempty"` - PrivateIP []string `json:"private_ip,omitempty"` - AvailabilityZone string `json:"availability_zone,omitempty"` - KernelId string `json:"kernel_id,omitempty"` + InstanceType string `json:"instance_type" required:"true"` + PublicIP []string `json:"public_ip" required:"true"` + PrivateIP []string `json:"private_ip" required:"true"` + AvailabilityZone string `json:"availability_zone" required:"true"` + KernelId string `json:"kernel_id" required:"true"` CloudRegion string `json:"cloud_region" required:"true"` - ResourceGroup string `json:"resource_group,omitempty"` - CpuMax float64 `json:"cpu_max,omitempty"` - CpuUsage float64 `json:"cpu_usage,omitempty"` - MemoryMax int64 `json:"memory_max,omitempty"` - MemoryUsage int64 `json:"memory_usage,omitempty"` - VulnerabilitiesCount int64 `json:"vulnerabilities_count"` - VulnerabilityScanStatus string `json:"vulnerability_scan_status"` - VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id"` - SecretsCount int64 `json:"secrets_count"` - SecretScanStatus string `json:"secret_scan_status"` - SecretLatestScanId string `json:"secret_latest_scan_id"` - MalwaresCount int64 `json:"malwares_count"` - MalwareScanStatus string `json:"malware_scan_status"` - MalwareLatestScanId string `json:"malware_latest_scan_id"` - CompliancesCount int64 `json:"compliances_count"` - ComplianceScanStatus string `json:"compliance_scan_status"` - ComplianceLatestScanId string `json:"compliance_latest_scan_id"` + ResourceGroup string `json:"resource_group" required:"true"` + CpuMax float64 `json:"cpu_max" required:"true"` + CpuUsage float64 `json:"cpu_usage" required:"true"` + MemoryMax int64 `json:"memory_max" required:"true"` + MemoryUsage int64 `json:"memory_usage" required:"true"` + VulnerabilitiesCount int64 `json:"vulnerabilities_count" required:"true"` + VulnerabilityScanStatus string `json:"vulnerability_scan_status" required:"true"` + VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id" required:"true"` + SecretsCount int64 `json:"secrets_count" required:"true"` + SecretScanStatus string `json:"secret_scan_status" required:"true"` + SecretLatestScanId string `json:"secret_latest_scan_id" required:"true"` + MalwaresCount int64 `json:"malwares_count" required:"true"` + MalwareScanStatus string `json:"malware_scan_status" required:"true"` + MalwareLatestScanId string `json:"malware_latest_scan_id" required:"true"` + CompliancesCount int64 `json:"compliances_count" required:"true"` + ComplianceScanStatus string `json:"compliance_scan_status" required:"true"` + ComplianceLatestScanId string `json:"compliance_latest_scan_id" required:"true"` } func (Host) NodeType() string { @@ -176,9 +158,9 @@ type Pod struct { Processes []Process `json:"processes" required:"true"` KubernetesClusterName string `json:"kubernetes_cluster_name" required:"true"` KubernetesClusterId string `json:"kubernetes_cluster_id" required:"true"` - KubernetesState string `json:"kubernetes_state"` - KubernetesIP string `json:"kubernetes_ip"` - KubernetesIsInHostNetwork string `json:"kubernetes_is_in_host_network"` + KubernetesState string `json:"kubernetes_state" required:"true"` + KubernetesIP string `json:"kubernetes_ip" required:"true"` + KubernetesIsInHostNetwork string `json:"kubernetes_is_in_host_network" required:"true"` } func (Pod) NodeType() string { @@ -209,28 +191,28 @@ type Container struct { Processes []Process `json:"processes" required:"true"` DockerLabels Metadata `json:"docker_labels" required:"true" nested_json:"true"` HostName string `json:"host_name" required:"true"` - DockerContainerCommand string `json:"docker_container_command,omitempty"` - DockerContainerState string `json:"docker_container_state,omitempty"` - DockerContainerStateHuman string `json:"docker_container_state_human,omitempty"` - DockerContainerNetworkMode string `json:"docker_container_network_mode,omitempty"` - DockerContainerNetworks string `json:"docker_container_networks,omitempty"` - DockerContainerIps []string `json:"docker_container_ips,omitempty"` - DockerContainerCreated string `json:"docker_container_created,omitempty"` - DockerContainerPorts string `json:"docker_container_ports,omitempty"` + DockerContainerCommand string `json:"docker_container_command" required:"true"` + DockerContainerState string `json:"docker_container_state" required:"true"` + DockerContainerStateHuman string `json:"docker_container_state_human" required:"true"` + DockerContainerNetworkMode string `json:"docker_container_network_mode" required:"true"` + DockerContainerNetworks string `json:"docker_container_networks" required:"true"` + DockerContainerIps []string `json:"docker_container_ips" required:"true"` + DockerContainerCreated string `json:"docker_container_created" required:"true"` + DockerContainerPorts string `json:"docker_container_ports" required:"true"` Uptime int `json:"uptime" required:"true"` - CpuMax float64 `json:"cpu_max,omitempty"` - CpuUsage float64 `json:"cpu_usage,omitempty"` - MemoryMax int64 `json:"memory_max,omitempty"` - MemoryUsage int64 `json:"memory_usage,omitempty"` - VulnerabilitiesCount int64 `json:"vulnerabilities_count"` - VulnerabilityScanStatus string `json:"vulnerability_scan_status"` - VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id"` - SecretsCount int64 `json:"secrets_count"` - SecretScanStatus string `json:"secret_scan_status"` - SecretLatestScanId string `json:"secret_latest_scan_id"` - MalwaresCount int64 `json:"malwares_count"` - MalwareScanStatus string `json:"malware_scan_status"` - MalwareLatestScanId string `json:"malware_latest_scan_id"` + CpuMax float64 `json:"cpu_max" required:"true"` + CpuUsage float64 `json:"cpu_usage" required:"true"` + MemoryMax int64 `json:"memory_max" required:"true"` + MemoryUsage int64 `json:"memory_usage" required:"true"` + VulnerabilitiesCount int64 `json:"vulnerabilities_count" required:"true"` + VulnerabilityScanStatus string `json:"vulnerability_scan_status" required:"true"` + VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id" required:"true"` + SecretsCount int64 `json:"secrets_count" required:"true"` + SecretScanStatus string `json:"secret_scan_status" required:"true"` + SecretLatestScanId string `json:"secret_latest_scan_id" required:"true"` + MalwaresCount int64 `json:"malwares_count" required:"true"` + MalwareScanStatus string `json:"malware_scan_status" required:"true"` + MalwareLatestScanId string `json:"malware_latest_scan_id" required:"true"` } func (Container) NodeType() string { @@ -260,11 +242,11 @@ type Process struct { Command string `json:"cmdline" required:"true"` PPID int `json:"ppid" required:"true"` ThreadNumber int `json:"threads" required:"true"` - CpuMax float64 `json:"cpu_max,omitempty"` - CpuUsage float64 `json:"cpu_usage,omitempty"` - MemoryMax int64 `json:"memory_max,omitempty"` - MemoryUsage int64 `json:"memory_usage,omitempty"` - OpenFilesCount int `json:"open_files_count,omitempty"` + CpuMax float64 `json:"cpu_max" required:"true"` + CpuUsage float64 `json:"cpu_usage" required:"true"` + MemoryMax int64 `json:"memory_max" required:"true"` + MemoryUsage int64 `json:"memory_usage" required:"true"` + OpenFilesCount int `json:"open_files_count" required:"true"` } func (Process) NodeType() string { @@ -293,19 +275,19 @@ type ContainerImage struct { Name string `json:"docker_image_name" required:"true"` Tag string `json:"docker_image_tag" required:"true"` Size string `json:"docker_image_size" required:"true"` - DockerImageCreatedAt string `json:"docker_image_created_at,omitempty"` - DockerImageVirtualSize string `json:"docker_image_virtual_size,omitempty"` - DockerImageID string `json:"docker_image_id,omitempty"` + DockerImageCreatedAt string `json:"docker_image_created_at" required:"true"` + DockerImageVirtualSize string `json:"docker_image_virtual_size" required:"true"` + DockerImageID string `json:"docker_image_id" required:"true"` Metadata Metadata `json:"metadata" required:"true" nested_json:"true"` - VulnerabilitiesCount int64 `json:"vulnerabilities_count"` - VulnerabilityScanStatus string `json:"vulnerability_scan_status"` - VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id"` - SecretsCount int64 `json:"secrets_count"` - SecretScanStatus string `json:"secret_scan_status"` - SecretLatestScanId string `json:"secret_latest_scan_id"` - MalwaresCount int64 `json:"malwares_count"` - MalwareScanStatus string `json:"malware_scan_status"` - MalwareLatestScanId string `json:"malware_latest_scan_id"` + VulnerabilitiesCount int64 `json:"vulnerabilities_count" required:"true"` + VulnerabilityScanStatus string `json:"vulnerability_scan_status" required:"true"` + VulnerabilityLatestScanId string `json:"vulnerability_latest_scan_id" required:"true"` + SecretsCount int64 `json:"secrets_count" required:"true"` + SecretScanStatus string `json:"secret_scan_status" required:"true"` + SecretLatestScanId string `json:"secret_latest_scan_id" required:"true"` + MalwaresCount int64 `json:"malwares_count" required:"true"` + MalwareScanStatus string `json:"malware_scan_status" required:"true"` + MalwareLatestScanId string `json:"malware_latest_scan_id" required:"true"` } func (ContainerImage) NodeType() string { diff --git a/deepfence_server/reporters/scan/scan_results.go b/deepfence_server/reporters/scan/scan_results.go deleted file mode 100644 index 61e9888f2e..0000000000 --- a/deepfence_server/reporters/scan/scan_results.go +++ /dev/null @@ -1,125 +0,0 @@ -package reporters_scan - -import ( - "context" - - "github.com/deepfence/ThreatMapper/deepfence_server/model" - "github.com/deepfence/ThreatMapper/deepfence_server/reporters" - "github.com/deepfence/golang_deepfence_sdk/utils/directory" - "github.com/deepfence/golang_deepfence_sdk/utils/utils" - "github.com/neo4j/neo4j-go-driver/v4/neo4j" - "github.com/rs/zerolog/log" -) - -type ScanStatus struct { - Status string - Count int64 - Id string -} - -func getScanStatuses[T reporters.Cypherable](tx neo4j.Transaction, scanType utils.Neo4jScanType, nodeIds []string) ([]ScanStatus, error) { - res := []ScanStatus{} - var dummy T - var r neo4j.Result - query := ` - MATCH (n:` + dummy.NodeType() + `) <-[:SCANNED]- (s:` + string(scanType) + `) - WHERE n.node_id IN $ids - WITH n, max(s.updated_at) as latest - MATCH (n) <-[:SCANNED]- (s:` + string(scanType) + `{updated_at: latest}) - OPTIONAL MATCH (s) -[:DETECTED]-> (v) - RETURN n.node_id, s.status, count(v), s.node_id - ORDER BY n.node_id` - - log.Info().Msgf("query: %s", query) - r, err := tx.Run(query, - map[string]interface{}{"ids": nodeIds}) - - if err != nil { - return res, err - } - - recs, err := r.Collect() - - if err != nil { - return res, err - } - - statuses := map[string]ScanStatus{} - for _, rec := range recs { - statuses[rec.Values[0].(string)] = ScanStatus{ - Status: rec.Values[1].(string), - Count: rec.Values[2].(int64), - Id: rec.Values[3].(string), - } - } - - for i := range nodeIds { - res = append(res, statuses[nodeIds[i]]) - } - - return res, nil -} - -func GetScanStatuses[T reporters.Cypherable](ctx context.Context, node_ids []string) ([]model.RegularScanStatus, error) { - res := []model.RegularScanStatus{} - - driver, err := directory.Neo4jClient(ctx) - if err != nil { - return res, err - } - - session, err := driver.Session(neo4j.AccessModeRead) - if err != nil { - return res, err - } - defer session.Close() - - tx, err := session.BeginTransaction() - if err != nil { - return res, err - } - defer tx.Close() - - vuln_status, err := getScanStatuses[T](tx, utils.NEO4J_VULNERABILITY_SCAN, node_ids) - if err != nil { - return res, err - } - secrets_status, err := getScanStatuses[T](tx, utils.NEO4J_SECRET_SCAN, node_ids) - if err != nil { - return res, err - } - malware_status, err := getScanStatuses[T](tx, utils.NEO4J_MALWARE_SCAN, node_ids) - if err != nil { - return res, err - } - compliance_status, err := getScanStatuses[T](tx, utils.NEO4J_COMPLIANCE_SCAN, node_ids) - if err != nil { - return res, err - } - cloud_compliance_status, err := getScanStatuses[T](tx, utils.NEO4J_CLOUD_COMPLIANCE_SCAN, node_ids) - if err != nil { - return res, err - } - - for i := range node_ids { - res = append(res, model.RegularScanStatus{ - VulnerabilitiesCount: vuln_status[i].Count, - VulnerabilityScanStatus: vuln_status[i].Status, - VulnerabilityLatestScanId: vuln_status[i].Id, - SecretsCount: secrets_status[i].Count, - SecretScanStatus: secrets_status[i].Status, - SecretLatestScanId: secrets_status[i].Id, - MalwaresCount: malware_status[i].Count, - MalwareScanStatus: malware_status[i].Status, - MalwareLatestScanId: malware_status[i].Id, - CompliancesCount: compliance_status[i].Count, - ComplianceScanStatus: compliance_status[i].Status, - ComplianceLatestScanId: compliance_status[i].Id, - CloudCompliancesCount: cloud_compliance_status[i].Count, - CloudComplianceScanStatus: cloud_compliance_status[i].Status, - CloudComplianceLatestScanId: cloud_compliance_status[i].Id, - }) - } - - return res, nil -}