From 3f6eb4f1abc77252213541d9f9ca4f98a61964cf Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Thu, 30 Nov 2023 11:06:18 +0000 Subject: [PATCH] add cve_node_id to runtime sbom --- deepfence_server/go.mod | 2 +- deepfence_server/model/scans.go | 1 + deepfence_worker/ingesters/common.go | 34 -------------- deepfence_worker/ingesters/malware.go | 3 +- deepfence_worker/ingesters/secrets.go | 3 +- deepfence_worker/ingesters/vulnerabilites.go | 10 ++--- deepfence_worker/tasks/sbom/scan_sbom.go | 47 +++++++++++++++++--- deepfence_worker/utils/utils.go | 42 +++++++++++++++++ 8 files changed, 91 insertions(+), 51 deletions(-) diff --git a/deepfence_server/go.mod b/deepfence_server/go.mod index eb06c61f0f..3cc4a3fb1e 100644 --- a/deepfence_server/go.mod +++ b/deepfence_server/go.mod @@ -43,6 +43,7 @@ require ( go.opentelemetry.io/otel/trace v1.18.0 golang.org/x/crypto v0.14.0 golang.org/x/mod v0.10.0 + golang.org/x/term v0.13.0 gotest.tools v2.2.0+incompatible k8s.io/api v0.28.0 k8s.io/apimachinery v0.28.0 @@ -133,7 +134,6 @@ require ( golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.8.0 // indirect diff --git a/deepfence_server/model/scans.go b/deepfence_server/model/scans.go index 5718a45428..f9d778d4cd 100644 --- a/deepfence_server/model/scans.go +++ b/deepfence_server/model/scans.go @@ -191,6 +191,7 @@ type SbomResponse struct { Licenses []string `json:"licenses,omitempty"` CveID string `json:"cve_id,omitempty"` Severity string `json:"severity,omitempty"` + CveNodeID string `json:"cve_node_id,omitempty"` } type ScanResultsReq struct { diff --git a/deepfence_worker/ingesters/common.go b/deepfence_worker/ingesters/common.go index 5b72c90fe4..fc322764fb 100644 --- a/deepfence_worker/ingesters/common.go +++ b/deepfence_worker/ingesters/common.go @@ -207,37 +207,3 @@ func anyCompleted(data []map[string]interface{}) bool { return complete } - -func getEntityIdFromScanID(scanId, scanType string, - tx neo4j.Transaction) (string, error) { - - entityId := "" - query := `MATCH (s:` + scanType + `{node_id:'` + scanId + `'}) - [:SCANNED] -> (n) - WITH labels(n) as label, n - RETURN - CASE - WHEN 'ContainerImage' IN label or 'Container' in label - THEN [(ci:ContainerImage{node_id:n.docker_image_id}) - [:IS] -> (cis) | cis.node_id] - ELSE [n.node_id] - END` - res, err := tx.Run(query, map[string]interface{}{}) - if err != nil { - return "", err - } - - rec, err := res.Single() - if err != nil { - return "", err - } - - values := rec.Values[0].([]interface{}) - if len(values) > 0 { - entityId = values[0].(string) - } - - if len(entityId) == 0 { - entityId = scanId - } - - return entityId, nil -} diff --git a/deepfence_worker/ingesters/malware.go b/deepfence_worker/ingesters/malware.go index 1395bd4278..54fde08fba 100644 --- a/deepfence_worker/ingesters/malware.go +++ b/deepfence_worker/ingesters/malware.go @@ -9,6 +9,7 @@ import ( "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" ingestersUtil "github.com/deepfence/ThreatMapper/deepfence_utils/utils/ingesters" + workerUtil "github.com/deepfence/ThreatMapper/deepfence_worker/utils" "github.com/neo4j/neo4j-go-driver/v4/neo4j" ) @@ -81,7 +82,7 @@ func malwareToMaps(data []ingestersUtil.Malware, if _, ok := malware["scan_id"]; ok { scanId := malware["scan_id"].(string) var err error - entityId, err = getEntityIdFromScanID(scanId, string(utils.NEO4JMalwareScan), tx) + entityId, err = workerUtil.GetEntityIdFromScanID(scanId, string(utils.NEO4JMalwareScan), tx) if err != nil { log.Error().Msgf("Error in getting entityId: %v", err) return nil, err diff --git a/deepfence_worker/ingesters/secrets.go b/deepfence_worker/ingesters/secrets.go index efb8e2f4dc..837b91b6cc 100644 --- a/deepfence_worker/ingesters/secrets.go +++ b/deepfence_worker/ingesters/secrets.go @@ -8,6 +8,7 @@ import ( "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" ingestersUtil "github.com/deepfence/ThreatMapper/deepfence_utils/utils/ingesters" + workerUtil "github.com/deepfence/ThreatMapper/deepfence_worker/utils" "github.com/neo4j/neo4j-go-driver/v4/neo4j" ) @@ -81,7 +82,7 @@ func secretsToMaps(data []ingestersUtil.Secret, if _, ok := secret["scan_id"]; ok { scanId := secret["scan_id"].(string) var err error - entityId, err = getEntityIdFromScanID(scanId, string(utils.NEO4JSecretScan), tx) + entityId, err = workerUtil.GetEntityIdFromScanID(scanId, string(utils.NEO4JSecretScan), tx) if err != nil { log.Error().Msgf("Error in getting entityId: %v", err) return nil, err diff --git a/deepfence_worker/ingesters/vulnerabilites.go b/deepfence_worker/ingesters/vulnerabilites.go index bdce1a978d..fb2e107018 100644 --- a/deepfence_worker/ingesters/vulnerabilites.go +++ b/deepfence_worker/ingesters/vulnerabilites.go @@ -7,6 +7,7 @@ import ( "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" ingestersUtil "github.com/deepfence/ThreatMapper/deepfence_utils/utils/ingesters" + workerUtil "github.com/deepfence/ThreatMapper/deepfence_worker/utils" "github.com/neo4j/neo4j-go-driver/v4/neo4j" ) @@ -64,22 +65,17 @@ func CVEsToMaps(ms []ingestersUtil.Vulnerability, for _, v := range ms { data, rule := v.Split() - entityId, err := getEntityIdFromScanID(v.ScanID, string(utils.NEO4JVulnerabilityScan), tx) + entityId, err := workerUtil.GetEntityIdFromScanID(v.ScanID, string(utils.NEO4JVulnerabilityScan), tx) if err != nil { log.Error().Msgf("Error in getting entityId: %v", err) return nil, err } - nodeId := data.CveCausedByPackage + rule.CveID - if len(entityId) > 0 { - nodeId = nodeId + "_" + entityId - } - res = append(res, map[string]interface{}{ "rule": utils.ToMap(rule), "data": utils.ToMap(data), "scan_id": v.ScanID, - "node_id": nodeId, + "node_id": workerUtil.GetVulnerabilityNodeID(data.CveCausedByPackage, rule.CveID, entityId), }) } return res, nil diff --git a/deepfence_worker/tasks/sbom/scan_sbom.go b/deepfence_worker/tasks/sbom/scan_sbom.go index 3e7af8579b..f174ea899f 100644 --- a/deepfence_worker/tasks/sbom/scan_sbom.go +++ b/deepfence_worker/tasks/sbom/scan_sbom.go @@ -17,6 +17,7 @@ import ( "github.com/deepfence/ThreatMapper/deepfence_utils/directory" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + workerUtil "github.com/deepfence/ThreatMapper/deepfence_worker/utils" "github.com/deepfence/golang_deepfence_sdk/utils/tasks" psOutput "github.com/deepfence/package-scanner/output" ps "github.com/deepfence/package-scanner/scanner" @@ -24,6 +25,7 @@ import ( psUtils "github.com/deepfence/package-scanner/utils" "github.com/hibiken/asynq" "github.com/minio/minio-go/v7" + "github.com/neo4j/neo4j-go-driver/v4/neo4j" "github.com/twmb/franz-go/pkg/kgo" ) @@ -214,8 +216,30 @@ func (s SbomParser) ScanSBOM(ctx context.Context, task *asynq.Task) error { } } + // generate runtime sbom needs entity Id + driver, err := directory.Neo4jClient(directory.NewContextWithNameSpace(directory.NamespaceID(tenantID))) + if err != nil { + return err + } + session := driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + if err != nil { + return err + } + defer session.Close() + + tx, err := session.BeginTransaction(neo4j.WithTxTimeout(30 * time.Second)) + if err != nil { + return err + } + defer tx.Close() + + entityID, err := workerUtil.GetEntityIdFromScanID(params.ScanID, string(utils.NEO4JVulnerabilityScan), tx) + if err != nil { + log.Error().Msgf("Error in getting entityId: %v", err) + } + // generate runtime sbom - runtimeSbom, err := generateRuntimeSBOM(sbomFilePath, report) + runtimeSbom, err := generateRuntimeSBOM(sbomFilePath, report, entityID) if err != nil { log.Error().Err(err).Msgf("failed to generate runtime sbom") return err @@ -256,21 +280,26 @@ func readSBOM(path string) (*sbom.SBOM, error) { } type cveInfo struct { - CveID string - Severity string + CveCausedByPackage string + CveID string + Severity string } // generate map of package:version with severity func mapVulnerabilities(vulnerabilities []ps.VulnerabilityScanReport) map[string]cveInfo { vMap := map[string]cveInfo{} for _, v := range vulnerabilities { - vMap[v.CveCausedByPackage] = cveInfo{CveID: v.CveID, Severity: v.CveSeverity} + vMap[v.CveCausedByPackage] = cveInfo{ + CveCausedByPackage: v.CveCausedByPackage, + CveID: v.CveID, + Severity: v.CveSeverity, + } } return vMap } // generate runtime sbom format -func generateRuntimeSBOM(path string, vulnerabilities []ps.VulnerabilityScanReport) (*[]model.SbomResponse, error) { +func generateRuntimeSBOM(path string, vulnerabilities []ps.VulnerabilityScanReport, entityID string) (*[]model.SbomResponse, error) { var ( runSBOM = make([]model.SbomResponse, 0) err error @@ -289,15 +318,19 @@ func generateRuntimeSBOM(path string, vulnerabilities []ps.VulnerabilityScanRepo for _, l := range item.Licenses.ToSlice() { licenses = append(licenses, l.Value) } - runSBOM = append(runSBOM, model.SbomResponse{ + entry := model.SbomResponse{ PackageName: item.Name, Version: item.Version, Locations: item.Locations.CoordinateSet().Paths(), Licenses: licenses, CveID: cveInfo.CveID, Severity: cveInfo.Severity, - }) + } + if len(cveInfo.CveID) > 0 { + entry.CveNodeID = workerUtil.GetVulnerabilityNodeID(cveInfo.CveCausedByPackage, cveInfo.CveID, entityID) + } + runSBOM = append(runSBOM, entry) } return &runSBOM, err diff --git a/deepfence_worker/utils/utils.go b/deepfence_worker/utils/utils.go index f6ffb5c257..f5b7f6e9a0 100644 --- a/deepfence_worker/utils/utils.go +++ b/deepfence_worker/utils/utils.go @@ -10,6 +10,7 @@ import ( "time" "github.com/deepfence/ThreatMapper/deepfence_server/reporters" + "github.com/neo4j/neo4j-go-driver/v4/neo4j" ) var ReportRetentionTime = 24 * time.Hour @@ -46,3 +47,44 @@ func RunCommand(cmd *exec.Cmd) (*bytes.Buffer, error) { } return &out, nil } + +func GetEntityIdFromScanID(scanId, scanType string, tx neo4j.Transaction) (string, error) { + + entityId := "" + query := `MATCH (s:` + scanType + `{node_id:'` + scanId + `'}) - [:SCANNED] -> (n) + WITH labels(n) as label, n + RETURN + CASE + WHEN 'ContainerImage' IN label or 'Container' in label + THEN [(ci:ContainerImage{node_id:n.docker_image_id}) - [:IS] -> (cis) | cis.node_id] + ELSE [n.node_id] + END` + res, err := tx.Run(query, map[string]interface{}{}) + if err != nil { + return "", err + } + + rec, err := res.Single() + if err != nil { + return "", err + } + + values := rec.Values[0].([]interface{}) + if len(values) > 0 { + entityId = values[0].(string) + } + + if len(entityId) == 0 { + entityId = scanId + } + + return entityId, nil +} + +func GetVulnerabilityNodeID(packageName, cveID, entityID string) string { + nodeId := packageName + cveID + if len(entityID) > 0 { + nodeId = nodeId + "_" + entityID + } + return nodeId +}