Skip to content

Commit

Permalink
fix container scans
Browse files Browse the repository at this point in the history
  • Loading branch information
noboruma committed Sep 12, 2024
1 parent df980f3 commit a820034
Show file tree
Hide file tree
Showing 7 changed files with 157 additions and 5 deletions.
2 changes: 1 addition & 1 deletion deepfence_agent/plugins/SecretScanner
Submodule SecretScanner updated 2 files
+9 −9 go.mod
+18 −18 go.sum
2 changes: 1 addition & 1 deletion deepfence_agent/plugins/cloud-scanner
Submodule cloud-scanner updated 61 files
+73 −0 Dockerfile
+5 −5 Dockerfile.steampipe
+12 −5 Makefile
+0 −10 README.md
+60 −44 cloud_resource_changes/cloud_resource_changes_aws/cloudtrail.go
+19 −8 cloud_resource_changes/cloud_resource_changes_aws/type.go
+12 −17 cloud_resource_changes/cloud_resource_changes_aws/util.go
+85 −0 cloudformation/deepfence-cloud-scanner-members.template
+109 −58 cloudformation/deepfence-cloud-scanner-org-common.template
+42 −55 cloudformation/deepfence-cloud-scanner-org-mgmt-console.template
+3 −14 cloudformation/deepfence-cloud-scanner-roles.template
+80 −91 cloudformation/deepfence-cloud-scanner.template
+4 −6 ...n/deepfence-managed/automated-deployment/deepfence-cloud-scanner-automated-organization-deployment.template
+2 −2 ...ormation/deepfence-managed/manual-deployment/deepfence-managed-cloud-scanner-organization-iam-role.template
+5 −5 ...ormation/deepfence-managed/manual-deployment/deepfence-managed-cloud-scanner-organization-stackset.template
+0 −35 ...rmation/deepfence-managed/single-account-deployment/deepfence-managed-cloud-scanner-single-account.template
+0 −53 .../self-hosted/eks-iam-roles/organization-eks-iam-role/deepfence-cloud-scanner-organization-iam-role.template
+0 −128 ...ted/eks-iam-roles/organization-eks-iam-role/deepfence-cloud-scanner-organization-stackset-iam-role.template
+0 −43 cloudformation/self-hosted/eks-iam-roles/single-account-eks-iam-role/README.md
+0 −58 ...f-hosted/eks-iam-roles/single-account-eks-iam-role/deepfence-cloud-scanner-single-account-iam-role.template
+0 −51 cloudformation/self-hosted/eks-iam-roles/single-account-eks-iam-role/main.tf
+0 −14 cloudformation/self-hosted/eks-iam-roles/single-account-eks-iam-role/output.tf
+0 −22 cloudformation/self-hosted/eks-iam-roles/single-account-eks-iam-role/variables.tf
+11 −0 entrypoint.sh
+4 −4 exportcontrols/aws/main_aws.go
+7 −7 exportcontrols/azure/main_azure.go
+3 −17 exportcontrols/gcp/main_gcp.go
+8 −58 go.mod
+17 −193 go.sum
+1 −1 golang_deepfence_sdk
+0 −2 helm-chart/.gitignore
+0 −8 helm-chart/README.md
+0 −23 helm-chart/deepfence-cloud-scanner/.helmignore
+0 −24 helm-chart/deepfence-cloud-scanner/Chart.yaml
+0 −3 helm-chart/deepfence-cloud-scanner/templates/NOTES.txt
+0 −62 helm-chart/deepfence-cloud-scanner/templates/_helpers.tpl
+0 −98 helm-chart/deepfence-cloud-scanner/templates/deployment.yaml
+0 −11 helm-chart/deepfence-cloud-scanner/templates/secret.yaml
+0 −13 helm-chart/deepfence-cloud-scanner/templates/serviceaccount.yaml
+0 −114 helm-chart/deepfence-cloud-scanner/values.yaml
+0 −14 helm-chart/index.yaml
+156 −34 internal/deepfence/client.go
+59 −0 internal/deepfence/diagnosis.go
+27 −0 internal/deepfence/util.go
+88 −58 main.go
+1 −1 output/file_output.go
+41 −27 output/output.go
+578 −847 query_resource/aws.go
+451 −440 query_resource/azure.go
+0 −51 query_resource/find_column_names/find_column_names.go
+168 −186 query_resource/gcp.go
+84 −80 query_resource/query.go
+0 −157 query_resource/query_service.go
+9 −9 scanner/parser.go
+123 −28 scanner/scanner.go
+246 −540 service/service.go
+0 −18 util/query_table_columns/README.md
+0 −32 util/query_table_columns/extract_cloud_resource_types.py
+0 −42 util/query_table_columns/format_cloud_resource_types.py
+75 −76 util/type.go
+21 −65 util/util.go
2 changes: 1 addition & 1 deletion deepfence_agent/plugins/package-scanner
2 changes: 1 addition & 1 deletion deepfence_agent/plugins/yara-rules
Submodule yara-rules updated 2 files
+1 −1 build-timestamp
+114,552 −28,086 malware.yar
119 changes: 119 additions & 0 deletions deepfence_server/handler/scan_reports.go
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,32 @@ func (h *Handler) StartVulnerabilityScanHandler(w http.ResponseWriter, r *http.R
return
}

scanTrigger := model.ScanTriggerCommon{}
for _, v := range reqs.ScanTriggerCommon.NodeIDs {
if v.NodeType == "container" {
imageID, err := GetImageFromContainerID(r.Context(), v.NodeID)
if err != nil {
log.Error().Err(err).Msg("Cannot start image scan for container")
continue
}
scanTrigger.NodeIDs = append(scanTrigger.NodeIDs, model.NodeIdentifier{
NodeID: imageID,
NodeType: "image",
})
}
}

_, _, err = StartMultiScan(r.Context(), true, utils.NEO4JVulnerabilityScan, scanTrigger, actionBuilder)
if err != nil {
if err.Error() == "Result contains no more records" {
h.respondError(&noNodesMatchedInNeo4jError, w)
return
}
log.Error().Msgf("%v", err)
h.respondError(err, w)
return
}

h.AuditUserActivity(r, EventVulnerabilityScan, ActionStart, reqs, true)

err = httpext.JSON(w, http.StatusAccepted, model.ScanTriggerResp{ScanIds: scanIDs, BulkScanID: bulkID})
Expand Down Expand Up @@ -400,6 +426,32 @@ func (h *Handler) StartSecretScanHandler(w http.ResponseWriter, r *http.Request)
return
}

scanTrigger := model.ScanTriggerCommon{}
for _, v := range reqs.ScanTriggerCommon.NodeIDs {
if v.NodeType == "container" {
imageID, err := GetImageFromContainerID(r.Context(), v.NodeID)
if err != nil {
log.Error().Err(err).Msg("Cannot start image scan for container")
continue
}
scanTrigger.NodeIDs = append(scanTrigger.NodeIDs, model.NodeIdentifier{
NodeID: imageID,
NodeType: "image",
})
}
}

_, _, err = StartMultiScan(r.Context(), true, utils.NEO4JSecretScan, scanTrigger, actionBuilder)
if err != nil {
if err.Error() == "Result contains no more records" {
h.respondError(&noNodesMatchedInNeo4jError, w)
return
}
log.Error().Msgf("%v", err)
h.respondError(err, w)
return
}

h.AuditUserActivity(r, EventSecretScan, ActionStart, reqs, true)

err = httpext.JSON(w, http.StatusAccepted, model.ScanTriggerResp{ScanIds: scanIDs, BulkScanID: bulkID})
Expand Down Expand Up @@ -520,6 +572,32 @@ func (h *Handler) StartMalwareScanHandler(w http.ResponseWriter, r *http.Request
return
}

scanTrigger := model.ScanTriggerCommon{}
for _, v := range reqs.ScanTriggerCommon.NodeIDs {
if v.NodeType == "container" {
imageID, err := GetImageFromContainerID(r.Context(), v.NodeID)
if err != nil {
log.Error().Err(err).Msg("Cannot start image scan for container")
continue
}
scanTrigger.NodeIDs = append(scanTrigger.NodeIDs, model.NodeIdentifier{
NodeID: imageID,
NodeType: "image",
})
}
}

_, _, err = StartMultiScan(r.Context(), true, utils.NEO4JMalwareScan, scanTrigger, actionBuilder)
if err != nil {
if err.Error() == "Result contains no more records" {
h.respondError(&noNodesMatchedInNeo4jError, w)
return
}
log.Error().Msgf("%v", err)
h.respondError(err, w)
return
}

h.AuditUserActivity(r, EventMalwareScan, ActionStart, reqs, true)

err = httpext.JSON(w, http.StatusAccepted, model.ScanTriggerResp{ScanIds: scanIDs, BulkScanID: bulkID})
Expand Down Expand Up @@ -2520,3 +2598,44 @@ func (h *Handler) rulesActionHandler(w http.ResponseWriter, r *http.Request, act
}
w.WriteHeader(http.StatusNoContent)
}

func GetImageFromContainerID(ctx context.Context, nodeID string) (string, error) {

ctx, span := telemetry.NewSpan(ctx, "scan-reports", "get-image-from-container-id")
defer span.End()

var name string

driver, err := directory.Neo4jClient(ctx)
if err != nil {
return name, err
}

session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead})
defer session.Close(ctx)

tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second))
if err != nil {
return name, err
}
defer tx.Close(ctx)

res, err := tx.Run(ctx, `
MATCH (n:Container{node_id:$node_id})
RETURN n.docker_image_id`,
map[string]interface{}{"node_id": nodeID})
if err != nil {
return name, err
}

rec, err := res.Single(ctx)
if err != nil {
return name, err
}

if vi, ok := rec.Get("n.docker_image_id"); ok && vi != nil {
name = vi.(string)
}

return name, nil
}
33 changes: 33 additions & 0 deletions deepfence_worker/cronjobs/neo4j.go
Original file line number Diff line number Diff line change
Expand Up @@ -846,6 +846,39 @@ func LinkNodes(ctx context.Context, task *asynq.Task) error {
return err
}

for _, scanType := range []utils.Neo4jScanType{
utils.NEO4JVulnerabilityScan,
utils.NEO4JSecretScan,
utils.NEO4JMalwareScan } {
scanNode := string(scanType)
statusField := ingestersUtil.ScanStatusField[scanType]
latestField := ingestersUtil.LatestScanIDField[scanType]
countField := ingestersUtil.ScanCountField[scanType]
if _, err = session.Run(ctx, `
MATCH (n:Container)
WHERE NOT EXISTS((n)<-[:SCANNED]-(:`+scanNode+`))
MATCH (i:ContainerImage{node_id:n.docker_image_id})
MATCH (i) -[:SCANNED]- (s:`+scanNode+`)
WITH max(s.updated_at) as latest, s, n
MATCH (s) -[r:DETECTED]- (v)
MERGE (news:`+scanNode+`{node_id:n.node_id+"-"+toString(TIMESTAMP())})
MERGE (news)-[:DETECTED{masked:r.masked}]-> (v)
MERGE (n) <-[:SCANNED]- (news)
WITH n, news, s, count(v) as cnt
SET news.status = s.status,
news.updated_at = TIMESTAMP(),
news.created_at = s.created_at,
news.is_priority = s.is_priority,
news.retries = s.retries,
news.status_message = s.status_message,
n.`+statusField+` = s.status,
n.`+latestField+` = news.node_id,
n.`+countField+` = cnt`,
map[string]interface{}{}, txConfig); err != nil {
return err
}
}

log.Debug().Msgf("Link Nodes task took: %v", time.Since(start))

return nil
Expand Down
2 changes: 1 addition & 1 deletion golang_deepfence_sdk

0 comments on commit a820034

Please sign in to comment.