Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Posture counts #1653

Merged
merged 3 commits into from
Oct 11, 2023
Merged

Posture counts #1653

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1c80604
fix: scan_count should be number of accounts which are scanned
gnmahanth Oct 10, 2023
5fb189c
trigger CachePostureProviders on detecting complliance scan success
gnmahanth Oct 10, 2023
897b4cd
add error checks for type conversion
gnmahanth Oct 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions deepfence_worker/cronjobs/cloud_compliance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ func AddCloudControls(ctx context.Context, task *asynq.Task) error {

func CachePostureProviders(ctx context.Context, task *asynq.Task) error {
log.Info().Msgf("Caching Posture Providers")
defer log.Info().Msgf("Caching Posture Providers - Done")
driver, err := directory.Neo4jClient(ctx)
if err != nil {
return err
Expand Down Expand Up @@ -237,9 +238,9 @@ func CachePostureProviders(ctx context.Context, task *asynq.Task) error {

scan_count_query = `
MATCH (n:` + string(neo4jNodeType) + `)
WHERE n.pseudo=false and n.active=true and n.agent_running=true
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the condition n.agent_running=true not required for linux posture count too?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hosts with agent_running=false are not allowed to scan so they will not appear here

WHERE n.pseudo=false
MATCH (n) <-[:SCANNED]- (m:` + string(utils.NEO4J_COMPLIANCE_SCAN) + `)
RETURN count(distinct m)`
RETURN count(distinct n)`

success_count_query = `
MATCH (n:` + string(neo4jNodeType) + `)
Expand Down Expand Up @@ -274,10 +275,9 @@ func CachePostureProviders(ctx context.Context, task *asynq.Task) error {

scan_count_query = `
MATCH (o:` + string(neo4jNodeType) + `{cloud_provider:$cloud_provider+'_org'}) -[:IS_CHILD]-> (m:` + string(neo4jNodeType) + `)
WHERE o.active=true
AND m.organization_id IS NOT NULL
MATCH (n:` + string(utils.NEO4J_CLOUD_COMPLIANCE_SCAN) + `)-[:SCANNED]->(m)
RETURN count(distinct n)`
RETURN count(distinct m)`

success_count_query = `
MATCH (o:` + string(neo4jNodeType) + `{cloud_provider:$cloud_provider+'_org'}) -[:IS_CHILD]-> (m:` + string(neo4jNodeType) + `)
Expand Down Expand Up @@ -307,9 +307,8 @@ func CachePostureProviders(ctx context.Context, task *asynq.Task) error {

scan_count_query = `
MATCH (m:` + string(neo4jNodeType) + `{cloud_provider: $cloud_provider})
WHERE m.active=true
MATCH (n:` + string(utils.NEO4J_CLOUD_COMPLIANCE_SCAN) + `)-[:SCANNED]->(m)
RETURN count(distinct n)`
RETURN count(distinct m)`

success_count_query = `
MATCH (m:` + string(neo4jNodeType) + `{cloud_provider: $cloud_provider})
Expand Down
37 changes: 31 additions & 6 deletions deepfence_worker/ingesters/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package ingesters

import (
"encoding/json"
"strconv"
"time"

"github.com/deepfence/ThreatMapper/deepfence_utils/directory"
Expand Down Expand Up @@ -79,12 +80,12 @@ func CommitFuncStatus[Status any](ts utils.Neo4jScanType) func(ns string, data [
return err
}

if ts != utils.NEO4J_COMPLIANCE_SCAN {
worker, err := directory.Worker(ctx)
if err != nil {
return err
}
worker, err := directory.Worker(ctx)
if err != nil {
return err
}

if ts != utils.NEO4J_COMPLIANCE_SCAN {
event := scans.UpdateScanEvent{
ScanType: ts,
RecordMap: recordMap,
Expand All @@ -97,7 +98,17 @@ func CommitFuncStatus[Status any](ts utils.Neo4jScanType) func(ns string, data [
if ts == utils.NEO4J_CLOUD_COMPLIANCE_SCAN {
task = utils.UpdateCloudResourceScanStatusTask
}
err = worker.Enqueue(task, b)
if err := worker.Enqueue(task, b); err != nil {
log.Error().Err(err).Msgf("failed to enqueue %s", task)
}
}

if (ts == utils.NEO4J_COMPLIANCE_SCAN || ts == utils.NEO4J_CLOUD_COMPLIANCE_SCAN) && anyCompleted(others) {
err := worker.Enqueue(utils.CachePostureProviders,
[]byte(strconv.FormatInt(utils.GetTimestamp(), 10)))
if err != nil {
log.Error().Err(err).Msgf("failed to enqueue %s", utils.CachePostureProviders)
}
}

return err
Expand Down Expand Up @@ -156,3 +167,17 @@ func ToMap[T any](data T) map[string]interface{} {
_ = json.Unmarshal(out, &bb)
return bb
}

func anyCompleted(data []map[string]interface{}) bool {

complete := false

for i := range data {
if data[i]["scan_status"].(string) == utils.SCAN_STATUS_SUCCESS {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to check if string conversion gave no error? Otherwise, might run into runtime error

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we dont need to check here as this field is guaranteed to have string data

complete = true
break
}
}

return complete
}