Skip to content

Commit

Permalink
ISSUE-1830-2: Rebasing changes with main
Browse files Browse the repository at this point in the history
  • Loading branch information
@varunsharma0286
varunsharma0286 committed Sep 18, 2023
1 parent 6bd22fb commit 9b12357
Show file tree
Hide file tree
Showing 6 changed files with 176 additions and 102 deletions.
46 changes: 34 additions & 12 deletions deepfence_server/handler/scan_reports.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1508,7 +1508,21 @@ func (h *Handler) scanResultActionHandler(w http.ResponseWriter, r *http.Request
}
h.AuditUserActivity(r, req.ScanType, ACTION_DELETE, req, true)
case "notify":
err = reporters_scan.NotifyScanResult(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, req.ResultIDs)
if req.NotifyIndividual {
for _, resultID := range req.ResultIDs {
err = reporters_scan.NotifyScanResult(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, []string{resultID})
if err != nil {
h.respondError(err, w)
return
}
}
} else {
err = reporters_scan.NotifyScanResult(r.Context(), utils.Neo4jScanType(req.ScanType), req.ScanID, req.ResultIDs)
if err != nil {
h.respondError(err, w)
return
}
}
h.AuditUserActivity(r, req.ScanType, ACTION_NOTIFY, req, true)
}
if err != nil {
Expand Down Expand Up @@ -1668,33 +1682,40 @@ func (h *Handler) BulkDeleteScans(w http.ResponseWriter, r *http.Request) {

log.Info().Msgf("bulk delete %s scans filters %+v", req.ScanType, req.Filters)

scanType := utils.DetectedNodeScanType[req.ScanType]
scansList, err := reporters_scan.GetScansList(r.Context(), scanType, nil, req.Filters, model.FetchWindow{})
err = h.bulkDeleteScanResults(r.Context(), req)
if err != nil {
h.respondError(&ValidatorError{err: err}, w)
h.respondError(err, w)
return
}

h.AuditUserActivity(r, ACTION_BULK, ACTION_DELETE, req, true)

w.WriteHeader(http.StatusNoContent)
}

func (h *Handler) bulkDeleteScanResults(ctx context.Context, req model.BulkDeleteScansRequest) error {
scanType := utils.DetectedNodeScanType[req.ScanType]
scansList, err := reporters_scan.GetScansList(ctx, scanType, nil, req.Filters, model.FetchWindow{})
if err != nil {
return err
}

for _, s := range scansList.ScansInfo {
log.Info().Msgf("delete scan %s %s", req.ScanType, s.ScanId)
err = reporters_scan.DeleteScan(r.Context(), scanType, s.ScanId, []string{})
err = reporters_scan.DeleteScan(ctx, scanType, s.ScanId, []string{})
if err != nil {
log.Error().Err(err).Msgf("failed to delete scan id %s", s.ScanId)
continue
}
}

if len(scansList.ScansInfo) > 0 && (scanType == utils.NEO4J_COMPLIANCE_SCAN || scanType == utils.NEO4J_CLOUD_COMPLIANCE_SCAN) {
err = h.CachePostureProviders(r.Context())
err = h.CachePostureProviders(ctx)
if err != nil {
h.respondError(err, w)
return
return err
}
}

h.AuditUserActivity(r, ACTION_BULK, ACTION_DELETE, req, true)

w.WriteHeader(http.StatusNoContent)
return nil
}

func (h *Handler) GetAllNodesInScanResultBulkHandler(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -2134,6 +2155,7 @@ func StartMultiScan(ctx context.Context,

func StartMultiCloudComplianceScan(ctx context.Context, reqs []model.NodeIdentifier, benchmarkTypes []string) ([]string, string, error) {
driver, err := directory.Neo4jClient(ctx)

if err != nil {
return nil, "", err
}
Expand Down
110 changes: 31 additions & 79 deletions deepfence_server/model/cloud_node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,7 +367,6 @@ func GetCloudComplianceNodesList(ctx context.Context, cloudProvider string, fw F
isOrgListing := false
neo4jNodeType := "CloudNode"
passStatus := []string{"ok", "info", "skip"}
scanType := utils.NEO4J_CLOUD_COMPLIANCE_SCAN
if cloudProvider == PostureProviderAWSOrg {
cloudProvider = PostureProviderAWS
isOrgListing = true
Expand All @@ -381,85 +380,38 @@ func GetCloudComplianceNodesList(ctx context.Context, cloudProvider string, fw F
passStatus = []string{"warn", "pass"}
}
var res neo4j.Result
if isOrgListing {
res, err = tx.Run(fmt.Sprintf(`
MATCH (m:%s{cloud_provider:$cloud_provider+'_org'}) -[:IS_CHILD]-> (n:%s{cloud_provider: $cloud_provider})
WITH DISTINCT m.node_id AS node_id, m.node_name AS node_name, m.updated_at AS updated_at, m.active AS active
UNWIND node_id AS x
OPTIONAL MATCH (m:%s{cloud_provider:$cloud_provider+'_org', node_id: x}) -[:IS_CHILD]-> (n:%s{cloud_provider: $cloud_provider})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c:CloudCompliance)
WITH x, node_name, updated_at, COUNT(c) AS total_compliance_count, active
OPTIONAL MATCH (m:%s{cloud_provider:$cloud_provider+'_org', node_id: x}) -[:IS_CHILD]-> (n:%s{cloud_provider: $cloud_provider})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c1:CloudCompliance)
WHERE c1.status IN $pass_status
WITH x, node_name, $cloud_provider+'_org' AS cloud_provider, CASE WHEN total_compliance_count = 0 THEN 0.0 ELSE COUNT(c1.status)*100.0/total_compliance_count END AS compliance_percentage, updated_at, active
CALL {
WITH x
OPTIONAL MATCH (m:%s{cloud_provider:$cloud_provider+'_org', node_id: x}) -[:IS_CHILD]-> (n:%s{cloud_provider: $cloud_provider})<-[:SCANNED]-(s1:%s)
RETURN s1.node_id AS last_scan_id, s1.status AS last_scan_status
ORDER BY s1.updated_at DESC LIMIT 1
}
RETURN x, node_name, cloud_provider, compliance_percentage, active, updated_at, COALESCE(last_scan_id, ''), COALESCE(last_scan_status, '')
ORDER BY updated_at`, neo4jNodeType, neo4jNodeType, neo4jNodeType, neo4jNodeType, scanType, neo4jNodeType,
neo4jNodeType, scanType, neo4jNodeType, scanType, utils.NodeTypeCloudNode)+fw.FetchWindow2CypherQuery(),
map[string]interface{}{"cloud_provider": cloudProvider, "pass_status": passStatus})
if err != nil {
return CloudNodeAccountsListResp{Total: 0}, err
}
} else if cloudProvider == PostureProviderKubernetes || cloudProvider == PostureProviderLinux {
scanType = utils.NEO4J_COMPLIANCE_SCAN
var query string
if cloudProvider == PostureProviderKubernetes || cloudProvider == PostureProviderLinux {
nonKubeFilter := ""
if cloudProvider == PostureProviderLinux {
nonKubeFilter = "{kubernetes_cluster_id:''}"
}
res, err = tx.Run(fmt.Sprintf(`
MATCH (n:%s%s)
query = `
MATCH (n:` + string(neo4jNodeType) + nonKubeFilter + `)
WHERE n.pseudo=false
WITH n.node_id AS node_id, n.node_name AS node_name, n.updated_at AS updated_at, n.active AS active
UNWIND node_id AS x
OPTIONAL MATCH (n:%s{node_id: x})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c:Compliance)
WITH x, node_name, updated_at, COUNT(c) AS total_compliance_count, active
OPTIONAL MATCH (n:%s{node_id: x})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c1:Compliance)
WHERE c1.status IN $pass_status
WITH x, node_name, CASE WHEN total_compliance_count = 0 THEN 0.0 ELSE COUNT(c1.status)*100.0/total_compliance_count END AS compliance_percentage, updated_at, active
CALL {
WITH x
OPTIONAL MATCH (n:%s{node_id: x})<-[:SCANNED]-(s1:%s)
RETURN s1.node_id AS last_scan_id, s1.status AS last_scan_status
ORDER BY s1.updated_at DESC LIMIT 1
}
RETURN x, node_name, $cloud_provider, compliance_percentage, active, updated_at, COALESCE(last_scan_id, ''), COALESCE(last_scan_status, '')
ORDER BY updated_at`, neo4jNodeType, nonKubeFilter, neo4jNodeType, scanType, neo4jNodeType, scanType, neo4jNodeType, scanType)+fw.FetchWindow2CypherQuery(),
map[string]interface{}{
"cloud_provider": cloudProvider,
"pass_status": passStatus,
})
if err != nil {
return CloudNodeAccountsListResp{Total: 0}, err
}
RETURN n.node_id, n.node_name, $cloud_provider, n.active, n.updated_at, COALESCE(n.compliance_latest_scan_id, ''), COALESCE(n.compliance_latest_scan_status, '')
ORDER BY n.updated_at` + fw.FetchWindow2CypherQuery()
} else if isOrgListing {
query = `
MATCH (m:` + string(neo4jNodeType) + `{cloud_provider:$cloud_provider+'_org'}) -[:IS_CHILD]-> (n:` + string(neo4jNodeType) + `)
RETURN n.node_id, n.node_name, $cloud_provider, n.active, n.updated_at, COALESCE(n.cloud_compliance_latest_scan_id, ''), COALESCE(n.cloud_compliance_latest_scan_status, '')
ORDER BY n.updated_at` + fw.FetchWindow2CypherQuery()
} else {
res, err = tx.Run(fmt.Sprintf(`
MATCH (n:%s{cloud_provider: $cloud_provider})
WITH n.node_id AS node_id, n.node_name AS node_name, n.cloud_provider AS cloud_provider, n.updated_at AS updated_at, n.active AS active
UNWIND node_id AS x
OPTIONAL MATCH (n:%s{cloud_provider: $cloud_provider, node_id: x})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c:CloudCompliance)
WITH x, node_name, cloud_provider, updated_at, COUNT(c) AS total_compliance_count, active
OPTIONAL MATCH (n:%s{cloud_provider: $cloud_provider, node_id: x})<-[:SCANNED]-(s:%s)-[:DETECTED]->(c1:CloudCompliance)
WHERE c1.status IN $pass_status
WITH x, node_name, cloud_provider, CASE WHEN total_compliance_count = 0 THEN 0.0 ELSE COUNT(c1.status)*100.0/total_compliance_count END AS compliance_percentage, updated_at, active
CALL {
WITH x
OPTIONAL MATCH (n:%s{cloud_provider: $cloud_provider, node_id: x})<-[:SCANNED]-(s1:%s)
RETURN s1.node_id AS last_scan_id, s1.status AS last_scan_status
ORDER BY s1.updated_at DESC LIMIT 1
}
RETURN x, node_name, cloud_provider, compliance_percentage, active, updated_at, COALESCE(last_scan_id, ''), COALESCE(last_scan_status, '')
ORDER BY updated_at`, neo4jNodeType, neo4jNodeType, scanType, neo4jNodeType, scanType, neo4jNodeType, scanType)+fw.FetchWindow2CypherQuery(),
map[string]interface{}{
"cloud_provider": cloudProvider,
"pass_status": passStatus,
})
if err != nil {
return CloudNodeAccountsListResp{Total: 0}, err
}
query = `
MATCH (n:` + string(neo4jNodeType) + `{cloud_provider: $cloud_provider})
RETURN n.node_id, n.node_name, $cloud_provider, n.active, n.updated_at, COALESCE(n.cloud_compliance_latest_scan_id, ''), COALESCE(n.cloud_compliance_latest_scan_status, '')
ORDER BY n.updated_at` + fw.FetchWindow2CypherQuery()
}

log.Debug().Msgf("posture query: %v", query)
res, err = tx.Run(
query,
map[string]interface{}{
"cloud_provider": cloudProvider,
"pass_status": passStatus},
)
if err != nil {
return CloudNodeAccountsListResp{Total: 0}, err
}

recs, err := res.Collect()
Expand All @@ -473,10 +425,10 @@ func GetCloudComplianceNodesList(ctx context.Context, cloudProvider string, fw F
NodeId: rec.Values[0].(string),
NodeName: rec.Values[1].(string),
CloudProvider: rec.Values[2].(string),
CompliancePercentage: rec.Values[3].(float64),
Active: rec.Values[4].(bool),
LastScanId: rec.Values[6].(string),
LastScanStatus: rec.Values[7].(string),
CompliancePercentage: 0,
Active: rec.Values[3].(bool),
LastScanId: rec.Values[5].(string),
LastScanStatus: rec.Values[6].(string),
}
cloud_node_accounts_info = append(cloud_node_accounts_info, tmp)
}
Expand Down Expand Up @@ -527,7 +479,7 @@ func GetActiveCloudControls(ctx context.Context, complianceTypes []string, cloud
var res neo4j.Result
res, err = tx.Run(`
MATCH (n:CloudComplianceBenchmark) -[:INCLUDES]-> (m:CloudComplianceControl)
WHERE m.active = true
WHERE m.active = true
AND m.compliance_type IN $compliance_types
AND n.cloud_provider = $cloud_provider
RETURN n.benchmark_id, n.compliance_type, collect(m.control_id)
Expand Down
8 changes: 4 additions & 4 deletions deepfence_server/model/scans.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,10 +136,10 @@ type ScanResultsMaskRequest struct {
}

type ScanResultsActionRequest struct {
ScanID string `json:"scan_id" validate:"required" required:"true"`
ResultIDs []string `json:"result_ids" validate:"required,gt=0,dive,min=1" required:"true"`
ScanType string `json:"scan_type" validate:"required,oneof=SecretScan VulnerabilityScan MalwareScan ComplianceScan CloudComplianceScan" required:"true" enum:"SecretScan,VulnerabilityScan,MalwareScan,ComplianceScan,CloudComplianceScan"`
//utils.Neo4jScanType
ScanID string `json:"scan_id" validate:"required" required:"true"`
ResultIDs []string `json:"result_ids" validate:"required,gt=0,dive,min=1" required:"true"`
ScanType string `json:"scan_type" validate:"required,oneof=SecretScan VulnerabilityScan MalwareScan ComplianceScan CloudComplianceScan" required:"true" enum:"SecretScan,VulnerabilityScan,MalwareScan,ComplianceScan,CloudComplianceScan"`
NotifyIndividual bool `json:"notify_individual"`
}

type DownloadReportResponse struct {
Expand Down
108 changes: 104 additions & 4 deletions deepfence_server/reporters/scan/scan_result_actions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,10 +278,10 @@ func StopScan(ctx context.Context, scanType, scanId string) error {

if _, err = tx.Run(fmt.Sprintf(query, scanType),
map[string]interface{}{
"scan_id": scanId,
"starting": utils.SCAN_STATUS_STARTING,
"in_progress": utils.SCAN_STATUS_INPROGRESS,
"cancel_pending": utils.SCAN_STATUS_CANCEL_PENDING,
"scan_id": scanId,
"starting": utils.SCAN_STATUS_STARTING,
"in_progress": utils.SCAN_STATUS_INPROGRESS,
"cancelling": utils.SCAN_STATUS_CANCEL_PENDING,
}); err != nil {
log.Error().Msgf("StopScan: Error in setting the state in neo4j: %v", err)
return err
Expand All @@ -291,5 +291,105 @@ func StopScan(ctx context.Context, scanType, scanId string) error {
}

func NotifyScanResult(ctx context.Context, scanType utils.Neo4jScanType, scanId string, scanIDs []string) error {
switch scanType {
case utils.NEO4J_VULNERABILITY_SCAN:
res, common, err := GetSelectedScanResults[model.Vulnerability](ctx, scanType, scanId, scanIDs)
if err != nil {
return err
}
if err := Notify[model.Vulnerability](ctx, res, common, string(scanType)); err != nil {
return err
}
case utils.NEO4J_SECRET_SCAN:
res, common, err := GetSelectedScanResults[model.Secret](ctx, scanType, scanId, scanIDs)
if err != nil {
return err
}
if err := Notify[model.Secret](ctx, res, common, string(scanType)); err != nil {
return err
}
case utils.NEO4J_MALWARE_SCAN:
res, common, err := GetSelectedScanResults[model.Malware](ctx, scanType, scanId, scanIDs)
if err != nil {
return err
}
if err := Notify[model.Malware](ctx, res, common, string(scanType)); err != nil {
return err
}
case utils.NEO4J_COMPLIANCE_SCAN:
res, common, err := GetSelectedScanResults[model.Compliance](ctx, scanType, scanId, scanIDs)
if err != nil {
return err
}
if err := Notify[model.Compliance](ctx, res, common, string(scanType)); err != nil {
return err
}
case utils.NEO4J_CLOUD_COMPLIANCE_SCAN:
res, common, err := GetSelectedScanResults[model.CloudCompliance](ctx, scanType, scanId, scanIDs)
if err != nil {
return err
}
if err := Notify[model.CloudCompliance](ctx, res, common, string(scanType)); err != nil {
return err
}
}

return nil
}

func GetSelectedScanResults[T any](ctx context.Context, scanType utils.Neo4jScanType, scanId string, scanIDs []string) ([]T, model.ScanResultsCommon, error) {
res := []T{}
common := model.ScanResultsCommon{}
driver, err := directory.Neo4jClient(ctx)
if err != nil {
return res, common, err
}
session := driver.NewSession(neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
defer session.Close()

tx, err := session.BeginTransaction(neo4j.WithTxTimeout(15 * time.Second))
if err != nil {
return res, common, err
}
defer tx.Close()

query := `MATCH (n:%s) -[:DETECTED]-> (m)
WHERE m.node_id IN $scan_ids
AND n.node_id = $scan_id
RETURN m{.*}`

result, err := tx.Run(fmt.Sprintf(query, scanType), map[string]interface{}{"scan_ids": scanIDs, "scan_id": scanId})
if err != nil {
log.Error().Msgf("NotifyScanResult: Error in getting the scan result nodes from neo4j: %v", err)
return res, common, err
}

recs, err := result.Collect()
if err != nil {
log.Error().Msgf("NotifyScanResult: Error in collecting the scan result nodes from neo4j: %v", err)
return res, common, err
}

for _, rec := range recs {
var tmp T
utils.FromMap(rec.Values[0].(map[string]interface{}), &tmp)
res = append(res, tmp)
}

ncommonres, err := tx.Run(`
MATCH (m:`+string(scanType)+`{node_id: $scan_id}) -[:SCANNED]-> (n)
RETURN n{.*, scan_id: m.node_id, updated_at:m.updated_at, created_at:m.created_at}`,
map[string]interface{}{"scan_id": scanId})
if err != nil {
return res, common, err
}

rec, err := ncommonres.Single()
if err != nil {
return res, common, err
}

utils.FromMap(rec.Values[0].(map[string]interface{}), &common)

return res, common, err
}
2 changes: 0 additions & 2 deletions deepfence_utils/utils/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ const (

// task names
const (
SetUpGraphDBTask = "set_up_graph_db"
CleanUpGraphDBTask = "clean_up_graph_db"
CleanUpPostgresqlTask = "clean_up_postgresql"
CleanupDiagnosisLogs = "clean_up_diagnosis_logs"
Expand Down Expand Up @@ -178,7 +177,6 @@ var Topics = []string{

// list of task names to create topics
var Tasks = []string{
SetUpGraphDBTask,
CleanUpGraphDBTask,
CleanUpPostgresqlTask,
CleanupDiagnosisLogs,
Expand Down
4 changes: 3 additions & 1 deletion deepfence_worker/tasks/sbom/generate_sbom.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@ func (s SbomGenerator) GenerateSbom(msg *message.Message) ([]*message.Message, e

defer func() {
log.Info().Msgf("remove auth directory %s", authFile)
if authFile == "" {
return
}
if err := os.RemoveAll(authFile); err != nil {
log.Error().Msg(err.Error())
}
Expand Down Expand Up @@ -166,7 +169,6 @@ func (s SbomGenerator) GenerateSbom(msg *message.Message) ([]*message.Message, e
log.Error().Msg(err.Error())
statusChan <- NewSbomScanStatus(params, utils.SCAN_STATUS_FAILED, err.Error(), nil)
}

return nil, nil
}

Expand Down

0 comments on commit 9b12357

Please sign in to comment.