From 345566485b31d6c97d51f079efa0f9c04cbbd5a4 Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Wed, 3 Apr 2024 16:55:55 +0000 Subject: [PATCH] WIP: Support deleting cloud accounts https://github.com/deepfence/ThreatMapper/issues/2044 --- deepfence_server/apiDocs/operation.go | 4 + deepfence_server/auth/policy.csv | 1 + deepfence_server/handler/cloud_node.go | 40 ++++++ deepfence_server/model/cloud_node.go | 4 + deepfence_server/router/router.go | 1 + deepfence_utils/utils/constants.go | 1 + .../tasks/scans/delete_cloud_accounts.go | 116 ++++++++++++++++++ deepfence_worker/worker.go | 2 + 8 files changed, 169 insertions(+) create mode 100644 deepfence_worker/tasks/scans/delete_cloud_accounts.go diff --git a/deepfence_server/apiDocs/operation.go b/deepfence_server/apiDocs/operation.go index d60f8ebf98..2251fde12e 100644 --- a/deepfence_server/apiDocs/operation.go +++ b/deepfence_server/apiDocs/operation.go @@ -412,6 +412,10 @@ func (d *OpenAPIDocs) AddCloudNodeOperations() { "Register Cloud Node Account", "Register Cloud Node Account and return any pending compliance scans from console", http.StatusOK, []string{tagCloudNodes}, bearerToken, new(CloudNodeAccountRegisterReq), new(CloudNodeAccountRegisterResp)) + d.AddOperation("deleteCloudNodeAccount", http.MethodDelete, "/deepfence/cloud-node/account", + "Delete Cloud Node Account", "Delete Cloud Node Account and related resources", + http.StatusAccepted, []string{tagCloudNodes}, bearerToken, new(CloudAccountDeleteReq), nil) + d.AddOperation("listCloudNodeAccount", http.MethodPost, "/deepfence/cloud-node/list/accounts", "List Cloud Node Accounts", "List Cloud Node Accounts registered with the console", http.StatusOK, []string{tagCloudNodes}, bearerToken, new(CloudNodeAccountsListReq), new(CloudNodeAccountsListResp)) diff --git a/deepfence_server/auth/policy.csv b/deepfence_server/auth/policy.csv index 175c2c11ce..c834d94503 100644 --- a/deepfence_server/auth/policy.csv +++ b/deepfence_server/auth/policy.csv @@ -37,6 +37,7 @@ p, admin, cloud-node, register p, admin, cloud-node, write p, admin, cloud-node, read p, admin, cloud-node, update +p, admin, cloud-node, delete p, standard-user, cloud-node, register p, standard-user, cloud-node, write p, standard-user, cloud-node, read diff --git a/deepfence_server/handler/cloud_node.go b/deepfence_server/handler/cloud_node.go index be7478e451..dc49dd30da 100644 --- a/deepfence_server/handler/cloud_node.go +++ b/deepfence_server/handler/cloud_node.go @@ -2,6 +2,7 @@ package handler import ( "context" + "encoding/json" "errors" "fmt" "net/http" @@ -324,3 +325,42 @@ func (h *Handler) CachePostureProviders(ctx context.Context) error { } return nil } + +func (h *Handler) DeleteCloudAccountHandler(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var req model.CloudAccountDeleteReq + err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req) + if err != nil { + log.Error().Msgf("%v", err) + h.respondError(&BadDecoding{err}, w) + return + } + + log.Info().Msgf("delete cloud accounts request: %v", req) + + if len(req.NodeIDs) > 0 { + worker, err := directory.Worker(r.Context()) + if err != nil { + log.Error().Msgf("%v", err) + h.respondError(&InternalServerError{err}, w) + return + } + + data, err := json.Marshal(req) + if err != nil { + log.Error().Err(err).Msg("failed to marshal cloud account delete request") + h.respondError(&InternalServerError{err}, w) + return + } + + if err := worker.Enqueue(utils.DeleteCloudAccounts, data, utils.CritialTaskOpts()...); err != nil { + log.Error().Err(err).Msg("failed enqueue task ddelete cloud accounts") + h.respondError(&InternalServerError{err}, w) + return + } + } + + h.AuditUserActivity(r, EventComplianceScan, ActionDelete, req, true) + + w.WriteHeader(http.StatusAccepted) +} diff --git a/deepfence_server/model/cloud_node.go b/deepfence_server/model/cloud_node.go index a2663a0b02..30fd5c8234 100644 --- a/deepfence_server/model/cloud_node.go +++ b/deepfence_server/model/cloud_node.go @@ -622,3 +622,7 @@ func (c *CloudAccountRefreshReq) GetCloudAccountRefresh(ctx context.Context) ([] } return updatedNodeIDs, tx.Commit(ctx) } + +type CloudAccountDeleteReq struct { + NodeIDs []string `json:"node_ids" validate:"required,gt=0" required:"true"` +} diff --git a/deepfence_server/router/router.go b/deepfence_server/router/router.go index d9eb1d0ac4..7e57a25b3f 100644 --- a/deepfence_server/router/router.go +++ b/deepfence_server/router/router.go @@ -352,6 +352,7 @@ func SetupRoutes(r *chi.Mux, serverPort string, serveOpenapiDocs bool, ingestC c r.Route("/cloud-node", func(r chi.Router) { r.Post("/account", dfHandler.AuthHandler(ResourceCloudNode, PermissionRegister, dfHandler.RegisterCloudNodeAccountHandler)) + r.Delete("/account", dfHandler.AuthHandler(ResourceCloudNode, PermissionDelete, dfHandler.DeleteCloudAccountHandler)) r.Post("/account/refresh", dfHandler.AuthHandler(ResourceCloudNode, PermissionWrite, dfHandler.RefreshCloudAccountHandler)) r.Post("/list/accounts", dfHandler.AuthHandler(ResourceCloudNode, PermissionRead, dfHandler.ListCloudNodeAccountHandler)) r.Get("/list/providers", dfHandler.AuthHandler(ResourceCloudNode, PermissionRead, dfHandler.ListCloudNodeProvidersHandler)) diff --git a/deepfence_utils/utils/constants.go b/deepfence_utils/utils/constants.go index da8b4e4e5d..80a3841896 100644 --- a/deepfence_utils/utils/constants.go +++ b/deepfence_utils/utils/constants.go @@ -57,6 +57,7 @@ const ( AutoFetchGenerativeAIIntegrations = "auto_fetch_generative_ai_integrations" AsynqDeleteAllArchivedTasks = "asynq_delete_all_archived_tasks" RedisRewriteAOF = "redis_rewrite_aof" + DeleteCloudAccounts = "delete_cloud_accounts" UpdateLicenseTask = "update_license" ReportLicenseUsageTask = "report_license_usage" diff --git a/deepfence_worker/tasks/scans/delete_cloud_accounts.go b/deepfence_worker/tasks/scans/delete_cloud_accounts.go new file mode 100644 index 0000000000..0075a3fbb3 --- /dev/null +++ b/deepfence_worker/tasks/scans/delete_cloud_accounts.go @@ -0,0 +1,116 @@ +package scans + +import ( + "context" + "encoding/json" + "time" + + "github.com/deepfence/ThreatMapper/deepfence_server/model" + "github.com/deepfence/ThreatMapper/deepfence_server/reporters" + reportersScan "github.com/deepfence/ThreatMapper/deepfence_server/reporters/scan" + "github.com/deepfence/ThreatMapper/deepfence_utils/directory" + "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" + "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + "github.com/neo4j/neo4j-go-driver/v5/neo4j" + + "github.com/deepfence/ThreatMapper/deepfence_utils/log" + "github.com/hibiken/asynq" +) + +func DeleteCloudAccounts(ctx context.Context, task *asynq.Task) error { + + log := log.WithCtx(ctx) + + var req model.CloudAccountDeleteReq + + if err := json.Unmarshal(task.Payload(), &req); err != nil { + log.Error().Err(err).Msg("failed to decode cloud account delete request") + return err + } + + log.Info().Msgf("delete cloud accounts payload: %v", req) + + // delete accounts + for _, accontID := range req.NodeIDs { + if err := deleteCloudAccount(ctx, accontID); err != nil { + log.Error().Err(err).Msgf("failed to delete cloud account %s", accontID) + } + } + + // recompute sice we are removing everything related to the account + worker, err := directory.Worker(ctx) + if err != nil { + return err + } + return worker.Enqueue(utils.CachePostureProviders, []byte{}, utils.CritialTaskOpts()...) +} + +func deleteCloudAccount(ctx context.Context, accountID string) error { + + ctx, span := telemetry.NewSpan(ctx, "scans", "delete-cloud-account") + defer span.End() + + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return err + } + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return err + } + defer tx.Close(ctx) + + // delete Cloud/ComplianceScan's related to the account first + nodeIDs := []model.NodeIdentifier{{NodeID: accountID, NodeType: "cloud_account"}} + filters := reporters.FieldsFilters{} + window := model.FetchWindow{Offset: 0, Size: 10000000} + + scans, err := reportersScan.GetScansList(ctx, utils.NEO4JCloudComplianceScan, nodeIDs, filters, window) + if err != nil { + log.Error().Err(err).Msgf("failed to list scans for cloud node %s", accountID) + } + + for _, s := range scans.ScansInfo { + err := reportersScan.DeleteScan(ctx, utils.NEO4JCloudComplianceScan, s.ScanID, []string{}) + if err != nil { + log.Error().Err(err).Msgf("failed to delete scan id %s", s.ScanID) + } + } + + // delete CloudResource's and CloudNode related to the acount + return deleteCloudResourceAndNode(ctx, accountID) +} + +func deleteCloudResourceAndNode(ctx context.Context, accountID string) error { + ctx, span := telemetry.NewSpan(ctx, "scans", "delete-cloud-resources-and-node") + defer span.End() + + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return err + } + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return err + } + defer tx.Close(ctx) + + deleteQuery := ` + MATCH (n:CloudNode{node_id:$node_id})-[:OWNS]-(r:CloudResource) + DETACH DELETE n,r + ` + + if _, err := tx.Run(ctx, deleteQuery, map[string]any{"node_id": accountID}); err != nil { + return err + } + + return tx.Commit(ctx) +} diff --git a/deepfence_worker/worker.go b/deepfence_worker/worker.go index 1d1c06bac9..0390db5a33 100644 --- a/deepfence_worker/worker.go +++ b/deepfence_worker/worker.go @@ -236,5 +236,7 @@ func NewWorker(ns directory.NamespaceID, cfg wtils.Config) (Worker, context.Canc worker.AddRetryableHandler(utils.ThreatIntelUpdateTask, cronjobs.FetchThreatIntel) + worker.AddRetryableHandler(utils.DeleteCloudAccounts, scans.DeleteCloudAccounts) + return worker, cancel, nil }