diff --git a/deepfence_server/apiDocs/operation.go b/deepfence_server/apiDocs/operation.go index d60f8ebf98..2251fde12e 100644 --- a/deepfence_server/apiDocs/operation.go +++ b/deepfence_server/apiDocs/operation.go @@ -412,6 +412,10 @@ func (d *OpenAPIDocs) AddCloudNodeOperations() { "Register Cloud Node Account", "Register Cloud Node Account and return any pending compliance scans from console", http.StatusOK, []string{tagCloudNodes}, bearerToken, new(CloudNodeAccountRegisterReq), new(CloudNodeAccountRegisterResp)) + d.AddOperation("deleteCloudNodeAccount", http.MethodDelete, "/deepfence/cloud-node/account", + "Delete Cloud Node Account", "Delete Cloud Node Account and related resources", + http.StatusAccepted, []string{tagCloudNodes}, bearerToken, new(CloudAccountDeleteReq), nil) + d.AddOperation("listCloudNodeAccount", http.MethodPost, "/deepfence/cloud-node/list/accounts", "List Cloud Node Accounts", "List Cloud Node Accounts registered with the console", http.StatusOK, []string{tagCloudNodes}, bearerToken, new(CloudNodeAccountsListReq), new(CloudNodeAccountsListResp)) diff --git a/deepfence_server/auth/policy.csv b/deepfence_server/auth/policy.csv index 175c2c11ce..c834d94503 100644 --- a/deepfence_server/auth/policy.csv +++ b/deepfence_server/auth/policy.csv @@ -37,6 +37,7 @@ p, admin, cloud-node, register p, admin, cloud-node, write p, admin, cloud-node, read p, admin, cloud-node, update +p, admin, cloud-node, delete p, standard-user, cloud-node, register p, standard-user, cloud-node, write p, standard-user, cloud-node, read diff --git a/deepfence_server/handler/cloud_node.go b/deepfence_server/handler/cloud_node.go index be7478e451..44a8ed7fd7 100644 --- a/deepfence_server/handler/cloud_node.go +++ b/deepfence_server/handler/cloud_node.go @@ -2,6 +2,7 @@ package handler import ( "context" + "encoding/json" "errors" "fmt" "net/http" @@ -324,3 +325,42 @@ func (h *Handler) CachePostureProviders(ctx context.Context) error { } return nil } + +func (h *Handler) DeleteCloudAccountHandler(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var req model.CloudAccountDeleteReq + err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req) + if err != nil { + log.Error().Msgf("%v", err) + h.respondError(&BadDecoding{err}, w) + return + } + + log.Info().Msgf("delete cloud accounts request: %v", req) + + if len(req.NodeIDs) > 0 { + worker, err := directory.Worker(r.Context()) + if err != nil { + log.Error().Msgf("%v", err) + h.respondError(&InternalServerError{err}, w) + return + } + + data, err := json.Marshal(req) + if err != nil { + log.Error().Err(err).Msg("failed to marshal cloud account delete request") + h.respondError(&InternalServerError{err}, w) + return + } + + if err := worker.Enqueue(utils.DeleteCloudAccounts, data, utils.CritialTaskOpts()...); err != nil { + log.Error().Err(err).Msg("failed enqueue task delete cloud accounts") + h.respondError(&InternalServerError{err}, w) + return + } + } + + h.AuditUserActivity(r, EventComplianceScan, ActionDelete, req, true) + + w.WriteHeader(http.StatusAccepted) +} diff --git a/deepfence_server/model/cloud_node.go b/deepfence_server/model/cloud_node.go index a2663a0b02..30fd5c8234 100644 --- a/deepfence_server/model/cloud_node.go +++ b/deepfence_server/model/cloud_node.go @@ -622,3 +622,7 @@ func (c *CloudAccountRefreshReq) GetCloudAccountRefresh(ctx context.Context) ([] } return updatedNodeIDs, tx.Commit(ctx) } + +type CloudAccountDeleteReq struct { + NodeIDs []string `json:"node_ids" validate:"required,gt=0" required:"true"` +} diff --git a/deepfence_server/router/router.go b/deepfence_server/router/router.go index d9eb1d0ac4..7e57a25b3f 100644 --- a/deepfence_server/router/router.go +++ b/deepfence_server/router/router.go @@ -352,6 +352,7 @@ func SetupRoutes(r *chi.Mux, serverPort string, serveOpenapiDocs bool, ingestC c r.Route("/cloud-node", func(r chi.Router) { r.Post("/account", dfHandler.AuthHandler(ResourceCloudNode, PermissionRegister, dfHandler.RegisterCloudNodeAccountHandler)) + r.Delete("/account", dfHandler.AuthHandler(ResourceCloudNode, PermissionDelete, dfHandler.DeleteCloudAccountHandler)) r.Post("/account/refresh", dfHandler.AuthHandler(ResourceCloudNode, PermissionWrite, dfHandler.RefreshCloudAccountHandler)) r.Post("/list/accounts", dfHandler.AuthHandler(ResourceCloudNode, PermissionRead, dfHandler.ListCloudNodeAccountHandler)) r.Get("/list/providers", dfHandler.AuthHandler(ResourceCloudNode, PermissionRead, dfHandler.ListCloudNodeProvidersHandler)) diff --git a/deepfence_utils/utils/constants.go b/deepfence_utils/utils/constants.go index da8b4e4e5d..80a3841896 100644 --- a/deepfence_utils/utils/constants.go +++ b/deepfence_utils/utils/constants.go @@ -57,6 +57,7 @@ const ( AutoFetchGenerativeAIIntegrations = "auto_fetch_generative_ai_integrations" AsynqDeleteAllArchivedTasks = "asynq_delete_all_archived_tasks" RedisRewriteAOF = "redis_rewrite_aof" + DeleteCloudAccounts = "delete_cloud_accounts" UpdateLicenseTask = "update_license" ReportLicenseUsageTask = "report_license_usage" diff --git a/deepfence_worker/ingesters/cloud_resource.go b/deepfence_worker/ingesters/cloud_resource.go index dfdef4bb3c..6078d670bb 100644 --- a/deepfence_worker/ingesters/cloud_resource.go +++ b/deepfence_worker/ingesters/cloud_resource.go @@ -153,6 +153,7 @@ func ResourceToMaps(ms []ingestersUtil.CloudResource) ([]map[string]interface{}, "instance_id": newmap["node_id"], "host_name": v.Name, "node_id": v.Name, + "account_id": newmap["account_id"], }) if k8sClusterName != "" { clusters = append(clusters, map[string]interface{}{ @@ -165,6 +166,7 @@ func ResourceToMaps(ms []ingestersUtil.CloudResource) ([]map[string]interface{}, "active": true, "cloud_provider": v.CloudProvider, "agent_running": false, + "account_id": newmap["account_id"], }) } } diff --git a/deepfence_worker/tasks/scans/delete_cloud_accounts.go b/deepfence_worker/tasks/scans/delete_cloud_accounts.go new file mode 100644 index 0000000000..158db591b7 --- /dev/null +++ b/deepfence_worker/tasks/scans/delete_cloud_accounts.go @@ -0,0 +1,283 @@ +package scans + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/deepfence/ThreatMapper/deepfence_server/model" + "github.com/deepfence/ThreatMapper/deepfence_server/reporters" + reportersScan "github.com/deepfence/ThreatMapper/deepfence_server/reporters/scan" + "github.com/deepfence/ThreatMapper/deepfence_utils/directory" + "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" + "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + "github.com/neo4j/neo4j-go-driver/v5/neo4j" + + "github.com/deepfence/ThreatMapper/deepfence_utils/log" + "github.com/hibiken/asynq" +) + +func DeleteCloudAccounts(ctx context.Context, task *asynq.Task) error { + + log := log.WithCtx(ctx) + + var req model.CloudAccountDeleteReq + + if err := json.Unmarshal(task.Payload(), &req); err != nil { + log.Error().Err(err).Msg("failed to decode cloud account delete request") + return err + } + + log.Info().Msgf("delete cloud accounts payload: %v", req) + + // delete accounts + for _, accontID := range req.NodeIDs { + if err := deleteCloudAccount(ctx, accontID); err != nil { + log.Error().Err(err).Msgf("failed to delete cloud account %s", accontID) + } + } + + // recompute sice we are removing everything related to the account + worker, err := directory.Worker(ctx) + if err != nil { + return err + } + return worker.Enqueue(utils.CachePostureProviders, []byte{}, utils.CritialTaskOpts()...) +} + +func deleteCloudAccount(ctx context.Context, accountID string) error { + + ctx, span := telemetry.NewSpan(ctx, "scans", "delete-cloud-account") + defer span.End() + + log := log.WithCtx(ctx) + + log.Info().Msgf("delete cloud account %s", accountID) + + org, err := isOrgAccount(ctx, accountID) + if err != nil { + log.Error().Err(err).Msgf("failed to determine if org account %s", accountID) + return err + } + if org { + children, err := listOrgChildAccounts(ctx, accountID) + if err != nil { + log.Error().Err(err).Msgf("failed to list child accounts for %s", accountID) + return err + } + log.Info().Msgf("org account %s has %d children", accountID, len(children)) + for _, childID := range children { + if err := deleteScans(ctx, childID); err != nil { + log.Error().Err(err).Msgf("failed to delete scans for account %s", childID) + } + if err := deleteCloudResourceAndNode(ctx, childID); err != nil { + log.Error().Err(err).Msgf("failed to delete resources for account %s", childID) + } + } + } + + // just single account + if err := deleteScans(ctx, accountID); err != nil { + log.Error().Err(err).Msgf("failed to delete scans for account %s", accountID) + return err + } + return deleteCloudResourceAndNode(ctx, accountID) + +} + +func deleteScans(ctx context.Context, accountID string) error { + + ctx, span := telemetry.NewSpan(ctx, "scans", "delete-scans") + defer span.End() + + log := log.WithCtx(ctx) + + // delete Cloud/ComplianceScan's related to the account first + nodeIDs := []model.NodeIdentifier{{NodeID: accountID, NodeType: "cloud_account"}} + filters := reporters.FieldsFilters{} + window := model.FetchWindow{Offset: 0, Size: 10000000} + + scans, err := reportersScan.GetScansList(ctx, utils.NEO4JCloudComplianceScan, nodeIDs, filters, window) + if err != nil { + log.Error().Err(err).Msgf("failed to list scans for cloud node %s", accountID) + } + + defer log.Info().Msgf("deleted %d scans for account %s", len(scans.ScansInfo), accountID) + + for _, s := range scans.ScansInfo { + err := reportersScan.DeleteScan(ctx, utils.NEO4JCloudComplianceScan, s.ScanID, []string{}) + if err != nil { + log.Error().Err(err).Msgf("failed to delete scan id %s", s.ScanID) + } + } + + return nil +} + +func deleteCloudResourceAndNode(ctx context.Context, accountID string) error { + + ctx, span := telemetry.NewSpan(ctx, "scans", "delete-cloud-resources-and-node") + defer span.End() + + log := log.WithCtx(ctx) + + defer log.Info().Msgf("deleted cloud node and resources for account %s", accountID) + + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return err + } + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return err + } + defer tx.Close(ctx) + + // delete cloud node and resources + deleteQuery := ` + MATCH (n:CloudNode{node_id: $node_id})-[:OWNS]->(r:CloudResource) + DETACH DELETE n,r + ` + + if _, err := tx.Run(ctx, deleteQuery, map[string]any{"node_id": accountID}); err != nil { + log.Error().Err(err).Msgf("failed to delete cloud node and resources for account %s", accountID) + return err + } + + // required in case where link cloud resources task has not yet run + deleteQuery1 := `MATCH (n:CloudNode{node_id: $node_id}) DETACH DELETE n` + deleteQuery2 := `MATCH (r:CloudResource{account_id: $node_id}) DETACH DELETE r` + + if _, err := tx.Run(ctx, deleteQuery1, map[string]any{"node_id": accountID}); err != nil { + log.Error().Err(err).Msgf("failed to delete cloud node account %s", accountID) + return err + } + + if _, err := tx.Run(ctx, deleteQuery2, map[string]any{"node_id": accountID}); err != nil { + log.Error().Err(err).Msgf("failed to delete cloud resources for account %s", accountID) + return err + } + + // delete hosts discovered from cloud + deleteHostsQuery := ` + MATCH (n:Node{account_id: $node_id}) + WHERE n.agent_running=false + DETACH DELETE n + ` + + if _, err := tx.Run(ctx, deleteHostsQuery, map[string]any{"node_id": accountID}); err != nil { + log.Error().Err(err).Msgf("failed to delete hosts for account %s", accountID) + return err + } + + // delete kube clusters discovered from cloud + deleteKubeClustersQuery := ` + MATCH (n:KubernetesCluster{account_id: $node_id}) + WHERE n.agent_running=false + DETACH DELETE n + ` + + if _, err := tx.Run(ctx, deleteKubeClustersQuery, map[string]any{"node_id": accountID}); err != nil { + log.Error().Err(err).Msgf("failed to delete kubernetes clusters for account %s", accountID) + return err + } + + return tx.Commit(ctx) +} + +func isOrgAccount(ctx context.Context, accountID string) (bool, error) { + ctx, span := telemetry.NewSpan(ctx, "scans", "check-org-account") + defer span.End() + + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return false, err + } + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return false, err + } + defer tx.Close(ctx) + + query := `MATCH (n:CloudNode{node_id: $node_id}) return n.cloud_provider as cloud_provider` + + result, err := tx.Run(ctx, query, map[string]any{"node_id": accountID}) + if err != nil { + log.Error().Err(err).Msgf("failed to delete cloud node and resources for account %s", accountID) + return false, err + } + + record, err := result.Single(ctx) + if err != nil { + return false, err + } + + cp, ok := record.Get("cloud_provider") + if !ok { + return false, fmt.Errorf("field not present in the result") + } + + switch cp.(string) { + case model.PostureProviderAWSOrg, model.PostureProviderGCPOrg: + return true, nil + default: + return false, nil + } + +} + +func listOrgChildAccounts(ctx context.Context, accountID string) ([]string, error) { + + ctx, span := telemetry.NewSpan(ctx, "scans", "list-org-child-accounts") + defer span.End() + + childern := []string{} + + driver, err := directory.Neo4jClient(ctx) + if err != nil { + return childern, err + } + + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + defer session.Close(ctx) + + tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second)) + if err != nil { + return childern, err + } + defer tx.Close(ctx) + + query := ` + MATCH (n:CloudNode{node_id: $node_id})-[:IS_CHILD]->(c:CloudNode) + return c.node_id as child_id + ` + + result, err := tx.Run(ctx, query, map[string]any{"node_id": accountID}) + if err != nil { + log.Error().Err(err).Msgf("failed to delete cloud node and resources for account %s", accountID) + return childern, err + } + + records, err := result.Collect(ctx) + if err != nil { + return childern, err + } + + for _, r := range records { + cid, ok := r.Get("child_id") + if ok { + childern = append(childern, cid.(string)) + } + } + + return childern, nil +} diff --git a/deepfence_worker/worker.go b/deepfence_worker/worker.go index 1d1c06bac9..0390db5a33 100644 --- a/deepfence_worker/worker.go +++ b/deepfence_worker/worker.go @@ -236,5 +236,7 @@ func NewWorker(ns directory.NamespaceID, cfg wtils.Config) (Worker, context.Canc worker.AddRetryableHandler(utils.ThreatIntelUpdateTask, cronjobs.FetchThreatIntel) + worker.AddRetryableHandler(utils.DeleteCloudAccounts, scans.DeleteCloudAccounts) + return worker, cancel, nil }