WIP: Support deleting cloud accounts

#2044

deepfence_server/apiDocs/operation.go

@@ -412,6 +412,10 @@ func (d *OpenAPIDocs) AddCloudNodeOperations() {
 		"Register Cloud Node Account", "Register Cloud Node Account and return any pending compliance scans from console",
 		http.StatusOK, []string{tagCloudNodes}, bearerToken, new(CloudNodeAccountRegisterReq), new(CloudNodeAccountRegisterResp))
 
+	d.AddOperation("deleteCloudNodeAccount", http.MethodDelete, "/deepfence/cloud-node/account",
+		"Delete Cloud Node Account", "Delete Cloud Node Account and related resources",
+		http.StatusAccepted, []string{tagCloudNodes}, bearerToken, new(CloudAccountDeleteReq), nil)
+
 	d.AddOperation("listCloudNodeAccount", http.MethodPost, "/deepfence/cloud-node/list/accounts",
 		"List Cloud Node Accounts", "List Cloud Node Accounts registered with the console",
 		http.StatusOK, []string{tagCloudNodes}, bearerToken, new(CloudNodeAccountsListReq), new(CloudNodeAccountsListResp))

deepfence_server/auth/policy.csv

@@ -37,6 +37,7 @@ p, admin, cloud-node, register
 p, admin, cloud-node, write
 p, admin, cloud-node, read
 p, admin, cloud-node, update
+p, admin, cloud-node, delete
 p, standard-user, cloud-node, register
 p, standard-user, cloud-node, write
 p, standard-user, cloud-node, read

deepfence_server/handler/cloud_node.go

@@ -2,6 +2,7 @@ package handler
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -324,3 +325,42 @@ func (h *Handler) CachePostureProviders(ctx context.Context) error {
 	}
 	return nil
 }
+
+func (h *Handler) DeleteCloudAccountHandler(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+	var req model.CloudAccountDeleteReq
+	err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req)
+	if err != nil {
+		log.Error().Msgf("%v", err)
+		h.respondError(&BadDecoding{err}, w)
+		return
+	}
+
+	log.Info().Msgf("delete cloud accounts request: %v", req)
+
+	if len(req.NodeIDs) > 0 {
+		worker, err := directory.Worker(r.Context())
+		if err != nil {
+			log.Error().Msgf("%v", err)
+			h.respondError(&InternalServerError{err}, w)
+			return
+		}
+
+		data, err := json.Marshal(req)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to marshal cloud account delete request")
+			h.respondError(&InternalServerError{err}, w)
+			return
+		}
+
+		if err := worker.Enqueue(utils.DeleteCloudAccounts, data, utils.CritialTaskOpts()...); err != nil {
+			log.Error().Err(err).Msg("failed enqueue task ddelete cloud accounts")
+			h.respondError(&InternalServerError{err}, w)
+			return
+		}
+	}
+
+	h.AuditUserActivity(r, EventComplianceScan, ActionDelete, req, true)
+
+	w.WriteHeader(http.StatusAccepted)
+}

deepfence_server/model/cloud_node.go

@@ -622,3 +622,7 @@ func (c *CloudAccountRefreshReq) GetCloudAccountRefresh(ctx context.Context) ([]
 	}
 	return updatedNodeIDs, tx.Commit(ctx)
 }
+
+type CloudAccountDeleteReq struct {
+	NodeIDs []string `json:"node_ids" validate:"required,gt=0" required:"true"`
+}

deepfence_server/router/router.go

@@ -352,6 +352,7 @@ func SetupRoutes(r *chi.Mux, serverPort string, serveOpenapiDocs bool, ingestC c
 
 			r.Route("/cloud-node", func(r chi.Router) {
 				r.Post("/account", dfHandler.AuthHandler(ResourceCloudNode, PermissionRegister, dfHandler.RegisterCloudNodeAccountHandler))
+				r.Delete("/account", dfHandler.AuthHandler(ResourceCloudNode, PermissionDelete, dfHandler.DeleteCloudAccountHandler))
 				r.Post("/account/refresh", dfHandler.AuthHandler(ResourceCloudNode, PermissionWrite, dfHandler.RefreshCloudAccountHandler))
 				r.Post("/list/accounts", dfHandler.AuthHandler(ResourceCloudNode, PermissionRead, dfHandler.ListCloudNodeAccountHandler))
 				r.Get("/list/providers", dfHandler.AuthHandler(ResourceCloudNode, PermissionRead, dfHandler.ListCloudNodeProvidersHandler))

deepfence_utils/utils/constants.go

@@ -57,6 +57,7 @@ const (
 	AutoFetchGenerativeAIIntegrations = "auto_fetch_generative_ai_integrations"
 	AsynqDeleteAllArchivedTasks       = "asynq_delete_all_archived_tasks"
 	RedisRewriteAOF                   = "redis_rewrite_aof"
+	DeleteCloudAccounts               = "delete_cloud_accounts"
 
 	UpdateLicenseTask      = "update_license"
 	ReportLicenseUsageTask = "report_license_usage"

deepfence_worker/tasks/scans/delete_cloud_accounts.go

@@ -0,0 +1,116 @@
+package scans
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"github.com/deepfence/ThreatMapper/deepfence_server/model"
+	"github.com/deepfence/ThreatMapper/deepfence_server/reporters"
+	reportersScan "github.com/deepfence/ThreatMapper/deepfence_server/reporters/scan"
+	"github.com/deepfence/ThreatMapper/deepfence_utils/directory"
+	"github.com/deepfence/ThreatMapper/deepfence_utils/telemetry"
+	"github.com/deepfence/ThreatMapper/deepfence_utils/utils"
+	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
+
+	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
+	"github.com/hibiken/asynq"
+)
+
+func DeleteCloudAccounts(ctx context.Context, task *asynq.Task) error {
+
+	log := log.WithCtx(ctx)
+
+	var req model.CloudAccountDeleteReq
+
+	if err := json.Unmarshal(task.Payload(), &req); err != nil {
+		log.Error().Err(err).Msg("failed to decode cloud account delete request")
+		return err
+	}
+
+	log.Info().Msgf("delete cloud accounts payload: %v", req)
+
+	// delete accounts
+	for _, accontID := range req.NodeIDs {
+		if err := deleteCloudAccount(ctx, accontID); err != nil {
+			log.Error().Err(err).Msgf("failed to delete cloud account %s", accontID)
+		}
+	}
+
+	// recompute sice we are removing everything related to the account
+	worker, err := directory.Worker(ctx)
+	if err != nil {
+		return err
+	}
+	return worker.Enqueue(utils.CachePostureProviders, []byte{}, utils.CritialTaskOpts()...)
+}
+
+func deleteCloudAccount(ctx context.Context, accountID string) error {
+
+	ctx, span := telemetry.NewSpan(ctx, "scans", "delete-cloud-account")
+	defer span.End()
+
+	driver, err := directory.Neo4jClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead})
+	defer session.Close(ctx)
+
+	tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second))
+	if err != nil {
+		return err
+	}
+	defer tx.Close(ctx)
+
+	// delete Cloud/ComplianceScan's related to the account first
+	nodeIDs := []model.NodeIdentifier{{NodeID: accountID, NodeType: "cloud_account"}}
+	filters := reporters.FieldsFilters{}
+	window := model.FetchWindow{Offset: 0, Size: 10000000}
+
+	scans, err := reportersScan.GetScansList(ctx, utils.NEO4JCloudComplianceScan, nodeIDs, filters, window)
+	if err != nil {
+		log.Error().Err(err).Msgf("failed to list scans for cloud node %s", accountID)
+	}
+
+	for _, s := range scans.ScansInfo {
+		err := reportersScan.DeleteScan(ctx, utils.NEO4JCloudComplianceScan, s.ScanID, []string{})
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to delete scan id %s", s.ScanID)
+		}
+	}
+
+	// delete CloudResource's and CloudNode related to the acount
+	return deleteCloudResourceAndNode(ctx, accountID)
+}
+
+func deleteCloudResourceAndNode(ctx context.Context, accountID string) error {
+	ctx, span := telemetry.NewSpan(ctx, "scans", "delete-cloud-resources-and-node")
+	defer span.End()
+
+	driver, err := directory.Neo4jClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
+	defer session.Close(ctx)
+
+	tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(30*time.Second))
+	if err != nil {
+		return err
+	}
+	defer tx.Close(ctx)
+
+	deleteQuery := `
+	MATCH (n:CloudNode{node_id:$node_id})-[:OWNS]-(r:CloudResource)
+	DETACH DELETE n,r
+	`
+
+	if _, err := tx.Run(ctx, deleteQuery, map[string]any{"node_id": accountID}); err != nil {
+		return err
+	}
+
+	return tx.Commit(ctx)
+}

deepfence_worker/worker.go

@@ -236,5 +236,7 @@ func NewWorker(ns directory.NamespaceID, cfg wtils.Config) (Worker, context.Canc
 
 	worker.AddRetryableHandler(utils.ThreatIntelUpdateTask, cronjobs.FetchThreatIntel)
 
+	worker.AddRetryableHandler(utils.DeleteCloudAccounts, scans.DeleteCloudAccounts)
+
 	return worker, cancel, nil
 }