From 5b0fc473b20ac3ee03ff739adecc6c818425999e Mon Sep 17 00:00:00 2001 From: shyam Date: Wed, 9 Oct 2024 22:26:25 +0530 Subject: [PATCH 1/4] Feature: [+] Added support for pasing in Service Account Key Credential content as base64 encoded env variable. [+] Does this break anything : No, All existing functionality works. ``` data "google_client_config" "current" {} resource "google_project_service" "cloud_resource_manager" { project = "" service = "cloudresourcemanager.googleapis.com" disable_on_destroy = false } module "cloud_scanner_example_multiple_project" { source = "deepfence/cloud-scanner/gcp//examples/gce-vm" version = "0.8.0" # gcp service account name name = "deepfence-cloud-scanner" # project_id example: dev1-123456 project_id = "" # org mode for multiple projects isOrganizationDeployment = true } resource "google_service_account_key" "cloud_scanner_key" { service_account_id = module.cloud_scanner_example_multiple_project.service_account_email } output "service_account_email" { value = module.cloud_scanner_example_multiple_project.service_account_email } output "service_account_key" { value = google_service_account_key.cloud_scanner_key.private_key sensitive = true } ``` --- docker-compose.yaml | 6 ++++- service/service.go | 61 ++++++++++++++++++++++++++++++++++++++++++--- util/type.go | 2 ++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 4c2e2cb..aba18f5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -60,6 +60,9 @@ services: # AZURE_CLIENT_SECRET: "" # AZURE_SUBSCRIPTION_ID: "" + # Provide base64 encoded Service Account Keys for GCP Scanner + # GCP_SERVICE_ACCOUNT_CREDENTIAL: "" + DEPLOYMENT_MODE: "docker" HOME_DIR: "/home/deepfence" DF_INSTALL_DIR: "/data/home/deepfence" @@ -73,4 +76,5 @@ services: volumes: cloud_scanner_data: - driver: local \ No newline at end of file + driver: local + diff --git a/service/service.go b/service/service.go index 87ff2ed..56c5df2 100644 --- a/service/service.go +++ b/service/service.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/base64" "encoding/json" "fmt" "net" @@ -25,6 +26,7 @@ import ( "github.com/deepfence/cloud-scanner/scanner" "github.com/deepfence/cloud-scanner/util" "google.golang.org/api/cloudresourcemanager/v1" + "google.golang.org/api/option" ) var ( @@ -191,12 +193,35 @@ func (c *ComplianceScanService) fetchGCPOrganizationProjects() ([]util.AccountsT } func (c *ComplianceScanService) fetchGCPProjects() ([]util.MonitoredAccount, error) { + log.Info().Msg("GCP Project is choosen") ctx := context.Background() - crm, err := cloudresourcemanager.NewService(ctx) - if err != nil { - log.Error().Err(err).Msg("failed to fetch GCP projects") - return nil, err + + var crm *cloudresourcemanager.Service + var err error + + if c.config.GCPCredentials != "" && strings.TrimSpace(c.config.GCPCredentials) != "" { + // Save the GCP credentials to a file + credentialFilePath, err := saveGCPCredentialsToFile(c.config.GCPCredentials) + if err != nil { + log.Error().Err(err).Msg("Failed to save GCP credentials to file, falling back to default authentication") + } else { + log.Info().Msgf("GCP credentials saved to file at: " + credentialFilePath) + clientOption := option.WithCredentialsFile(credentialFilePath) + crm, err = cloudresourcemanager.NewService(ctx, clientOption) + if err != nil { + log.Error().Err(err).Msg("Failed to create GCP client with provided credentials, falling back to default authentication") + } + } + } + + if crm == nil { + crm, err = cloudresourcemanager.NewService(ctx) + if err != nil { + log.Error().Err(err).Msg("failed to create GCP client with default authentication") + return nil, err + } } + projectsRequest := crm.Projects.List().PageSize(1000) projectsResponse, err := projectsRequest.Do() if err != nil { @@ -212,9 +237,36 @@ func (c *ComplianceScanService) fetchGCPProjects() ([]util.MonitoredAccount, err NodeID: util.GetNodeID(c.config.CloudProvider, project.ProjectId), } } + return organizationAccountIDs, nil } +func saveGCPCredentialsToFile(credentials string) (string, error) { + + configDir := "/home/deepfence/.config/gcloud/" + credentialFilePath := configDir + "application_default_credentials.json" + + // Check if the directory exists, create it if not + if _, err := os.Stat(configDir); os.IsNotExist(err) { + err = os.MkdirAll(configDir, 0700) + if err != nil { + return "", fmt.Errorf("failed to create directory: %w", err) + } + } + + credBytes, err := base64.StdEncoding.DecodeString(credentials) + if err != nil { + return "", fmt.Errorf("failed to decode GCP credentials: %w", err) + } + + err = os.WriteFile(credentialFilePath, credBytes, 0600) + if err != nil { + return "", fmt.Errorf("failed to write credentials to file: %w", err) + } + + return credentialFilePath, nil +} + func (c *ComplianceScanService) fetchAzureTenantSubscriptions() ([]util.MonitoredAccount, error) { cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { @@ -770,3 +822,4 @@ func (c *ComplianceScanService) handleRequest(conn net.Conn) { } } } + diff --git a/util/type.go b/util/type.go index aec086a..289c659 100644 --- a/util/type.go +++ b/util/type.go @@ -70,6 +70,7 @@ type Config struct { ScanInactiveThreshold int `envconfig:"SCAN_INACTIVE_THRESHOLD" default:"21600" json:"scan_inactive_threshold"` CloudScannerPolicy string `envconfig:"CLOUD_SCANNER_POLICY" json:"cloud_scanner_policy"` DeploymentMode string `envconfig:"DEPLOYMENT_MODE" json:"deployment_mode"` + GCPCredentials string `envconfig:"GCP_SERVICE_ACCOUNT_CREDENTIAL" json:"gcp_service_account_credential"` CloudMetadata cloudmetadata.CloudMetadata `ignored:"true" json:"cloud_metadata"` NodeID string `ignored:"true" json:"-"` @@ -200,3 +201,4 @@ func init() { SteampipeInstallDirectory = "/home/deepfence/.steampipe" } } + From 6b1bd0f4acd4e5213ce6c29129560b6f8dff86a5 Mon Sep 17 00:00:00 2001 From: sam <67697492+shyam0904a@users.noreply.github.com> Date: Wed, 9 Oct 2024 22:52:34 +0530 Subject: [PATCH 2/4] Update values.yaml --- helm-chart/deepfence-cloud-scanner/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm-chart/deepfence-cloud-scanner/values.yaml b/helm-chart/deepfence-cloud-scanner/values.yaml index fb1eaf2..00a445e 100644 --- a/helm-chart/deepfence-cloud-scanner/values.yaml +++ b/helm-chart/deepfence-cloud-scanner/values.yaml @@ -96,6 +96,7 @@ env_vars: {} # AZURE_CLIENT_ID : # AZURE_CLIENT_SECRET: # AZURE_SUBSCRIPTION_ID: + # GCP_SERVICE_ACCOUNT_CREDENTIAL: imagePullSecrets: [] nameOverride: "" From ebb21ca9bca99714b3c7d6b6650a218d5f1124bd Mon Sep 17 00:00:00 2001 From: shyam Date: Wed, 9 Oct 2024 22:56:04 +0530 Subject: [PATCH 3/4] Changes hardcoded Home Dir Path --- service/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/service.go b/service/service.go index 56c5df2..dddb439 100644 --- a/service/service.go +++ b/service/service.go @@ -243,7 +243,7 @@ func (c *ComplianceScanService) fetchGCPProjects() ([]util.MonitoredAccount, err func saveGCPCredentialsToFile(credentials string) (string, error) { - configDir := "/home/deepfence/.config/gcloud/" + configDir := util.HomeDirectory+"/.config/gcloud/" credentialFilePath := configDir + "application_default_credentials.json" // Check if the directory exists, create it if not From fbfc3f5ed7bcd9bb10d855a6014406e8b217f9fe Mon Sep 17 00:00:00 2001 From: sam <67697492+shyam0904a@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:57:48 +0530 Subject: [PATCH 4/4] Added support for GCP Single Project deployment using env credentials --- service/service.go | 59 +++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/service/service.go b/service/service.go index dddb439..2e7c42d 100644 --- a/service/service.go +++ b/service/service.go @@ -169,55 +169,59 @@ func (c *ComplianceScanService) fetchGCPOrganizationProjects() ([]util.AccountsT return nil, err } - c.organizationAccountIDsLock.Lock() + if c.config.IsOrganizationDeployment { + c.organizationAccountIDsLock.Lock() - accountsMap := make(map[string]struct{}, len(c.organizationAccountIDs)) - for _, account := range c.organizationAccountIDs { - accountsMap[account.AccountID] = struct{}{} - } + accountsMap := make(map[string]struct{}, len(c.organizationAccountIDs)) + for _, account := range c.organizationAccountIDs { + accountsMap[account.AccountID] = struct{}{} + } - var newAccounts []util.AccountsToRefresh - for _, account := range projects { - if _, ok := accountsMap[account.AccountID]; !ok { - newAccounts = append(newAccounts, util.AccountsToRefresh{ - AccountID: account.AccountID, - NodeID: account.NodeID, - }) + var newAccounts []util.AccountsToRefresh + for _, account := range projects { + if _, ok := accountsMap[account.AccountID]; !ok { + newAccounts = append(newAccounts, util.AccountsToRefresh{ + AccountID: account.AccountID, + NodeID: account.NodeID, + }) + } } - } - c.organizationAccountIDs = projects - c.organizationAccountIDsLock.Unlock() + c.organizationAccountIDs = projects + c.organizationAccountIDsLock.Unlock() - return newAccounts, nil + return newAccounts, nil + } + + return nil, nil } func (c *ComplianceScanService) fetchGCPProjects() ([]util.MonitoredAccount, error) { - log.Info().Msg("GCP Project is choosen") + log.Info().Msg("Fetching GCP projects") ctx := context.Background() var crm *cloudresourcemanager.Service var err error if c.config.GCPCredentials != "" && strings.TrimSpace(c.config.GCPCredentials) != "" { - // Save the GCP credentials to a file + // Save the GCP credentials to a file credentialFilePath, err := saveGCPCredentialsToFile(c.config.GCPCredentials) if err != nil { log.Error().Err(err).Msg("Failed to save GCP credentials to file, falling back to default authentication") } else { - log.Info().Msgf("GCP credentials saved to file at: " + credentialFilePath) - clientOption := option.WithCredentialsFile(credentialFilePath) - crm, err = cloudresourcemanager.NewService(ctx, clientOption) + log.Info().Msgf("GCP credentials saved to file at: %s", credentialFilePath) + clientOption := option.WithCredentialsFile(credentialFilePath) + crm, err = cloudresourcemanager.NewService(ctx, clientOption) if err != nil { log.Error().Err(err).Msg("Failed to create GCP client with provided credentials, falling back to default authentication") } } - } + } if crm == nil { crm, err = cloudresourcemanager.NewService(ctx) if err != nil { - log.Error().Err(err).Msg("failed to create GCP client with default authentication") + log.Error().Err(err).Msg("Failed to create GCP client with default authentication") return nil, err } } @@ -225,7 +229,7 @@ func (c *ComplianceScanService) fetchGCPProjects() ([]util.MonitoredAccount, err projectsRequest := crm.Projects.List().PageSize(1000) projectsResponse, err := projectsRequest.Do() if err != nil { - log.Error().Err(err).Msg("failed to fetch GCP projects") + log.Error().Err(err).Msg("Failed to fetch GCP projects") return nil, err } @@ -241,6 +245,7 @@ func (c *ComplianceScanService) fetchGCPProjects() ([]util.MonitoredAccount, err return organizationAccountIDs, nil } + func saveGCPCredentialsToFile(credentials string) (string, error) { configDir := util.HomeDirectory+"/.config/gcloud/" @@ -403,6 +408,12 @@ func (c *ComplianceScanService) RunRegisterServices() error { return err } } + }else { + // Handle individual project credentials + if _, err := c.fetchGCPOrganizationProjects(); err != nil { + log.Error().Msg(err.Error()) + return err + } } processGcpCredentials(c) case util.CloudProviderAzure: