GCP Service Account Credential as docker env variable for cloud scann…

…er (#32)

deepfence_utils

@@ -1 +1 @@
-../../../deepfence_utils
+../ThreatMapper/deepfence_utils

docker-compose.yaml

@@ -1,7 +1,7 @@
 services:
   cloud-scanner:
     container_name: deepfence-cloud-scanner
-    image: ${IMAGE_REPOSITORY:-quay.io/deepfenceio}/cloud_scanner_ce:${DF_IMG_TAG:-2.4.0}
+    image: ${IMAGE_REPOSITORY:-quay.io/deepfenceio}/cloud_scanner_ce:${DF_IMG_TAG:-3.0.0}
     restart: unless-stopped
     environment:
       # Deepfence management console url and port
@@ -60,6 +60,9 @@ services:
       # AZURE_CLIENT_SECRET: ""
       # AZURE_SUBSCRIPTION_ID: ""
 
+      # Provide base64 encoded Service Account Keys for GCP Scanner
+      # GCP_SERVICE_ACCOUNT_CREDENTIAL: ""
+
       DEPLOYMENT_MODE: "docker"
       HOME_DIR: "/home/deepfence"
       DF_INSTALL_DIR: "/data/home/deepfence"
@@ -73,4 +76,4 @@ services:
 
 volumes:
   cloud_scanner_data:
-    driver: local
+    driver: local

helm-chart/deepfence-cloud-scanner/values.yaml

@@ -96,6 +96,7 @@ env_vars: {}
   # AZURE_CLIENT_ID :
   # AZURE_CLIENT_SECRET:
   # AZURE_SUBSCRIPTION_ID: 
+  # GCP_SERVICE_ACCOUNT_CREDENTIAL:
 
 imagePullSecrets: []
 nameOverride: ""

main.go

@@ -106,6 +106,8 @@ func main() {
 		if config.IsOrganizationDeployment && config.RoleName == "" {
 			log.Fatal().Msgf("ROLE_NAME is required in aws installation")
 		}
+	case util.CloudProviderGCP:
+		config.GCPCredentials = strings.TrimSpace(config.GCPCredentials)
 	default:
 		config.AWSCredentialSource = ""
 	}

query_resource/aws.go

@@ -109,16 +109,6 @@ var (
     ],
     "id_column": "api_id"
   },
-  {
-    "table": "aws_appautoscaling_target",
-    "columns": [
-      "region",
-      "resource_id",
-      "service_namespace",
-      "title"
-    ],
-    "id_column": "resource_id"
-  },
   {
     "table": "aws_appsync_graphql_api",
     "columns": [
@@ -1407,7 +1397,6 @@ var (
     "table": "aws_ssm_document",
     "columns": [
       "account_id",
-      "account_ids",
       "arn",
       "name",
       "owner_type",
@@ -1427,19 +1416,6 @@ var (
     ],
     "id_column": "arn"
   },
-  {
-    "table": "aws_ssm_managed_instance_compliance",
-    "columns": [
-      "account_id",
-      "compliance_type",
-      "id",
-      "region",
-      "resource_id",
-      "status",
-      "title"
-    ],
-    "id_column": "title"
-  },
   {
     "table": "aws_vpc",
     "columns": [

service/service.go

@@ -2,12 +2,14 @@ package service
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"net"
 	"net/http"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
@@ -25,6 +27,7 @@ import (
 	"github.com/deepfence/cloud-scanner/scanner"
 	"github.com/deepfence/cloud-scanner/util"
 	"google.golang.org/api/cloudresourcemanager/v1"
+	"google.golang.org/api/option"
 )
 
 var (
@@ -191,16 +194,32 @@ func (c *ComplianceScanService) fetchGCPOrganizationProjects() ([]util.AccountsT
 }
 
 func (c *ComplianceScanService) fetchGCPProjects() ([]util.MonitoredAccount, error) {
+	log.Info().Msg("Fetching GCP projects")
 	ctx := context.Background()
-	crm, err := cloudresourcemanager.NewService(ctx)
-	if err != nil {
-		log.Error().Err(err).Msg("failed to fetch GCP projects")
-		return nil, err
+
+	var crm *cloudresourcemanager.Service
+	var err error
+
+	if _, err = os.Stat(util.GCPCredentialFilePath); err == nil {
+		clientOption := option.WithCredentialsFile(util.GCPCredentialFilePath)
+		crm, err = cloudresourcemanager.NewService(ctx, clientOption)
+		if err != nil {
+			log.Error().Err(err).Msg("Failed to create GCP client with provided credentials, falling back to default authentication")
+		}
 	}
+
+	if crm == nil {
+		crm, err = cloudresourcemanager.NewService(ctx)
+		if err != nil {
+			log.Error().Err(err).Msg("Failed to create GCP client with default authentication")
+			return nil, err
+		}
+	}
+
 	projectsRequest := crm.Projects.List().PageSize(1000)
 	projectsResponse, err := projectsRequest.Do()
 	if err != nil {
-		log.Error().Err(err).Msg("failed to fetch GCP projects")
+		log.Error().Err(err).Msg("Failed to fetch GCP projects")
 		return nil, err
 	}
 
@@ -212,9 +231,34 @@ func (c *ComplianceScanService) fetchGCPProjects() ([]util.MonitoredAccount, err
 			NodeID:      util.GetNodeID(c.config.CloudProvider, project.ProjectId),
 		}
 	}
+
 	return organizationAccountIDs, nil
 }
 
+func saveGCPCredentialsToFile(credentials string) error {
+	configDir := filepath.Dir(util.GCPCredentialFilePath)
+
+	// Check if the directory exists, create it if not
+	if _, err := os.Stat(configDir); os.IsNotExist(err) {
+		err = os.MkdirAll(configDir, 0700)
+		if err != nil {
+			return fmt.Errorf("failed to create directory: %w", err)
+		}
+	}
+
+	credBytes, err := base64.StdEncoding.DecodeString(credentials)
+	if err != nil {
+		return fmt.Errorf("failed to decode GCP credentials: %w", err)
+	}
+
+	err = os.WriteFile(util.GCPCredentialFilePath, credBytes, 0600)
+	if err != nil {
+		return fmt.Errorf("failed to write credentials to file: %w", err)
+	}
+	log.Info().Msgf("GCP credentials saved to file at: %s", util.GCPCredentialFilePath)
+	return nil
+}
+
 func (c *ComplianceScanService) fetchAzureTenantSubscriptions() ([]util.MonitoredAccount, error) {
 	cred, err := azidentity.NewDefaultAzureCredential(nil)
 	if err != nil {
@@ -319,6 +363,12 @@ func (c *ComplianceScanService) RunRegisterServices() error {
 		}
 		processAwsCredentials(c)
 	case util.CloudProviderGCP:
+		if c.config.GCPCredentials != "" {
+			err = saveGCPCredentialsToFile(c.config.GCPCredentials)
+			if err != nil {
+				log.Fatal().Msgf(err.Error())
+			}
+		}
 		if c.config.IsOrganizationDeployment {
 			projects, err := c.fetchGCPOrganizationProjects()
 			if err != nil || len(projects) == 0 {

util/type.go

@@ -70,6 +70,7 @@ type Config struct {
 	ScanInactiveThreshold    int      `envconfig:"SCAN_INACTIVE_THRESHOLD" default:"21600" json:"scan_inactive_threshold"`
 	CloudScannerPolicy       string   `envconfig:"CLOUD_SCANNER_POLICY" json:"cloud_scanner_policy"`
 	DeploymentMode           string   `envconfig:"DEPLOYMENT_MODE" json:"deployment_mode"`
+	GCPCredentials           string   `envconfig:"GCP_SERVICE_ACCOUNT_CREDENTIAL" json:"gcp_service_account_credential"`
 
 	CloudMetadata                cloudmetadata.CloudMetadata `ignored:"true" json:"cloud_metadata"`
 	NodeID                       string                      `ignored:"true" json:"-"`
@@ -182,6 +183,8 @@ var (
 	SteampipeGCPPluginVersion     = fmt.Sprintf("gcp@%s", os.Getenv("STEAMPIPE_GCP_PLUGIN_VERSION"))
 	SteampipeAzurePluginVersion   = fmt.Sprintf("azure@%s", os.Getenv("STEAMPIPE_AZURE_PLUGIN_VERSION"))
 	SteampipeAzureADPluginVersion = fmt.Sprintf("azuread@%s", os.Getenv("STEAMPIPE_AZURE_AD_PLUGIN_VERSION"))
+
+	GCPCredentialFilePath string
 )
 
 func init() {
@@ -199,4 +202,6 @@ func init() {
 	if SteampipeInstallDirectory == "" {
 		SteampipeInstallDirectory = "/home/deepfence/.steampipe"
 	}
+
+	GCPCredentialFilePath = HomeDirectory + "/.config/gcloud/application_default_credentials.json"
 }