Azure tenant - multiple subscriptions

helm-chart/deepfence-cloud-scanner/templates/deployment.yaml

@@ -50,6 +50,8 @@ spec:
             value: "{{ .Values.cloudAccount.region }}"
           - name: CLOUD_ACCOUNT_ID
             value: "{{ .Values.cloudAccount.accountID }}"
+          - name: CLOUD_ORGANIZATION_ID
+            value: "{{ .Values.cloudAccount.organizationAccountID }}"
           - name: ORGANIZATION_DEPLOYMENT
             value: "{{ .Values.cloudAccount.isOrganizationDeployment }}"
           - name: ROLE_NAME

helm-chart/deepfence-cloud-scanner/values.yaml

@@ -29,6 +29,7 @@ cloudAccount:
   region: ""
   # Is this organization deployment or single account deployment?
   isOrganizationDeployment: false
+  organizationAccountID: ""
   # Role name. The name should be same across all accounts in the Organization deployment.
   # Role ARN example: arn:aws:iam::123456789012:role/deepfence-managed-cloud-scanner-role
   # Role name in this case is deepfence-managed-cloud-scanner-role

helm-chart/index.yaml

@@ -3,12 +3,12 @@ entries:
   deepfence-cloud-scanner:
   - apiVersion: v2
     appVersion: 2.3.0
-    created: "2024-06-06T11:06:36.139253+05:30"
+    created: "2024-06-18T19:12:20.637723+05:30"
     description: Deepfence Cloud Scanner
-    digest: f4a52c6d6bced63954c001dd534fc8706bfd504b8defbd32e21c07bd89f02c57
+    digest: f60582daf0ba69673788177432defc318b61f65bc1c86c334b38db2ba8fb3818
     name: deepfence-cloud-scanner
     type: application
     urls:
     - deepfence-cloud-scanner-1.0.0.tgz
     version: 1.0.0
-generated: "2024-06-06T11:06:36.138791+05:30"
+generated: "2024-06-18T19:12:20.636193+05:30"

internal/deepfence/client.go

@@ -76,7 +76,7 @@ func (c *Client) RegisterCloudAccount(monitoredAccountIDs []string) error {
 				IsOrganizationDeployment: &c.config.IsOrganizationDeployment,
 				MonitoredAccountIds:      monitoredAccounts,
 				NodeId:                   nodeId,
-				OrganizationAccountId:    &c.config.AccountID,
+				OrganizationAccountId:    &c.config.OrganizationID,
 				Version:                  c.config.Version,
 			},
 		)
@@ -93,14 +93,14 @@ func (c *Client) RegisterCloudAccount(monitoredAccountIDs []string) error {
 		)
 	}
 
-	log.Debug().Msgf("Before CloudNodesAPI.RegisterCloudNodeAccountExecute")
+	log.Debug().Msgf("Registering on management console")
 	_, err := c.client.Client().CloudNodesAPI.RegisterCloudNodeAccountExecute(req)
 	if err != nil {
 		log.Error().Msgf("Request errored on registering on management console: %s", err.Error())
 		return err
 	}
 
-	log.Info().Msgf("RegisterCloudAccount complete")
+	log.Info().Msgf("Register cloud account complete")
 	return nil
 }
 

main.go

@@ -82,6 +82,12 @@ func main() {
 		deepfence.SendSuccessfulDeploymentSignal(config.SuccessSignalUrl)
 	}
 
+	if config.IsOrganizationDeployment {
+		if config.OrganizationID == "" {
+			log.Fatal().Msgf("ORGANIZATION_ID is required in organization deployment")
+		}
+	}
+
 	switch config.CloudProvider {
 	case util.CloudProviderAWS:
 		if config.AWSCredentialSource != "EcsContainer" && config.AWSCredentialSource != "Ec2InstanceMetadata" && config.AWSCredentialSource != "Environment" {

query_resource/query.go

@@ -132,10 +132,7 @@ func QueryAndUpdateResources(config util.Config, cloudResourceTypesToRefresh map
 	count := 0
 	var errs = make([]error, 0)
 	for accountID, resourceTypesToRefresh := range cloudResourceTypesToRefresh {
-		accountIDPrefix := ""
-		if accountID != config.CloudMetadata.ID {
-			accountIDPrefix = config.CloudProvider + "_" + accountID + "."
-		}
+		accountIDPrefix := config.CloudProvider + "_" + accountID + "."
 
 		for _, cloudResourceInfo := range cloudProviderToResourceMap[config.CloudProvider] {
 			if !util.InSlice(cloudResourceInfo.Table, resourceTypesToRefresh) {

scanner/scanner.go

@@ -120,16 +120,14 @@ func (c *CloudComplianceScan) RunComplianceScanBenchmark(ctx context.Context,
 
 	var cmdStr string
 	switch c.CloudProvider {
-	case util.CloudProviderAWS:
-		cmdStr = fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --search-path=%s_%s --export=%s %s", cloudProviderPath[c.CloudProvider], c.CloudProvider, strings.Replace(accountId, "-", "", -1), tempFileName, benchmark.Id)
 	case util.CloudProviderGCP:
 		if c.IsOrganizationDeployment {
 			cmdStr = fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --search-path=%s_%s --export=%s %s", cloudProviderPath[c.CloudProvider], c.CloudProvider, strings.Replace(accountId, "-", "", -1), tempFileName, benchmark.Id)
 		} else {
 			cmdStr = fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --export=%s %s", cloudProviderPath[c.CloudProvider], tempFileName, benchmark.Id)
 		}
 	default:
-		cmdStr = fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --export=%s %s", cloudProviderPath[c.CloudProvider], tempFileName, benchmark.Id)
+		cmdStr = fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --search-path=%s_%s --export=%s %s", cloudProviderPath[c.CloudProvider], c.CloudProvider, strings.Replace(accountId, "-", "", -1), tempFileName, benchmark.Id)
 	}
 
 	log.Debug().Msgf("Steampipe command: %s", cmdStr)

service/service.go

@@ -24,7 +24,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	ctl "github.com/deepfence/ThreatMapper/deepfence_utils/controls"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
-	cloud_metadata "github.com/deepfence/cloud-scanner/cloud-metadata"
 	"github.com/deepfence/cloud-scanner/cloud_resource_changes"
 	"github.com/deepfence/cloud-scanner/internal/deepfence"
 	"github.com/deepfence/cloud-scanner/query_resource"
@@ -147,7 +146,7 @@ func getAWSCredentialsConfig(ctx context.Context, accountID, region, roleName st
 	return cfg, err
 }
 
-func (c *ComplianceScanService) fetchOrganizationAccountIDs() error {
+func (c *ComplianceScanService) fetchAWSOrganizationAccountIDs() error {
 	organizationAccountIDs := []string{}
 
 	ctx := context.Background()
@@ -195,28 +194,27 @@ func (c *ComplianceScanService) RunRegisterServices() error {
 	if c.config.HttpServerRequired {
 		go c.runHttpServer()
 	}
-	if c.config.IsOrganizationDeployment {
-		err := c.fetchOrganizationAccountIDs()
-		if err != nil {
-			log.Warn().Msg(err.Error())
+	var err error
+	switch c.config.CloudProvider {
+	case util.CloudProviderAWS:
+		if c.config.IsOrganizationDeployment {
+			err = c.fetchAWSOrganizationAccountIDs()
+			if err != nil {
+				log.Warn().Msg(err.Error())
+			}
 		}
-	}
-	if c.config.CloudProvider == cloud_metadata.CloudProviderAWS {
 		processAwsCredentials(c)
-	} else if c.config.CloudProvider == cloud_metadata.CloudProviderGCP {
-		err := processGcpCredentials(c)
-		if err != nil {
-			log.Fatal().Msgf("%+v", err)
-		}
-	} else if c.config.CloudProvider == cloud_metadata.CloudProviderAzure {
-		processAzureCredentials()
+	case util.CloudProviderGCP:
+		processGcpCredentials(c)
+	case util.CloudProviderAzure:
+		processAzureCredentials(c)
 	}
 
 	log.Info().Msgf("Restarting the steampipe service")
 	util.RestartSteampipeService()
 
 	log.Info().Msgf("CloudResourceChanges Initialization started")
-	err := c.CloudResourceChanges.Initialize()
+	err = c.CloudResourceChanges.Initialize()
 	if err != nil {
 		log.Warn().Msgf("%+v", err)
 	}
@@ -238,14 +236,26 @@ func (c *ComplianceScanService) RunRegisterServices() error {
 	return nil
 }
 
-func processAzureCredentials() {
-	err := saveFileOverwrite(HomeDirectory+"/.steampipe/config/azure.spc",
-		"\nconnection \"azure\" {\n  plugin = \"azure\"\n "+
-			"  subscription_id = \""+os.Getenv("AZURE_SUBSCRIPTION_ID")+"\"\n"+
-			"  tenant_id = \""+os.Getenv("AZURE_TENANT_ID")+"\"\n"+
-			"  client_id = \""+os.Getenv("AZURE_CLIENT_ID")+"\"\n"+
-			"  client_secret = \""+os.Getenv("AZURE_CLIENT_SECRET")+"\"\n"+
-			"  ignore_error_codes = [\"AccessDenied\", \"AccessDeniedException\", \"NotAuthorized\", \"UnauthorizedOperation\", \"AuthorizationError\"]\n}\n")
+func processAzureCredentials(c *ComplianceScanService) {
+	steampipeConfigFile := "connection \"azure_all\" {\n  type = \"aggregator\" \n plugin      = \"azure\" \n  connections = [\"azure_*\"] \n} \n"
+	if c.config.IsOrganizationDeployment {
+		for _, accountID := range c.GetOrganizationAccountIDs() {
+			steampipeConfigFile += "\nconnection \"azure_" + strings.Replace(accountID, "-", "", -1) + "\" {\n  plugin = \"azure\"\n " +
+				"  subscription_id = \"" + accountID + "\"\n" +
+				"  tenant_id = \"" + c.config.OrganizationID + "\"\n" +
+				"  client_id = \"" + os.Getenv("AZURE_CLIENT_ID") + "\"\n" +
+				"  client_secret = \"" + os.Getenv("AZURE_CLIENT_SECRET") + "\"\n" +
+				"  ignore_error_codes = [\"AccessDenied\", \"AccessDeniedException\", \"NotAuthorized\", \"UnauthorizedOperation\", \"AuthorizationError\"]\n}\n"
+		}
+	} else {
+		steampipeConfigFile += "\nconnection \"azure_" + strings.Replace(c.config.AccountID, "-", "", -1) + "\" {\n  plugin = \"azure\"\n " +
+			"  subscription_id = \"" + c.config.AccountID + "\"\n" +
+			"  tenant_id = \"" + c.config.OrganizationID + "\"\n" +
+			"  client_id = \"" + os.Getenv("AZURE_CLIENT_ID") + "\"\n" +
+			"  client_secret = \"" + os.Getenv("AZURE_CLIENT_SECRET") + "\"\n" +
+			"  ignore_error_codes = [\"AccessDenied\", \"AccessDeniedException\", \"NotAuthorized\", \"UnauthorizedOperation\", \"AuthorizationError\"]\n}\n"
+	}
+	err := saveFileOverwrite(HomeDirectory+"/.steampipe/config/azure.spc", steampipeConfigFile)
 	if err != nil {
 		log.Fatal().Msgf(err.Error())
 	}
@@ -287,19 +297,19 @@ func processAwsCredentials(c *ComplianceScanService) {
 	}
 }
 
-func processGcpCredentials(c *ComplianceScanService) error {
-	organizationAccountIDs := c.GetOrganizationAccountIDs()
-	if len(organizationAccountIDs) > 0 {
-		steampipeConfigFile := "connection \"gcp_all\" {\n  type = \"aggregator\" \n plugin      = \"gcp\" \n  connections = [\"gcp_*\"] \n} \n"
-		for _, accountID := range organizationAccountIDs {
+func processGcpCredentials(c *ComplianceScanService) {
+	steampipeConfigFile := "connection \"gcp_all\" {\n  type = \"aggregator\" \n plugin      = \"gcp\" \n  connections = [\"gcp_*\"] \n} \n"
+	if c.config.IsOrganizationDeployment {
+		for _, accountID := range c.GetOrganizationAccountIDs() {
 			steampipeConfigFile += "connection \"gcp_" + strings.Replace(accountID, "-", "", -1) + "\" {\n  plugin  = \"gcp\"\n  project = \"" + accountID + "\"\n}\n"
 		}
-		err := saveFileOverwrite(HomeDirectory+"/.steampipe/config/gcp.spc", steampipeConfigFile)
-		if err != nil {
-			log.Fatal().Msgf(err.Error())
-		}
+	} else {
+		steampipeConfigFile += "connection \"gcp_" + strings.Replace(c.config.AccountID, "-", "", -1) + "\" {\n  plugin  = \"gcp\"\n  project = \"" + c.config.AccountID + "\"\n}\n"
+	}
+	err := saveFileOverwrite(HomeDirectory+"/.steampipe/config/gcp.spc", steampipeConfigFile)
+	if err != nil {
+		log.Fatal().Msgf(err.Error())
 	}
-	return nil
 }
 
 func saveFileOverwrite(fileName string, fileContents string) error {
@@ -321,20 +331,19 @@ func (c *ComplianceScanService) refreshOrganizationAccountIDs() {
 	for {
 		select {
 		case <-ticker.C:
-			err := c.fetchOrganizationAccountIDs()
-			if err != nil {
-				log.Warn().Msg(err.Error())
-				continue
-			}
-			if c.config.CloudProvider == cloud_metadata.CloudProviderAWS {
-				processAwsCredentials(c)
-			} else if c.config.CloudProvider == cloud_metadata.CloudProviderGCP {
-				err := processGcpCredentials(c)
-				if err != nil {
-					log.Fatal().Msgf("%+v", err)
+			var err error
+			if c.config.CloudProvider == util.CloudProviderAWS {
+				if c.config.IsOrganizationDeployment {
+					err = c.fetchAWSOrganizationAccountIDs()
+					if err != nil {
+						log.Warn().Msg(err.Error())
+					}
 				}
-			} else if c.config.CloudProvider == cloud_metadata.CloudProviderAzure {
-				processAzureCredentials()
+				processAwsCredentials(c)
+			} else if c.config.CloudProvider == util.CloudProviderGCP {
+				processGcpCredentials(c)
+			} else if c.config.CloudProvider == util.CloudProviderAzure {
+				processAzureCredentials(c)
 			}
 		}
 	}

util/type.go

@@ -46,6 +46,7 @@ type Config struct {
 	CloudProvider            string   `envconfig:"CLOUD_PROVIDER" json:"cloud_provider"`
 	CloudRegion              string   `envconfig:"CLOUD_REGION" json:"cloud_region"`
 	AccountID                string   `envconfig:"CLOUD_ACCOUNT_ID" json:"account_id"`
+	OrganizationID           string   `envconfig:"CLOUD_ORGANIZATION_ID" json:"organization_id"`
 	IsOrganizationDeployment bool     `envconfig:"ORGANIZATION_DEPLOYMENT" default:"false" json:"is_organization_deployment"`
 	RoleName                 string   `envconfig:"ROLE_NAME" json:"role_name"`
 	AWSCredentialSource      string   `envconfig:"AWS_CREDENTIAL_SOURCE" json:"aws_credential_source"`