Azure tenant - multiple subscriptions

cloud_resource_changes/cloud_resource_changes_aws/cloudtrail.go

@@ -195,7 +195,7 @@ func (c *CloudResourceChangesAWS) listAndProcessS3Objects(regionalFilePrefix str
 		EncodingType: aws.String("url"),
 		Prefix:       aws.String(regionalFilePrefix),
 	}
-	if accId != c.config.CloudMetadata.ID {
+	if accId != c.config.AccountID {
 		params = params.SetExpectedBucketOwner(accId)
 	}
 	err = svc.ListObjectsV2Pages(params, func(resp *s3.ListObjectsV2Output, lastPage bool) bool {
@@ -257,7 +257,7 @@ func (c *CloudResourceChangesAWS) processCloudtrailEventLogFile(fileName string,
 		Bucket: aws.String(s3Bucket),
 		Key:    aws.String(*key.Key),
 	}
-	if accId != c.config.CloudMetadata.ID {
+	if accId != c.config.AccountID {
 		s3ObjectInput.SetExpectedBucketOwner(accId)
 	}
 	_, err = downloader.Download(file, &s3ObjectInput)

cloudformation/self-hosted/organization-deployment/deepfence-cloud-scanner-org-mgmt-console.template

@@ -306,6 +306,8 @@ Resources:
           Value: !Ref 'AWS::Region'
         - Name: CLOUD_ACCOUNT_ID
           Value: !Ref OrgAccId
+        - Name: CLOUD_ORGANIZATION_ID
+          Value: !Ref OrgAccId
         - Name: ORGANIZATION_DEPLOYMENT
           Value: "true"
         - Name: ROLE_NAME

helm-chart/deepfence-cloud-scanner/templates/deployment.yaml

@@ -50,6 +50,8 @@ spec:
             value: "{{ .Values.cloudAccount.region }}"
           - name: CLOUD_ACCOUNT_ID
             value: "{{ .Values.cloudAccount.accountID }}"
+          - name: CLOUD_ORGANIZATION_ID
+            value: "{{ .Values.cloudAccount.organizationAccountID }}"
           - name: ORGANIZATION_DEPLOYMENT
             value: "{{ .Values.cloudAccount.isOrganizationDeployment }}"
           - name: ROLE_NAME

helm-chart/deepfence-cloud-scanner/values.yaml

@@ -29,6 +29,7 @@ cloudAccount:
   region: ""
   # Is this organization deployment or single account deployment?
   isOrganizationDeployment: false
+  organizationAccountID: ""
   # Role name. The name should be same across all accounts in the Organization deployment.
   # Role ARN example: arn:aws:iam::123456789012:role/deepfence-managed-cloud-scanner-role
   # Role name in this case is deepfence-managed-cloud-scanner-role

helm-chart/index.yaml

@@ -3,12 +3,12 @@ entries:
   deepfence-cloud-scanner:
   - apiVersion: v2
     appVersion: 2.3.0
-    created: "2024-06-06T11:06:36.139253+05:30"
+    created: "2024-06-18T19:12:20.637723+05:30"
     description: Deepfence Cloud Scanner
-    digest: f4a52c6d6bced63954c001dd534fc8706bfd504b8defbd32e21c07bd89f02c57
+    digest: f60582daf0ba69673788177432defc318b61f65bc1c86c334b38db2ba8fb3818
     name: deepfence-cloud-scanner
     type: application
     urls:
     - deepfence-cloud-scanner-1.0.0.tgz
     version: 1.0.0
-generated: "2024-06-06T11:06:36.138791+05:30"
+generated: "2024-06-18T19:12:20.636193+05:30"

internal/deepfence/client.go

@@ -58,32 +58,38 @@ func NewClient(config util.Config) (*Client, error) {
 	return &Client{client: client, config: config}, nil
 }
 
-func (c *Client) RegisterCloudAccount(monitoredAccountIDs []string) error {
+func (c *Client) RegisterCloudAccount(monitoredOrganizationAccounts []util.OrganizationMonitoredAccount) error {
 	nodeId := util.GetNodeId(c.config.CloudProvider, c.config.AccountID)
 
 	req := c.client.Client().CloudNodesAPI.RegisterCloudNodeAccount(context.Background())
 	if c.config.IsOrganizationDeployment {
-		monitoredAccounts := map[string]string{}
-		for _, accountID := range monitoredAccountIDs {
-			monitoredAccounts[accountID] = util.GetNodeId(c.config.CloudProvider, accountID)
+		monitoredAccounts := make([]client.ModelCloudNodeMonitoredAccount, len(monitoredOrganizationAccounts))
+		for _, account := range monitoredOrganizationAccounts {
+			monitoredAccounts = append(monitoredAccounts, client.ModelCloudNodeMonitoredAccount{
+				AccountId:   account.AccountId,
+				AccountName: account.AccountName,
+				NodeId:      account.NodeId,
+			})
 		}
 
 		req = req.ModelCloudNodeAccountRegisterReq(
 			client.ModelCloudNodeAccountRegisterReq{
 				AccountId:                c.config.AccountID,
+				AccountName:              "",
 				CloudProvider:            c.config.CloudProvider,
 				HostNodeId:               c.config.NodeID,
 				IsOrganizationDeployment: &c.config.IsOrganizationDeployment,
-				MonitoredAccountIds:      monitoredAccounts,
+				MonitoredAccounts:        monitoredAccounts,
 				NodeId:                   nodeId,
-				OrganizationAccountId:    &c.config.AccountID,
+				OrganizationAccountId:    &c.config.OrganizationID,
 				Version:                  c.config.Version,
 			},
 		)
 	} else {
 		req = req.ModelCloudNodeAccountRegisterReq(
 			client.ModelCloudNodeAccountRegisterReq{
 				AccountId:                c.config.AccountID,
+				AccountName:              "",
 				CloudProvider:            c.config.CloudProvider,
 				HostNodeId:               c.config.NodeID,
 				IsOrganizationDeployment: &c.config.IsOrganizationDeployment,
@@ -93,14 +99,14 @@ func (c *Client) RegisterCloudAccount(monitoredAccountIDs []string) error {
 		)
 	}
 
-	log.Debug().Msgf("Before CloudNodesAPI.RegisterCloudNodeAccountExecute")
+	log.Debug().Msgf("Registering on management console")
 	_, err := c.client.Client().CloudNodesAPI.RegisterCloudNodeAccountExecute(req)
 	if err != nil {
 		log.Error().Msgf("Request errored on registering on management console: %s", err.Error())
 		return err
 	}
 
-	log.Info().Msgf("RegisterCloudAccount complete")
+	log.Info().Msgf("Register cloud account complete")
 	return nil
 }
 

main.go

@@ -82,6 +82,12 @@ func main() {
 		deepfence.SendSuccessfulDeploymentSignal(config.SuccessSignalUrl)
 	}
 
+	if config.IsOrganizationDeployment {
+		if config.OrganizationID == "" {
+			log.Fatal().Msgf("CLOUD_ORGANIZATION_ID is required in organization deployment")
+		}
+	}
+
 	switch config.CloudProvider {
 	case util.CloudProviderAWS:
 		if config.AWSCredentialSource != "EcsContainer" && config.AWSCredentialSource != "Ec2InstanceMetadata" && config.AWSCredentialSource != "Environment" {

query_resource/query.go

@@ -132,10 +132,7 @@ func QueryAndUpdateResources(config util.Config, cloudResourceTypesToRefresh map
 	count := 0
 	var errs = make([]error, 0)
 	for accountID, resourceTypesToRefresh := range cloudResourceTypesToRefresh {
-		accountIDPrefix := ""
-		if accountID != config.CloudMetadata.ID {
-			accountIDPrefix = config.CloudProvider + "_" + accountID + "."
-		}
+		accountIDPrefix := config.CloudProvider + "_" + accountID + "."
 
 		for _, cloudResourceInfo := range cloudProviderToResourceMap[config.CloudProvider] {
 			if !util.InSlice(cloudResourceInfo.Table, resourceTypesToRefresh) {
@@ -161,20 +158,7 @@ func QueryAndUpdateResources(config util.Config, cloudResourceTypesToRefresh map
 func queryResources(accountId string, cloudResourceInfo CloudResourceInfo, config util.Config, cloudResourcesFile *os.File) (int, error) {
 	log.Debug().Msgf("Querying resources for %s", cloudResourceInfo.Table)
 
-	var query string
-	switch config.CloudProvider {
-	case util.CloudProviderAWS:
-		query = "steampipe query --output json \"select \\\"" + strings.Join(cloudResourceInfo.Columns[:], "\\\" , \\\"") + "\\\" from aws_" + accountId + "." + cloudResourceInfo.Table + " \""
-	case util.CloudProviderGCP:
-		if config.IsOrganizationDeployment {
-			query = "steampipe query --output json \"select \\\"" + strings.Join(cloudResourceInfo.Columns[:], "\\\" , \\\"") + "\\\" from gcp_" + strings.Replace(accountId, "-", "", -1) + "." + cloudResourceInfo.Table + " \""
-		} else {
-			query = "steampipe query --output json \"select \\\"" + strings.Join(cloudResourceInfo.Columns[:], "\\\" , \\\"") + "\\\" from " + cloudResourceInfo.Table + " \""
-		}
-	default:
-		query = "steampipe query --output json \"select \\\"" + strings.Join(cloudResourceInfo.Columns[:], "\\\" , \\\"") + "\\\" from " + cloudResourceInfo.Table + " \""
-	}
-
+	query := "steampipe query --output json \"select \\\"" + strings.Join(cloudResourceInfo.Columns[:], "\\\" , \\\"") + "\\\" from " + config.CloudProvider + "_" + strings.Replace(accountId, "-", "", -1) + "." + cloudResourceInfo.Table + " \""
 	var stdOut []byte
 	var stdErr error
 	for i := 0; i <= 3; i++ {

scanner/scanner.go

@@ -68,70 +68,14 @@ func NewCloudComplianceScan(config util.Config) (*CloudComplianceScan, error) {
 	}, nil
 }
 
-func (c *CloudComplianceScan) RunComplianceScan() (util.ComplianceGroup, error) {
-	tempFileName := fmt.Sprintf("/tmp/%s.json", util.RandomString(12))
-	defer os.Remove(tempFileName)
-	cmd := fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --export=%s %s", cloudProviderPath[c.CloudProvider], tempFileName, c.ComplianceBenchmark)
-
-	var stdOut []byte
-	var stdErr error
-	for i := 0; i <= 3; i++ {
-		stdOut, stdErr = exec.Command("bash", "-c", cmd).CombinedOutput()
-		if stdErr != nil {
-			log.Error().Msgf("Steampipe check error: %v for query: %s", stdErr, cmd)
-			log.Error().Msgf(string(stdOut))
-			if strings.Contains(string(stdOut), util.ErrSteampipeDB) || strings.Contains(string(stdOut), util.ErrSteampipeInvalidClientTokenID) {
-				util.RestartSteampipeService()
-			} else {
-				time.Sleep(util.SleepTime)
-			}
-			os.Remove(tempFileName)
-			continue
-		} else {
-			break
-		}
-	}
-
-	var complianceResults util.ComplianceGroup
-	if _, err := os.Stat(tempFileName); errors.Is(err, os.ErrNotExist) {
-		return complianceResults, fmt.Errorf("%s: %v", stdOut, stdErr)
-	}
-	tempFile, err := os.Open(tempFileName)
-	if err != nil {
-		return complianceResults, err
-	}
-	results, err := io.ReadAll(tempFile)
-	if err != nil {
-		return complianceResults, err
-	}
-	err = json.Unmarshal(results, &complianceResults)
-	if err != nil {
-		return complianceResults, err
-	}
-	return complianceResults, nil
-}
-
 func (c *CloudComplianceScan) RunComplianceScanBenchmark(ctx context.Context,
 	benchmark ctl.CloudComplianceScanBenchmark, accountId string) (*util.ComplianceGroup, error) {
 
 	tempFileName := fmt.Sprintf("/tmp/%s.json", util.RandomString(12))
 	defer os.Remove(tempFileName)
 	log.Debug().Msgf("Account ID: %s, config cloud metadata id: %s", accountId, c.CloudMetadata.ID)
 
-	var cmdStr string
-	switch c.CloudProvider {
-	case util.CloudProviderAWS:
-		cmdStr = fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --search-path=%s_%s --export=%s %s", cloudProviderPath[c.CloudProvider], c.CloudProvider, strings.Replace(accountId, "-", "", -1), tempFileName, benchmark.Id)
-	case util.CloudProviderGCP:
-		if c.IsOrganizationDeployment {
-			cmdStr = fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --search-path=%s_%s --export=%s %s", cloudProviderPath[c.CloudProvider], c.CloudProvider, strings.Replace(accountId, "-", "", -1), tempFileName, benchmark.Id)
-		} else {
-			cmdStr = fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --export=%s %s", cloudProviderPath[c.CloudProvider], tempFileName, benchmark.Id)
-		}
-	default:
-		cmdStr = fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --export=%s %s", cloudProviderPath[c.CloudProvider], tempFileName, benchmark.Id)
-	}
-
+	cmdStr := fmt.Sprintf("cd %s && steampipe check --progress=false --output=none --search-path=%s_%s --export=%s %s", cloudProviderPath[c.CloudProvider], c.CloudProvider, strings.Replace(accountId, "-", "", -1), tempFileName, benchmark.Id)
 	log.Debug().Msgf("Steampipe command: %s", cmdStr)
 	cmd := exec.CommandContext(ctx, "bash", "-c", cmdStr)
 	//cmd.Env = os.Environ()