Exporter: phase 1 of support for Account-level exports (#2205)

Implemented export of users/groups/service principals on account level, including
generation of corresponding role objects to support `account_admin` role.

To support role export, was need to make roles exclusion for users/service principals
configurable.

Current limitation: not all users & service principals could be exported - will require
addition of the dedicated `List` implementation for these resources, as `account users`
group isn't returned by the API.

docs/guides/experimental-exporter.md

@@ -82,6 +82,7 @@ Exporter aims to generate HCL code for the most of resources within the Databric
 | [databricks_group](../resources/group.md) | Yes |
 | [databricks_group_instance_profile](../resources/group_instance_profile.md) | Yes |
 | [databricks_group_member](../resources/group_member.md) | Yes |
+| [databricks_group_role](../resources/group_role.md) | Yes |
 | [databricks_instance_pool](../resources/instance_pool.md) | Yes |
 | [databricks_instance_profile](../resources/instance_profile.md) | Yes |
 | [databricks_ip_access_list](../resources/ip_access_list.md) | Yes |
@@ -98,6 +99,7 @@ Exporter aims to generate HCL code for the most of resources within the Databric
 | [databricks_secret_acl](../resources/secret_acl.md) | Yes |
 | [databricks_secret_scope](../resources/secret_scope.md) | Yes |
 | [databricks_service_principal](../resources/service_principal.md) | Yes |
+| [databricks_service_principal_role](../resources/service_principal_role.md) | Yes |
 | [databricks_sql_alert](../resources/sql_alert.md) | Yes |
 | [databricks_sql_dashboard](../resources/sql_dashboard.md) | Yes |
 | [databricks_sql_endpoint](../resources/sql_endpoint.md) | Yes |
@@ -109,4 +111,5 @@ Exporter aims to generate HCL code for the most of resources within the Databric
 | [databricks_token](../resources/token.md) | Not Applicable |
 | [databricks_user](../resources/user.md) | Yes |
 | [databricks_user_instance_profile](../resources/user_instance_profile.md) | No (Deprecated) |
+| [databricks_user_role](../resources/user_role.md) | Yes |
 | [databricks_workspace_conf](../resources/workspace_conf.md) | Yes (partial) |

exporter/context.go

@@ -80,6 +80,7 @@ type importContext struct {
 	generateDeclaration bool
 	meAdmin             bool
 	prefix              string
+	accountLevel        bool
 }
 
 type mount struct {
@@ -161,16 +162,23 @@ func (ic *importContext) Run() error {
 	if err != nil {
 		return err
 	}
-	me, err := w.CurrentUser.Me(ic.Context)
-	if err != nil {
-		return err
-	}
-	for _, g := range me.Groups {
-		if g.Display == "admins" {
-			ic.meAdmin = true
-			break
+
+	ic.accountLevel = ic.Client.Config.IsAccountClient()
+	if ic.accountLevel {
+		ic.meAdmin = true
+	} else {
+		me, err := w.CurrentUser.Me(ic.Context)
+		if err != nil {
+			return err
+		}
+		for _, g := range me.Groups {
+			if g.Display == "admins" {
+				ic.meAdmin = true
+				break
+			}
 		}
 	}
+
 	for resourceName, ir := range ic.Importables {
 		if ir.List == nil {
 			continue
@@ -180,6 +188,10 @@ func (ic *importContext) Run() error {
 				resourceName, ir.Service)
 			continue
 		}
+		if ic.accountLevel && !ir.AccountLevel {
+			log.Printf("[DEBUG] %s (%s service) is not account level", resourceName, ir.Service)
+			continue
+		}
 		if err := ir.List(ic); err != nil {
 			log.Printf("[ERROR] %s (%s service) listing failed: %s",
 				resourceName, ir.Service, err)
@@ -474,6 +486,12 @@ func (ic *importContext) Emit(r *resource) {
 		log.Printf("[ERROR] %s is not available for import", r)
 		return
 	}
+	if ic.accountLevel && !ir.AccountLevel {
+		log.Printf("[DEBUG] %s (%s service) is not part of the account level export",
+			r.Resource, ir.Service)
+		return
+
+	}
 	if !strings.Contains(ic.services, ir.Service) {
 		log.Printf("[DEBUG] %s (%s service) is not part of the import",
 			r.Resource, ir.Service)

exporter/exporter_test.go

@@ -360,7 +360,7 @@ func TestImportingUsersGroupsSecretScopes(t *testing.T) {
 			},
 			{
 				Method:   "GET",
-				Resource: "/api/2.0/preview/scim/v2/ServicePrincipals?excludedAttributes=roles&filter=applicationId%20eq%20%27spn%27",
+				Resource: "/api/2.0/preview/scim/v2/ServicePrincipals?filter=applicationId%20eq%20%27spn%27",
 				Response: scim.User{ID: "321", DisplayName: "spn", ApplicationID: "spn",
 					Groups: []scim.ComplexValue{
 						{Display: "admins", Value: "a", Ref: "Groups/a", Type: "direct"},
@@ -426,7 +426,7 @@ func TestImportingUsersGroupsSecretScopes(t *testing.T) {
 			},
 			{
 				Method:   "GET",
-				Resource: "/api/2.0/preview/scim/v2/Users?excludedAttributes=roles&filter=userName%20eq%20%27test%40test.com%27",
+				Resource: "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27test%40test.com%27",
 				Response: scim.UserList{
 					Resources: []scim.User{
 						{ID: "123", DisplayName: "test@test.com", UserName: "test@test.com"},
@@ -1316,7 +1316,7 @@ func TestImportingUser(t *testing.T) {
 			{
 				Method:       "GET",
 				ReuseRequest: true,
-				Resource:     "/api/2.0/preview/scim/v2/Users?excludedAttributes=roles&filter=userName%20eq%20%27me%27",
+				Resource:     "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27me%27",
 				Response: scim.UserList{
 					Resources: []scim.User{
 						{
@@ -1658,7 +1658,7 @@ func TestImportingDLTPipelines(t *testing.T) {
 			},
 			{
 				Method:   "GET",
-				Resource: "/api/2.0/preview/scim/v2/Users?excludedAttributes=roles&filter=userName%20eq%20%27user%40domain.com%27",
+				Resource: "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27user%40domain.com%27",
 				Response: scim.UserList{
 					Resources: []scim.User{
 						{ID: "123", DisplayName: "user@domain.com", UserName: "user@domain.com"},

exporter/importables.go

@@ -196,12 +196,29 @@ var resourcesMap map[string]importable = map[string]importable{
 		},
 	},
 	"databricks_group_role": {
-		Service: "access",
+		Service:      "access",
+		AccountLevel: true,
 		Depends: []reference{
 			{Path: "group_id", Resource: "databricks_group"},
 			{Path: "role", Resource: "databricks_instance_profile", Match: "instance_profile_arn"},
 		},
 	},
+	"databricks_user_role": {
+		Service:      "access",
+		AccountLevel: true,
+		Depends: []reference{
+			{Path: "user_id", Resource: "databricks_user"},
+			{Path: "role", Resource: "databricks_instance_profile", Match: "instance_profile_arn"},
+		},
+	},
+	"databricks_service_principal_role": {
+		Service:      "access",
+		AccountLevel: true,
+		Depends: []reference{
+			{Path: "service_principal_id", Resource: "databricks_group"},
+			{Path: "role", Resource: "databricks_instance_profile", Match: "instance_profile_arn"},
+		},
+	},
 	"databricks_library": {
 		Service: "compute",
 		Depends: []reference{
@@ -531,7 +548,8 @@ var resourcesMap map[string]importable = map[string]importable{
 		Body: resourceOrDataBlockBody,
 	},
 	"databricks_group": {
-		Service: "groups",
+		Service:      "groups",
+		AccountLevel: true,
 		Name: func(ic *importContext, d *schema.ResourceData) string {
 			return d.Get("display_name").(string) + "_" + d.Id()
 		},
@@ -590,16 +608,7 @@ var resourcesMap map[string]importable = map[string]importable{
 				if r.ID != g.ID {
 					continue
 				}
-				for _, instanceProfile := range g.Roles {
-					ic.Emit(&resource{
-						Resource: "databricks_instance_profile",
-						ID:       instanceProfile.Value,
-					})
-					ic.Emit(&resource{
-						Resource: "databricks_group_role",
-						ID:       fmt.Sprintf("%s|%s", g.ID, instanceProfile.Value),
-					})
-				}
+				ic.emitRoles("group", g.ID, g.Roles)
 				if g.DisplayName == "users" && !ic.importAllUsers {
 					log.Printf("[INFO] Skipping import of entire user directory ...")
 					continue
@@ -656,7 +665,8 @@ var resourcesMap map[string]importable = map[string]importable{
 		Body: resourceOrDataBlockBody,
 	},
 	"databricks_group_member": {
-		Service: "groups",
+		Service:      "groups",
+		AccountLevel: true,
 		Depends: []reference{
 			{Path: "group_id", Resource: "databricks_group"},
 			{Path: "member_id", Resource: "databricks_user"},
@@ -665,7 +675,8 @@ var resourcesMap map[string]importable = map[string]importable{
 		},
 	},
 	"databricks_user": {
-		Service: "users",
+		Service:      "users",
+		AccountLevel: true,
 		Name: func(ic *importContext, d *schema.ResourceData) string {
 			s := d.Get("user_name").(string)
 			// if CLI argument includeUserDomains is set then it includes domain portion as well
@@ -694,11 +705,13 @@ var resourcesMap map[string]importable = map[string]importable{
 				userName = u.UserName
 			}
 			ic.emitGroups(u, userName)
+			ic.emitRoles("user", u.ID, u.Roles)
 			return nil
 		},
 	},
 	"databricks_service_principal": {
-		Service: "users",
+		Service:      "users",
+		AccountLevel: true,
 		Name: func(ic *importContext, d *schema.ResourceData) string {
 			name := d.Get("display_name").(string)
 			if name == "" {
@@ -738,6 +751,7 @@ var resourcesMap map[string]importable = map[string]importable{
 				spnName = u.ApplicationID
 			}
 			ic.emitGroups(u, spnName)
+			ic.emitRoles("service_principal", u.ID, u.Roles)
 			return nil
 		},
 	},

exporter/importables_test.go

@@ -113,6 +113,7 @@ func TestGroup(t *testing.T) {
 			Roles: []scim.ComplexValue{
 				{
 					Value: "abc",
+					Type:  "direct",
 				},
 			},
 			Members: []scim.ComplexValue{
@@ -444,7 +445,7 @@ func TestUserSearchFails(t *testing.T) {
 		{
 			ReuseRequest: true,
 			Method:       "GET",
-			Resource:     "/api/2.0/preview/scim/v2/Users?excludedAttributes=roles&filter=userName%20eq%20%27dbc%27",
+			Resource:     "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27dbc%27",
 			Status:       404,
 			Response:     apierr.NotFound("nope"),
 		},
@@ -473,7 +474,7 @@ func TestSpnSearchFails(t *testing.T) {
 		{
 			ReuseRequest: true,
 			Method:       "GET",
-			Resource:     "/api/2.0/preview/scim/v2/ServicePrincipals?excludedAttributes=roles&filter=applicationId%20eq%20%27dbc%27",
+			Resource:     "/api/2.0/preview/scim/v2/ServicePrincipals?filter=applicationId%20eq%20%27dbc%27",
 			Status:       404,
 			Response:     apierr.NotFound("nope"),
 		},
@@ -502,7 +503,7 @@ func TestSpnSearchSuccess(t *testing.T) {
 		{
 			ReuseRequest: true,
 			Method:       "GET",
-			Resource:     "/api/2.0/preview/scim/v2/ServicePrincipals?excludedAttributes=roles&filter=applicationId%20eq%20%27dbc%27",
+			Resource:     "/api/2.0/preview/scim/v2/ServicePrincipals?filter=applicationId%20eq%20%27dbc%27",
 			Response: scim.UserList{Resources: []scim.User{
 				{ID: "321", DisplayName: "spn", ApplicationID: "dbc"},
 			}},
@@ -556,7 +557,7 @@ func TestUserImportSkipNonDirectGroups(t *testing.T) {
 		{
 			ReuseRequest: true,
 			Method:       "GET",
-			Resource:     "/api/2.0/preview/scim/v2/Users?excludedAttributes=roles&filter=userName%20eq%20%27dbc%27",
+			Resource:     "/api/2.0/preview/scim/v2/Users?filter=userName%20eq%20%27dbc%27",
 			Response: scim.UserList{
 				Resources: []scim.User{
 					{

exporter/model.go

@@ -54,6 +54,8 @@ type importable struct {
 	ShouldOmitField func(ic *importContext, pathString string, as *schema.Schema, d *schema.ResourceData) bool
 	// Defines which API version should be used for this specific resource
 	ApiVersion common.ApiVersion
+	// Defines if specific service is account level
+	AccountLevel bool
 }
 
 type MatchType string

exporter/util.go

@@ -166,6 +166,25 @@ func (ic *importContext) emitGroups(u scim.User, principal string) {
 	}
 }
 
+func (ic *importContext) emitRoles(objType string, id string, roles []scim.ComplexValue) {
+	log.Printf("[DEBUG] emitting roles for object type: %s, ID: %s, roles: %v", objType, id, roles)
+	for _, role := range roles {
+		if role.Type != "direct" {
+			continue
+		}
+		if !ic.accountLevel {
+			ic.Emit(&resource{
+				Resource: "databricks_instance_profile",
+				ID:       role.Value,
+			})
+		}
+		ic.Emit(&resource{
+			Resource: fmt.Sprintf("databricks_%s_role", objType),
+			ID:       fmt.Sprintf("%s|%s", id, role.Value),
+		})
+	}
+}
+
 func (ic *importContext) importLibraries(d *schema.ResourceData, s map[string]*schema.Schema) error {
 	var cll libraries.ClusterLibraryList
 	common.DataToStructPointer(d, s, &cll)
@@ -212,7 +231,7 @@ func (ic *importContext) cacheGroups() error {
 
 func (ic *importContext) findUserByName(name string) (u scim.User, err error) {
 	a := scim.NewUsersAPI(ic.Context, ic.Client)
-	users, err := a.Filter(fmt.Sprintf("userName eq '%s'", name))
+	users, err := a.Filter(fmt.Sprintf("userName eq '%s'", name), false)
 	if err != nil {
 		return
 	}
@@ -226,7 +245,7 @@ func (ic *importContext) findUserByName(name string) (u scim.User, err error) {
 
 func (ic *importContext) findSpnByAppID(applicationID string) (u scim.User, err error) {
 	a := scim.NewServicePrincipalsAPI(ic.Context, ic.Client)
-	users, err := a.Filter(fmt.Sprintf("applicationId eq '%s'", strings.ReplaceAll(applicationID, "'", "")))
+	users, err := a.Filter(fmt.Sprintf("applicationId eq '%s'", strings.ReplaceAll(applicationID, "'", "")), false)
 	if err != nil {
 		return
 	}

scim/data_service_principal.go

@@ -23,7 +23,7 @@ func DataSourceServicePrincipal() *schema.Resource {
 	return common.DataResource(spnData{}, func(ctx context.Context, e any, c *common.DatabricksClient) error {
 		response := e.(*spnData)
 		spnAPI := NewServicePrincipalsAPI(ctx, c)
-		spList, err := spnAPI.Filter(fmt.Sprintf("applicationId eq '%s'", response.ApplicationID))
+		spList, err := spnAPI.Filter(fmt.Sprintf("applicationId eq '%s'", response.ApplicationID), true)
 		if err != nil {
 			return err
 		}

scim/data_service_principals.go

@@ -24,7 +24,7 @@ func DataSourceServicePrincipals() *schema.Resource {
 		if response.DisplayNameContains != "" {
 			filter = fmt.Sprintf("displayName co '%s'", response.DisplayNameContains)
 		}
-		spList, err := spnAPI.Filter(filter)
+		spList, err := spnAPI.Filter(filter, true)
 		if err != nil {
 			return err
 		}

scim/data_user.go

@@ -13,7 +13,7 @@ func getUser(usersAPI UsersAPI, id, name string) (user User, err error) {
 	if id != "" {
 		return usersAPI.Read(id, "userName,displayName,externalId,applicationId")
 	}
-	userList, err := usersAPI.Filter(fmt.Sprintf("userName eq '%s'", name))
+	userList, err := usersAPI.Filter(fmt.Sprintf("userName eq '%s'", name), true)
 	if err != nil {
 		return
 	}

scim/resource_service_principal.go

@@ -39,14 +39,16 @@ func (a ServicePrincipalsAPI) Read(servicePrincipalID string) (sp User, err erro
 	return
 }
 
-func (a ServicePrincipalsAPI) Filter(filter string) (u []User, err error) {
+func (a ServicePrincipalsAPI) Filter(filter string, excludeRoles bool) (u []User, err error) {
 	var sps UserList
 	req := map[string]string{}
 	if filter != "" {
 		req["filter"] = filter
 	}
 	// We exclude roles to reduce load on the scim service
-	req["excludedAttributes"] = "roles"
+	if excludeRoles {
+		req["excludedAttributes"] = "roles"
+	}
 	err = a.client.Scim(a.context, http.MethodGet, "/preview/scim/v2/ServicePrincipals", req, &sps)
 	if err != nil {
 		return
@@ -211,7 +213,7 @@ func createForceOverridesManuallyAddedServicePrincipal(err error, d *schema.Reso
 	if !slices.Contains(knownErrs, err.Error()) {
 		return err
 	}
-	spList, err := spAPI.Filter(fmt.Sprintf("applicationId eq '%s'", strings.ReplaceAll(u.ApplicationID, "'", "")))
+	spList, err := spAPI.Filter(fmt.Sprintf("applicationId eq '%s'", strings.ReplaceAll(u.ApplicationID, "'", "")), true)
 	if err != nil {
 		return err
 	}

scim/resource_user.go

@@ -137,7 +137,7 @@ func createForceOverridesManuallyAddedUser(err error, d *schema.ResourceData, us
 	if (err.Error() != userExistsErrorMessage(userName, false)) && (err.Error() != userExistsErrorMessage(userName, true)) {
 		return err
 	}
-	userList, err := usersAPI.Filter(fmt.Sprintf("userName eq '%s'", userName))
+	userList, err := usersAPI.Filter(fmt.Sprintf("userName eq '%s'", userName), true)
 	if err != nil {
 		return err
 	}