Add support for Databricks Repos

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Version changelog
 
+## 0.3.8
+
+* Added `databricks_repo` resource to manage [Databricks Repos](https://docs.databricks.com/repos.html) ([#771](https://github.com/databrickslabs/terraform-provider-databricks/pull/771))
+
 ## 0.3.7
 
 * Added `databricks_obo_token` resource to create On-Behalf-Of tokens for a Service Principal in Databricks workspaces on AWS. It is very useful, when you want to provision resources within a workspace through narrowly-scoped service principal, that has no access to other workspaces within the same Databricks Account ([#736](https://github.com/databrickslabs/terraform-provider-databricks/pull/736))

README.md

@@ -43,6 +43,7 @@
 | [databricks_obo_token](docs/resources/obo_token.md)
 | [databricks_permissions](docs/resources/permissions.md)
 | [databricks_pipeline](docs/resources/pipeline.md)
+| [databricks_repo](docs/resources/repo.md)
 | [databricks_secret](docs/resources/secret.md)
 | [databricks_secret_acl](docs/resources/secret_acl.md)
 | [databricks_secret_scope](docs/resources/secret_scope.md)

access/resource_permissions.go

@@ -238,6 +238,7 @@ func permissionsResourceIDFields(ctx context.Context) []permissionsIDFieldMappin
 		{"notebook_path", "notebook", "notebooks", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, PATH},
 		{"directory_id", "directory", "directories", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, SIMPLE},
 		{"directory_path", "directory", "directories", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, PATH},
+		{"repo_id", "repo", "repos", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, SIMPLE},
 		{"authorization", "tokens", "authorization", []string{"CAN_USE"}, SIMPLE},
 		{"authorization", "passwords", "authorization", []string{"CAN_USE"}, SIMPLE},
 		{"sql_endpoint_id", "endpoints", "sql/endpoints", []string{"CAN_USE", "CAN_MANAGE"}, SIMPLE},

common/http.go

@@ -487,10 +487,10 @@ func (c *DatabricksClient) genericQuery(ctx context.Context, method, requestURL
 
 func makeRequestBody(method string, requestURL *string, data interface{}, marshalJSON bool) ([]byte, error) {
 	var requestBody []byte
+	if data == nil && (method == "DELETE" || method == "GET") {
+		return requestBody, nil
+	}
 	if method == "GET" {
-		if data == nil {
-			return requestBody, nil
-		}
 		inputVal := reflect.ValueOf(data)
 		inputType := reflect.TypeOf(data)
 		switch inputType.Kind() {

docs/data-sources/user.md

@@ -42,4 +42,5 @@ Data source exposes the following attributes:
 - `user_name` - Name of the [user](../resources/user.md), e.g. `mr.foo@example.com`.
 - `display_name` - Display name of the [user](../resources/user.md), e.g. `Mr Foo`.
 - `home` - Home folder of the [user](../resources/user.md), e.g. `/Users/mr.foo@example.com`.
+- `repos` - Personal Repos location of the [user](../resources/user.md), e.g. `/Repos/mr.foo@example.com`.
 - `alphanumeric` - Alphanumeric representation of user local name. e.g. `mr_foo`.

docs/index.md

@@ -19,6 +19,7 @@ Compute resources
 * Speedup job & cluster startup with [databricks_instance_pool](resources/instance_pool.md)
 * Customize clusters with [databricks_global_init_script](resources/global_init_script.md)
 * Manage few [databricks_notebook](resources/notebook.md), and even [list them](data-sources/notebook_paths.md)
+* Manage [databricks_repo](resources/repo.md)
 
 Storage
 * Manage JAR, Wheel & Egg libraries through [databricks_dbfs_file](resources/dbfs_file.md)

docs/resources/permissions.md

@@ -267,6 +267,43 @@ resource "databricks_permissions" "folder_usage" {
 }
 ```
 
+## Repos usage
+
+Valid [permission levels](https://docs.databricks.com/security/access-control/workspace-acl.html) for [databricks_repo](repo.md) are: `CAN_READ`, `CAN_RUN`, `CAN_EDIT`, and `CAN_MANAGE`.
+
+```hcl
+resource "databricks_group" "auto" {
+    display_name = "Automation"
+}
+
+resource "databricks_group" "eng" {
+    display_name = "Engineering"
+}
+
+resource "databricks_repo" "this" {
+  url = "https://github.com/user/demo.git"
+}
+
+resource "databricks_permissions" "repo_usage" {
+    repo_id = databricks_repo.this.id
+
+    access_control {
+        group_name = "users"
+        permission_level = "CAN_READ"
+    }
+
+    access_control {
+        group_name = databricks_group.auto.display_name
+        permission_level = "CAN_RUN"
+    }
+
+    access_control {
+        group_name = databricks_group.eng.display_name
+        permission_level = "CAN_EDIT"
+    }
+}
+```
+
 ## Passwords usage
 
 By default on AWS deployments, all admin users can sign in to Databricks using either SSO or their username and password, and all API users can authenticate to the Databricks REST APIs using their username and password. As an admin, you [can limit](https://docs.databricks.com/administration-guide/users-groups/single-sign-on/index.html#optional-configure-password-access-control) admin users’ and API users’ ability to authenticate with their username and password by configuring `CAN_USE` permissions using password access control.

docs/resources/repo.md

@@ -0,0 +1,52 @@
+---
+subcategory: "Workspace"
+---
+# databricks_repo Resource
+
+This resource allows you to manage [Databricks Repos](https://docs.databricks.com/repos.html).
+
+## Example Usage
+
+You can declare Terraform-managed Repo by specifying `url` attribute of Git repository. In addition to that you may need to specify `git_provider` attribute if Git provider doesn't belong to cloud Git providers (Github, GitLab, ...).  If `path` attribute isn't provided, then repo will be created in the user's repo directory (`/Repos/<username>/...`):
+
+
+```hcl
+resource "databricks_repo" "nutter_in_home" {
+  url = "https://github.com/user/demo.git"
+}
+
+```
+
+## Argument Reference
+
+-> **Note** Repo in Databricks workspace would only be changed, if Terraform stage did change. This means that any manual changes to managed repository won't be overwritten by Terraform, if there's no local changes to configuration. If Repo in Databricks workspace is modifying, application of configuration changes will fail.
+
+The following arguments are supported:
+
+* `url` -  (Required) The URL of the Git Repository to clone from. If value changes, repo is re-created
+* `git_provider` - (Optional, if it's possible to detect Git provider by host name) case insensitive name of the Git provider.  Following values are supported right now (maybe a subject for change, consult [Repos API documentation](https://docs.databricks.com/dev-tools/api/latest/repos.html)): `gitHub`, `gitHubEnterprise`, `bitbucketCloud`, `bitbucketServer`, `azureDevOpsServices`, `gitLab`, `gitLabEnterpriseEdition`
+* `path` - (Optional) path to put the checked out Repo. If not specified, then repo will be created in the user's repo directory (`/Repos/<username>/...`).  If value changes, repo is re-created
+* `branch` - (Optional) name of the branch for initial checkout. If not specified, the default branch of the repository will be used.  Conflicts with `tag`.  If `branch` is removed, and `tag` isn't specified, then the repository will stay at the previously checked out state.
+* `tag` - (Optional) name of the tag for initial checkout.  Conflicts with `branch`
+
+## Attribute Reference
+
+In addition to all arguments above, the following attributes are exported:
+
+* `id` -  Repo identifier
+* `commit_hash` - Hash of the HEAD commit at time of the last executed operation. It won't change if you manually perform pull operation via UI or API
+
+## Access Control
+
+* [databricks_permissions](permissions.md#Repos-usage) can control which groups or individual users can access repos.
+
+## Import
+
+The resource Repo can be imported using the Repo ID (obtained via UI or using API)
+
+```bash
+$ terraform import databricks_repo.this repo_id
+```
+
+
+

exporter/exporter_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/databrickslabs/terraform-provider-databricks/compute"
 	"github.com/databrickslabs/terraform-provider-databricks/identity"
 	"github.com/databrickslabs/terraform-provider-databricks/qa"
+	"github.com/databrickslabs/terraform-provider-databricks/workspace"
 	"github.com/hashicorp/hcl/v2/hclwrite"
 
 	"github.com/stretchr/testify/assert"
@@ -198,10 +199,18 @@ var meAdminFixture = qa.HTTPFixture{
 	},
 }
 
+var repoListFixture = qa.HTTPFixture{
+	Method:       "GET",
+	ReuseRequest: true,
+	Resource:     "/api/2.0/repos?",
+	Response:     workspace.ReposListResponse{},
+}
+
 func TestImportingUsersGroupsSecretScopes(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/preview/scim/v2/Groups?",
@@ -354,6 +363,7 @@ func TestImportingNoResourcesError(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/preview/scim/v2/Groups?",
@@ -404,6 +414,7 @@ func TestImportingClusters(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/preview/scim/v2/Groups?",
@@ -552,6 +563,7 @@ func TestImportingJobs_JobList(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/jobs/list",
@@ -768,6 +780,7 @@ func TestImportingSecrets(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/preview/scim/v2/Groups?",
@@ -845,6 +858,7 @@ func TestImportingGlobalInitScripts(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:       "GET",
 				Resource:     "/api/2.0/global-init-scripts",
@@ -924,3 +938,50 @@ func TestEitherString(t *testing.T) {
 	assert.Equal(t, "a", eitherString(nil, "a"))
 	assert.Equal(t, "", eitherString(nil, nil))
 }
+
+func TestImportingRepos(t *testing.T) {
+	resp := workspace.ReposInformation{
+		ID:           121232342,
+		Url:          "https://github.com/user/test.git",
+		Provider:     "gitHub",
+		Path:         "/Repos/user@domain/test",
+		HeadCommitID: "1124323423abc23424",
+		Branch:       "releases",
+	}
+
+	qa.HTTPFixturesApply(t,
+		[]qa.HTTPFixture{
+			meAdminFixture,
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/repos?",
+				Response: workspace.ReposListResponse{
+					Repos: []workspace.ReposInformation{
+						resp,
+					},
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/repos/121232342",
+				Response: resp,
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/permissions/repos/121232342",
+				Response: getJSONObject("test-data/get-repo-permissions.json"),
+			},
+		},
+		func(ctx context.Context, client *common.DatabricksClient) {
+			tmpDir := fmt.Sprintf("/tmp/tf-%s", qa.RandomName())
+			defer os.RemoveAll(tmpDir)
+
+			ic := newImportContext(client)
+			ic.Directory = tmpDir
+			ic.listing = "repos"
+			ic.services = "repos,access"
+
+			err := ic.Run()
+			assert.NoError(t, err)
+		})
+}

exporter/importables.go

@@ -817,7 +817,7 @@ var resourcesMap map[string]importable = map[string]importable{
 					Resource: "databricks_global_init_script",
 					ID:       gis.ScriptID,
 				})
-				log.Printf("[INFO] Scanned %d of %d clusters", offset+1, len(globalInitScripts))
+				log.Printf("[INFO] Scanned %d of %d global init scripts", offset+1, len(globalInitScripts))
 			}
 			return nil
 		},
@@ -856,4 +856,57 @@ var resourcesMap map[string]importable = map[string]importable{
 			return nil
 		},
 	},
+	"databricks_repo": {
+		Service: "repos",
+		Name: func(d *schema.ResourceData) string {
+			name := d.Get("path").(string)
+			if name == "" {
+				return d.Id()
+			} else {
+				name = strings.TrimPrefix(name, "/")
+			}
+			re := regexp.MustCompile(`[^0-9A-Za-z_]`)
+			return re.ReplaceAllString(name, "_")
+		},
+		List: func(ic *importContext) error {
+			repoList, err := workspace.NewReposAPI(ic.Context, ic.Client).ListAll()
+			if err != nil {
+				return err
+			}
+			for offset, repo := range repoList {
+				if repo.Url != "" {
+					ic.Emit(&resource{
+						Resource: "databricks_repo",
+						ID:       fmt.Sprintf("%d", repo.ID),
+					})
+				}
+				log.Printf("[INFO] Scanned %d of %d repos", offset+1, len(repoList))
+			}
+			return nil
+		},
+		Import: func(ic *importContext, r *resource) error {
+			if ic.meAdmin {
+				ic.Emit(&resource{
+					Resource: "databricks_permissions",
+					ID:       fmt.Sprintf("/repos/%s", r.ID),
+					Name:     "repo_" + ic.Importables["databricks_repo"].Name(r.Data),
+				})
+			}
+			return nil
+		},
+		Body: func(ic *importContext, body *hclwrite.Body, r *resource) error {
+			b := body.AppendNewBlock("resource", []string{r.Resource, r.Name}).Body()
+			b.SetAttributeValue("url", cty.StringVal(r.Data.Get("url").(string)))
+			b.SetAttributeValue("git_provider", cty.StringVal(r.Data.Get("git_provider").(string)))
+			t := r.Data.Get("branch").(string)
+			if t != "" {
+				b.SetAttributeValue("branch", cty.StringVal(t))
+			}
+			t = r.Data.Get("path").(string)
+			if t != "" {
+				b.SetAttributeValue("path", cty.StringVal(t))
+			}
+			return nil
+		},
+	},
 }

exporter/test-data/get-repo-permissions.json

@@ -0,0 +1,30 @@
+{
+  "access_control_list": [
+    {
+      "all_permissions": [
+        {
+          "inherited": true,
+          "inherited_from_object": [
+            "/directories/167168545015994"
+          ],
+          "permission_level": "CAN_MANAGE"
+        }
+      ],
+      "user_name": "test@test.com"
+    },
+    {
+      "all_permissions": [
+        {
+          "inherited": true,
+          "inherited_from_object": [
+            "/directories/"
+          ],
+          "permission_level": "CAN_MANAGE"
+        }
+      ],
+      "group_name": "admins"
+    }
+  ],
+  "object_id": "/repos/121232342",
+  "object_type": "repo"
+}

identity/data_current_user.go

@@ -24,6 +24,10 @@ func DataSourceCurrentUser() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+			"repos": {
+				Type:     schema.TypeString,
+				Computed: true,
+			},
 			"alphanumeric": {
 				Type:     schema.TypeString,
 				Computed: true,
@@ -37,6 +41,7 @@ func DataSourceCurrentUser() *schema.Resource {
 			}
 			d.Set("user_name", me.UserName)
 			d.Set("home", fmt.Sprintf("/Users/%s", me.UserName))
+			d.Set("repos", fmt.Sprintf("/Repos/%s", me.UserName))
 			splits := strings.Split(me.UserName, "@")
 			norm := nonAlphanumeric.ReplaceAllLiteralString(splits[0], "_")
 			norm = strings.ToLower(norm)

identity/data_current_user_test.go

@@ -29,5 +29,6 @@ func TestDataSourceCurrentUser(t *testing.T) {
 	assert.Equal(t, "123", d.Id())
 	assert.Equal(t, d.Get("user_name"), "mr.test@example.com")
 	assert.Equal(t, d.Get("home"), "/Users/mr.test@example.com")
+	assert.Equal(t, d.Get("repos"), "/Repos/mr.test@example.com")
 	assert.Equal(t, d.Get("alphanumeric"), "mr_test")
 }