Add exporting of repos

exporter/exporter_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/databrickslabs/terraform-provider-databricks/compute"
 	"github.com/databrickslabs/terraform-provider-databricks/identity"
 	"github.com/databrickslabs/terraform-provider-databricks/qa"
+	"github.com/databrickslabs/terraform-provider-databricks/workspace"
 	"github.com/hashicorp/hcl/v2/hclwrite"
 
 	"github.com/stretchr/testify/assert"
@@ -198,10 +199,18 @@ var meAdminFixture = qa.HTTPFixture{
 	},
 }
 
+var repoListFixture = qa.HTTPFixture{
+	Method:       "GET",
+	ReuseRequest: true,
+	Resource:     "/api/2.0/repos?",
+	Response:     workspace.ReposListResponse{},
+}
+
 func TestImportingUsersGroupsSecretScopes(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/preview/scim/v2/Groups?",
@@ -354,6 +363,7 @@ func TestImportingNoResourcesError(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/preview/scim/v2/Groups?",
@@ -404,6 +414,7 @@ func TestImportingClusters(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/preview/scim/v2/Groups?",
@@ -552,6 +563,7 @@ func TestImportingJobs_JobList(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/jobs/list",
@@ -768,6 +780,7 @@ func TestImportingSecrets(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:   "GET",
 				Resource: "/api/2.0/preview/scim/v2/Groups?",
@@ -845,6 +858,7 @@ func TestImportingGlobalInitScripts(t *testing.T) {
 	qa.HTTPFixturesApply(t,
 		[]qa.HTTPFixture{
 			meAdminFixture,
+			repoListFixture,
 			{
 				Method:       "GET",
 				Resource:     "/api/2.0/global-init-scripts",
@@ -924,3 +938,50 @@ func TestEitherString(t *testing.T) {
 	assert.Equal(t, "a", eitherString(nil, "a"))
 	assert.Equal(t, "", eitherString(nil, nil))
 }
+
+func TestImportingRepos(t *testing.T) {
+	resp := workspace.ReposResponse{
+		Id:           121232342,
+		Url:          "https://github.com/user/test.git",
+		Provider:     "gitHub",
+		Path:         "/Repos/user@domain/test",
+		HeadCommitId: "1124323423abc23424",
+		Branch:       "releases",
+	}
+
+	qa.HTTPFixturesApply(t,
+		[]qa.HTTPFixture{
+			meAdminFixture,
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/repos?",
+				Response: workspace.ReposListResponse{
+					Repos: []workspace.ReposResponse{
+						resp,
+					},
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/repos/121232342",
+				Response: resp,
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/permissions/repos/121232342",
+				Response: getJSONObject("test-data/get-repo-permissions.json"),
+			},
+		},
+		func(ctx context.Context, client *common.DatabricksClient) {
+			tmpDir := fmt.Sprintf("/tmp/tf-%s", qa.RandomName())
+			defer os.RemoveAll(tmpDir)
+
+			ic := newImportContext(client)
+			ic.Directory = tmpDir
+			ic.listing = "repos"
+			ic.services = "repos,access"
+
+			err := ic.Run()
+			assert.NoError(t, err)
+		})
+}

exporter/importables.go

@@ -817,7 +817,7 @@ var resourcesMap map[string]importable = map[string]importable{
 					Resource: "databricks_global_init_script",
 					ID:       gis.ScriptID,
 				})
-				log.Printf("[INFO] Scanned %d of %d clusters", offset+1, len(globalInitScripts))
+				log.Printf("[INFO] Scanned %d of %d global init scripts", offset+1, len(globalInitScripts))
 			}
 			return nil
 		},
@@ -856,4 +856,57 @@ var resourcesMap map[string]importable = map[string]importable{
 			return nil
 		},
 	},
+	"databricks_repo": {
+		Service: "repos",
+		Name: func(d *schema.ResourceData) string {
+			name := d.Get("path").(string)
+			if name == "" {
+				return d.Id()
+			} else {
+				name = strings.TrimPrefix(name, "/")
+			}
+			re := regexp.MustCompile(`[^0-9A-Za-z_]`)
+			return re.ReplaceAllString(name, "_")
+		},
+		List: func(ic *importContext) error {
+			repoList, err := workspace.NewReposAPI(ic.Context, ic.Client).ListAll()
+			if err != nil {
+				return err
+			}
+			for offset, repo := range repoList {
+				if repo.Url != "" {
+					ic.Emit(&resource{
+						Resource: "databricks_repo",
+						ID:       fmt.Sprintf("%d", repo.Id),
+					})
+				}
+				log.Printf("[INFO] Scanned %d of %d repos", offset+1, len(repoList))
+			}
+			return nil
+		},
+		Import: func(ic *importContext, r *resource) error {
+			if ic.meAdmin {
+				ic.Emit(&resource{
+					Resource: "databricks_permissions",
+					ID:       fmt.Sprintf("/repos/%s", r.ID),
+					Name:     "repo_" + ic.Importables["databricks_repo"].Name(r.Data),
+				})
+			}
+			return nil
+		},
+		Body: func(ic *importContext, body *hclwrite.Body, r *resource) error {
+			b := body.AppendNewBlock("resource", []string{r.Resource, r.Name}).Body()
+			b.SetAttributeValue("url", cty.StringVal(r.Data.Get("url").(string)))
+			b.SetAttributeValue("git_provider", cty.StringVal(r.Data.Get("git_provider").(string)))
+			t := r.Data.Get("branch").(string)
+			if t != "" {
+				b.SetAttributeValue("branch", cty.StringVal(t))
+			}
+			t = r.Data.Get("path").(string)
+			if t != "" {
+				b.SetAttributeValue("path", cty.StringVal(t))
+			}
+			return nil
+		},
+	},
 }

exporter/test-data/get-repo-permissions.json

@@ -0,0 +1,30 @@
+{
+  "access_control_list": [
+    {
+      "all_permissions": [
+        {
+          "inherited": true,
+          "inherited_from_object": [
+            "/directories/167168545015994"
+          ],
+          "permission_level": "CAN_MANAGE"
+        }
+      ],
+      "user_name": "test@test.com"
+    },
+    {
+      "all_permissions": [
+        {
+          "inherited": true,
+          "inherited_from_object": [
+            "/directories/"
+          ],
+          "permission_level": "CAN_MANAGE"
+        }
+      ],
+      "group_name": "admins"
+    }
+  ],
+  "object_id": "/repos/121232342",
+  "object_type": "repo"
+}

workspace/resource_repo.go

@@ -57,6 +57,36 @@ func (a ReposAPI) Read(id string) (ReposResponse, error) {
 	return resp, err
 }
 
+type ReposListResponse struct {
+	NextPageToken string          `json:"next_page_token,omitempty"`
+	Repos         []ReposResponse `json:"repos"`
+}
+
+func (a ReposAPI) List(prefix string) ([]ReposResponse, error) {
+	req := map[string]string{}
+	if prefix != "" {
+		req["path_prefix"] = prefix
+	}
+	reposList := []ReposResponse{}
+	for {
+		var resp ReposListResponse
+		err := a.client.Get(a.context, "/repos", req, &resp)
+		if err != nil {
+			return nil, err
+		}
+		reposList = append(reposList, resp.Repos...)
+		if resp.NextPageToken == "" {
+			break
+		}
+		req["next_page_token"] = resp.NextPageToken
+	}
+	return reposList, nil
+}
+
+func (a ReposAPI) ListAll() ([]ReposResponse, error) {
+	return a.List("")
+}
+
 var gitProvidersMap = map[string]string{
 	"github.com":    "gitHub",
 	"dev.azure.com": "azureDevOpsServices",

workspace/resource_repo_test.go

@@ -1,11 +1,13 @@
 package workspace
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/databrickslabs/terraform-provider-databricks/common"
 
@@ -329,3 +331,75 @@ func TestResourceReposUpdateSwitchToBranch(t *testing.T) {
 	assert.NoError(t, err, err)
 	assert.Equal(t, "releases", d.Get("branch"))
 }
+
+func TestReposListAll(t *testing.T) {
+	resp := ReposResponse{
+		Id:           121232342,
+		Url:          "https://github.com/user/test.git",
+		Provider:     "gitHub",
+		Path:         "/Repos/user@domain/test",
+		HeadCommitId: "1124323423abc23424",
+		Branch:       "releases",
+	}
+
+	client, server, err := qa.HttpFixtureClient(t, []qa.HTTPFixture{
+		{
+			Method:   "GET",
+			Resource: "/api/2.0/repos?",
+			Response: ReposListResponse{
+				NextPageToken: "12312423442343242343",
+				Repos: []ReposResponse{
+					resp,
+				},
+			},
+		},
+		{
+			Method:   "GET",
+			Resource: "/api/2.0/repos?next_page_token=12312423442343242343",
+			Response: ReposListResponse{
+				Repos: []ReposResponse{
+					resp,
+				},
+			},
+		},
+	})
+	defer server.Close()
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	reposList, err := NewReposAPI(ctx, client).ListAll()
+	require.NoError(t, err)
+	assert.Equal(t, len(reposList), 2)
+	assert.Equal(t, resp.Branch, reposList[1].Branch)
+}
+
+func TestReposListWithPrefix(t *testing.T) {
+	resp := ReposResponse{
+		Id:           121232342,
+		Url:          "https://github.com/user/test.git",
+		Provider:     "gitHub",
+		Path:         "/Repos/user@domain/test",
+		HeadCommitId: "1124323423abc23424",
+		Branch:       "releases",
+	}
+
+	client, server, err := qa.HttpFixtureClient(t, []qa.HTTPFixture{
+		{
+			Method:   "GET",
+			Resource: "/api/2.0/repos?path_prefix=%2FRepos%2Fabc",
+			Response: ReposListResponse{
+				Repos: []ReposResponse{
+					resp,
+				},
+			},
+		},
+	})
+	defer server.Close()
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	reposList, err := NewReposAPI(ctx, client).List("/Repos/abc")
+	require.NoError(t, err)
+	assert.Equal(t, len(reposList), 1)
+	assert.Equal(t, resp.Branch, reposList[0].Branch)
+}