diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index da70fc1d36..7dcafec94e 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -14,6 +14,7 @@ import ( "github.com/databrickslabs/terraform-provider-databricks/compute" "github.com/databrickslabs/terraform-provider-databricks/identity" "github.com/databrickslabs/terraform-provider-databricks/qa" + "github.com/databrickslabs/terraform-provider-databricks/workspace" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/stretchr/testify/assert" @@ -198,10 +199,18 @@ var meAdminFixture = qa.HTTPFixture{ }, } +var repoListFixture = qa.HTTPFixture{ + Method: "GET", + ReuseRequest: true, + Resource: "/api/2.0/repos?", + Response: workspace.ReposListResponse{}, +} + func TestImportingUsersGroupsSecretScopes(t *testing.T) { qa.HTTPFixturesApply(t, []qa.HTTPFixture{ meAdminFixture, + repoListFixture, { Method: "GET", Resource: "/api/2.0/preview/scim/v2/Groups?", @@ -354,6 +363,7 @@ func TestImportingNoResourcesError(t *testing.T) { qa.HTTPFixturesApply(t, []qa.HTTPFixture{ meAdminFixture, + repoListFixture, { Method: "GET", Resource: "/api/2.0/preview/scim/v2/Groups?", @@ -404,6 +414,7 @@ func TestImportingClusters(t *testing.T) { qa.HTTPFixturesApply(t, []qa.HTTPFixture{ meAdminFixture, + repoListFixture, { Method: "GET", Resource: "/api/2.0/preview/scim/v2/Groups?", @@ -552,6 +563,7 @@ func TestImportingJobs_JobList(t *testing.T) { qa.HTTPFixturesApply(t, []qa.HTTPFixture{ meAdminFixture, + repoListFixture, { Method: "GET", Resource: "/api/2.0/jobs/list", @@ -768,6 +780,7 @@ func TestImportingSecrets(t *testing.T) { qa.HTTPFixturesApply(t, []qa.HTTPFixture{ meAdminFixture, + repoListFixture, { Method: "GET", Resource: "/api/2.0/preview/scim/v2/Groups?", @@ -845,6 +858,7 @@ func TestImportingGlobalInitScripts(t *testing.T) { qa.HTTPFixturesApply(t, []qa.HTTPFixture{ meAdminFixture, + repoListFixture, { Method: "GET", Resource: "/api/2.0/global-init-scripts", @@ -924,3 +938,50 @@ func TestEitherString(t *testing.T) { assert.Equal(t, "a", eitherString(nil, "a")) assert.Equal(t, "", eitherString(nil, nil)) } + +func TestImportingRepos(t *testing.T) { + resp := workspace.ReposResponse{ + Id: 121232342, + Url: "https://github.com/user/test.git", + Provider: "gitHub", + Path: "/Repos/user@domain/test", + HeadCommitId: "1124323423abc23424", + Branch: "releases", + } + + qa.HTTPFixturesApply(t, + []qa.HTTPFixture{ + meAdminFixture, + { + Method: "GET", + Resource: "/api/2.0/repos?", + Response: workspace.ReposListResponse{ + Repos: []workspace.ReposResponse{ + resp, + }, + }, + }, + { + Method: "GET", + Resource: "/api/2.0/repos/121232342", + Response: resp, + }, + { + Method: "GET", + Resource: "/api/2.0/permissions/repos/121232342", + Response: getJSONObject("test-data/get-repo-permissions.json"), + }, + }, + func(ctx context.Context, client *common.DatabricksClient) { + tmpDir := fmt.Sprintf("/tmp/tf-%s", qa.RandomName()) + defer os.RemoveAll(tmpDir) + + ic := newImportContext(client) + ic.Directory = tmpDir + ic.listing = "repos" + ic.services = "repos,access" + + err := ic.Run() + assert.NoError(t, err) + }) +} diff --git a/exporter/importables.go b/exporter/importables.go index 9a253132af..ec5d21c372 100644 --- a/exporter/importables.go +++ b/exporter/importables.go @@ -817,7 +817,7 @@ var resourcesMap map[string]importable = map[string]importable{ Resource: "databricks_global_init_script", ID: gis.ScriptID, }) - log.Printf("[INFO] Scanned %d of %d clusters", offset+1, len(globalInitScripts)) + log.Printf("[INFO] Scanned %d of %d global init scripts", offset+1, len(globalInitScripts)) } return nil }, @@ -856,4 +856,57 @@ var resourcesMap map[string]importable = map[string]importable{ return nil }, }, + "databricks_repo": { + Service: "repos", + Name: func(d *schema.ResourceData) string { + name := d.Get("path").(string) + if name == "" { + return d.Id() + } else { + name = strings.TrimPrefix(name, "/") + } + re := regexp.MustCompile(`[^0-9A-Za-z_]`) + return re.ReplaceAllString(name, "_") + }, + List: func(ic *importContext) error { + repoList, err := workspace.NewReposAPI(ic.Context, ic.Client).ListAll() + if err != nil { + return err + } + for offset, repo := range repoList { + if repo.Url != "" { + ic.Emit(&resource{ + Resource: "databricks_repo", + ID: fmt.Sprintf("%d", repo.Id), + }) + } + log.Printf("[INFO] Scanned %d of %d repos", offset+1, len(repoList)) + } + return nil + }, + Import: func(ic *importContext, r *resource) error { + if ic.meAdmin { + ic.Emit(&resource{ + Resource: "databricks_permissions", + ID: fmt.Sprintf("/repos/%s", r.ID), + Name: "repo_" + ic.Importables["databricks_repo"].Name(r.Data), + }) + } + return nil + }, + Body: func(ic *importContext, body *hclwrite.Body, r *resource) error { + b := body.AppendNewBlock("resource", []string{r.Resource, r.Name}).Body() + b.SetAttributeValue("url", cty.StringVal(r.Data.Get("url").(string))) + b.SetAttributeValue("git_provider", cty.StringVal(r.Data.Get("git_provider").(string))) + t := r.Data.Get("branch").(string) + if t != "" { + b.SetAttributeValue("branch", cty.StringVal(t)) + } + t = r.Data.Get("path").(string) + if t != "" { + b.SetAttributeValue("path", cty.StringVal(t)) + } + return nil + }, + }, } diff --git a/exporter/test-data/get-repo-permissions.json b/exporter/test-data/get-repo-permissions.json new file mode 100644 index 0000000000..7af32da387 --- /dev/null +++ b/exporter/test-data/get-repo-permissions.json @@ -0,0 +1,30 @@ +{ + "access_control_list": [ + { + "all_permissions": [ + { + "inherited": true, + "inherited_from_object": [ + "/directories/167168545015994" + ], + "permission_level": "CAN_MANAGE" + } + ], + "user_name": "test@test.com" + }, + { + "all_permissions": [ + { + "inherited": true, + "inherited_from_object": [ + "/directories/" + ], + "permission_level": "CAN_MANAGE" + } + ], + "group_name": "admins" + } + ], + "object_id": "/repos/121232342", + "object_type": "repo" +} diff --git a/workspace/resource_repo.go b/workspace/resource_repo.go index 84abd2f741..b849633d7c 100644 --- a/workspace/resource_repo.go +++ b/workspace/resource_repo.go @@ -57,6 +57,36 @@ func (a ReposAPI) Read(id string) (ReposResponse, error) { return resp, err } +type ReposListResponse struct { + NextPageToken string `json:"next_page_token,omitempty"` + Repos []ReposResponse `json:"repos"` +} + +func (a ReposAPI) List(prefix string) ([]ReposResponse, error) { + req := map[string]string{} + if prefix != "" { + req["path_prefix"] = prefix + } + reposList := []ReposResponse{} + for { + var resp ReposListResponse + err := a.client.Get(a.context, "/repos", req, &resp) + if err != nil { + return nil, err + } + reposList = append(reposList, resp.Repos...) + if resp.NextPageToken == "" { + break + } + req["next_page_token"] = resp.NextPageToken + } + return reposList, nil +} + +func (a ReposAPI) ListAll() ([]ReposResponse, error) { + return a.List("") +} + var gitProvidersMap = map[string]string{ "github.com": "gitHub", "dev.azure.com": "azureDevOpsServices", diff --git a/workspace/resource_repo_test.go b/workspace/resource_repo_test.go index a4441f036f..02b45e6383 100644 --- a/workspace/resource_repo_test.go +++ b/workspace/resource_repo_test.go @@ -1,11 +1,13 @@ package workspace import ( + "context" "fmt" "net/http" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/databrickslabs/terraform-provider-databricks/common" @@ -329,3 +331,75 @@ func TestResourceReposUpdateSwitchToBranch(t *testing.T) { assert.NoError(t, err, err) assert.Equal(t, "releases", d.Get("branch")) } + +func TestReposListAll(t *testing.T) { + resp := ReposResponse{ + Id: 121232342, + Url: "https://github.com/user/test.git", + Provider: "gitHub", + Path: "/Repos/user@domain/test", + HeadCommitId: "1124323423abc23424", + Branch: "releases", + } + + client, server, err := qa.HttpFixtureClient(t, []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.0/repos?", + Response: ReposListResponse{ + NextPageToken: "12312423442343242343", + Repos: []ReposResponse{ + resp, + }, + }, + }, + { + Method: "GET", + Resource: "/api/2.0/repos?next_page_token=12312423442343242343", + Response: ReposListResponse{ + Repos: []ReposResponse{ + resp, + }, + }, + }, + }) + defer server.Close() + require.NoError(t, err) + + ctx := context.Background() + reposList, err := NewReposAPI(ctx, client).ListAll() + require.NoError(t, err) + assert.Equal(t, len(reposList), 2) + assert.Equal(t, resp.Branch, reposList[1].Branch) +} + +func TestReposListWithPrefix(t *testing.T) { + resp := ReposResponse{ + Id: 121232342, + Url: "https://github.com/user/test.git", + Provider: "gitHub", + Path: "/Repos/user@domain/test", + HeadCommitId: "1124323423abc23424", + Branch: "releases", + } + + client, server, err := qa.HttpFixtureClient(t, []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.0/repos?path_prefix=%2FRepos%2Fabc", + Response: ReposListResponse{ + Repos: []ReposResponse{ + resp, + }, + }, + }, + }) + defer server.Close() + require.NoError(t, err) + + ctx := context.Background() + reposList, err := NewReposAPI(ctx, client).List("/Repos/abc") + require.NoError(t, err) + assert.Equal(t, len(reposList), 1) + assert.Equal(t, resp.Branch, reposList[0].Branch) +}