Merge branch 'master' into mb-fix-gs

.github/workflows/push.yml

@@ -3,6 +3,7 @@ name: build
 on:
   pull_request:
     types: [opened, synchronize]
+    paths-ignore: ['**.md']
   push:
     branches: [master]
 

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Version changelog
 
+## 0.5.9
+
+* Added warning section for debug mode ([#1325](https://github.com/databrickslabs/terraform-provider-databricks/pull/1325)).
+* Added ability to specify tags for `databricks_job` ([#1337](https://github.com/databrickslabs/terraform-provider-databricks/pull/1337)).
+* Upgraded AWS provider for AWS guides. Added examples for account-level identities ([#1332](https://github.com/databrickslabs/terraform-provider-databricks/pull/1332)).
+* Updated docs to use `application_id` as privilege for `databricks_service_principal` ([#1336](https://github.com/databrickslabs/terraform-provider-databricks/pull/1336)).
+* Added `databricks_service_principal_role` resource ([#1340](https://github.com/databrickslabs/terraform-provider-databricks/pull/1340)).
+* Fixed itegration testing image ([#1342](https://github.com/databrickslabs/terraform-provider-databricks/pull/1342), [#1343](https://github.com/databrickslabs/terraform-provider-databricks/pull/1343)).
+* Added `skip_validation` for `databricks_external_location` ([#1330](https://github.com/databrickslabs/terraform-provider-databricks/pull/1330)).
+* Added `alert_on_last_attempt` to `databricks_job` ([#1341](https://github.com/databrickslabs/terraform-provider-databricks/pull/1341)).
+* Skip `make test` on doc-only changes ([#1339](https://github.com/databrickslabs/terraform-provider-databricks/pull/1339)).
+* Improve common package test coverage ([#1344](https://github.com/databrickslabs/terraform-provider-databricks/pull/1344)).
+* Re-create purged cluster for `databricks_mount` for AWS S3 ([#1345](https://github.com/databrickslabs/terraform-provider-databricks/pull/1345)).
+
+Updated dependency versions:
+
+* Bump google.golang.org/api from 0.79.0 to 0.80.0
+* Bump github.com/Azure/go-autorest/autorest/adal from 0.9.19 to 0.9.20
+
 ## 0.5.8
 
 * Update `aws_iam_policy_document` in `databricks_mws_customer_managed_keys` docs to restrict KMS policy to caller AWS account ([#1309](https://github.com/databrickslabs/terraform-provider-databricks/pull/1309)).

README.md

@@ -82,7 +82,7 @@ terraform {
   required_providers {
     databricks = {
       source  = "databrickslabs/databricks"
-      version = "0.5.8"
+      version = "0.5.9"
     }
   }
 }

aws/resource_service_principal_role.go

@@ -0,0 +1,33 @@
+package aws
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databrickslabs/terraform-provider-databricks/common"
+	"github.com/databrickslabs/terraform-provider-databricks/scim"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+// ResourceServicePrincipalRole binds service principal and instance profile
+func ResourceServicePrincipalRole() *schema.Resource {
+	r := common.NewPairID("service_principal_id", "role").BindResource(common.BindResource{
+		CreateContext: func(ctx context.Context, servicePrincipalID, role string, c *common.DatabricksClient) error {
+			return scim.NewServicePrincipalsAPI(ctx, c).Patch(servicePrincipalID, scim.PatchRequest("add", "roles", role))
+		},
+		ReadContext: func(ctx context.Context, servicePrincipalID, roleARN string, c *common.DatabricksClient) error {
+			servicePrincipal, err := scim.NewServicePrincipalsAPI(ctx, c).Read(servicePrincipalID)
+			hasRole := scim.ComplexValues(servicePrincipal.Roles).HasValue(roleARN)
+			if err == nil && !hasRole {
+				return common.NotFound("Service Principal has no role")
+			}
+			return err
+		},
+		DeleteContext: func(ctx context.Context, servicePrincipalID, roleARN string, c *common.DatabricksClient) error {
+			return scim.NewServicePrincipalsAPI(ctx, c).Patch(servicePrincipalID, scim.PatchRequest(
+				"remove", fmt.Sprintf(`roles[value eq "%s"]`, roleARN), ""))
+		},
+	})
+	return r
+}

aws/resource_service_principal_role_test.go

@@ -0,0 +1,135 @@
+package aws
+
+import (
+	"testing"
+
+	"github.com/databrickslabs/terraform-provider-databricks/common"
+
+	"github.com/databrickslabs/terraform-provider-databricks/scim"
+
+	"github.com/databrickslabs/terraform-provider-databricks/qa"
+)
+
+func TestResourceServicePrincipalRoleCreate(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "PATCH",
+				Resource: "/api/2.0/preview/scim/v2/ServicePrincipals/abc",
+				ExpectedRequest: scim.PatchRequest(
+					"add",
+					"roles",
+					"arn:aws:iam::999999999999:instance-profile/my-fake-instance-profile"),
+				Response: scim.User{
+					ID: "abc",
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/scim/v2/ServicePrincipals/abc",
+				Response: scim.User{
+					Schemas:     []scim.URN{scim.ServicePrincipalSchema},
+					DisplayName: "ABC SP",
+					Roles: []scim.ComplexValue{
+						{
+							Value: "arn:aws:iam::999999999999:instance-profile/my-fake-instance-profile",
+						},
+					},
+					ID: "abc",
+				},
+			},
+		},
+		Resource: ResourceServicePrincipalRole(),
+		State: map[string]interface{}{
+			"service_principal_id": "abc",
+			"role":                 "arn:aws:iam::999999999999:instance-profile/my-fake-instance-profile",
+		},
+		Create: true,
+	}.ApplyAndExpectData(t, map[string]interface{}{"id": "abc|arn:aws:iam::999999999999:instance-profile/my-fake-instance-profile"})
+}
+
+func TestResourceServicePrincipalRoleCreate_Error(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "PATCH",
+				Resource: "/api/2.0/preview/scim/v2/ServicePrincipals/abc",
+				Response: common.APIErrorBody{
+					ErrorCode: "INVALID_REQUEST",
+					Message:   "Internal error happened",
+				},
+				Status: 400,
+			},
+		},
+		Resource: ResourceServicePrincipalRole(),
+		State: map[string]interface{}{
+			"service_principal_id": "abc",
+			"role":                 "arn:aws:iam::999999999999:instance-profile/my-fake-instance-profile",
+		},
+		Create: true,
+	}.ExpectError(t, "Internal error happened")
+}
+
+func TestResourceServicePrincipalRoleRead(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/scim/v2/ServicePrincipals/abc",
+				Response: scim.User{
+					Schemas:     []scim.URN{scim.ServicePrincipalSchema},
+					DisplayName: "ABC SP",
+					Roles: []scim.ComplexValue{
+						{
+							Value: "arn:aws:iam::999999999999:instance-profile/my-fake-instance-profile",
+						},
+					},
+					ID: "abc",
+				},
+			},
+		},
+		Resource: ResourceServicePrincipalRole(),
+		Read:     true,
+		ID:       "abc|arn:aws:iam::999999999999:instance-profile/my-fake-instance-profile",
+	}.ApplyAndExpectData(t, map[string]interface{}{"id": "abc|arn:aws:iam::999999999999:instance-profile/my-fake-instance-profile"})
+}
+
+func TestResourceServicePrincipalRoleRead_NoRole(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/scim/v2/ServicePrincipals/abc",
+				Response: scim.User{
+					Schemas:     []scim.URN{scim.ServicePrincipalSchema},
+					DisplayName: "ABC SP",
+					ID:          "abc",
+				},
+			},
+		},
+		Resource: ResourceServicePrincipalRole(),
+		Read:     true,
+		Removed:  true,
+		ID:       "abc|arn:aws:iam::999999999999:instance-profile/my-fake-instance-profile",
+	}.ApplyNoError(t)
+}
+
+func TestResourceServicePrincipalRoleRead_NotFound(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/preview/scim/v2/ServicePrincipals/abc",
+				Response: common.APIErrorBody{
+					ErrorCode: "NOT_FOUND",
+					Message:   "Item not found",
+				},
+				Status: 404,
+			},
+		},
+		Resource: ResourceServicePrincipalRole(),
+		Read:     true,
+		Removed:  true,
+		ID:       "abc|arn:aws:iam::999999999999:instance-profile/my-fake-instance-profile",
+	}.ApplyNoError(t)
+}

catalog/resource_external_location.go

@@ -22,6 +22,7 @@ type ExternalLocationInfo struct {
 	URL            string `json:"url"`
 	CredentialName string `json:"credential_name"`
 	Comment        string `json:"comment,omitempty"`
+	SkipValidation bool   `json:"skip_validation,omitempty"`
 	Owner          string `json:"owner,omitempty" tf:"computed"`
 	MetastoreID    string `json:"metastore_id,omitempty" tf:"computed"`
 }
@@ -74,6 +75,7 @@ func ResourceExternalLocation() *schema.Resource {
 				Name:           d.Id(),
 				URL:            el.URL,
 				CredentialName: el.CredentialName,
+				SkipValidation: el.SkipValidation,
 				Comment:        el.Comment,
 				Owner:          el.Owner,
 			})

commands/commands.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/databrickslabs/terraform-provider-databricks/clusters"
 	"github.com/databrickslabs/terraform-provider-databricks/common"
-	"github.com/databrickslabs/terraform-provider-databricks/internal"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 )
@@ -52,7 +51,7 @@ func (a CommandsAPI) Execute(clusterID, language, commandStr string) common.Comm
 			Summary:    fmt.Sprintf("Cluster %s has to be running or resizing, but is %s", clusterID, cluster.State),
 		}
 	}
-	commandStr = internal.TrimLeadingWhitespace(commandStr)
+	commandStr = TrimLeadingWhitespace(commandStr)
 	log.Printf("[INFO] Executing %s command on %s:\n%s", language, clusterID, commandStr)
 	context, err := a.createContext(language, clusterID)
 	if err != nil {

internal/utils.go → commands/leading_whitespace.go

@@ -1,10 +1,11 @@
-package internal
+package commands
 
 import (
 	"strings"
 )
 
-// TrimLeadingWhitespace removes leading whitespace
+// TrimLeadingWhitespace removes leading whitespace, so that Python code blocks
+// that are embedded into Go code still could be interpreted properly.
 func TrimLeadingWhitespace(commandStr string) (newCommand string) {
 	lines := strings.Split(strings.ReplaceAll(commandStr, "\t", "    "), "\n")
 	leadingWhitespace := 1<<31 - 1

internal/utils_test.go → commands/leading_whitespace_test.go

@@ -1,4 +1,4 @@
-package internal
+package commands
 
 import (
 	"testing"

common/client.go

@@ -320,6 +320,8 @@ func (c *DatabricksClient) niceAuthError(message string) error {
 		}
 		info = ". " + strings.Join(infos, ". ")
 	}
+	info = strings.TrimSuffix(info, ".")
+	message = strings.TrimSuffix(message, ".")
 	docUrl := "https://registry.terraform.io/providers/databrickslabs/databricks/latest/docs#authentication"
 	return fmt.Errorf("%s%s. Please check %s for details", message, info, docUrl)
 }