Added initial support for Multi Task Jobs

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## 0.3.9
 
+* Added initial support for multiple task orchestration in `databricks_job` [#853](https://github.com/databrickslabs/terraform-provider-databricks/pull/853)
 * Fixed provider crash for new terraform states related to bug [#813](https://github.com/hashicorp/terraform-plugin-sdk/issues/813) in Terraform SDK v2.8.0 ([#854](https://github.com/databrickslabs/terraform-provider-databricks/issues/854))
 
 Updated dependency versions:

access/resource_sql_permissions_test.go

@@ -431,5 +431,5 @@ func TestResourceSqlPermissions_Delete(t *testing.T) {
 }
 
 func TestResourceSqlPermissions_CornerCases(t *testing.T) {
-	qa.ResourceCornerCases(t, ResourceSqlPermissions(), "database/foo")
+	qa.ResourceCornerCases(t, ResourceSqlPermissions(), qa.CornerCaseID("database/foo"))
 }

common/http.go

@@ -244,7 +244,7 @@ func (c *DatabricksClient) checkHTTPRetry(ctx context.Context, resp *http.Respon
 
 // Get on path
 func (c *DatabricksClient) Get(ctx context.Context, path string, request interface{}, response interface{}) error {
-	body, err := c.authenticatedQuery(ctx, http.MethodGet, path, request, c.api2)
+	body, err := c.authenticatedQuery(ctx, http.MethodGet, path, request, c.completeUrl)
 	if err != nil {
 		return err
 	}
@@ -253,7 +253,7 @@ func (c *DatabricksClient) Get(ctx context.Context, path string, request interfa
 
 // Post on path
 func (c *DatabricksClient) Post(ctx context.Context, path string, request interface{}, response interface{}) error {
-	body, err := c.authenticatedQuery(ctx, http.MethodPost, path, request, c.api2)
+	body, err := c.authenticatedQuery(ctx, http.MethodPost, path, request, c.completeUrl)
 	if err != nil {
 		return err
 	}
@@ -262,19 +262,19 @@ func (c *DatabricksClient) Post(ctx context.Context, path string, request interf
 
 // Delete on path
 func (c *DatabricksClient) Delete(ctx context.Context, path string, request interface{}) error {
-	_, err := c.authenticatedQuery(ctx, http.MethodDelete, path, request, c.api2)
+	_, err := c.authenticatedQuery(ctx, http.MethodDelete, path, request, c.completeUrl)
 	return err
 }
 
 // Patch on path
 func (c *DatabricksClient) Patch(ctx context.Context, path string, request interface{}) error {
-	_, err := c.authenticatedQuery(ctx, http.MethodPatch, path, request, c.api2)
+	_, err := c.authenticatedQuery(ctx, http.MethodPatch, path, request, c.completeUrl)
 	return err
 }
 
 // Put on path
 func (c *DatabricksClient) Put(ctx context.Context, path string, request interface{}) error {
-	_, err := c.authenticatedQuery(ctx, http.MethodPut, path, request, c.api2)
+	_, err := c.authenticatedQuery(ctx, http.MethodPut, path, request, c.completeUrl)
 	return err
 }
 
@@ -298,28 +298,26 @@ func (c *DatabricksClient) unmarshall(path string, body []byte, response interfa
 	}
 }
 
-func (c *DatabricksClient) api2(r *http.Request) error {
+type ApiVersion string
+
+const (
+	API_1_2 ApiVersion = "1.2"
+	API_2_0 ApiVersion = "2.0"
+	API_2_1 ApiVersion = "2.1"
+)
+
+func (c *DatabricksClient) completeUrl(r *http.Request) error {
 	if r.URL == nil {
 		return fmt.Errorf("no URL found in request")
 	}
-	r.URL.Path = fmt.Sprintf("/api/2.0%s", r.URL.Path)
-	r.Header.Set("Content-Type", "application/json")
 
-	url, err := url.Parse(c.Host)
-	if err != nil {
-		return err
+	ctx := r.Context()
+	av, ok := ctx.Value(Api).(ApiVersion)
+	if !ok {
+		av = API_2_0
 	}
-	r.URL.Host = url.Host
-	r.URL.Scheme = url.Scheme
 
-	return nil
-}
-
-func (c *DatabricksClient) api12(r *http.Request) error {
-	if r.URL == nil {
-		return fmt.Errorf("no URL found in request")
-	}
-	r.URL.Path = fmt.Sprintf("/api/1.2%s", r.URL.Path)
+	r.URL.Path = fmt.Sprintf("/api/%s%s", av, r.URL.Path)
 	r.Header.Set("Content-Type", "application/json")
 
 	url, err := url.Parse(c.Host)
@@ -334,7 +332,7 @@ func (c *DatabricksClient) api12(r *http.Request) error {
 
 // Scim sets SCIM headers
 func (c *DatabricksClient) Scim(ctx context.Context, method, path string, request interface{}, response interface{}) error {
-	body, err := c.authenticatedQuery(ctx, method, path, request, c.api2, func(r *http.Request) error {
+	body, err := c.authenticatedQuery(ctx, method, path, request, c.completeUrl, func(r *http.Request) error {
 		r.Header.Set("Content-Type", "application/scim+json")
 		if c.isAccountsClient() && c.AccountID != "" {
 			// until `/preview` is there for workspace scim
@@ -348,15 +346,6 @@ func (c *DatabricksClient) Scim(ctx context.Context, method, path string, reques
 	return c.unmarshall(path, body, &response)
 }
 
-// OldAPI performs call on context api
-func (c *DatabricksClient) OldAPI(ctx context.Context, method, path string, request interface{}, response interface{}) error {
-	body, err := c.authenticatedQuery(ctx, method, path, request, c.api12)
-	if err != nil {
-		return err
-	}
-	return c.unmarshall(path, body, &response)
-}
-
 func (c *DatabricksClient) authenticatedQuery(ctx context.Context, method, requestURL string,
 	data interface{}, visitors ...func(*http.Request) error) (body []byte, err error) {
 	err = c.Authenticate(ctx)
@@ -462,7 +451,7 @@ func (c *DatabricksClient) genericQuery(ctx context.Context, method, requestURL
 			headers += "\n"
 		}
 	}
-	log.Printf("[DEBUG] %s %s %s%v", method, requestURL, headers, c.redactedDump(requestBody)) // lgtm[go/clear-text-logging]
+	log.Printf("[DEBUG] %s %s %s%v", method, request.URL.Path, headers, c.redactedDump(requestBody)) // lgtm[go/clear-text-logging]
 
 	r, err := retryablehttp.FromRequest(request)
 	if err != nil {
@@ -486,7 +475,7 @@ func (c *DatabricksClient) genericQuery(ctx context.Context, method, requestURL
 	if err != nil {
 		return nil, err
 	}
-	log.Printf("[DEBUG] %s %v <- %s %s", resp.Status, c.redactedDump(body), method, requestURL)
+	log.Printf("[DEBUG] %s %v <- %s %s", resp.Status, c.redactedDump(body), method, request.URL.Path)
 	return body, nil
 }
 

common/http_test.go

@@ -288,12 +288,12 @@ func TestUnmarshall(t *testing.T) {
 
 func TestAPI2(t *testing.T) {
 	ws := DatabricksClient{Host: "ht_tp://example.com/"}
-	err := ws.api2(&http.Request{})
+	err := ws.completeUrl(&http.Request{})
 	require.Error(t, err)
 	assert.True(t, strings.HasPrefix(err.Error(), "no URL found in request"),
 		"Actual message: %s", err.Error())
 
-	err = ws.api2(&http.Request{
+	err = ws.completeUrl(&http.Request{
 		Header: http.Header{},
 		URL: &url.URL{
 			Path: "/x/y/x",
@@ -314,15 +314,6 @@ func TestScim(t *testing.T) {
 	require.NoError(t, err)
 }
 
-func TestOldAPI(t *testing.T) {
-	ws, server := singleRequestServer(t, "GET", "/api/1.2/imaginary/endpoint", `{"a": "b"}`)
-	defer server.Close()
-
-	var resp map[string]string
-	err := ws.OldAPI(context.Background(), "GET", "/imaginary/endpoint", nil, &resp)
-	require.NoError(t, err)
-}
-
 func TestMakeRequestBody(t *testing.T) {
 	type x struct {
 		Scope string `json:"scope" url:"scope"`

common/reflect_resource.go

@@ -66,10 +66,18 @@ func SchemaPath(s map[string]*schema.Schema, path ...string) (*schema.Schema, er
 	return nil, fmt.Errorf("%v does not compute", path)
 }
 
+func MustSchemaPath(s map[string]*schema.Schema, path ...string) *schema.Schema {
+	sch, err := SchemaPath(s, path...)
+	if err != nil {
+		panic(err)
+	}
+	return sch
+}
+
 // StructToSchema makes schema from a struct type & applies customizations from callback given
 func StructToSchema(v interface{}, customize func(map[string]*schema.Schema) map[string]*schema.Schema) map[string]*schema.Schema {
 	rv := reflect.ValueOf(v)
-	scm := typeToSchema(rv, rv.Type())
+	scm := typeToSchema(rv, rv.Type(), []string{})
 	if customize != nil {
 		scm = customize(scm)
 	}
@@ -128,7 +136,9 @@ func chooseFieldName(typeField reflect.StructField) string {
 	return strings.Split(jsonTag, ",")[0]
 }
 
-func typeToSchema(v reflect.Value, t reflect.Type) map[string]*schema.Schema {
+// typeToSchema converts struct into terraform schema. `path` is used for block suppressions
+// special path element `"0"` is used to denote either arrays or sets of elements
+func typeToSchema(v reflect.Value, t reflect.Type, path []string) map[string]*schema.Schema {
 	scm := map[string]*schema.Schema{}
 	rk := v.Kind()
 	if rk != reflect.Struct {
@@ -180,8 +190,14 @@ func typeToSchema(v reflect.Value, t reflect.Type) map[string]*schema.Schema {
 			scm[fieldName].Type = schema.TypeList
 			elem := typeField.Type.Elem()
 			sv := reflect.New(elem).Elem()
+			if strings.Contains(tfTag, "suppress_diff") {
+				// TODO: we may also suppress count diffs on all json:"..,omitempty" without tf:"force_new"
+				// find . -type f -name '*.go' -not -path "vendor/*" | xargs grep ',omitempty' | grep '*'
+				blockCount := strings.Join(append(path, fieldName, "#"), ".")
+				scm[fieldName].DiffSuppressFunc = makeEmptyBlockSuppressFunc(blockCount)
+			}
 			scm[fieldName].Elem = &schema.Resource{
-				Schema: typeToSchema(sv, elem),
+				Schema: typeToSchema(sv, elem, append(path, fieldName, "0")),
 			}
 		case reflect.Slice:
 			ft := schema.TypeList
@@ -202,7 +218,7 @@ func typeToSchema(v reflect.Value, t reflect.Type) map[string]*schema.Schema {
 			case reflect.Struct:
 				sv := reflect.New(elem).Elem()
 				scm[fieldName].Elem = &schema.Resource{
-					Schema: typeToSchema(sv, elem),
+					Schema: typeToSchema(sv, elem, append(path, fieldName, "0")),
 				}
 			}
 		default:

common/reflect_resource_test.go

@@ -194,7 +194,7 @@ type Dummy struct {
 	Unique      []Address         `json:"unique,omitempty" tf:"slice_set"`
 	Things      []string          `json:"things,omitempty" tf:"slice_set"`
 	Tags        map[string]string `json:"tags,omitempty" tf:"max_items:5"`
-	Home        *Address          `json:"home,omitempty" tf:"group:v"`
+	Home        *Address          `json:"home,omitempty" tf:"group:v,suppress_diff"`
 	House       *Address          `json:"house,omitempty" tf:"group:v"`
 }
 
@@ -368,6 +368,10 @@ func TestStructToData(t *testing.T) {
 	assert.Equal(t, false, d.Get("enabled"))
 	assert.Equal(t, 2, d.Get("addresses.#"))
 
+	assert.NotNil(t, s["home"].DiffSuppressFunc)
+	assert.True(t, s["home"].DiffSuppressFunc("home.#", "1", "0", d))
+	assert.False(t, s["home"].DiffSuppressFunc("home.#", "1", "1", d))
+
 	{
 		//lint:ignore SA1019 Empty optional string should not be set.
 		_, ok := d.GetOkExists("addresses.0.optional_string")

common/resource.go

@@ -3,6 +3,8 @@ package common
 import (
 	"context"
 	"log"
+	"regexp"
+	"strings"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
@@ -96,11 +98,18 @@ func (r Resource) ToResource() *schema.Resource {
 	}
 }
 
-func MakeEmptyBlockSuppressFunc(name string) func(k, old, new string, d *schema.ResourceData) bool {
+func MustCompileKeyRE(name string) *regexp.Regexp {
+	regexFromName := strings.ReplaceAll(name, ".", "\\.")
+	regexFromName = strings.ReplaceAll(regexFromName, ".0", ".\\d+")
+	return regexp.MustCompile(regexFromName)
+}
+
+func makeEmptyBlockSuppressFunc(name string) func(k, old, new string, d *schema.ResourceData) bool {
+	re := MustCompileKeyRE(name)
 	return func(k, old, new string, d *schema.ResourceData) bool {
-		log.Printf("[DEBUG] k='%v', old='%v', new='%v'", k, old, new)
-		if k == name && old == "1" && new == "0" {
-			log.Printf("[DEBUG] Disable removal of empty block")
+		log.Printf("[DEBUG] name=%s k='%v', old='%v', new='%v'", name, k, old, new)
+		if re.Match([]byte(name)) && old == "1" && new == "0" {
+			log.Printf("[DEBUG] Suppressing diff for name=%s k=%#v old=%#v new=%#v", name, k, old, new)
 			return true
 		}
 		return false

common/resource_test.go

@@ -56,12 +56,16 @@ func TestUpdate(t *testing.T) {
 		},
 	}.ToResource()
 
+	client := &DatabricksClient{}
+	ctx := context.Background()
 	d := r.TestResourceData()
-	datas, err := r.Importer.StateContext(
-		context.Background(), d,
-		&DatabricksClient{})
+	datas, err := r.Importer.StateContext(ctx, d, client)
 	require.NoError(t, err)
 	assert.Len(t, datas, 1)
 	assert.False(t, r.Schema["foo"].ForceNew)
 	assert.Equal(t, "", d.Id())
+
+	diags := r.UpdateContext(ctx, d, client)
+	assert.True(t, diags.HasError())
+	assert.Equal(t, "nope", diags[0].Summary)
 }

common/version.go

@@ -12,6 +12,8 @@ var (
 	Current contextKey = 3
 	// If current resource is data
 	IsData contextKey = 4
+	// apiVersion
+	Api contextKey = 5
 )
 
 type contextKey int