From 5f227ab30fa75f982d19dfb3d28a588c809059fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Cie=C5=9Blak?= Date: Wed, 10 Jan 2024 09:40:21 +0100 Subject: [PATCH] Fix external tables --- Makefile | 1 - docs/resources/external_table.md | 1 + pkg/resources/external_table.go | 60 +-- .../external_table_acceptance_test.go | 413 ++++++++++++++++-- .../test.tf | 35 ++ .../variables.tf | 6 +- .../test.tf | 29 ++ .../variables.tf | 23 + .../TestAcc_ExternalTable_DeltaLake/test.tf | 31 ++ .../variables.tf | 23 + .../TestAcc_ExternalTable_basic/1/test.tf | 8 + .../1/variables.tf | 23 + .../TestAcc_ExternalTable_basic/2/test.tf | 29 ++ .../2/variables.tf | 23 + .../TestAcc_ExternalTable_basic/test.tf | 33 -- pkg/sdk/external_tables.go | 27 +- pkg/sdk/external_tables_dto.go | 2 - pkg/sdk/external_tables_dto_builders_gen.go | 11 +- pkg/sdk/external_tables_test.go | 9 +- .../external_tables_integration_test.go | 2 - 20 files changed, 656 insertions(+), 133 deletions(-) create mode 100644 pkg/resources/testdata/TestAcc_ExternalTable_CanCreateWithPartitions/test.tf rename pkg/resources/testdata/{TestAcc_ExternalTable_basic => TestAcc_ExternalTable_CanCreateWithPartitions}/variables.tf (71%) create mode 100644 pkg/resources/testdata/TestAcc_ExternalTable_CorrectDataTypes/test.tf create mode 100644 pkg/resources/testdata/TestAcc_ExternalTable_CorrectDataTypes/variables.tf create mode 100644 pkg/resources/testdata/TestAcc_ExternalTable_DeltaLake/test.tf create mode 100644 pkg/resources/testdata/TestAcc_ExternalTable_DeltaLake/variables.tf create mode 100644 pkg/resources/testdata/TestAcc_ExternalTable_basic/1/test.tf create mode 100644 pkg/resources/testdata/TestAcc_ExternalTable_basic/1/variables.tf create mode 100644 pkg/resources/testdata/TestAcc_ExternalTable_basic/2/test.tf create mode 100644 pkg/resources/testdata/TestAcc_ExternalTable_basic/2/variables.tf delete mode 100644 pkg/resources/testdata/TestAcc_ExternalTable_basic/test.tf diff --git a/Makefile b/Makefile index d4a9d88bbb..10a75d8c23 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ export SKIP_EMAIL_INTEGRATION_TESTS=true -export SKIP_EXTERNAL_TABLE_TEST=true export SKIP_NOTIFICATION_INTEGRATION_TESTS=true export SKIP_SAML_INTEGRATION_TESTS=true export SKIP_STREAM_TEST=true diff --git a/docs/resources/external_table.md b/docs/resources/external_table.md index 364f7fb3a9..d0c43febea 100644 --- a/docs/resources/external_table.md +++ b/docs/resources/external_table.md @@ -53,6 +53,7 @@ resource "snowflake_external_table" "external_table" { - `partition_by` (List of String) Specifies any partition columns to evaluate for the external table. - `pattern` (String) Specifies the file names and/or paths on the external stage to match. - `refresh_on_create` (Boolean) Specifies weather to refresh when an external table is created. +- `table_format` (String) Identifies the external table table type. For now, only "delta" for Delta Lake table format is supported. - `tag` (Block List, Deprecated) Definitions of a tag to associate with the resource. (see [below for nested schema](#nestedblock--tag)) ### Read-Only diff --git a/pkg/resources/external_table.go b/pkg/resources/external_table.go index cbcf598976..f9d73c4237 100644 --- a/pkg/resources/external_table.go +++ b/pkg/resources/external_table.go @@ -6,6 +6,8 @@ import ( "fmt" "log" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/helpers" "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -30,13 +32,12 @@ var externalTableSchema = map[string]*schema.Schema{ ForceNew: true, Description: "The database in which to create the external table.", }, - // TODO: Could be a string that we would validate as always "delta" (could be easy to add another type if snowflake introduces one) - "table_format_delta": { - Type: schema.TypeBool, - Required: true, + "table_format": { + Type: schema.TypeString, + Optional: true, ForceNew: true, - Description: `Identifies the external table as referencing a Delta Lake on the cloud storage location. A Delta Lake on Amazon S3, Google Cloud Storage, or Microsoft Azure cloud storage is supported.`, - RequiredWith: []string{"user_specified_partitions"}, + Description: `Identifies the external table table type. For now, only "delta" for Delta Lake table format is supported.`, + ValidateFunc: validation.StringInSlice([]string{"delta"}, true), }, "column": { Type: schema.TypeList, @@ -92,18 +93,11 @@ var externalTableSchema = map[string]*schema.Schema{ ForceNew: true, Description: "Specifies the aws sns topic for the external table.", }, - "user_specified_partitions": { - Type: schema.TypeBool, + "partition_by": { + Type: schema.TypeList, Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, ForceNew: true, - Description: "Enables to manage partitions manually and perform updates instead of recreating table on partition_by change.", - }, - "partition_by": { - Type: schema.TypeList, - Optional: true, - Elem: &schema.Schema{Type: schema.TypeString}, - //ForceNew: true, - // TODO: Update on user_specified_partitions = true and force new on false Description: "Specifies any partition columns to evaluate for the external table.", }, "refresh_on_create": { @@ -175,15 +169,11 @@ func CreateExternalTable(d *schema.ResourceData, meta any) error { for key, val := range col.(map[string]any) { columnDef[key] = val.(string) } - - name := columnDef["name"] - dataTypeString := columnDef["type"] - dataType, err := sdk.ToDataType(dataTypeString) - if err != nil { - return fmt.Errorf(`failed to parse datatype: %s`, dataTypeString) - } - as := columnDef["as"] - columnRequests[i] = sdk.NewExternalTableColumnRequest(name, dataType, as) + columnRequests[i] = sdk.NewExternalTableColumnRequest( + columnDef["name"], + sdk.DataType(columnDef["type"]), + columnDef["as"], + ) } autoRefresh := sdk.Bool(d.Get("auto_refresh").(bool)) refreshOnCreate := sdk.Bool(d.Get("refresh_on_create").(bool)) @@ -219,30 +209,15 @@ func CreateExternalTable(d *schema.ResourceData, meta any) error { } switch { - case d.Get("table_format_delta").(bool): + case d.Get("table_format").(string) == "delta": err := client.ExternalTables.CreateDeltaLake( ctx, sdk.NewCreateDeltaLakeExternalTableRequest(id, location). - WithRawFileFormat(&fileFormat). WithColumns(columnRequests). WithPartitionBy(partitionBy). WithRefreshOnCreate(refreshOnCreate). WithAutoRefresh(autoRefresh). - WithCopyGrants(copyGrants). - WithComment(comment). - WithTag(tagAssociationRequests), - ) - if err != nil { - return err - } - case d.Get("user_specified_partitions").(bool): - err := client.ExternalTables.CreateWithManualPartitioning( - ctx, - sdk.NewCreateWithManualPartitioningExternalTableRequest(id, location). WithRawFileFormat(&fileFormat). - WithColumns(columnRequests). - WithPartitionBy(partitionBy). - WithRawFileFormat(sdk.String(fileFormat)). WithCopyGrants(copyGrants). WithComment(comment). WithTag(tagAssociationRequests), @@ -254,13 +229,12 @@ func CreateExternalTable(d *schema.ResourceData, meta any) error { err := client.ExternalTables.Create( ctx, sdk.NewCreateExternalTableRequest(id, location). - WithRawFileFormat(&fileFormat). WithColumns(columnRequests). WithPartitionBy(partitionBy). WithRefreshOnCreate(refreshOnCreate). WithAutoRefresh(autoRefresh). WithPattern(pattern). - WithRawFileFormat(sdk.String(fileFormat)). + WithRawFileFormat(&fileFormat). WithAwsSnsTopic(awsSnsTopic). WithCopyGrants(copyGrants). WithComment(comment). diff --git a/pkg/resources/external_table_acceptance_test.go b/pkg/resources/external_table_acceptance_test.go index b990adc75d..556eb49b70 100644 --- a/pkg/resources/external_table_acceptance_test.go +++ b/pkg/resources/external_table_acceptance_test.go @@ -3,11 +3,17 @@ package resources_test import ( "context" "database/sql" + "encoding/json" "fmt" + "log" "os" + "slices" "strings" "testing" + "github.com/hashicorp/terraform-plugin-testing/plancheck" + "github.com/stretchr/testify/require" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" "github.com/hashicorp/terraform-plugin-testing/config" "github.com/hashicorp/terraform-plugin-testing/terraform" @@ -19,27 +25,251 @@ import ( ) func TestAcc_ExternalTable_basic(t *testing.T) { - env := os.Getenv("SKIP_EXTERNAL_TABLE_TEST") - if env != "" { - t.Skip("Skipping TestAcc_ExternalTable") + shouldSkip, awsBucketURL, awsKeyId, awsSecretKey := externalTableTestEnvs() + if shouldSkip { + t.Skip("Skipping TestAcc_ExternalTable_basic") + } + + name := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) + resourceName := "snowflake_external_table.test_table" + + innerDirectory := "/external_tables_test_data/" + configVariables := map[string]config.Variable{ + "name": config.StringVariable(name), + "location": config.StringVariable(awsBucketURL), + "aws_key_id": config.StringVariable(awsKeyId), + "aws_secret_key": config.StringVariable(awsSecretKey), + "database": config.StringVariable(acc.TestDatabaseName), + "schema": config.StringVariable(acc.TestSchemaName), + } + + data, err := json.Marshal([]struct { + Name string `json:"name"` + Age int `json:"age"` + }{ + { + Name: "one", + Age: 11, + }, + { + Name: "two", + Age: 22, + }, + { + Name: "three", + Age: 33, + }, + }) + require.NoError(t, err) + + resource.Test(t, resource.TestCase{ + ProtoV6ProviderFactories: acc.TestAccProtoV6ProviderFactories, + PreCheck: func() { acc.TestAccPreCheck(t) }, + TerraformVersionChecks: []tfversion.TerraformVersionCheck{ + tfversion.RequireAbove(tfversion.Version1_5_0), + }, + CheckDestroy: testAccCheckExternalTableDestroy, + Steps: []resource.TestStep{ + { + ConfigDirectory: config.TestStepDirectory(), + ConfigVariables: configVariables, + }, + { + PreConfig: func() { + publishExternalTablesTestData(sdk.NewSchemaObjectIdentifier(acc.TestDatabaseName, acc.TestSchemaName, name), data) + }, + ConfigDirectory: config.TestStepDirectory(), + ConfigVariables: configVariables, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr(resourceName, "name", name), + resource.TestCheckResourceAttr(resourceName, "database", acc.TestDatabaseName), + resource.TestCheckResourceAttr(resourceName, "schema", acc.TestSchemaName), + resource.TestCheckResourceAttr(resourceName, "location", fmt.Sprintf(`@"%s"."%s"."%s"%s`, acc.TestDatabaseName, acc.TestSchemaName, name, innerDirectory)), + resource.TestCheckResourceAttr(resourceName, "file_format", "TYPE = JSON, STRIP_OUTER_ARRAY = TRUE"), + resource.TestCheckResourceAttr(resourceName, "comment", "Terraform acceptance test"), + resource.TestCheckResourceAttr(resourceName, "column.#", "2"), + resource.TestCheckResourceAttr(resourceName, "column.0.name", "name"), + resource.TestCheckResourceAttr(resourceName, "column.0.type", "string"), + resource.TestCheckResourceAttr(resourceName, "column.0.as", "value:name::string"), + resource.TestCheckResourceAttr(resourceName, "column.1.name", "age"), + resource.TestCheckResourceAttr(resourceName, "column.1.type", "number"), + resource.TestCheckResourceAttr(resourceName, "column.1.as", "value:age::number"), + ), + }, + { + ConfigDirectory: acc.ConfigurationSameAsStepN(2), + ConfigVariables: configVariables, + Check: externalTableContainsData(name, func(rows []map[string]*any) bool { + expectedNames := []string{"one", "two", "three"} + names := make([]string, 3) + for i, row := range rows { + nameValue, ok := row["NAME"] + if !ok { + return false + } + + if nameValue == nil { + return false + } + + nameStringValue, ok := (*nameValue).(string) + if !ok { + return false + } + + names[i] = nameStringValue + } + + return !slices.ContainsFunc(expectedNames, func(expectedName string) bool { + return !slices.Contains(names, expectedName) + }) + }), + }, + }, + }) +} + +// proves https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/2310 is fixed +func TestAcc_ExternalTable_CorrectDataTypes(t *testing.T) { + shouldSkip, awsBucketURL, awsKeyId, awsSecretKey := externalTableTestEnvs() + if shouldSkip { + t.Skip("Skipping TestAcc_ExternalTable_CorrectDataTypes") + } + + name := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) + resourceName := "snowflake_external_table.test_table" + + innerDirectory := "/external_tables_test_data/" + configVariables := map[string]config.Variable{ + "name": config.StringVariable(name), + "location": config.StringVariable(awsBucketURL), + "aws_key_id": config.StringVariable(awsKeyId), + "aws_secret_key": config.StringVariable(awsSecretKey), + "database": config.StringVariable(acc.TestDatabaseName), + "schema": config.StringVariable(acc.TestSchemaName), + } + + resource.Test(t, resource.TestCase{ + ProtoV6ProviderFactories: acc.TestAccProtoV6ProviderFactories, + PreCheck: func() { acc.TestAccPreCheck(t) }, + TerraformVersionChecks: []tfversion.TerraformVersionCheck{ + tfversion.RequireAbove(tfversion.Version1_5_0), + }, + CheckDestroy: testAccCheckExternalTableDestroy, + Steps: []resource.TestStep{ + { + ConfigDirectory: config.TestNameDirectory(), + ConfigVariables: configVariables, + ConfigPlanChecks: resource.ConfigPlanChecks{ + PostApplyPostRefresh: []plancheck.PlanCheck{ + plancheck.ExpectEmptyPlan(), + }, + }, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr(resourceName, "name", name), + resource.TestCheckResourceAttr(resourceName, "database", acc.TestDatabaseName), + resource.TestCheckResourceAttr(resourceName, "schema", acc.TestSchemaName), + resource.TestCheckResourceAttr(resourceName, "location", fmt.Sprintf(`@"%s"."%s"."%s"%s`, acc.TestDatabaseName, acc.TestSchemaName, name, innerDirectory)), + resource.TestCheckResourceAttr(resourceName, "file_format", "TYPE = JSON, STRIP_OUTER_ARRAY = TRUE"), + resource.TestCheckResourceAttr(resourceName, "comment", "Terraform acceptance test"), + resource.TestCheckResourceAttr(resourceName, "column.#", "2"), + resource.TestCheckResourceAttr(resourceName, "column.0.name", "name"), + resource.TestCheckResourceAttr(resourceName, "column.0.type", "varchar(200)"), + resource.TestCheckResourceAttr(resourceName, "column.0.as", "value:name::string"), + resource.TestCheckResourceAttr(resourceName, "column.1.name", "age"), + resource.TestCheckResourceAttr(resourceName, "column.1.type", "number(2, 2)"), + resource.TestCheckResourceAttr(resourceName, "column.1.as", "value:age::number"), + expectTableToHaveColumnDataTypes(name, []sdk.DataType{ + sdk.DataTypeVariant, + "VARCHAR(200)", + "NUMBER(2,2)", + }), + ), + }, + }, + }) +} + +// proves https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/2293 is fixed +func TestAcc_ExternalTable_CanCreateWithPartitions(t *testing.T) { + shouldSkip, awsBucketURL, awsKeyId, awsSecretKey := externalTableTestEnvs() + if shouldSkip { + t.Skip("Skipping TestAcc_ExternalTable_CanCreateWithPartitions") } + name := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) - bucketURL := os.Getenv("AWS_EXTERNAL_BUCKET_URL") - if bucketURL == "" { - t.Skip("Skipping TestAcc_ExternalTable") + resourceName := "snowflake_external_table.test_table" + + innerDirectory := "/external_tables_test_data/" + configVariables := map[string]config.Variable{ + "name": config.StringVariable(name), + "location": config.StringVariable(awsBucketURL), + "aws_key_id": config.StringVariable(awsKeyId), + "aws_secret_key": config.StringVariable(awsSecretKey), + "database": config.StringVariable(acc.TestDatabaseName), + "schema": config.StringVariable(acc.TestSchemaName), } - roleName := os.Getenv("AWS_EXTERNAL_ROLE_NAME") - if roleName == "" { - t.Skip("Skipping TestAcc_ExternalTable") + + resource.Test(t, resource.TestCase{ + ProtoV6ProviderFactories: acc.TestAccProtoV6ProviderFactories, + PreCheck: func() { acc.TestAccPreCheck(t) }, + TerraformVersionChecks: []tfversion.TerraformVersionCheck{ + tfversion.RequireAbove(tfversion.Version1_5_0), + }, + CheckDestroy: testAccCheckExternalTableDestroy, + Steps: []resource.TestStep{ + { + ConfigDirectory: config.TestNameDirectory(), + ConfigVariables: configVariables, + ConfigPlanChecks: resource.ConfigPlanChecks{ + PostApplyPostRefresh: []plancheck.PlanCheck{ + plancheck.ExpectEmptyPlan(), + }, + }, + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr(resourceName, "name", name), + resource.TestCheckResourceAttr(resourceName, "database", acc.TestDatabaseName), + resource.TestCheckResourceAttr(resourceName, "schema", acc.TestSchemaName), + resource.TestCheckResourceAttr(resourceName, "location", fmt.Sprintf(`@"%s"."%s"."%s"%s`, acc.TestDatabaseName, acc.TestSchemaName, name, innerDirectory)), + resource.TestCheckResourceAttr(resourceName, "file_format", "TYPE = JSON, STRIP_OUTER_ARRAY = TRUE"), + resource.TestCheckResourceAttr(resourceName, "comment", "Terraform acceptance test"), + resource.TestCheckResourceAttr(resourceName, "partition_by.#", "1"), + resource.TestCheckResourceAttr(resourceName, "partition_by.0", "filename"), + resource.TestCheckResourceAttr(resourceName, "column.#", "3"), + resource.TestCheckResourceAttr(resourceName, "column.0.name", "filename"), + resource.TestCheckResourceAttr(resourceName, "column.0.type", "string"), + resource.TestCheckResourceAttr(resourceName, "column.0.as", "metadata$filename"), + resource.TestCheckResourceAttr(resourceName, "column.1.name", "name"), + resource.TestCheckResourceAttr(resourceName, "column.1.type", "varchar(200)"), + resource.TestCheckResourceAttr(resourceName, "column.1.as", "value:name::string"), + resource.TestCheckResourceAttr(resourceName, "column.2.name", "age"), + resource.TestCheckResourceAttr(resourceName, "column.2.type", "number(2, 2)"), + resource.TestCheckResourceAttr(resourceName, "column.2.as", "value:age::number"), + expectTableDDLContains(name, "partition by (FILENAME)"), + ), + }, + }, + }) +} + +// proves https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/1564 is implemented +func TestAcc_ExternalTable_DeltaLake(t *testing.T) { + shouldSkip, awsBucketURL, awsKeyId, awsSecretKey := externalTableTestEnvs() + if shouldSkip { + t.Skip("Skipping TestAcc_ExternalTable_DeltaLake") } + + name := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) resourceName := "snowflake_external_table.test_table" + innerDirectory := "/external_tables_test_data/" configVariables := map[string]config.Variable{ - "name": config.StringVariable(name), - "location": config.StringVariable(bucketURL), - "aws_arn": config.StringVariable(roleName), - "database": config.StringVariable(acc.TestDatabaseName), - "schema": config.StringVariable(acc.TestSchemaName), + "name": config.StringVariable(name), + "location": config.StringVariable(awsBucketURL), + "aws_key_id": config.StringVariable(awsKeyId), + "aws_secret_key": config.StringVariable(awsSecretKey), + "database": config.StringVariable(acc.TestDatabaseName), + "schema": config.StringVariable(acc.TestSchemaName), } resource.Test(t, resource.TestCase{ @@ -53,29 +283,162 @@ func TestAcc_ExternalTable_basic(t *testing.T) { { ConfigDirectory: config.TestNameDirectory(), ConfigVariables: configVariables, + ConfigPlanChecks: resource.ConfigPlanChecks{ + PostApplyPostRefresh: []plancheck.PlanCheck{ + plancheck.ExpectEmptyPlan(), + }, + }, Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr(resourceName, "name", name), resource.TestCheckResourceAttr(resourceName, "database", acc.TestDatabaseName), resource.TestCheckResourceAttr(resourceName, "schema", acc.TestSchemaName), - resource.TestCheckResourceAttr(resourceName, "location", fmt.Sprintf(`@"%s"."%s"."%s"`, acc.TestDatabaseName, acc.TestSchemaName, name)), - resource.TestCheckResourceAttr(resourceName, "file_format", "TYPE = CSV"), + resource.TestCheckResourceAttr(resourceName, "location", fmt.Sprintf(`@"%s"."%s"."%s"%s`, acc.TestDatabaseName, acc.TestSchemaName, name, innerDirectory)), + resource.TestCheckResourceAttr(resourceName, "file_format", "TYPE = PARQUET"), resource.TestCheckResourceAttr(resourceName, "comment", "Terraform acceptance test"), + resource.TestCheckResourceAttr(resourceName, "table_format", "delta"), + resource.TestCheckResourceAttr(resourceName, "partition_by.#", "1"), + resource.TestCheckResourceAttr(resourceName, "partition_by.0", "filename"), resource.TestCheckResourceAttr(resourceName, "column.#", "2"), - resource.TestCheckResourceAttr(resourceName, "column[0].name", "column1"), - resource.TestCheckResourceAttr(resourceName, "column[0].type", "STRING"), - resource.TestCheckResourceAttr(resourceName, "column[0].as", "TO_VARCHAR(TO_TIMESTAMP_NTZ(value:unix_timestamp_property::NUMBER, 3), 'yyyy-mm-dd-hh')"), - resource.TestCheckResourceAttr(resourceName, "column[1].name", "column2"), - resource.TestCheckResourceAttr(resourceName, "column[1].type", "TIMESTAMP_NTZ(9)"), - resource.TestCheckResourceAttr(resourceName, "column[1].as", "($1:\"CreatedDate\"::timestamp)"), + resource.TestCheckResourceAttr(resourceName, "column.0.name", "filename"), + resource.TestCheckResourceAttr(resourceName, "column.0.type", "string"), + resource.TestCheckResourceAttr(resourceName, "column.0.as", "metadata$filename"), + resource.TestCheckResourceAttr(resourceName, "column.1.name", "name"), + resource.TestCheckResourceAttr(resourceName, "column.1.type", "string"), + resource.TestCheckResourceAttr(resourceName, "column.1.as", "value:name::string"), + func(state *terraform.State) error { + client := sdk.NewClientFromDB(acc.TestAccProvider.Meta().(*sql.DB)) + ctx := context.Background() + id := sdk.NewSchemaObjectIdentifier(acc.TestDatabaseName, acc.TestSchemaName, name) + result, err := client.ExternalTables.ShowByID(ctx, sdk.NewShowExternalTableByIDRequest(id)) + if err != nil { + return err + } + if result.TableFormat != "DELTA" { + return fmt.Errorf("expeted table_format: DELTA, got: %s", result.TableFormat) + } + return nil + }, ), }, }, }) } -// TODO TEST: with partitionBy set (check with select get_ddl) - https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/2293 -// TODO: support table_format = delta - https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/1564 -// TODO: invalid column types - https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/2310 +func externalTableTestEnvs() (bool, string, string, string) { + shouldSkip := os.Getenv("SKIP_EXTERNAL_TABLE_TEST") + awsBucketURL := os.Getenv("AWS_EXTERNAL_BUCKET_URL") + awsKeyId := os.Getenv("AWS_EXTERNAL_KEY_ID") + awsSecretKey := os.Getenv("AWS_EXTERNAL_SECRET_KEY") + return shouldSkip != "" || awsBucketURL == "" || awsKeyId == "" || awsSecretKey == "", awsBucketURL, awsKeyId, awsSecretKey +} + +func externalTableContainsData(name string, contains func(rows []map[string]*any) bool) func(state *terraform.State) error { + return func(state *terraform.State) error { + client := sdk.NewClientFromDB(acc.TestAccProvider.Meta().(*sql.DB)) + ctx := context.Background() + id := sdk.NewSchemaObjectIdentifier(acc.TestDatabaseName, acc.TestSchemaName, name) + rows, err := client.QueryUnsafe(ctx, fmt.Sprintf("select * from %s", id.FullyQualifiedName())) + if err != nil { + return err + } + + jsonRows, err := json.MarshalIndent(rows, "", " ") + if err != nil { + return err + } + log.Printf("Retrieved rows for %s: %v", id.FullyQualifiedName(), string(jsonRows)) + + if !contains(rows) { + return fmt.Errorf("unexpected data returned by external table %s", id.FullyQualifiedName()) + } + + return nil + } +} + +func publishExternalTablesTestData(stageName sdk.SchemaObjectIdentifier, data []byte) { + client, err := sdk.NewDefaultClient() + if err != nil { + log.Fatal(err) + } + ctx := context.Background() + + _, err = client.ExecForTests(ctx, fmt.Sprintf(`copy into @%s/external_tables_test_data/test_data from (select parse_json('%s')) overwrite = true`, stageName.FullyQualifiedName(), string(data))) + if err != nil { + log.Fatal(err) + } +} + +func expectTableToHaveColumnDataTypes(tableName string, expectedDataTypes []sdk.DataType) func(s *terraform.State) error { + return func(s *terraform.State) error { + client := sdk.NewClientFromDB(acc.TestAccProvider.Meta().(*sql.DB)) + ctx := context.Background() + id := sdk.NewSchemaObjectIdentifier(acc.TestDatabaseName, acc.TestSchemaName, tableName) + columnsDesc, err := client.ExternalTables.DescribeColumns(ctx, sdk.NewDescribeExternalTableColumnsRequest(id)) + if err != nil { + return err + } + + actualTableDataTypes := make([]sdk.DataType, len(columnsDesc)) + for i, desc := range columnsDesc { + actualTableDataTypes[i] = desc.Type + } + + slices.SortFunc(expectedDataTypes, func(a, b sdk.DataType) int { + return strings.Compare(string(a), string(b)) + }) + slices.SortFunc(actualTableDataTypes, func(a, b sdk.DataType) int { + return strings.Compare(string(a), string(b)) + }) + + if !slices.Equal(expectedDataTypes, actualTableDataTypes) { + return fmt.Errorf("expected table %s to have columns with data types: %v, got: %v", tableName, expectedDataTypes, actualTableDataTypes) + } + + return nil + } +} + +func expectTableDDLContains(tableName string, substr string) func(s *terraform.State) error { + return func(s *terraform.State) error { + client := sdk.NewClientFromDB(acc.TestAccProvider.Meta().(*sql.DB)) + ctx := context.Background() + id := sdk.NewSchemaObjectIdentifier(acc.TestDatabaseName, acc.TestSchemaName, tableName) + + rows, err := client.QueryUnsafe(ctx, fmt.Sprintf("select get_ddl('table', '%s')", id.FullyQualifiedName())) + if err != nil { + return err + } + + if len(rows) != 1 { + return fmt.Errorf("unexpectedly returned more than one row: %d", len(rows)) + } + + row := rows[0] + + if len(row) != 1 { + return fmt.Errorf("unexpectedly returned more than one columns: %d", len(row)) + } + + for _, v := range row { + if v == nil { + return fmt.Errorf("unexpectedly row value of ddl is nil") + } + + ddl, ok := (*v).(string) + + if !ok { + return fmt.Errorf("unexpectedly ddl is not type string") + } + + if !strings.Contains(ddl, substr) { + return fmt.Errorf("expected '%s' to be a substring of '%s'", substr, ddl) + } + } + + return nil + } +} func testAccCheckExternalTableDestroy(s *terraform.State) error { db := acc.TestAccProvider.Meta().(*sql.DB) diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_CanCreateWithPartitions/test.tf b/pkg/resources/testdata/TestAcc_ExternalTable_CanCreateWithPartitions/test.tf new file mode 100644 index 0000000000..978684a35a --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_CanCreateWithPartitions/test.tf @@ -0,0 +1,35 @@ +resource "snowflake_stage" "test" { + name = var.name + url = var.location + database = var.database + schema = var.schema + credentials = "aws_key_id = '${var.aws_key_id}' aws_secret_key = '${var.aws_secret_key}'" + file_format = "TYPE = JSON NULL_IF = []" +} + +resource "snowflake_external_table" "test_table" { + name = var.name + database = var.database + schema = var.schema + comment = "Terraform acceptance test" + column { + name = "filename" + type = "string" + as = "metadata$filename" + } + column { + name = "name" + type = "varchar(200)" + as = "value:name::string" + } + column { + name = "age" + type = "number(2, 2)" + as = "value:age::number" + } + partition_by = ["filename"] + auto_refresh = false + refresh_on_create = true + file_format = "TYPE = JSON, STRIP_OUTER_ARRAY = TRUE" + location = "@\"${var.database}\".\"${var.schema}\".\"${snowflake_stage.test.name}\"/external_tables_test_data/" +} diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_basic/variables.tf b/pkg/resources/testdata/TestAcc_ExternalTable_CanCreateWithPartitions/variables.tf similarity index 71% rename from pkg/resources/testdata/TestAcc_ExternalTable_basic/variables.tf rename to pkg/resources/testdata/TestAcc_ExternalTable_CanCreateWithPartitions/variables.tf index a447ded368..9badff675d 100644 --- a/pkg/resources/testdata/TestAcc_ExternalTable_basic/variables.tf +++ b/pkg/resources/testdata/TestAcc_ExternalTable_CanCreateWithPartitions/variables.tf @@ -6,7 +6,11 @@ variable "location" { type = string } -variable "aws_arn" { +variable "aws_key_id" { + type = string +} + +variable "aws_secret_key" { type = string } diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_CorrectDataTypes/test.tf b/pkg/resources/testdata/TestAcc_ExternalTable_CorrectDataTypes/test.tf new file mode 100644 index 0000000000..bcd409e0e6 --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_CorrectDataTypes/test.tf @@ -0,0 +1,29 @@ +resource "snowflake_stage" "test" { + name = var.name + url = var.location + database = var.database + schema = var.schema + credentials = "aws_key_id = '${var.aws_key_id}' aws_secret_key = '${var.aws_secret_key}'" + file_format = "TYPE = JSON NULL_IF = []" +} + +resource "snowflake_external_table" "test_table" { + name = var.name + database = var.database + schema = var.schema + comment = "Terraform acceptance test" + column { + name = "name" + type = "varchar(200)" + as = "value:name::string" + } + column { + name = "age" + type = "number(2, 2)" + as = "value:age::number" + } + auto_refresh = false + refresh_on_create = true + file_format = "TYPE = JSON, STRIP_OUTER_ARRAY = TRUE" + location = "@\"${var.database}\".\"${var.schema}\".\"${snowflake_stage.test.name}\"/external_tables_test_data/" +} diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_CorrectDataTypes/variables.tf b/pkg/resources/testdata/TestAcc_ExternalTable_CorrectDataTypes/variables.tf new file mode 100644 index 0000000000..9badff675d --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_CorrectDataTypes/variables.tf @@ -0,0 +1,23 @@ +variable "name" { + type = string +} + +variable "location" { + type = string +} + +variable "aws_key_id" { + type = string +} + +variable "aws_secret_key" { + type = string +} + +variable "database" { + type = string +} + +variable "schema" { + type = string +} diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_DeltaLake/test.tf b/pkg/resources/testdata/TestAcc_ExternalTable_DeltaLake/test.tf new file mode 100644 index 0000000000..205c58648b --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_DeltaLake/test.tf @@ -0,0 +1,31 @@ +resource "snowflake_stage" "test" { + name = var.name + url = var.location + database = var.database + schema = var.schema + credentials = "aws_key_id = '${var.aws_key_id}' aws_secret_key = '${var.aws_secret_key}'" + file_format = "TYPE = PARQUET NULL_IF = []" +} + +resource "snowflake_external_table" "test_table" { + name = var.name + database = var.database + schema = var.schema + comment = "Terraform acceptance test" + table_format = "delta" + column { + name = "filename" + type = "string" + as = "metadata$filename" + } + column { + name = "name" + type = "string" + as = "value:name::string" + } + partition_by = ["filename"] + auto_refresh = false + refresh_on_create = false + file_format = "TYPE = PARQUET" + location = "@\"${var.database}\".\"${var.schema}\".\"${snowflake_stage.test.name}\"/external_tables_test_data/" +} diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_DeltaLake/variables.tf b/pkg/resources/testdata/TestAcc_ExternalTable_DeltaLake/variables.tf new file mode 100644 index 0000000000..9badff675d --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_DeltaLake/variables.tf @@ -0,0 +1,23 @@ +variable "name" { + type = string +} + +variable "location" { + type = string +} + +variable "aws_key_id" { + type = string +} + +variable "aws_secret_key" { + type = string +} + +variable "database" { + type = string +} + +variable "schema" { + type = string +} diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_basic/1/test.tf b/pkg/resources/testdata/TestAcc_ExternalTable_basic/1/test.tf new file mode 100644 index 0000000000..b56902890c --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_basic/1/test.tf @@ -0,0 +1,8 @@ +resource "snowflake_stage" "test" { + name = var.name + url = var.location + database = var.database + schema = var.schema + credentials = "aws_key_id = '${var.aws_key_id}' aws_secret_key = '${var.aws_secret_key}'" + file_format = "TYPE = JSON NULL_IF = []" +} diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_basic/1/variables.tf b/pkg/resources/testdata/TestAcc_ExternalTable_basic/1/variables.tf new file mode 100644 index 0000000000..9badff675d --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_basic/1/variables.tf @@ -0,0 +1,23 @@ +variable "name" { + type = string +} + +variable "location" { + type = string +} + +variable "aws_key_id" { + type = string +} + +variable "aws_secret_key" { + type = string +} + +variable "database" { + type = string +} + +variable "schema" { + type = string +} diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_basic/2/test.tf b/pkg/resources/testdata/TestAcc_ExternalTable_basic/2/test.tf new file mode 100644 index 0000000000..f8fa2ac2fc --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_basic/2/test.tf @@ -0,0 +1,29 @@ +resource "snowflake_stage" "test" { + name = var.name + url = var.location + database = var.database + schema = var.schema + credentials = "aws_key_id = '${var.aws_key_id}' aws_secret_key = '${var.aws_secret_key}'" + file_format = "TYPE = JSON NULL_IF = []" +} + +resource "snowflake_external_table" "test_table" { + name = var.name + database = var.database + schema = var.schema + comment = "Terraform acceptance test" + column { + name = "name" + type = "string" + as = "value:name::string" + } + column { + name = "age" + type = "number" + as = "value:age::number" + } + auto_refresh = false + refresh_on_create = true + file_format = "TYPE = JSON, STRIP_OUTER_ARRAY = TRUE" + location = "@\"${var.database}\".\"${var.schema}\".\"${snowflake_stage.test.name}\"/external_tables_test_data/" +} diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_basic/2/variables.tf b/pkg/resources/testdata/TestAcc_ExternalTable_basic/2/variables.tf new file mode 100644 index 0000000000..9badff675d --- /dev/null +++ b/pkg/resources/testdata/TestAcc_ExternalTable_basic/2/variables.tf @@ -0,0 +1,23 @@ +variable "name" { + type = string +} + +variable "location" { + type = string +} + +variable "aws_key_id" { + type = string +} + +variable "aws_secret_key" { + type = string +} + +variable "database" { + type = string +} + +variable "schema" { + type = string +} diff --git a/pkg/resources/testdata/TestAcc_ExternalTable_basic/test.tf b/pkg/resources/testdata/TestAcc_ExternalTable_basic/test.tf deleted file mode 100644 index c8efacb28f..0000000000 --- a/pkg/resources/testdata/TestAcc_ExternalTable_basic/test.tf +++ /dev/null @@ -1,33 +0,0 @@ -resource "snowflake_storage_integration" "i" { - name = var.name - storage_allowed_locations = [var.location] - storage_provider = "S3" - storage_aws_role_arn = var.aws_arn -} - -resource "snowflake_stage" "test" { - name = var.name - url = var.location - database = var.database - schema = var.schema - storage_integration = snowflake_storage_integration.i.name -} - -resource "snowflake_external_table" "test_table" { - name = var.name - database = var.database - schema = var.schema - comment = "Terraform acceptance test" - column { - name = "column1" - type = "STRING" - as = "TO_VARCHAR(TO_TIMESTAMP_NTZ(value:unix_timestamp_property::NUMBER, 3), 'yyyy-mm-dd-hh')" - } - column { - name = "column2" - type = "TIMESTAMP_NTZ(9)" - as = "($1:\"CreatedDate\"::timestamp)" - } - file_format = "TYPE = CSV" - location = "@\"${var.database}\".\"${var.schema}\".\"${snowflake_stage.test.name}\"" -} diff --git a/pkg/sdk/external_tables.go b/pkg/sdk/external_tables.go index b3171eff4c..e6ebff779e 100644 --- a/pkg/sdk/external_tables.go +++ b/pkg/sdk/external_tables.go @@ -225,24 +225,23 @@ type CreateWithManualPartitioningExternalTableOptions struct { // CreateDeltaLakeExternalTableOptions based on https://docs.snowflake.com/en/sql-reference/sql/create-external-table type CreateDeltaLakeExternalTableOptions struct { - create bool `ddl:"static" sql:"CREATE"` - OrReplace *bool `ddl:"keyword" sql:"OR REPLACE"` - externalTable bool `ddl:"static" sql:"EXTERNAL TABLE"` - IfNotExists *bool `ddl:"keyword" sql:"IF NOT EXISTS"` - name SchemaObjectIdentifier `ddl:"identifier"` - Columns []ExternalTableColumn `ddl:"list,parentheses"` - CloudProviderParams *CloudProviderParams - PartitionBy []string `ddl:"keyword,parentheses" sql:"PARTITION BY"` - Location string `ddl:"parameter" sql:"LOCATION"` - RefreshOnCreate *bool `ddl:"parameter" sql:"REFRESH_ON_CREATE"` - AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` - userSpecifiedPartitionType bool `ddl:"static" sql:"PARTITION_TYPE = USER_SPECIFIED"` - FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` + create bool `ddl:"static" sql:"CREATE"` + OrReplace *bool `ddl:"keyword" sql:"OR REPLACE"` + externalTable bool `ddl:"static" sql:"EXTERNAL TABLE"` + IfNotExists *bool `ddl:"keyword" sql:"IF NOT EXISTS"` + name SchemaObjectIdentifier `ddl:"identifier"` + Columns []ExternalTableColumn `ddl:"list,parentheses"` + CloudProviderParams *CloudProviderParams + PartitionBy []string `ddl:"keyword,parentheses" sql:"PARTITION BY"` + Location string `ddl:"parameter" sql:"LOCATION"` + RefreshOnCreate *bool `ddl:"parameter" sql:"REFRESH_ON_CREATE"` + AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` + FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` // RawFileFormat was introduced, because of the decision taken during https://github.com/Snowflake-Labs/terraform-provider-snowflake/pull/2228 // that for now the snowflake_external_table resource should continue on using raw file format, which wasn't previously supported by the new SDK. // In the future it should most likely be replaced by a more structured version FileFormat RawFileFormat *RawFileFormat `ddl:"list,parentheses" sql:"FILE_FORMAT ="` - DeltaTableFormat *bool `ddl:"keyword" sql:"TABLE_FORMAT = DELTA"` + deltaTableFormat bool `ddl:"static" sql:"TABLE_FORMAT = DELTA"` CopyGrants *bool `ddl:"keyword" sql:"COPY GRANTS"` Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` RowAccessPolicy *TableRowAccessPolicy `ddl:"keyword"` diff --git a/pkg/sdk/external_tables_dto.go b/pkg/sdk/external_tables_dto.go index 0d5bf56d78..5aceccbc24 100644 --- a/pkg/sdk/external_tables_dto.go +++ b/pkg/sdk/external_tables_dto.go @@ -365,7 +365,6 @@ type CreateDeltaLakeExternalTableRequest struct { autoRefresh *bool rawFileFormat *string fileFormat *ExternalTableFileFormatRequest - deltaTableFormat *bool copyGrants *bool comment *string rowAccessPolicy *RowAccessPolicyRequest @@ -421,7 +420,6 @@ func (v *CreateDeltaLakeExternalTableRequest) toOpts() *CreateDeltaLakeExternalT AutoRefresh: v.autoRefresh, RawFileFormat: rawFileFormat, FileFormat: fileFormat, - DeltaTableFormat: v.deltaTableFormat, CopyGrants: v.copyGrants, Comment: v.comment, RowAccessPolicy: rowAccessPolicy, diff --git a/pkg/sdk/external_tables_dto_builders_gen.go b/pkg/sdk/external_tables_dto_builders_gen.go index 8192700862..a466ac8b54 100644 --- a/pkg/sdk/external_tables_dto_builders_gen.go +++ b/pkg/sdk/external_tables_dto_builders_gen.go @@ -2,6 +2,8 @@ package sdk +import () + func NewCreateExternalTableRequest( name SchemaObjectIdentifier, location string, @@ -99,8 +101,8 @@ func NewExternalTableColumnRequest( return &s } -func (s *ExternalTableColumnRequest) WithNotNull() *ExternalTableColumnRequest { - s.notNull = Bool(true) +func (s *ExternalTableColumnRequest) WithNotNull(notNull *bool) *ExternalTableColumnRequest { + s.notNull = notNull return s } @@ -395,11 +397,6 @@ func (s *CreateDeltaLakeExternalTableRequest) WithFileFormat(fileFormat *Externa return s } -func (s *CreateDeltaLakeExternalTableRequest) WithDeltaTableFormat(deltaTableFormat *bool) *CreateDeltaLakeExternalTableRequest { - s.deltaTableFormat = deltaTableFormat - return s -} - func (s *CreateDeltaLakeExternalTableRequest) WithCopyGrants(copyGrants *bool) *CreateDeltaLakeExternalTableRequest { s.copyGrants = copyGrants return s diff --git a/pkg/sdk/external_tables_test.go b/pkg/sdk/external_tables_test.go index 57eee6fa61..325bd87557 100644 --- a/pkg/sdk/external_tables_test.go +++ b/pkg/sdk/external_tables_test.go @@ -180,7 +180,7 @@ func TestExternalTablesCreateWithManualPartitioning(t *testing.T) { }, Comment: String("some_comment"), } - assertOptsValidAndSQLEquals(t, opts, `CREATE OR REPLACE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) INTEGRATION = '123' LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON) COPY GRANTS COMMENT = 'some_comment' ROW ACCESS POLICY "db"."schema"."row_access_policy" ON (value1, value2) TAG ("tag1" = 'value1', "tag2" = 'value2')`) + assertOptsValidAndSQLEquals(t, opts, `CREATE OR REPLACE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) INTEGRATION = '123' LOCATION = @s1/logs/ PARTITION_TYPE = USER_SPECIFIED FILE_FORMAT = (TYPE = JSON) COPY GRANTS COMMENT = 'some_comment' ROW ACCESS POLICY "db"."schema"."row_access_policy" ON (value1, value2) TAG ("tag1" = 'value1', "tag2" = 'value2')`) }) t.Run("invalid options", func(t *testing.T) { @@ -216,7 +216,7 @@ func TestExternalTablesCreateWithManualPartitioning(t *testing.T) { Location: "@s1/logs/", RawFileFormat: &RawFileFormat{Format: "TYPE = JSON"}, } - assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) LOCATION = @s1/logs/ PARTITION_TYPE = USER_SPECIFIED FILE_FORMAT = (TYPE = JSON)`) }) t.Run("validation: neither raw file format is set, nor file format", func(t *testing.T) { @@ -263,8 +263,7 @@ func TestExternalTablesCreateDeltaLake(t *testing.T) { Name: String("JSON"), }, }, - DeltaTableFormat: Bool(true), - CopyGrants: Bool(true), + CopyGrants: Bool(true), RowAccessPolicy: &TableRowAccessPolicy{ Name: NewSchemaObjectIdentifier("db", "schema", "row_access_policy"), On: []string{"value1", "value2"}, @@ -317,7 +316,7 @@ func TestExternalTablesCreateDeltaLake(t *testing.T) { Location: "@s1/logs/", RawFileFormat: &RawFileFormat{Format: "TYPE = JSON"}, } - assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON)`) + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL TABLE "db"."schema"."external_table" (column varchar AS (value::column::varchar) NOT NULL CONSTRAINT my_constraint UNIQUE) LOCATION = @s1/logs/ FILE_FORMAT = (TYPE = JSON) TABLE_FORMAT = DELTA`) }) t.Run("validation: neither raw file format is set, nor file format", func(t *testing.T) { diff --git a/pkg/sdk/testint/external_tables_integration_test.go b/pkg/sdk/testint/external_tables_integration_test.go index cbe21326d4..45542dede1 100644 --- a/pkg/sdk/testint/external_tables_integration_test.go +++ b/pkg/sdk/testint/external_tables_integration_test.go @@ -51,7 +51,6 @@ func TestInt_ExternalTables(t *testing.T) { WithFileFormat(sdk.NewExternalTableFileFormatRequest().WithFileFormatType(&sdk.ExternalTableFileFormatTypeJSON)). WithOrReplace(sdk.Bool(true)). WithColumns(columnsWithPartition). - WithUserSpecifiedPartitionType(sdk.Bool(true)). WithPartitionBy([]string{"part_date"}). WithCopyGrants(sdk.Bool(true)). WithComment(sdk.String("some_comment")). @@ -158,7 +157,6 @@ func TestInt_ExternalTables(t *testing.T) { WithOrReplace(sdk.Bool(true)). WithColumns(columnsWithPartition). WithPartitionBy([]string{"filename"}). - WithDeltaTableFormat(sdk.Bool(true)). WithAutoRefresh(sdk.Bool(false)). WithRefreshOnCreate(sdk.Bool(false)). WithCopyGrants(sdk.Bool(true)).