From 6595eeb52ef817981bfa44602a211c5c8b8de29a Mon Sep 17 00:00:00 2001 From: gary-beautypie <71257526+gary-beautypie@users.noreply.github.com> Date: Thu, 4 Aug 2022 17:51:02 +0100 Subject: [PATCH] fix: Remove validate_utf8 parameter from file_format (#1166) * Fix - Remove validate_utf8 from file_formats * docs * fix test Co-authored-by: Jason Lin --- docs/resources/file_format.md | 1 - .../file_formats_acceptance_test.go | 1 - pkg/resources/file_format.go | 26 ------------------- pkg/resources/file_format_acceptance_test.go | 2 -- pkg/resources/file_format_test.go | 9 +++---- pkg/snowflake/file_format.go | 7 ----- pkg/snowflake/file_format_test.go | 8 +----- 7 files changed, 5 insertions(+), 49 deletions(-) diff --git a/docs/resources/file_format.md b/docs/resources/file_format.md index 772abc801a..3d80e3ac59 100644 --- a/docs/resources/file_format.md +++ b/docs/resources/file_format.md @@ -64,7 +64,6 @@ resource "snowflake_file_format" "example_file_format" { - `time_format` (String) Defines the format of time values in the data files (data loading) or table (data unloading). - `timestamp_format` (String) Defines the format of timestamp values in the data files (data loading) or table (data unloading). - `trim_space` (Boolean) Boolean that specifies whether to remove white space from fields. -- `validate_utf8` (Boolean) Boolean that specifies whether to validate UTF-8 character encoding in string column data. ### Read-Only diff --git a/pkg/datasources/file_formats_acceptance_test.go b/pkg/datasources/file_formats_acceptance_test.go index fe73cd4380..3d93c902f8 100644 --- a/pkg/datasources/file_formats_acceptance_test.go +++ b/pkg/datasources/file_formats_acceptance_test.go @@ -65,7 +65,6 @@ func fileFormats(databaseName string, schemaName string, fileFormatName string) null_if = ["NULL"] error_on_column_count_mismatch = true replace_invalid_characters = true - validate_utf8 = true empty_field_as_null = false skip_byte_order_mark = false encoding = "UTF-16" diff --git a/pkg/resources/file_format.go b/pkg/resources/file_format.go index b265155fc9..ea9a15be55 100644 --- a/pkg/resources/file_format.go +++ b/pkg/resources/file_format.go @@ -39,7 +39,6 @@ var formatTypeOptions = map[string][]string{ "null_if", "error_on_column_count_mismatch", "replace_invalid_characters", - "validate_utf8", "empty_field_as_null", "skip_byte_order_mark", "encoding", @@ -199,11 +198,6 @@ var fileFormatSchema = map[string]*schema.Schema{ Optional: true, Description: "Boolean that specifies whether to replace invalid UTF-8 characters with the Unicode replacement character (�).", }, - "validate_utf8": { - Type: schema.TypeBool, - Optional: true, - Description: "Boolean that specifies whether to validate UTF-8 character encoding in string column data.", - }, "empty_field_as_null": { Type: schema.TypeBool, Optional: true, @@ -425,12 +419,6 @@ func CreateFileFormat(d *schema.ResourceData, meta interface{}) error { return err } - if v, ok, err := getFormatTypeOption(d, formatType, "validate_utf8"); ok && err == nil { - builder.WithValidateUTF8(v.(bool)) - } else if err != nil { - return err - } - if v, ok, err := getFormatTypeOption(d, formatType, "empty_field_as_null"); ok && err == nil { builder.WithEmptyFieldAsNull(v.(bool)) } else if err != nil { @@ -664,11 +652,6 @@ func ReadFileFormat(d *schema.ResourceData, meta interface{}) error { return err } - err = d.Set("validate_utf8", opts.ValidateUTF8) - if err != nil { - return err - } - err = d.Set("empty_field_as_null", opts.EmptyFieldAsNull) if err != nil { return err @@ -920,15 +903,6 @@ func UpdateFileFormat(d *schema.ResourceData, meta interface{}) error { } } - if d.HasChange("validate_utf8") { - change := d.Get("validate_utf8") - q := builder.ChangeValidateUTF8(change.(bool)) - err := snowflake.Exec(db, q) - if err != nil { - return errors.Wrapf(err, "error updating file format validate_utf8 on %v", d.Id()) - } - } - if d.HasChange("empty_field_as_null") { change := d.Get("empty_field_as_null") q := builder.ChangeEmptyFieldAsNull(change.(bool)) diff --git a/pkg/resources/file_format_acceptance_test.go b/pkg/resources/file_format_acceptance_test.go index 16088ab0b5..d1a3074b64 100644 --- a/pkg/resources/file_format_acceptance_test.go +++ b/pkg/resources/file_format_acceptance_test.go @@ -40,7 +40,6 @@ func TestAcc_FileFormatCSV(t *testing.T) { resource.TestCheckResourceAttr("snowflake_file_format.test", "null_if.0", "NULL"), resource.TestCheckResourceAttr("snowflake_file_format.test", "error_on_column_count_mismatch", "true"), resource.TestCheckResourceAttr("snowflake_file_format.test", "replace_invalid_characters", "true"), - resource.TestCheckResourceAttr("snowflake_file_format.test", "validate_utf8", "true"), resource.TestCheckResourceAttr("snowflake_file_format.test", "empty_field_as_null", "false"), resource.TestCheckResourceAttr("snowflake_file_format.test", "skip_byte_order_mark", "false"), resource.TestCheckResourceAttr("snowflake_file_format.test", "encoding", "UTF-16"), @@ -223,7 +222,6 @@ resource "snowflake_file_format" "test" { null_if = ["NULL"] error_on_column_count_mismatch = true replace_invalid_characters = true - validate_utf8 = true empty_field_as_null = false skip_byte_order_mark = false encoding = "UTF-16" diff --git a/pkg/resources/file_format_test.go b/pkg/resources/file_format_test.go index a459b7920b..42ddce52d2 100644 --- a/pkg/resources/file_format_test.go +++ b/pkg/resources/file_format_test.go @@ -27,7 +27,6 @@ func TestFileFormatCreate(t *testing.T) { "schema": "test_schema", "format_type": "CSV", "null_if": []interface{}{"NULL"}, - "validate_utf8": true, "error_on_column_count_mismatch": true, "comment": "great comment", } @@ -36,7 +35,7 @@ func TestFileFormatCreate(t *testing.T) { WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { mock.ExpectExec( - `^CREATE FILE FORMAT "test_db"."test_schema"."test_file_format" TYPE = 'CSV' NULL_IF = \('NULL'\) SKIP_BLANK_LINES = false TRIM_SPACE = false ERROR_ON_COLUMN_COUNT_MISMATCH = true REPLACE_INVALID_CHARACTERS = false VALIDATE_UTF8 = true EMPTY_FIELD_AS_NULL = false SKIP_BYTE_ORDER_MARK = false COMMENT = 'great comment'$`, + `^CREATE FILE FORMAT "test_db"."test_schema"."test_file_format" TYPE = 'CSV' NULL_IF = \('NULL'\) SKIP_BLANK_LINES = false TRIM_SPACE = false ERROR_ON_COLUMN_COUNT_MISMATCH = true REPLACE_INVALID_CHARACTERS = false EMPTY_FIELD_AS_NULL = false SKIP_BYTE_ORDER_MARK = false COMMENT = 'great comment'$`, ).WillReturnResult(sqlmock.NewResult(1, 1)) expectReadFileFormat(mock) err := resources.CreateFileFormat(d, db) @@ -53,7 +52,7 @@ func TestFileFormatCreateInvalidOptions(t *testing.T) { "schema": "test_schema", "format_type": "JSON", "null_if": []interface{}{"NULL"}, - "validate_utf8": true, + "field_delimiter": ",", "comment": "great comment", } d := schema.TestResourceDataRaw(t, resources.FileFormat().Schema, in) @@ -61,13 +60,13 @@ func TestFileFormatCreateInvalidOptions(t *testing.T) { WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { err := resources.CreateFileFormat(d, db) - r.EqualError(err, "validate_utf8 is an invalid format type option for format type JSON") + r.EqualError(err, "field_delimiter is an invalid format type option for format type JSON") }) } func expectReadFileFormat(mock sqlmock.Sqlmock) { rows := sqlmock.NewRows([]string{ "created_on", "name", "database_name", "schema_name", "type", "owner", "comment", "format_options"}, - ).AddRow("2019-12-23 17:20:50.088 +0000", "test_file_format", "test_db", "test_schema", "CSV", "test", "great comment", `{"TYPE":"CSV","RECORD_DELIMITER":"\n","FIELD_DELIMITER":",","FILE_EXTENSION":null,"SKIP_HEADER":0,"DATE_FORMAT":"AUTO","TIME_FORMAT":"AUTO","TIMESTAMP_FORMAT":"AUTO","BINARY_FORMAT":"HEX","ESCAPE":"NONE","ESCAPE_UNENCLOSED_FIELD":"\\","TRIM_SPACE":false,"FIELD_OPTIONALLY_ENCLOSED_BY":"NONE","NULL_IF":["\\N"],"COMPRESSION":"AUTO","ERROR_ON_COLUMN_COUNT_MISMATCH":false,"VALIDATE_UTF8":false,"SKIP_BLANK_LINES":false,"REPLACE_INVALID_CHARACTERS":false,"EMPTY_FIELD_AS_NULL":false,"SKIP_BYTE_ORDER_MARK":false,"ENCODING":"UTF8"}`) + ).AddRow("2019-12-23 17:20:50.088 +0000", "test_file_format", "test_db", "test_schema", "CSV", "test", "great comment", `{"TYPE":"CSV","RECORD_DELIMITER":"\n","FIELD_DELIMITER":",","FILE_EXTENSION":null,"SKIP_HEADER":0,"DATE_FORMAT":"AUTO","TIME_FORMAT":"AUTO","TIMESTAMP_FORMAT":"AUTO","BINARY_FORMAT":"HEX","ESCAPE":"NONE","ESCAPE_UNENCLOSED_FIELD":"\\","TRIM_SPACE":false,"FIELD_OPTIONALLY_ENCLOSED_BY":"NONE","NULL_IF":["\\N"],"COMPRESSION":"AUTO","ERROR_ON_COLUMN_COUNT_MISMATCH":false,"FIELD_DELIMETER":",","SKIP_BLANK_LINES":false,"REPLACE_INVALID_CHARACTERS":false,"EMPTY_FIELD_AS_NULL":false,"SKIP_BYTE_ORDER_MARK":false,"ENCODING":"UTF8"}`) mock.ExpectQuery(`^SHOW FILE FORMATS LIKE 'test_file_format' IN SCHEMA "test_db"."test_schema"$`).WillReturnRows(rows) } diff --git a/pkg/snowflake/file_format.go b/pkg/snowflake/file_format.go index aed2960240..b9425e9ed6 100644 --- a/pkg/snowflake/file_format.go +++ b/pkg/snowflake/file_format.go @@ -349,7 +349,6 @@ func (ffb *FileFormatBuilder) Create() string { q.WriteString(fmt.Sprintf(` TRIM_SPACE = %v`, ffb.trimSpace)) q.WriteString(fmt.Sprintf(` ERROR_ON_COLUMN_COUNT_MISMATCH = %v`, ffb.errorOnColumnCountMismatch)) q.WriteString(fmt.Sprintf(` REPLACE_INVALID_CHARACTERS = %v`, ffb.replaceInvalidCharacters)) - q.WriteString(fmt.Sprintf(` VALIDATE_UTF8 = %v`, ffb.validateUTF8)) q.WriteString(fmt.Sprintf(` EMPTY_FIELD_AS_NULL = %v`, ffb.emptyFieldAsNull)) q.WriteString(fmt.Sprintf(` SKIP_BYTE_ORDER_MARK = %v`, ffb.skipByteOrderMark)) } else if ffb.formatType == "JSON" { @@ -427,11 +426,6 @@ func (ffb *FileFormatBuilder) ChangeErrorOnColumnCountMismatch(c bool) string { return fmt.Sprintf(`ALTER FILE FORMAT %v SET ERROR_ON_COLUMN_COUNT_MISMATCH = %v`, ffb.QualifiedName(), c) } -// ChangeValidateUTF8 returns the SQL query that will update the error_on_column_count_mismatch on the file format. -func (ffb *FileFormatBuilder) ChangeValidateUTF8(c bool) string { - return fmt.Sprintf(`ALTER FILE FORMAT %v SET VALIDATE_UTF8 = %v`, ffb.QualifiedName(), c) -} - // ChangeEmptyFieldAsNull returns the SQL query that will update the error_on_column_count_mismatch on the file format. func (ffb *FileFormatBuilder) ChangeEmptyFieldAsNull(c bool) string { return fmt.Sprintf(`ALTER FILE FORMAT %v SET EMPTY_FIELD_AS_NULL = %v`, ffb.QualifiedName(), c) @@ -594,7 +588,6 @@ type fileFormatOptions struct { FieldOptionallyEnclosedBy string `json:"FIELD_OPTIONALLY_ENCLOSED_BY,omitempty"` NullIf []string `json:"NULL_IF,omitempty"` ErrorOnColumnCountMismatch bool `json:"ERROR_ON_COLUMN_COUNT_MISMATCH,omitempty"` - ValidateUTF8 bool `json:"VALIDATE_UTF8,omitempty"` SkipBlankLines bool `json:"SKIP_BLANK_LINES,omitempty"` ReplaceInvalidCharacters bool `json:"REPLACE_INVALID_CHARACTERS,omitempty"` EmptyFieldAsNull bool `json:"EMPTY_FIELD_AS_NULL,omitempty"` diff --git a/pkg/snowflake/file_format_test.go b/pkg/snowflake/file_format_test.go index 50dbee85b6..cd29014552 100644 --- a/pkg/snowflake/file_format_test.go +++ b/pkg/snowflake/file_format_test.go @@ -34,7 +34,7 @@ func TestFileFormatCreateCSV(t *testing.T) { f.WithSkipByteOrderMark(true) f.WithEncoding("UTF8") - r.Equal(`CREATE FILE FORMAT "test_db"."test_schema"."test_file_format" TYPE = 'CSV' COMPRESSION = 'AUTO' RECORD_DELIMITER = '\n' FIELD_DELIMITER = ',' FILE_EXTENSION = '.CSV' SKIP_HEADER = 1 DATE_FORMAT = 'AUTO' TIME_FORMAT = 'AUTO' TIMESTAMP_FORMAT = 'AUTO' BINARY_FORMAT = 'HEX' ESCAPE = 'None' ESCAPE_UNENCLOSED_FIELD = 'None' FIELD_OPTIONALLY_ENCLOSED_BY = '"' NULL_IF = () ENCODING = 'UTF8' SKIP_BLANK_LINES = true TRIM_SPACE = true ERROR_ON_COLUMN_COUNT_MISMATCH = false REPLACE_INVALID_CHARACTERS = false VALIDATE_UTF8 = false EMPTY_FIELD_AS_NULL = true SKIP_BYTE_ORDER_MARK = true`, f.Create()) + r.Equal(`CREATE FILE FORMAT "test_db"."test_schema"."test_file_format" TYPE = 'CSV' COMPRESSION = 'AUTO' RECORD_DELIMITER = '\n' FIELD_DELIMITER = ',' FILE_EXTENSION = '.CSV' SKIP_HEADER = 1 DATE_FORMAT = 'AUTO' TIME_FORMAT = 'AUTO' TIMESTAMP_FORMAT = 'AUTO' BINARY_FORMAT = 'HEX' ESCAPE = 'None' ESCAPE_UNENCLOSED_FIELD = 'None' FIELD_OPTIONALLY_ENCLOSED_BY = '"' NULL_IF = () ENCODING = 'UTF8' SKIP_BLANK_LINES = true TRIM_SPACE = true ERROR_ON_COLUMN_COUNT_MISMATCH = false REPLACE_INVALID_CHARACTERS = false EMPTY_FIELD_AS_NULL = true SKIP_BYTE_ORDER_MARK = true`, f.Create()) } func TestFileFormatCreateJSON(t *testing.T) { @@ -112,12 +112,6 @@ func TestFileFormatChangeBinaryFormat(t *testing.T) { r.Equal(`ALTER FILE FORMAT "test_db"."test_schema"."test_file_format" SET BINARY_FORMAT = 'AUTO'`, f.ChangeBinaryFormat("AUTO")) } -func TestFileFormatChangeValidateUTF8(t *testing.T) { - r := require.New(t) - f := FileFormat("test_file_format", "test_db", "test_schema") - r.Equal(`ALTER FILE FORMAT "test_db"."test_schema"."test_file_format" SET VALIDATE_UTF8 = true`, f.ChangeValidateUTF8(true)) -} - func TestFileFormatChangeEmptyFieldAsNull(t *testing.T) { r := require.New(t) f := FileFormat("test_file_format", "test_db", "test_schema")