fix: Remove validate_utf8 parameter from file_format (#1166)

* Fix - Remove validate_utf8 from file_formats

* docs

* fix test

Co-authored-by: Jason Lin <jason.lin@snowflake.com>

docs/resources/file_format.md

@@ -64,7 +64,6 @@ resource "snowflake_file_format" "example_file_format" {
 - `time_format` (String) Defines the format of time values in the data files (data loading) or table (data unloading).
 - `timestamp_format` (String) Defines the format of timestamp values in the data files (data loading) or table (data unloading).
 - `trim_space` (Boolean) Boolean that specifies whether to remove white space from fields.
-- `validate_utf8` (Boolean) Boolean that specifies whether to validate UTF-8 character encoding in string column data.
 
 ### Read-Only
 

pkg/datasources/file_formats_acceptance_test.go

@@ -65,7 +65,6 @@ func fileFormats(databaseName string, schemaName string, fileFormatName string)
 		null_if = ["NULL"]
 		error_on_column_count_mismatch = true
 		replace_invalid_characters = true
-		validate_utf8 = true
 		empty_field_as_null = false 
 		skip_byte_order_mark = false
 		encoding = "UTF-16"

pkg/resources/file_format.go

@@ -39,7 +39,6 @@ var formatTypeOptions = map[string][]string{
 		"null_if",
 		"error_on_column_count_mismatch",
 		"replace_invalid_characters",
-		"validate_utf8",
 		"empty_field_as_null",
 		"skip_byte_order_mark",
 		"encoding",
@@ -199,11 +198,6 @@ var fileFormatSchema = map[string]*schema.Schema{
 		Optional:    true,
 		Description: "Boolean that specifies whether to replace invalid UTF-8 characters with the Unicode replacement character (�).",
 	},
-	"validate_utf8": {
-		Type:        schema.TypeBool,
-		Optional:    true,
-		Description: "Boolean that specifies whether to validate UTF-8 character encoding in string column data.",
-	},
 	"empty_field_as_null": {
 		Type:        schema.TypeBool,
 		Optional:    true,
@@ -425,12 +419,6 @@ func CreateFileFormat(d *schema.ResourceData, meta interface{}) error {
 		return err
 	}
 
-	if v, ok, err := getFormatTypeOption(d, formatType, "validate_utf8"); ok && err == nil {
-		builder.WithValidateUTF8(v.(bool))
-	} else if err != nil {
-		return err
-	}
-
 	if v, ok, err := getFormatTypeOption(d, formatType, "empty_field_as_null"); ok && err == nil {
 		builder.WithEmptyFieldAsNull(v.(bool))
 	} else if err != nil {
@@ -664,11 +652,6 @@ func ReadFileFormat(d *schema.ResourceData, meta interface{}) error {
 		return err
 	}
 
-	err = d.Set("validate_utf8", opts.ValidateUTF8)
-	if err != nil {
-		return err
-	}
-
 	err = d.Set("empty_field_as_null", opts.EmptyFieldAsNull)
 	if err != nil {
 		return err
@@ -920,15 +903,6 @@ func UpdateFileFormat(d *schema.ResourceData, meta interface{}) error {
 		}
 	}
 
-	if d.HasChange("validate_utf8") {
-		change := d.Get("validate_utf8")
-		q := builder.ChangeValidateUTF8(change.(bool))
-		err := snowflake.Exec(db, q)
-		if err != nil {
-			return errors.Wrapf(err, "error updating file format validate_utf8 on %v", d.Id())
-		}
-	}
-
 	if d.HasChange("empty_field_as_null") {
 		change := d.Get("empty_field_as_null")
 		q := builder.ChangeEmptyFieldAsNull(change.(bool))

pkg/resources/file_format_acceptance_test.go

@@ -40,7 +40,6 @@ func TestAcc_FileFormatCSV(t *testing.T) {
 					resource.TestCheckResourceAttr("snowflake_file_format.test", "null_if.0", "NULL"),
 					resource.TestCheckResourceAttr("snowflake_file_format.test", "error_on_column_count_mismatch", "true"),
 					resource.TestCheckResourceAttr("snowflake_file_format.test", "replace_invalid_characters", "true"),
-					resource.TestCheckResourceAttr("snowflake_file_format.test", "validate_utf8", "true"),
 					resource.TestCheckResourceAttr("snowflake_file_format.test", "empty_field_as_null", "false"),
 					resource.TestCheckResourceAttr("snowflake_file_format.test", "skip_byte_order_mark", "false"),
 					resource.TestCheckResourceAttr("snowflake_file_format.test", "encoding", "UTF-16"),
@@ -223,7 +222,6 @@ resource "snowflake_file_format" "test" {
 	null_if = ["NULL"]
 	error_on_column_count_mismatch = true
 	replace_invalid_characters = true
-	validate_utf8 = true
 	empty_field_as_null = false 
 	skip_byte_order_mark = false
 	encoding = "UTF-16"

pkg/resources/file_format_test.go

@@ -27,7 +27,6 @@ func TestFileFormatCreate(t *testing.T) {
 		"schema":                         "test_schema",
 		"format_type":                    "CSV",
 		"null_if":                        []interface{}{"NULL"},
-		"validate_utf8":                  true,
 		"error_on_column_count_mismatch": true,
 		"comment":                        "great comment",
 	}
@@ -36,7 +35,7 @@ func TestFileFormatCreate(t *testing.T) {
 
 	WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) {
 		mock.ExpectExec(
-			`^CREATE FILE FORMAT "test_db"."test_schema"."test_file_format" TYPE = 'CSV' NULL_IF = \('NULL'\) SKIP_BLANK_LINES = false TRIM_SPACE = false ERROR_ON_COLUMN_COUNT_MISMATCH = true REPLACE_INVALID_CHARACTERS = false VALIDATE_UTF8 = true EMPTY_FIELD_AS_NULL = false SKIP_BYTE_ORDER_MARK = false COMMENT = 'great comment'$`,
+			`^CREATE FILE FORMAT "test_db"."test_schema"."test_file_format" TYPE = 'CSV' NULL_IF = \('NULL'\) SKIP_BLANK_LINES = false TRIM_SPACE = false ERROR_ON_COLUMN_COUNT_MISMATCH = true REPLACE_INVALID_CHARACTERS = false EMPTY_FIELD_AS_NULL = false SKIP_BYTE_ORDER_MARK = false COMMENT = 'great comment'$`,
 		).WillReturnResult(sqlmock.NewResult(1, 1))
 		expectReadFileFormat(mock)
 		err := resources.CreateFileFormat(d, db)
@@ -53,21 +52,21 @@ func TestFileFormatCreateInvalidOptions(t *testing.T) {
 		"schema":        "test_schema",
 		"format_type":   "JSON",
 		"null_if":       []interface{}{"NULL"},
-		"validate_utf8": true,
+		"field_delimiter": ",",
 		"comment":       "great comment",
 	}
 	d := schema.TestResourceDataRaw(t, resources.FileFormat().Schema, in)
 	r.NotNil(d)
 
 	WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) {
 		err := resources.CreateFileFormat(d, db)
-		r.EqualError(err, "validate_utf8 is an invalid format type option for format type JSON")
+		r.EqualError(err, "field_delimiter is an invalid format type option for format type JSON")
 	})
 }
 
 func expectReadFileFormat(mock sqlmock.Sqlmock) {
 	rows := sqlmock.NewRows([]string{
 		"created_on", "name", "database_name", "schema_name", "type", "owner", "comment", "format_options"},
-	).AddRow("2019-12-23 17:20:50.088 +0000", "test_file_format", "test_db", "test_schema", "CSV", "test", "great comment", `{"TYPE":"CSV","RECORD_DELIMITER":"\n","FIELD_DELIMITER":",","FILE_EXTENSION":null,"SKIP_HEADER":0,"DATE_FORMAT":"AUTO","TIME_FORMAT":"AUTO","TIMESTAMP_FORMAT":"AUTO","BINARY_FORMAT":"HEX","ESCAPE":"NONE","ESCAPE_UNENCLOSED_FIELD":"\\","TRIM_SPACE":false,"FIELD_OPTIONALLY_ENCLOSED_BY":"NONE","NULL_IF":["\\N"],"COMPRESSION":"AUTO","ERROR_ON_COLUMN_COUNT_MISMATCH":false,"VALIDATE_UTF8":false,"SKIP_BLANK_LINES":false,"REPLACE_INVALID_CHARACTERS":false,"EMPTY_FIELD_AS_NULL":false,"SKIP_BYTE_ORDER_MARK":false,"ENCODING":"UTF8"}`)
+	).AddRow("2019-12-23 17:20:50.088 +0000", "test_file_format", "test_db", "test_schema", "CSV", "test", "great comment", `{"TYPE":"CSV","RECORD_DELIMITER":"\n","FIELD_DELIMITER":",","FILE_EXTENSION":null,"SKIP_HEADER":0,"DATE_FORMAT":"AUTO","TIME_FORMAT":"AUTO","TIMESTAMP_FORMAT":"AUTO","BINARY_FORMAT":"HEX","ESCAPE":"NONE","ESCAPE_UNENCLOSED_FIELD":"\\","TRIM_SPACE":false,"FIELD_OPTIONALLY_ENCLOSED_BY":"NONE","NULL_IF":["\\N"],"COMPRESSION":"AUTO","ERROR_ON_COLUMN_COUNT_MISMATCH":false,"FIELD_DELIMETER":",","SKIP_BLANK_LINES":false,"REPLACE_INVALID_CHARACTERS":false,"EMPTY_FIELD_AS_NULL":false,"SKIP_BYTE_ORDER_MARK":false,"ENCODING":"UTF8"}`)
 	mock.ExpectQuery(`^SHOW FILE FORMATS LIKE 'test_file_format' IN SCHEMA "test_db"."test_schema"$`).WillReturnRows(rows)
 }

pkg/snowflake/file_format.go

@@ -349,7 +349,6 @@ func (ffb *FileFormatBuilder) Create() string {
 		q.WriteString(fmt.Sprintf(` TRIM_SPACE = %v`, ffb.trimSpace))
 		q.WriteString(fmt.Sprintf(` ERROR_ON_COLUMN_COUNT_MISMATCH = %v`, ffb.errorOnColumnCountMismatch))
 		q.WriteString(fmt.Sprintf(` REPLACE_INVALID_CHARACTERS = %v`, ffb.replaceInvalidCharacters))
-		q.WriteString(fmt.Sprintf(` VALIDATE_UTF8 = %v`, ffb.validateUTF8))
 		q.WriteString(fmt.Sprintf(` EMPTY_FIELD_AS_NULL = %v`, ffb.emptyFieldAsNull))
 		q.WriteString(fmt.Sprintf(` SKIP_BYTE_ORDER_MARK = %v`, ffb.skipByteOrderMark))
 	} else if ffb.formatType == "JSON" {
@@ -427,11 +426,6 @@ func (ffb *FileFormatBuilder) ChangeErrorOnColumnCountMismatch(c bool) string {
 	return fmt.Sprintf(`ALTER FILE FORMAT %v SET ERROR_ON_COLUMN_COUNT_MISMATCH = %v`, ffb.QualifiedName(), c)
 }
 
-// ChangeValidateUTF8 returns the SQL query that will update the error_on_column_count_mismatch on the file format.
-func (ffb *FileFormatBuilder) ChangeValidateUTF8(c bool) string {
-	return fmt.Sprintf(`ALTER FILE FORMAT %v SET VALIDATE_UTF8 = %v`, ffb.QualifiedName(), c)
-}
-
 // ChangeEmptyFieldAsNull returns the SQL query that will update the error_on_column_count_mismatch on the file format.
 func (ffb *FileFormatBuilder) ChangeEmptyFieldAsNull(c bool) string {
 	return fmt.Sprintf(`ALTER FILE FORMAT %v SET EMPTY_FIELD_AS_NULL = %v`, ffb.QualifiedName(), c)
@@ -594,7 +588,6 @@ type fileFormatOptions struct {
 	FieldOptionallyEnclosedBy  string   `json:"FIELD_OPTIONALLY_ENCLOSED_BY,omitempty"`
 	NullIf                     []string `json:"NULL_IF,omitempty"`
 	ErrorOnColumnCountMismatch bool     `json:"ERROR_ON_COLUMN_COUNT_MISMATCH,omitempty"`
-	ValidateUTF8               bool     `json:"VALIDATE_UTF8,omitempty"`
 	SkipBlankLines             bool     `json:"SKIP_BLANK_LINES,omitempty"`
 	ReplaceInvalidCharacters   bool     `json:"REPLACE_INVALID_CHARACTERS,omitempty"`
 	EmptyFieldAsNull           bool     `json:"EMPTY_FIELD_AS_NULL,omitempty"`

pkg/snowflake/file_format_test.go

@@ -34,7 +34,7 @@ func TestFileFormatCreateCSV(t *testing.T) {
 	f.WithSkipByteOrderMark(true)
 	f.WithEncoding("UTF8")
 
-	r.Equal(`CREATE FILE FORMAT "test_db"."test_schema"."test_file_format" TYPE = 'CSV' COMPRESSION = 'AUTO' RECORD_DELIMITER = '\n' FIELD_DELIMITER = ',' FILE_EXTENSION = '.CSV' SKIP_HEADER = 1 DATE_FORMAT = 'AUTO' TIME_FORMAT = 'AUTO' TIMESTAMP_FORMAT = 'AUTO' BINARY_FORMAT = 'HEX' ESCAPE = 'None' ESCAPE_UNENCLOSED_FIELD = 'None' FIELD_OPTIONALLY_ENCLOSED_BY = '"' NULL_IF = () ENCODING = 'UTF8' SKIP_BLANK_LINES = true TRIM_SPACE = true ERROR_ON_COLUMN_COUNT_MISMATCH = false REPLACE_INVALID_CHARACTERS = false VALIDATE_UTF8 = false EMPTY_FIELD_AS_NULL = true SKIP_BYTE_ORDER_MARK = true`, f.Create())
+	r.Equal(`CREATE FILE FORMAT "test_db"."test_schema"."test_file_format" TYPE = 'CSV' COMPRESSION = 'AUTO' RECORD_DELIMITER = '\n' FIELD_DELIMITER = ',' FILE_EXTENSION = '.CSV' SKIP_HEADER = 1 DATE_FORMAT = 'AUTO' TIME_FORMAT = 'AUTO' TIMESTAMP_FORMAT = 'AUTO' BINARY_FORMAT = 'HEX' ESCAPE = 'None' ESCAPE_UNENCLOSED_FIELD = 'None' FIELD_OPTIONALLY_ENCLOSED_BY = '"' NULL_IF = () ENCODING = 'UTF8' SKIP_BLANK_LINES = true TRIM_SPACE = true ERROR_ON_COLUMN_COUNT_MISMATCH = false REPLACE_INVALID_CHARACTERS = false EMPTY_FIELD_AS_NULL = true SKIP_BYTE_ORDER_MARK = true`, f.Create())
 }
 
 func TestFileFormatCreateJSON(t *testing.T) {
@@ -112,12 +112,6 @@ func TestFileFormatChangeBinaryFormat(t *testing.T) {
 	r.Equal(`ALTER FILE FORMAT "test_db"."test_schema"."test_file_format" SET BINARY_FORMAT = 'AUTO'`, f.ChangeBinaryFormat("AUTO"))
 }
 
-func TestFileFormatChangeValidateUTF8(t *testing.T) {
-	r := require.New(t)
-	f := FileFormat("test_file_format", "test_db", "test_schema")
-	r.Equal(`ALTER FILE FORMAT "test_db"."test_schema"."test_file_format" SET VALIDATE_UTF8 = true`, f.ChangeValidateUTF8(true))
-}
-
 func TestFileFormatChangeEmptyFieldAsNull(t *testing.T) {
 	r := require.New(t)
 	f := FileFormat("test_file_format", "test_db", "test_schema")