From 7ebbe364c5829546a70843d249629f8fcc3cdcf6 Mon Sep 17 00:00:00 2001 From: Artur Sawicki Date: Fri, 20 Dec 2024 14:56:09 +0100 Subject: [PATCH] chore: Generate marshal json for each model (#3307) - generate marshal and `depends_on` for each resource and data source model - replace deprecated `FromModel` with the new one utilizing the aforementioned marshaling - fix test with explicit empty list (and add it to special variables) - fix tests with explicit null value (and add it to special variables) - generalize multiline handling and test it - use multiline strings for definitions in functions and procedures config builders - simplify function and procedure definitions - format whitespace in function and procedure definitions For the next PRs (added to the issue): - current config building for functions/procedures (especially definitions) is complex: - definition is build using multiline golang strong and sprintf - whitespace is formatted - this is added in the config builder wrapped in special multiline marker - hcl-compatible json is generated - json converted to hcl - hcl formatted using custom formatters This should be refined and made simpler (e.g. newline replacement, encoding problems, etc.) - marking in config builders generators which fields should be handled as multiline by default (because now e.g. SQL function requires the definition, so it's generated in basic builder as a string input, and not the multiline one; because of that it has to be added again with `WithFunctionDefinitionValue(...)` which is confusing) --- .../bettertestspoc/config/config.go | 49 +---- .../bettertestspoc/config/config_test.go | 10 + .../bettertestspoc/config/custom_variables.go | 47 ++++- .../datasourcemodel/accounts_model_gen.go | 45 ++++- .../datasourcemodel/database_model_ext.go | 17 -- .../datasourcemodel/database_model_gen.go | 22 +++ .../datasourcemodel/databases_model_ext.go | 19 -- .../datasourcemodel/databases_model_gen.go | 22 +++ .../config/datasourcemodel/gen/templates.go | 10 +- .../gen/templates/marshal_json.tmpl | 24 +++ .../config/hcl_config_provider.go | 21 ++- .../config/hcl_config_provider_test.go | 25 ++- .../config/json_config_provider_test.go | 38 ++++ .../config/model/account_model_gen.go | 22 +++ .../model/account_parameter_model_gen.go | 22 +++ .../config/model/database_model_gen.go | 22 +++ .../config/model/database_role_model_gen.go | 22 +++ .../config/model/function_java_model_ext.go | 15 +- .../config/model/function_java_model_gen.go | 22 +++ .../model/function_javascript_model_ext.go | 24 +-- .../model/function_javascript_model_gen.go | 22 +++ .../config/model/function_python_model_ext.go | 17 +- .../config/model/function_python_model_gen.go | 22 +++ .../config/model/function_scala_model_ext.go | 18 +- .../config/model/function_scala_model_gen.go | 22 +++ .../config/model/function_sql_model_ext.go | 24 +-- .../config/model/function_sql_model_gen.go | 33 ++-- .../bettertestspoc/config/model/gen/model.go | 2 +- .../config/model/gen/templates.go | 10 +- .../model/gen/templates/marshal_json.tmpl | 24 +++ .../model/legacy_service_user_model_gen.go | 22 +++ .../config/model/masking_policy_model_ext.go | 3 +- .../config/model/masking_policy_model_gen.go | 22 +++ ...ntegration_for_custom_clients_model_gen.go | 30 +++ ...tion_for_partner_applications_model_gen.go | 22 +++ .../model/primary_connection_model_gen.go | 22 +++ .../config/model/procedure_java_model_ext.go | 19 +- .../config/model/procedure_java_model_gen.go | 22 +++ .../model/procedure_javascript_model_ext.go | 17 +- .../model/procedure_javascript_model_gen.go | 22 +++ .../model/procedure_python_model_ext.go | 16 +- .../model/procedure_python_model_gen.go | 22 +++ .../config/model/procedure_scala_model_ext.go | 16 +- .../config/model/procedure_scala_model_gen.go | 22 +++ .../config/model/procedure_sql_model_ext.go | 17 +- .../config/model/procedure_sql_model_gen.go | 22 +++ .../model/resource_monitor_model_gen.go | 22 +++ .../model/row_access_policy_model_gen.go | 22 +++ .../config/model/schema_model_gen.go | 22 +++ .../model/secondary_connection_model_gen.go | 22 +++ ...with_authorization_code_grant_model_gen.go | 22 +++ ...ret_with_basic_authentication_model_gen.go | 22 +++ ...ecret_with_client_credentials_model_gen.go | 22 +++ .../secret_with_generic_string_model_gen.go | 22 +++ .../config/model/service_user_model_gen.go | 22 +++ .../stream_on_directory_table_model_gen.go | 22 +++ .../stream_on_external_table_model_gen.go | 22 +++ .../config/model/stream_on_table_model_gen.go | 22 +++ .../config/model/stream_on_view_model_gen.go | 22 +++ .../config/model/tag_association_model_gen.go | 22 +++ .../config/model/tag_model_gen.go | 22 +++ .../config/model/task_model_gen.go | 22 +++ .../config/model/user_model_ext.go | 2 +- .../config/model/user_model_gen.go | 22 +++ .../config/model/view_model_ext.go | 18 -- .../config/model/view_model_gen.go | 22 +++ .../config/model/warehouse_model_gen.go | 22 +++ .../bettertestspoc/config/placeholders.go | 5 +- .../providermodel/snowflake_model_ext.go | 3 +- .../config/resource_model_test.go | 19 ++ pkg/acceptance/helpers/function_client.go | 50 ++--- pkg/acceptance/helpers/procedure_client.go | 47 ++--- .../connections_acceptance_test.go | 16 +- pkg/datasources/secrets_acceptance_test.go | 23 +-- pkg/datasources/streams_acceptance_test.go | 18 +- pkg/datasources/tags_acceptance_test.go | 10 +- .../usage_tracking_acceptance_test.go | 23 +-- pkg/datasources/users_acceptance_test.go | 26 +-- pkg/provider/provider_acceptance_test.go | 2 +- pkg/resources/account_acceptance_test.go | 45 +++-- .../account_parameter_acceptance_test.go | 12 +- .../database_role_acceptance_test.go | 26 ++- .../function_javascript_acceptance_test.go | 6 +- .../function_python_acceptance_test.go | 2 +- .../legacy_service_user_acceptance_test.go | 34 ++-- ...tion_for_custom_clients_acceptance_test.go | 22 +-- ...or_partner_applications_acceptance_test.go | 23 +-- .../object_renaming_acceptance_test.go | 37 ++-- .../primary_connection_acceptance_test.go | 21 ++- .../resource_monitor_acceptance_test.go | 84 ++++----- .../secondary_connection_acceptance_test.go | 9 +- ...th_basic_authentication_acceptance_test.go | 19 +- ...ret_with_generic_string_acceptance_test.go | 19 +- ...uthorization_code_grant_acceptance_test.go | 31 ++-- ...auth_client_credentials_acceptance_test.go | 29 +-- pkg/resources/service_user_acceptance_test.go | 34 ++-- ...ream_on_directory_table_acceptance_test.go | 35 ++-- ...tream_on_external_table_acceptance_test.go | 41 +++-- .../stream_on_table_acceptance_test.go | 41 +++-- .../stream_on_view_acceptance_test.go | 41 +++-- pkg/resources/tag_acceptance_test.go | 30 +-- .../tag_association_acceptance_test.go | 16 +- pkg/resources/task_acceptance_test.go | 36 ++-- .../usage_tracking_acceptance_test.go | 22 +-- pkg/resources/user_acceptance_test.go | 174 +++++++++--------- pkg/resources/view_acceptance_test.go | 12 +- pkg/resources/warehouse_acceptance_test.go | 5 +- .../testint/procedures_integration_test.go | 2 +- 108 files changed, 1738 insertions(+), 858 deletions(-) delete mode 100644 pkg/acceptance/bettertestspoc/config/datasourcemodel/database_model_ext.go create mode 100644 pkg/acceptance/bettertestspoc/config/datasourcemodel/gen/templates/marshal_json.tmpl create mode 100644 pkg/acceptance/bettertestspoc/config/model/gen/templates/marshal_json.tmpl diff --git a/pkg/acceptance/bettertestspoc/config/config.go b/pkg/acceptance/bettertestspoc/config/config.go index 3174d20fca..9a7a900bda 100644 --- a/pkg/acceptance/bettertestspoc/config/config.go +++ b/pkg/acceptance/bettertestspoc/config/config.go @@ -1,8 +1,6 @@ package config import ( - "encoding/json" - "fmt" "reflect" "strings" "testing" @@ -67,7 +65,8 @@ func ProviderFromModel(t *testing.T, model ProviderModel) string { return hcl } -// FromModels allows to combine multiple models. +// FromModels should be used in terraform acceptance tests for Config attribute to get string config from all models. +// FromModels allows to combine multiple model types. // TODO [SNOW-1501905]: introduce some common interface for all three existing models (ResourceModel, DatasourceModel, and ProviderModel) func FromModels(t *testing.T, models ...any) string { t.Helper() @@ -91,50 +90,6 @@ func FromModels(t *testing.T, models ...any) string { return sb.String() } -// FromModel should be used in terraform acceptance tests for Config attribute to get string config from ResourceModel. -// Current implementation is really straightforward but it could be improved and tested. It may not handle all cases (like objects, lists, sets) correctly. -// TODO [SNOW-1501905]: use reflection to build config directly from model struct (or some other different way) -// TODO [SNOW-1501905]: add support for config.TestStepConfigFunc (to use as ConfigFile); the naive implementation would be to just create a tmp directory and save file there -// TODO [SNOW-1501905]: add generating MarshalJSON() function -// TODO [SNOW-1501905]: migrate resources to new config generation method (above needed first) -// Use ResourceFromModel, DatasourceFromModel, ProviderFromModel, and FromModels instead. -func FromModel(t *testing.T, model ResourceModel) string { - t.Helper() - - b, err := json.Marshal(model) - require.NoError(t, err) - - var objMap map[string]json.RawMessage - err = json.Unmarshal(b, &objMap) - require.NoError(t, err) - - var sb strings.Builder - sb.WriteString(fmt.Sprintf(`resource "%s" "%s" {`, model.Resource(), model.ResourceName())) - sb.WriteRune('\n') - for k, v := range objMap { - sb.WriteString(fmt.Sprintf("\t%s = %s\n", k, v)) - } - if len(model.DependsOn()) > 0 { - sb.WriteString(fmt.Sprintf("\tdepends_on = [%s]\n", strings.Join(model.DependsOn(), ", "))) - } - sb.WriteString(`}`) - sb.WriteRune('\n') - s := sb.String() - t.Logf("Generated config:\n%s", s) - return s -} - -// FromModelsDeprecated allows to combine multiple resource models. -// Use FromModels instead. -func FromModelsDeprecated(t *testing.T, models ...ResourceModel) string { - t.Helper() - var sb strings.Builder - for _, model := range models { - sb.WriteString(FromModel(t, model) + "\n") - } - return sb.String() -} - // ConfigVariablesFromModel constructs config.Variables needed in acceptance tests that are using ConfigVariables in // combination with ConfigDirectory. It's necessary for cases not supported by FromModel, like lists of objects. // Use ResourceFromModel, DatasourceFromModel, ProviderFromModel, and FromModels instead. diff --git a/pkg/acceptance/bettertestspoc/config/config_test.go b/pkg/acceptance/bettertestspoc/config/config_test.go index f061058adc..5c6959012d 100644 --- a/pkg/acceptance/bettertestspoc/config/config_test.go +++ b/pkg/acceptance/bettertestspoc/config/config_test.go @@ -35,6 +35,9 @@ resource "snowflake_share" "test" { Item{IntField: 2, StringField: "second item"}, ). WithSingleObject("one", 2). + WithTextFieldExplicitNull(). + WithListFieldEmpty(). + WithMultilineField("some\nmultiline\ncontent"). WithDependsOn("some_other_resource.some_name", "other_resource.some_other_name", "third_resource.third_name") expectedOutput := strings.TrimPrefix(` resource "snowflake_share" "test" { @@ -54,6 +57,13 @@ resource "snowflake_share" "test" { a = "one" b = 2 } + text_field = null + list_field = [] + multiline_field = <