diff --git a/docs/resources/api_integration.md b/docs/resources/api_integration.md new file mode 100644 index 0000000000..f672da9219 --- /dev/null +++ b/docs/resources/api_integration.md @@ -0,0 +1,57 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "snowflake_api_integration Resource - terraform-provider-snowflake" +subcategory: "" +description: |- + +--- + +# snowflake_api_integration (Resource) + + + +## Example Usage + +```terraform +resource "snowflake_api_integration" "api_integration" { + name = "aws_integration" + api_provider = "aws_api_gateway" + api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" + api_allowed_prefixes = ["https://123456.execute-api.us-west-2.amazonaws.com/prod/"] + enabled = true +} +``` + + +## Schema + +### Required + +- **api_allowed_prefixes** (List of String) Explicitly limits external functions that use the integration to reference one or more HTTPS proxy service endpoints and resources within those proxies. +- **api_provider** (String) Specifies the HTTPS proxy service type. +- **name** (String) Specifies the name of the API integration. This name follows the rules for Object Identifiers. The name should be unique among api integrations in your account. + +### Optional + +- **api_aws_role_arn** (String) ARN of a cloud platform role. +- **api_blocked_prefixes** (List of String) Lists the endpoints and resources in the HTTPS proxy service that are not allowed to be called from Snowflake. +- **azure_ad_application_id** (String) The 'Application (client) id' of the Azure AD app for your remote service. +- **azure_tenant_id** (String) Specifies the ID for your Office 365 tenant that all Azure API Management instances belong to. +- **enabled** (Boolean) Specifies whether this API integration is enabled or disabled. If the API integration is disabled, any external function that relies on it will not work. +- **id** (String) The ID of this resource. + +### Read-Only + +- **api_aws_external_id** (String) The external ID that Snowflake will use when assuming the AWS role. +- **api_aws_iam_user_arn** (String) The Snowflake user that will attempt to assume the AWS role. +- **azure_consent_url** (String) +- **azure_multi_tenant_app_name** (String) +- **created_on** (String) Date and time when the API integration was created. + +## Import + +Import is supported using the following syntax: + +```shell +terraform import snowflake_api_integration.example name +``` diff --git a/docs/resources/external_function.md b/docs/resources/external_function.md new file mode 100644 index 0000000000..362ddb093a --- /dev/null +++ b/docs/resources/external_function.md @@ -0,0 +1,88 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "snowflake_external_function Resource - terraform-provider-snowflake" +subcategory: "" +description: |- + +--- + +# snowflake_external_function (Resource) + + + +## Example Usage + +```terraform +resource "snowflake_external_function" "test_ext_func" { + name = "my_function" + database = "my_test_db" + schema = "my_test_schema" + arg { + name = "arg1" + type = "varchar" + } + arg { + name = "arg2" + type = "varchar" + } + return_type = "varchar" + return_behavior = "IMMUTABLE" + api_integration = "api_integration_name" + url_of_proxy_and_resource = "https://123456.execute-api.us-west-2.amazonaws.com/prod/test_func" +} +``` + + +## Schema + +### Required + +- **api_integration** (String) The name of the API integration object that should be used to authenticate the call to the proxy service. +- **database** (String) The database in which to create the external function. +- **name** (String) Specifies the identifier for the external function. The identifier can contain the schema name and database name, as well as the function name. The function's signature (name and argument data types) must be unique within the schema. +- **return_behavior** (String) Specifies the behavior of the function when returning results +- **return_type** (String) Specifies the data type returned by the external function. +- **schema** (String) The schema in which to create the external function. +- **url_of_proxy_and_resource** (String) This is the invocation URL of the proxy service and resource through which Snowflake calls the remote service. + +### Optional + +- **arg** (Block List) Specifies the arguments/inputs for the external function. These should correspond to the arguments that the remote service expects. (see [below for nested schema](#nestedblock--arg)) +- **comment** (String) A description of the external function. +- **compression** (String) If specified, the JSON payload is compressed when sent from Snowflake to the proxy service, and when sent back from the proxy service to Snowflake. +- **context_headers** (List of String) Binds Snowflake context function results to HTTP headers. +- **header** (Block List) Allows users to specify key-value metadata that is sent with every request as HTTP headers. (see [below for nested schema](#nestedblock--header)) +- **id** (String) The ID of this resource. +- **max_batch_rows** (Number) This specifies the maximum number of rows in each batch sent to the proxy service. +- **null_input_behavior** (String) Specifies the behavior of the external function when called with null inputs. +- **return_null_allowed** (Boolean) Indicates whether the function can return NULL values or must return only NON-NULL values. + +### Read-Only + +- **created_on** (String) Date and time when the external function was created. + + +### Nested Schema for `arg` + +Required: + +- **name** (String) Argument name +- **type** (String) Argument type, e.g. VARCHAR + + + +### Nested Schema for `header` + +Required: + +- **name** (String) Header name +- **value** (String) Header value + +## Import + +Import is supported using the following syntax: + +```shell +# format is database name | schema name | external function name | +terraform import snowflake_external_function.example 'dbName|schemaName|externalFunctionName|varchar-varchar-varchar' +``` diff --git a/examples/resources/snowflake_api_integration/import.sh b/examples/resources/snowflake_api_integration/import.sh new file mode 100644 index 0000000000..44c1faf658 --- /dev/null +++ b/examples/resources/snowflake_api_integration/import.sh @@ -0,0 +1 @@ +terraform import snowflake_api_integration.example name diff --git a/examples/resources/snowflake_api_integration/resource.tf b/examples/resources/snowflake_api_integration/resource.tf new file mode 100644 index 0000000000..98bbf7a974 --- /dev/null +++ b/examples/resources/snowflake_api_integration/resource.tf @@ -0,0 +1,7 @@ +resource "snowflake_api_integration" "api_integration" { + name = "aws_integration" + api_provider = "aws_api_gateway" + api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" + api_allowed_prefixes = ["https://123456.execute-api.us-west-2.amazonaws.com/prod/"] + enabled = true +} \ No newline at end of file diff --git a/examples/resources/snowflake_external_function/import.sh b/examples/resources/snowflake_external_function/import.sh new file mode 100644 index 0000000000..5210458738 --- /dev/null +++ b/examples/resources/snowflake_external_function/import.sh @@ -0,0 +1,2 @@ +# format is database name | schema name | external function name | +terraform import snowflake_external_function.example 'dbName|schemaName|externalFunctionName|varchar-varchar-varchar' diff --git a/examples/resources/snowflake_external_function/resource.tf b/examples/resources/snowflake_external_function/resource.tf new file mode 100644 index 0000000000..7c4bfafdda --- /dev/null +++ b/examples/resources/snowflake_external_function/resource.tf @@ -0,0 +1,17 @@ +resource "snowflake_external_function" "test_ext_func" { + name = "my_function" + database = "my_test_db" + schema = "my_test_schema" + arg { + name = "arg1" + type = "varchar" + } + arg { + name = "arg2" + type = "varchar" + } + return_type = "varchar" + return_behavior = "IMMUTABLE" + api_integration = "api_integration_name" + url_of_proxy_and_resource = "https://123456.execute-api.us-west-2.amazonaws.com/prod/test_func" +} \ No newline at end of file diff --git a/go.sum b/go.sum index 777aad2c32..5acd58f0e7 100644 --- a/go.sum +++ b/go.sum @@ -649,8 +649,6 @@ github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/snowflakedb/gosnowflake v1.4.0 h1:tt3nMrv+qQ4DZtTjasornB6nD8siIU5VpTI+qEyK4sc= -github.com/snowflakedb/gosnowflake v1.4.0/go.mod h1:6nfka9aTXkUNha1p1cjeeyjDvcyh7jfjp0l8kGpDBok= github.com/snowflakedb/gosnowflake v1.4.1 h1:5Yu1Pi0wh6gyebzxtwmngd63VtUIps1HvrmLwxtpAEI= github.com/snowflakedb/gosnowflake v1.4.1/go.mod h1:6nfka9aTXkUNha1p1cjeeyjDvcyh7jfjp0l8kGpDBok= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -671,6 +669,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 8bd3d1a6d2..fefbf2e482 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -106,7 +106,9 @@ func GetGrantResources() resources.TerraformGrantResources { func getResources() map[string]*schema.Resource { others := map[string]*schema.Resource{ + "snowflake_api_integration": resources.APIIntegration(), "snowflake_database": resources.Database(), + "snowflake_external_function": resources.ExternalFunction(), "snowflake_managed_account": resources.ManagedAccount(), "snowflake_masking_policy": resources.MaskingPolicy(), "snowflake_materialized_view": resources.MaterializedView(), diff --git a/pkg/resources/api_integration.go b/pkg/resources/api_integration.go new file mode 100644 index 0000000000..b0c461c821 --- /dev/null +++ b/pkg/resources/api_integration.go @@ -0,0 +1,324 @@ +package resources + +import ( + "database/sql" + "fmt" + "log" + "strings" + + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/snowflake" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" +) + +var apiIntegrationSchema = map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "Specifies the name of the API integration. This name follows the rules for Object Identifiers. The name should be unique among api integrations in your account.", + }, + "api_provider": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringInSlice([]string{"aws_api_gateway", "aws_private_api_gateway", "azure_api_management"}, false), + Description: "Specifies the HTTPS proxy service type.", + }, + "api_aws_role_arn": { + Type: schema.TypeString, + Optional: true, + Default: "", + Description: "ARN of a cloud platform role.", + }, + // Computed. Info you get by issuing a 'DESCRIBE INTEGRATION ' command (API_AWS_IAM_USER_ARN) + "api_aws_iam_user_arn": { + Type: schema.TypeString, + Computed: true, + Description: "The Snowflake user that will attempt to assume the AWS role.", + }, + // Computed. Info you get by issuing a 'DESCRIBE INTEGRATION ' command (API_AWS_EXTERNAL_ID) + "api_aws_external_id": { + Type: schema.TypeString, + Computed: true, + Description: "The external ID that Snowflake will use when assuming the AWS role.", + }, + "azure_tenant_id": { + Type: schema.TypeString, + Optional: true, + Default: "", + Description: "Specifies the ID for your Office 365 tenant that all Azure API Management instances belong to.", + }, + "azure_ad_application_id": { + Type: schema.TypeString, + Optional: true, + Default: "", + Description: "The 'Application (client) id' of the Azure AD app for your remote service.", + }, + // Computed. Info you get by issuing a 'DESCRIBE INTEGRATION ' command (AZURE_MULTI_TENANT_APP_NAME) + "azure_multi_tenant_app_name": { + Type: schema.TypeString, + Computed: true, + }, + // Computed. Info you get by issuing a 'DESCRIBE INTEGRATION ' command (AZURE_CONSENT_URL) + "azure_consent_url": { + Type: schema.TypeString, + Computed: true, + }, + "api_allowed_prefixes": { + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Required: true, + Description: "Explicitly limits external functions that use the integration to reference one or more HTTPS proxy service endpoints and resources within those proxies.", + MinItems: 1, + }, + "api_blocked_prefixes": { + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + Description: "Lists the endpoints and resources in the HTTPS proxy service that are not allowed to be called from Snowflake.", + }, + "enabled": { + Type: schema.TypeBool, + Optional: true, + Default: true, + Description: "Specifies whether this API integration is enabled or disabled. If the API integration is disabled, any external function that relies on it will not work.", + }, + "created_on": { + Type: schema.TypeString, + Computed: true, + Description: "Date and time when the API integration was created.", + }, +} + +// APIIntegration returns a pointer to the resource representing an api integration +func APIIntegration() *schema.Resource { + return &schema.Resource{ + Create: CreateAPIIntegration, + Read: ReadAPIIntegration, + Update: UpdateAPIIntegration, + Delete: DeleteAPIIntegration, + + Schema: apiIntegrationSchema, + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + } +} + +// CreateAPIIntegration implements schema.CreateFunc +func CreateAPIIntegration(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + name := d.Get("name").(string) + + stmt := snowflake.ApiIntegration(name).Create() + + // Set required fields + stmt.SetBool(`ENABLED`, d.Get("enabled").(bool)) + + stmt.SetStringList("API_ALLOWED_PREFIXES", expandStringList(d.Get("api_allowed_prefixes").([]interface{}))) + + // Set optional fields + if _, ok := d.GetOk("api_blocked_prefixes"); ok { + stmt.SetStringList("API_BLOCKED_PREFIXES", expandStringList(d.Get("api_blocked_prefixes").([]interface{}))) + } + + // Now, set the API provider + err := setAPIProviderSettings(d, stmt) + if err != nil { + return err + } + + err = snowflake.Exec(db, stmt.Statement()) + if err != nil { + return fmt.Errorf("error creating api integration: %w", err) + } + + d.SetId(name) + + return ReadAPIIntegration(d, meta) +} + +// ReadAPIIntegration implements schema.ReadFunc +func ReadAPIIntegration(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + id := d.Id() + + stmt := snowflake.ApiIntegration(id).Show() + row := snowflake.QueryRow(db, stmt) + + // Some properties can come from the SHOW INTEGRATION call + + s, err := snowflake.ScanApiIntegration(row) + if err != nil { + return fmt.Errorf("Could not show api integration: %w", err) + } + + // Note: category must be API or something is broken + if c := s.Category.String; c != "API" { + return fmt.Errorf("Expected %v to be an api integration, got %v", id, c) + } + + if err := d.Set("name", s.Name.String); err != nil { + return err + } + + if err := d.Set("created_on", s.CreatedOn.String); err != nil { + return err + } + + if err := d.Set("enabled", s.Enabled.Bool); err != nil { + return err + } + + // Some properties come from the DESCRIBE INTEGRATION call + // We need to grab them in a loop + var k, pType string + var v, unused interface{} + stmt = snowflake.ApiIntegration(id).Describe() + rows, err := db.Query(stmt) + if err != nil { + return fmt.Errorf("Could not describe api integration: %w", err) + } + defer rows.Close() + for rows.Next() { + if err := rows.Scan(&k, &pType, &v, &unused); err != nil { + return err + } + switch k { + case "ENABLED": + // We set this using the SHOW INTEGRATION call so let's ignore it here + case "API_ALLOWED_PREFIXES": + if err = d.Set("api_allowed_prefixes", strings.Split(v.(string), ",")); err != nil { + return err + } + case "API_BLOCKED_PREFIXES": + if val := v.(string); val != "" { + if err = d.Set("api_blocked_prefixes", strings.Split(val, ",")); err != nil { + return err + } + } + case "API_AWS_IAM_USER_ARN": + if err = d.Set("api_aws_iam_user_arn", v.(string)); err != nil { + return err + } + case "API_AWS_ROLE_ARN": + if err = d.Set("api_aws_role_arn", v.(string)); err != nil { + return err + } + case "API_AWS_EXTERNAL_ID": + if err = d.Set("api_aws_external_id", v.(string)); err != nil { + return err + } + case "AZURE_CONSENT_URL": + if err = d.Set("azure_consent_url", v.(string)); err != nil { + return err + } + case "AZURE_MULTI_TENANT_APP_NAME": + if err = d.Set("azure_multi_tenant_app_name", v.(string)); err != nil { + return err + } + default: + log.Printf("[WARN] unexpected api integration property %v returned from Snowflake", k) + } + } + + return err +} + +// UpdateAPIIntegration implements schema.UpdateFunc +func UpdateAPIIntegration(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + id := d.Id() + + stmt := snowflake.ApiIntegration(id).Alter() + + var runSetStatement bool + + if d.HasChange("enabled") { + runSetStatement = true + stmt.SetBool(`ENABLED`, d.Get("enabled").(bool)) + } + + if d.HasChange("api_allowed_prefixes") { + runSetStatement = true + stmt.SetStringList("API_ALLOWED_PREFIXES", expandStringList(d.Get("api_allowed_prefixes").([]interface{}))) + } + + // We need to UNSET this if we remove all api blocked prefixes. + if d.HasChange("api_blocked_prefixes") { + v := d.Get("api_blocked_prefixes").([]interface{}) + if len(v) == 0 { + err := snowflake.Exec(db, fmt.Sprintf(`ALTER API INTEGRATION %v UNSET API_BLOCKED_PREFIXES`, id)) + if err != nil { + return fmt.Errorf("error unsetting api_blocked_prefixes: %w", err) + } + } else { + runSetStatement = true + stmt.SetStringList("API_BLOCKED_PREFIXES", expandStringList(v)) + } + } + + if d.HasChange("api_provider") { + runSetStatement = true + err := setAPIProviderSettings(d, stmt) + if err != nil { + return err + } + } else { + if d.HasChange("api_aws_role_arn") { + runSetStatement = true + stmt.SetString("API_AWS_ROLE_ARN", d.Get("api_aws_role_arn").(string)) + } + if d.HasChange("azure_tenant_id") { + runSetStatement = true + stmt.SetString("AZURE_TENANT_ID", d.Get("azure_tenant_id").(string)) + } + if d.HasChange("azure_ad_application_id") { + runSetStatement = true + stmt.SetString("AZURE_AD_APPLICATION_ID", d.Get("azure_ad_application_id").(string)) + } + } + + if runSetStatement { + if err := snowflake.Exec(db, stmt.Statement()); err != nil { + return fmt.Errorf("error updating api integration: %w", err) + } + } + + return ReadAPIIntegration(d, meta) +} + +// DeleteAPIIntegration implements schema.DeleteFunc +func DeleteAPIIntegration(d *schema.ResourceData, meta interface{}) error { + return DeleteResource("", snowflake.ApiIntegration)(d, meta) +} + +func setAPIProviderSettings(data *schema.ResourceData, stmt snowflake.SettingBuilder) error { + apiProvider := data.Get("api_provider").(string) + stmt.SetRaw("API_PROVIDER=" + apiProvider) + + switch apiProvider { + case "aws_api_gateway", "aws_private_api_gateway": + v, ok := data.GetOk("api_aws_role_arn") + if !ok { + return fmt.Errorf("If you use AWS api provider you must specify an api_aws_role_arn") + } + stmt.SetString(`API_AWS_ROLE_ARN`, v.(string)) + case "azure_api_management": + v, ok := data.GetOk("azure_tenant_id") + if !ok { + return fmt.Errorf("If you use the Azure api provider you must specify an azure_tenant_id") + } + stmt.SetString(`AZURE_TENANT_ID`, v.(string)) + + v, ok = data.GetOk("azure_ad_application_id") + if !ok { + return fmt.Errorf("If you use the Azure api provider you must specify an azure_ad_application_id") + } + stmt.SetString(`AZURE_AD_APPLICATION_ID`, v.(string)) + default: + return fmt.Errorf("Unexpected provider %v", apiProvider) + } + + return nil +} diff --git a/pkg/resources/api_integration_acceptance_test.go b/pkg/resources/api_integration_acceptance_test.go new file mode 100644 index 0000000000..cafd087110 --- /dev/null +++ b/pkg/resources/api_integration_acceptance_test.go @@ -0,0 +1,71 @@ +package resources_test + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestAcc_ApiIntegration(t *testing.T) { + if _, ok := os.LookupEnv("SKIP_API_INTEGRATION_TESTS"); ok { + t.Skip("Skipping TestAccApiIntegration") + } + + apiIntName := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) + apiIntName2 := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) + + resource.Test(t, resource.TestCase{ + Providers: providers(), + Steps: []resource.TestStep{ + { + Config: apiIntegrationConfig_aws(apiIntName, []string{"https://123456.execute-api.us-west-2.amazonaws.com/prod/"}), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("snowflake_api_integration.test_aws_int", "name", apiIntName), + resource.TestCheckResourceAttr("snowflake_api_integration.test_aws_int", "api_provider", "aws_api_gateway"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_aws_int", "created_on"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_aws_int", "api_aws_iam_user_arn"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_aws_int", "api_aws_external_id"), + ), + }, + { + Config: apiIntegrationConfig_azure(apiIntName2, []string{"https://apim-hello-world.azure-api.net/"}), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("snowflake_api_integration.test_azure_int", "name", apiIntName2), + resource.TestCheckResourceAttr("snowflake_api_integration.test_azure_int", "api_provider", "azure_api_management"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_azure_int", "created_on"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_azure_int", "azure_multi_tenant_app_name"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_azure_int", "azure_consent_url"), + ), + }, + }, + }) +} + +func apiIntegrationConfig_aws(name string, prefixes []string) string { + return fmt.Sprintf(` + resource "snowflake_api_integration" "test_aws_int" { + name = "%s" + api_provider = "aws_api_gateway" + api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" + api_allowed_prefixes = %q + enabled = true + } + `, name, prefixes) +} + +func apiIntegrationConfig_azure(name string, prefixes []string) string { + return fmt.Sprintf(` + resource "snowflake_api_integration" "test_azure_int" { + name = "%s" + api_provider = "azure_api_management" + azure_tenant_id = "123456" + azure_ad_application_id = "7890" + api_allowed_prefixes = %q + enabled = true + } + `, name, prefixes) +} diff --git a/pkg/resources/api_integration_test.go b/pkg/resources/api_integration_test.go new file mode 100644 index 0000000000..ab91ff31cd --- /dev/null +++ b/pkg/resources/api_integration_test.go @@ -0,0 +1,84 @@ +package resources_test + +import ( + "database/sql" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/provider" + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/resources" + . "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/testhelpers" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/require" +) + +func TestAPIIntegration(t *testing.T) { + r := require.New(t) + err := resources.APIIntegration().InternalValidate(provider.Provider().Schema, true) + r.NoError(err) +} + +func TestAPIIntegrationCreate(t *testing.T) { + r := require.New(t) + + in := map[string]interface{}{ + "name": "test_api_integration", + "api_allowed_prefixes": []interface{}{"https://123456.execute-api.us-west-2.amazonaws.com/prod/"}, + "api_provider": "aws_api_gateway", + "api_aws_role_arn": "arn:aws:iam::000000000001:/role/test", + } + d := schema.TestResourceDataRaw(t, resources.APIIntegration().Schema, in) + r.NotNil(d) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `^CREATE API INTEGRATION "test_api_integration" API_PROVIDER=aws_api_gateway API_AWS_ROLE_ARN='arn:aws:iam::000000000001:/role/test' API_ALLOWED_PREFIXES=\('https://123456.execute-api.us-west-2.amazonaws.com/prod/'\) ENABLED=true$`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + expectReadAPIIntegration(mock) + + err := resources.CreateAPIIntegration(d, db) + r.NoError(err) + }) +} + +func TestAPIIntegrationRead(t *testing.T) { + r := require.New(t) + + d := apiIntegration(t, "test_api_integration", map[string]interface{}{"name": "test_api_integration"}) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + expectReadAPIIntegration(mock) + + err := resources.ReadAPIIntegration(d, db) + r.NoError(err) + }) +} + +func TestAPIIntegrationDelete(t *testing.T) { + r := require.New(t) + + d := apiIntegration(t, "drop_it", map[string]interface{}{"name": "drop_it"}) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec(`DROP API INTEGRATION "drop_it"`).WillReturnResult(sqlmock.NewResult(1, 1)) + err := resources.DeleteAPIIntegration(d, db) + r.NoError(err) + }) +} + +func expectReadAPIIntegration(mock sqlmock.Sqlmock) { + showRows := sqlmock.NewRows([]string{ + "name", "type", "category", "enabled", "created_on"}, + ).AddRow("test_api_integration", "EXTERNAL_API", "API", true, "now") + mock.ExpectQuery(`^SHOW API INTEGRATIONS LIKE 'test_api_integration'$`).WillReturnRows(showRows) + + descRows := sqlmock.NewRows([]string{ + "property", "property_type", "property_value", "property_default", + }).AddRow("ENABLED", "Boolean", true, false). + AddRow("API_ALLOWED_PREFIXES", "List", "https://123456.execute-api.us-west-2.amazonaws.com/prod/,https://123456.execute-api.us-west-2.amazonaws.com/staging/", nil). + AddRow("API_AWS_IAM_USER_ARN", "String", "arn:aws:iam::000000000000:/user/test", nil). + AddRow("API_AWS_ROLE_ARN", "String", "arn:aws:iam::000000000001:/role/test", nil). + AddRow("API_AWS_EXTERNAL_ID", "String", "AGreatExternalID", nil) + + mock.ExpectQuery(`DESCRIBE API INTEGRATION "test_api_integration"$`).WillReturnRows(descRows) +} diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go new file mode 100644 index 0000000000..a76bcb709d --- /dev/null +++ b/pkg/resources/external_function.go @@ -0,0 +1,500 @@ +package resources + +import ( + "bytes" + "database/sql" + "encoding/csv" + "fmt" + "log" + "regexp" + "strconv" + "strings" + + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/snowflake" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" + "github.com/pkg/errors" +) + +const ( + externalFunctionIDDelimiter = '|' +) + +var externalFunctionSchema = map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "Specifies the identifier for the external function. The identifier can contain the schema name and database name, as well as the function name. The function's signature (name and argument data types) must be unique within the schema.", + }, + "schema": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The schema in which to create the external function.", + }, + "database": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The database in which to create the external function.", + }, + "arg": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Description: "Specifies the arguments/inputs for the external function. These should correspond to the arguments that the remote service expects.", + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + // Suppress the diff shown if the values are equal when both compared in lower case. + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + return strings.EqualFold(strings.ToLower(old), strings.ToLower(new)) + }, + Description: "Argument name", + }, + "type": { + Type: schema.TypeString, + Required: true, + // Suppress the diff shown if the values are equal when both compared in lower case. + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + return strings.EqualFold(strings.ToLower(old), strings.ToLower(new)) + }, + Description: "Argument type, e.g. VARCHAR", + }, + }, + }, + }, + "null_input_behavior": { + Type: schema.TypeString, + Optional: true, + Default: "CALLED ON NULL INPUT", + ForceNew: true, + ValidateFunc: validation.StringInSlice([]string{"CALLED ON NULL INPUT", "RETURNS NULL ON NULL INPUT", "STRICT"}, false), + Description: "Specifies the behavior of the external function when called with null inputs.", + }, + "return_type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + // Suppress the diff shown if the values are equal when both compared in lower case. + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + return strings.EqualFold(strings.ToLower(old), strings.ToLower(new)) + }, + Description: "Specifies the data type returned by the external function.", + }, + "return_null_allowed": { + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + Description: "Indicates whether the function can return NULL values or must return only NON-NULL values.", + }, + "return_behavior": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validation.StringInSlice([]string{"VOLATILE", "IMMUTABLE"}, false), + Description: "Specifies the behavior of the function when returning results", + }, + "api_integration": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The name of the API integration object that should be used to authenticate the call to the proxy service.", + }, + "header": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Description: "Allows users to specify key-value metadata that is sent with every request as HTTP headers.", + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + Description: "Header name", + }, + "value": { + Type: schema.TypeString, + Required: true, + Description: "Header value", + }, + }, + }, + }, + "context_headers": { + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + ForceNew: true, + // Suppress the diff shown if the values are equal when both compared in lower case. + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + return strings.EqualFold(strings.ToLower(old), strings.ToLower(new)) + }, + Description: "Binds Snowflake context function results to HTTP headers.", + }, + "max_batch_rows": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + Description: "This specifies the maximum number of rows in each batch sent to the proxy service.", + }, + "compression": { + Type: schema.TypeString, + Optional: true, + Default: "AUTO", + ForceNew: true, + ValidateFunc: validation.StringInSlice([]string{"NONE", "AUTO", "GZIP", "DEFLATE"}, false), + Description: "If specified, the JSON payload is compressed when sent from Snowflake to the proxy service, and when sent back from the proxy service to Snowflake.", + }, + "url_of_proxy_and_resource": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "This is the invocation URL of the proxy service and resource through which Snowflake calls the remote service.", + }, + "comment": { + Type: schema.TypeString, + Optional: true, + Default: "user-defined function", + ForceNew: true, + Description: "A description of the external function.", + }, + "created_on": { + Type: schema.TypeString, + Computed: true, + Description: "Date and time when the external function was created.", + }, +} + +// ExternalFunction returns a pointer to the resource representing an external function +func ExternalFunction() *schema.Resource { + return &schema.Resource{ + Create: CreateExternalFunction, + Read: ReadExternalFunction, + Delete: DeleteExternalFunction, + + Schema: externalFunctionSchema, + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + } +} + +type externalFunctionID struct { + DatabaseName string + SchemaName string + ExternalFunctionName string + ExternalFunctionArgTypes string +} + +func (si *externalFunctionID) String() (string, error) { + var buf bytes.Buffer + csvWriter := csv.NewWriter(&buf) + csvWriter.Comma = externalFunctionIDDelimiter + err := csvWriter.WriteAll([][]string{{si.DatabaseName, si.SchemaName, si.ExternalFunctionName, si.ExternalFunctionArgTypes}}) + if err != nil { + return "", err + } + + return strings.TrimSpace(buf.String()), nil +} + +func externalFunctionIDFromString(stringID string) (*externalFunctionID, error) { + reader := csv.NewReader(strings.NewReader(stringID)) + reader.Comma = externalFunctionIDDelimiter + lines, err := reader.ReadAll() + if err != nil { + return nil, fmt.Errorf("Not CSV compatible") + } + + if len(lines) != 1 { + return nil, fmt.Errorf("1 line at a time") + } + if len(lines[0]) != 4 { + return nil, fmt.Errorf("4 fields allowed") + } + + return &externalFunctionID{ + DatabaseName: lines[0][0], + SchemaName: lines[0][1], + ExternalFunctionName: lines[0][2], + ExternalFunctionArgTypes: lines[0][3], + }, nil +} + +// CreateExternalFunction implements schema.CreateFunc +func CreateExternalFunction(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + database := d.Get("database").(string) + dbSchema := d.Get("schema").(string) + name := d.Get("name").(string) + var argtypes string + + builder := snowflake.ExternalFunction(name, database, dbSchema) + builder.WithReturnType(d.Get("return_type").(string)) + builder.WithReturnBehavior(d.Get("return_behavior").(string)) + builder.WithAPIIntegration(d.Get("api_integration").(string)) + builder.WithURLOfProxyAndResource(d.Get("url_of_proxy_and_resource").(string)) + + // Set optionals + if _, ok := d.GetOk("arg"); ok { + var types []string + args := []map[string]string{} + for _, arg := range d.Get("arg").([]interface{}) { + argDef := map[string]string{} + for key, val := range arg.(map[string]interface{}) { + argDef[key] = val.(string) + + if key == "type" { + // Also store arg types in distinct array as list of types is required for some Snowflake commands (DESC, DROP) + types = append(types, argDef[key]) + } + } + args = append(args, argDef) + } + + // Use '-' as a separator between arg types as the result will end in the Terraform resource id + argtypes = strings.Join(types, "-") + + builder.WithArgs(args) + builder.WithArgTypes(argtypes) + } + + if v, ok := d.GetOk("return_null_allowed"); ok { + builder.WithReturnNullAllowed(v.(bool)) + } + + if v, ok := d.GetOk("null_input_behavior"); ok { + builder.WithNullInputBehavior(v.(string)) + } + + if v, ok := d.GetOk("comment"); ok { + builder.WithComment(v.(string)) + } + + if _, ok := d.GetOk("header"); ok { + headers := []map[string]string{} + for _, header := range d.Get("header").([]interface{}) { + headerDef := map[string]string{} + for key, val := range header.(map[string]interface{}) { + headerDef[key] = val.(string) + } + headers = append(headers, headerDef) + } + + builder.WithHeaders(headers) + } + + if v, ok := d.GetOk("context_headers"); ok { + contextHeaders := expandStringList(v.([]interface{})) + builder.WithContextHeaders(contextHeaders) + } + + if v, ok := d.GetOk("max_batch_rows"); ok { + builder.WithMaxBatchRows(v.(int)) + } + + if v, ok := d.GetOk("compression"); ok { + builder.WithCompression(v.(string)) + } + + stmt := builder.Create() + err := snowflake.Exec(db, stmt) + if err != nil { + return errors.Wrapf(err, "error creating external function %v", name) + } + + externalFunctionID := &externalFunctionID{ + DatabaseName: database, + SchemaName: dbSchema, + ExternalFunctionName: name, + ExternalFunctionArgTypes: argtypes, + } + dataIDInput, err := externalFunctionID.String() + if err != nil { + return err + } + d.SetId(dataIDInput) + + return ReadExternalFunction(d, meta) +} + +// ReadExternalFunction implements schema.ReadFunc +func ReadExternalFunction(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + externalFunctionID, err := externalFunctionIDFromString(d.Id()) + if err != nil { + return err + } + + dbName := externalFunctionID.DatabaseName + dbSchema := externalFunctionID.SchemaName + name := externalFunctionID.ExternalFunctionName + argtypes := externalFunctionID.ExternalFunctionArgTypes + + // Some properties can come from the SHOW EXTERNAL FUNCTION call + stmt := snowflake.ExternalFunction(name, dbName, dbSchema).Show() + row := snowflake.QueryRow(db, stmt) + externalFunction, err := snowflake.ScanExternalFunction(row) + if err != nil { + return err + } + + // Note: 'language' must be EXTERNAL and 'is_external_function' set to Y + if externalFunction.Language.String != "EXTERNAL" || externalFunction.IsExternalFunction.String != "Y" { + return fmt.Errorf("Expected %v to be an external function, got 'language=%v' and 'is_external_function=%v'", d.Id(), externalFunction.Language.String, externalFunction.IsExternalFunction.String) + } + + if err := d.Set("name", externalFunction.ExternalFunctionName.String); err != nil { + return err + } + + if err := d.Set("schema", externalFunction.SchemaName.String); err != nil { + return err + } + + if err := d.Set("database", externalFunction.DatabaseName.String); err != nil { + return err + } + + if err := d.Set("comment", externalFunction.Comment.String); err != nil { + return err + } + + if err := d.Set("created_on", externalFunction.CreatedOn.String); err != nil { + return err + } + + // Some properties come from the DESCRIBE FUNCTION call + stmt = snowflake.ExternalFunction(name, dbName, dbSchema).WithArgTypes(argtypes).Describe() + externalFunctionDescriptionRows, err := snowflake.Query(db, stmt) + if err != nil { + return err + } + + externalFunctionDescription, err := snowflake.ScanExternalFunctionDescription(externalFunctionDescriptionRows) + if err != nil { + return err + } + + for _, desc := range externalFunctionDescription { + switch desc.Property.String { + case "signature": + // Format in Snowflake DB is: (argName argType, argName argType, ...) + args := strings.ReplaceAll(strings.ReplaceAll(desc.Value.String, "(", ""), ")", "") + + if args != "" { // Do nothing for functions without arguments + argPairs := strings.Split(args, ", ") + args := []interface{}{} + + for _, argPair := range argPairs { + argItem := strings.Split(argPair, " ") + + arg := map[string]interface{}{} + arg["name"] = argItem[0] + arg["type"] = argItem[1] + args = append(args, arg) + } + + if err = d.Set("arg", args); err != nil { + return err + } + } + case "returns": + // Format in Snowflake DB is returnType() + re := regexp.MustCompile(`^(.*)\([0-9]*\)$`) + match := re.FindStringSubmatch(desc.Value.String) + if err = d.Set("return_type", match[1]); err != nil { + return err + } + case "null handling": + if err = d.Set("null_input_behavior", desc.Value.String); err != nil { + return err + } + case "volatility": + if err = d.Set("return_behavior", desc.Value.String); err != nil { + return err + } + case "headers": + if desc.Value.Valid && desc.Value.String != "null" { + // Format in Snowflake DB is: {"head1":"val1","head2":"val2"} + headerPairs := strings.Split(strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(desc.Value.String, "{", ""), "}", ""), "\"", ""), ",") + headers := []interface{}{} + + for _, headerPair := range headerPairs { + headerItem := strings.Split(headerPair, ":") + + header := map[string]interface{}{} + header["name"] = headerItem[0] + header["value"] = headerItem[1] + headers = append(headers, header) + } + + if err = d.Set("header", headers); err != nil { + return err + } + } + case "context_headers": + if desc.Value.Valid && desc.Value.String != "null" { + // Format in Snowflake DB is: ["CONTEXT_FUNCTION_1","CONTEXT_FUNCTION_2"] + contextHeaders := strings.Split(strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(desc.Value.String, "[", ""), "]", ""), "\"", ""), ",") + + if err = d.Set("context_headers", contextHeaders); err != nil { + return err + } + } + case "max_batch_rows": + if desc.Value.String != "not set" { + i, err := strconv.ParseInt(desc.Value.String, 10, 64) + if err != nil { + return err + } + + if err = d.Set("max_batch_rows", i); err != nil { + return err + } + } + case "compression": + if err = d.Set("compression", desc.Value.String); err != nil { + return err + } + case "body": + if err = d.Set("url_of_proxy_and_resource", desc.Value.String); err != nil { + return err + } + case "language": + // To ignore + default: + log.Printf("[WARN] unexpected external function property %v returned from Snowflake", desc.Property.String) + } + } + + return nil +} + +// DeleteExternalFunction implements schema.DeleteFunc +func DeleteExternalFunction(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + externalFunctionID, err := externalFunctionIDFromString(d.Id()) + if err != nil { + return err + } + + dbName := externalFunctionID.DatabaseName + dbSchema := externalFunctionID.SchemaName + name := externalFunctionID.ExternalFunctionName + argtypes := externalFunctionID.ExternalFunctionArgTypes + + q := snowflake.ExternalFunction(name, dbName, dbSchema).WithArgTypes(argtypes).Drop() + + err = snowflake.Exec(db, q) + if err != nil { + return errors.Wrapf(err, "error deleting external function %v", d.Id()) + } + + d.SetId("") + return nil +} diff --git a/pkg/resources/external_function_acceptance_test.go b/pkg/resources/external_function_acceptance_test.go new file mode 100644 index 0000000000..b091a4a432 --- /dev/null +++ b/pkg/resources/external_function_acceptance_test.go @@ -0,0 +1,91 @@ +package resources_test + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestAcc_ExternalFunction(t *testing.T) { + if _, ok := os.LookupEnv("SKIP_EXTERNAL_FUNCTION_TESTS"); ok { + t.Skip("Skipping TestAccExternalFunction") + } + + accName := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) + + resource.Test(t, resource.TestCase{ + Providers: providers(), + Steps: []resource.TestStep{ + { + Config: externalFunctionConfig(accName, []string{"https://123456.execute-api.us-west-2.amazonaws.com/prod/"}, "https://123456.execute-api.us-west-2.amazonaws.com/prod/test_func"), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("snowflake_external_function.test_func", "name", accName), + resource.TestCheckResourceAttr("snowflake_external_function.test_func", "comment", "Terraform acceptance test"), + resource.TestCheckResourceAttrSet("snowflake_external_function.test_func", "created_on"), + ), + }, + }, + }) +} + +func externalFunctionConfig(name string, prefixes []string, url string) string { + return fmt.Sprintf(` + resource "snowflake_database" "test_database" { + name = "%s" + comment = "Terraform acceptance test" + } + + resource "snowflake_schema" "test_schema" { + name = "%s" + database = snowflake_database.test_database.name + comment = "Terraform acceptance test" + } + + resource "snowflake_api_integration" "test_api_int" { + name = "%s" + api_provider = "aws_api_gateway" + api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" + api_allowed_prefixes = %q + enabled = true + } + + resource "snowflake_external_function" "test_func" { + name = "%s" + database = snowflake_database.test_database.name + schema = snowflake_schema.test_schema.name + arg { + name = "arg1" + type = "varchar" + } + arg { + name = "arg2" + type = "varchar" + } + comment = "Terraform acceptance test" + return_type = "varchar" + return_behavior = "IMMUTABLE" + api_integration = snowflake_api_integration.test_api_int.name + url_of_proxy_and_resource = "%s" + } + + resource "snowflake_external_function" "test_func_2" { + name = "%s" + database = snowflake_database.test_database.name + schema = snowflake_schema.test_schema.name + comment = "Terraform acceptance test" + return_type = "varchar" + return_behavior = "IMMUTABLE" + api_integration = snowflake_api_integration.test_api_int.name + header { + name = "x-custom-header" + value = "snowflake" + } + max_batch_rows = 500 + url_of_proxy_and_resource = "%s" + } + `, name, name, name, prefixes, name, url, name, url+"_2") +} diff --git a/pkg/resources/external_function_test.go b/pkg/resources/external_function_test.go new file mode 100644 index 0000000000..8a51aad0d8 --- /dev/null +++ b/pkg/resources/external_function_test.go @@ -0,0 +1,121 @@ +package resources_test + +import ( + "database/sql" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/provider" + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/resources" + . "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/testhelpers" + "github.com/stretchr/testify/require" +) + +func TestExternalFunction(t *testing.T) { + r := require.New(t) + err := resources.ExternalFunction().InternalValidate(provider.Provider().Schema, true) + r.NoError(err) +} + +func TestExternalFunctionCreate(t *testing.T) { + r := require.New(t) + + in := map[string]interface{}{ + "name": "my_test_function", + "database": "database_name", + "schema": "schema_name", + "arg": []interface{}{map[string]interface{}{"name": "data", "type": "varchar"}}, + "return_type": "varchar", + "return_behavior": "IMMUTABLE", + "api_integration": "test_api_integration_01", + "header": []interface{}{map[string]interface{}{"name": "x-custom-header", "value": "snowflake"}}, + "context_headers": []interface{}{"current_timestamp"}, + "url_of_proxy_and_resource": "https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function", + } + d := externalFunction(t, "database_name|schema_name|my_test_function|varchar", in) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec(`CREATE EXTERNAL FUNCTION "database_name"."schema_name"."my_test_function" \(data varchar\) RETURNS varchar NULL CALLED ON NULL INPUT IMMUTABLE COMMENT = 'user-defined function' API_INTEGRATION = 'test_api_integration_01' HEADERS = \('x-custom-header' = 'snowflake'\) CONTEXT_HEADERS = \(current_timestamp\) COMPRESSION = 'AUTO' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function'`).WillReturnResult(sqlmock.NewResult(1, 1)) + + expectExternalFunctionRead(mock) + err := resources.CreateExternalFunction(d, db) + r.NoError(err) + r.Equal("my_test_function", d.Get("name").(string)) + }) +} + +func expectExternalFunctionRead(mock sqlmock.Sqlmock) { + rows := sqlmock.NewRows([]string{"created_on", "name", "schema_name", "is_builtin", "is_aggregate", "is_ansi", "min_num_arguments", "max_num_arguments", "arguments", "description", "catalog_name", "is_table_function", "valid_for_clustering", "is_secure", "is_external_function", "language"}).AddRow("now", "my_test_function", "schema_name", "N", "N", "N", "1", "1", "MY_TEST_FUNCTION(VARCHAR) RETURN VARCHAR", "mock comment", "database_name", "N", "N", "N", "Y", "EXTERNAL") + mock.ExpectQuery(`SHOW EXTERNAL FUNCTIONS LIKE 'my_test_function' IN SCHEMA "database_name"."schema_name"`).WillReturnRows(rows) + + describeRows := sqlmock.NewRows([]string{"property", "value"}). + AddRow("returns", "VARCHAR(123456789)"). // This is how return type is stored in Snowflake DB + AddRow("null handling", "CALLED ON NULL INPUT"). + AddRow("volatility", "IMMUTABLE"). + AddRow("body", "https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function"). + AddRow("headers", "{\"x-custom-header\":\"snowflake\""). + AddRow("context_headers", "[\"CURRENT_TIMESTAMP\"]"). + AddRow("max_batch_rows", "not set"). + AddRow("compression", "AUTO") + + mock.ExpectQuery(`DESCRIBE FUNCTION "database_name"."schema_name"."my_test_function" \(varchar\)`).WillReturnRows(describeRows) +} + +func TestExternalFunctionRead(t *testing.T) { + r := require.New(t) + + d := externalFunction(t, "database_name|schema_name|my_test_function|varchar", map[string]interface{}{"name": "my_test_function", "arg": []interface{}{map[string]interface{}{"name": "data", "type": "varchar"}}, "return_type": "varchar", "comment": "mock comment"}) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + expectExternalFunctionRead(mock) + + err := resources.ReadExternalFunction(d, db) + r.NoError(err) + r.Equal("my_test_function", d.Get("name").(string)) + r.Equal("mock comment", d.Get("comment").(string)) + r.Equal("VARCHAR", d.Get("return_type").(string)) + + args := d.Get("arg").([]interface{}) + r.Len(args, 1) + test_func_args := args[0].(map[string]interface{}) + r.Len(test_func_args, 2) + r.Equal("data", test_func_args["name"].(string)) + r.Equal("varchar", test_func_args["type"].(string)) + + headers := d.Get("header").([]interface{}) + r.Len(headers, 1) + test_func_headers := headers[0].(map[string]interface{}) + r.Len(test_func_headers, 2) + r.Equal("x-custom-header", test_func_headers["name"].(string)) + r.Equal("snowflake", test_func_headers["value"].(string)) + + context_headers := d.Get("context_headers").([]interface{}) + r.Len(context_headers, 1) + test_func_context_headers := expandStringList(context_headers) + r.Len(test_func_context_headers, 1) + r.Equal("CURRENT_TIMESTAMP", test_func_context_headers[0]) + }) +} + +func TestExternalFunctionDelete(t *testing.T) { + r := require.New(t) + + d := externalFunction(t, "database_name|schema_name|drop_it|", map[string]interface{}{"name": "drop_it"}) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec(`DROP FUNCTION "database_name"."schema_name"."drop_it" ()`).WillReturnResult(sqlmock.NewResult(1, 1)) + err := resources.DeleteExternalFunction(d, db) + r.NoError(err) + }) +} + +func expandStringList(configured []interface{}) []string { + vs := make([]string, 0, len(configured)) + for _, v := range configured { + val, ok := v.(string) + if ok && val != "" { + vs = append(vs, val) + } + } + return vs +} diff --git a/pkg/resources/helpers_test.go b/pkg/resources/helpers_test.go index 0f4f2fbab2..057dc7420b 100644 --- a/pkg/resources/helpers_test.go +++ b/pkg/resources/helpers_test.go @@ -176,6 +176,22 @@ func roleGrants(t *testing.T, id string, params map[string]interface{}) *schema. return d } +func apiIntegration(t *testing.T, id string, params map[string]interface{}) *schema.ResourceData { + r := require.New(t) + d := schema.TestResourceDataRaw(t, resources.APIIntegration().Schema, params) + r.NotNil(d) + d.SetId(id) + return d +} + +func externalFunction(t *testing.T, id string, params map[string]interface{}) *schema.ResourceData { + r := require.New(t) + d := schema.TestResourceDataRaw(t, resources.ExternalFunction().Schema, params) + r.NotNil(d) + d.SetId(id) + return d +} + func storageIntegration(t *testing.T, id string, params map[string]interface{}) *schema.ResourceData { r := require.New(t) d := schema.TestResourceDataRaw(t, resources.StorageIntegration().Schema, params) diff --git a/pkg/snowflake/api_integration.go b/pkg/snowflake/api_integration.go new file mode 100644 index 0000000000..70ca8cd810 --- /dev/null +++ b/pkg/snowflake/api_integration.go @@ -0,0 +1,38 @@ +package snowflake + +import ( + "database/sql" + + "github.com/jmoiron/sqlx" +) + +// ApiIntegration returns a pointer to a Builder that abstracts the DDL operations for an api integration. +// +// Supported DDL operations are: +// - CREATE API INTEGRATION +// - ALTER API INTEGRATION +// - DROP INTEGRATION +// - SHOW INTEGRATIONS +// - DESCRIBE INTEGRATION +// +// [Snowflake Reference](https://docs.snowflake.com/en/sql-reference/ddl-user-security.html#api-integrations) +func ApiIntegration(name string) *Builder { + return &Builder{ + entityType: ApiIntegrationType, + name: name, + } +} + +type apiIntegration struct { + Name sql.NullString `db:"name"` + Category sql.NullString `db:"category"` + IntegrationType sql.NullString `db:"type"` + CreatedOn sql.NullString `db:"created_on"` + Enabled sql.NullBool `db:"enabled"` +} + +func ScanApiIntegration(row *sqlx.Row) (*apiIntegration, error) { + r := &apiIntegration{} + err := row.StructScan(r) + return r, err +} diff --git a/pkg/snowflake/api_integration_test.go b/pkg/snowflake/api_integration_test.go new file mode 100644 index 0000000000..d8399bcdcc --- /dev/null +++ b/pkg/snowflake/api_integration_test.go @@ -0,0 +1,27 @@ +package snowflake_test + +import ( + "testing" + + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/snowflake" + "github.com/stretchr/testify/require" +) + +func TestApiIntegration(t *testing.T) { + r := require.New(t) + builder := snowflake.ApiIntegration("aws_api") + r.NotNil(builder) + + q := builder.Show() + r.Equal("SHOW API INTEGRATIONS LIKE 'aws_api'", q) + + c := builder.Create() + + c.SetRaw(`API_PROVIDER=aws_private_api_gateway`) + c.SetString(`api_aws_role_arn`, "arn:aws:iam::xxxx:role/snowflake-execute-externalfunc-privendpoint-role") + c.SetStringList(`api_allowed_prefixes`, []string{"https://123456.execute-api.us-west-2.amazonaws.com/prod/", "https://123456.execute-api.us-west-2.amazonaws.com/test/"}) + c.SetBool(`enabled`, true) + q = c.Statement() + + r.Equal(`CREATE API INTEGRATION "aws_api" API_PROVIDER=aws_private_api_gateway API_AWS_ROLE_ARN='arn:aws:iam::xxxx:role/snowflake-execute-externalfunc-privendpoint-role' API_ALLOWED_PREFIXES=('https://123456.execute-api.us-west-2.amazonaws.com/prod/', 'https://123456.execute-api.us-west-2.amazonaws.com/test/') ENABLED=true`, q) +} diff --git a/pkg/snowflake/exec.go b/pkg/snowflake/exec.go index 6f676366c9..fd5ae17ee7 100644 --- a/pkg/snowflake/exec.go +++ b/pkg/snowflake/exec.go @@ -44,6 +44,7 @@ func QueryRow(db *sql.DB, stmt string) *sqlx.Row { // [DB.Unsafe](https://godoc.org/github.com/jmoiron/sqlx#DB.Unsafe) so that we can scan to structs // without worrying about newly introduced columns func Query(db *sql.DB, stmt string) (*sqlx.Rows, error) { + log.Print("[DEBUG] query stmt ", stmt) sdb := sqlx.NewDb(db, "snowflake").Unsafe() return sdb.Queryx(stmt) } diff --git a/pkg/snowflake/external_function.go b/pkg/snowflake/external_function.go new file mode 100644 index 0000000000..60f7bc0e43 --- /dev/null +++ b/pkg/snowflake/external_function.go @@ -0,0 +1,267 @@ +package snowflake + +import ( + "database/sql" + "fmt" + "strings" + + "github.com/jmoiron/sqlx" +) + +// ExternalFunctionBuilder abstracts the creation of SQL queries for a Snowflake schema +type ExternalFunctionBuilder struct { + name string + db string + schema string + args []map[string]string + argtypes string // only used for 'DESC FUNCTION' & 'DROP FUNCTION' commands as of today (list of args types is required) + nullInputBehavior string + returnType string + returnNullAllowed bool + returnBehavior string + apiIntegration string + headers []map[string]string + contextHeaders []string + maxBatchRows int + compression string + urlOfProxyAndResource string + comment string +} + +// QualifiedName prepends the db and schema if set and escapes everything nicely +func (fb *ExternalFunctionBuilder) QualifiedName() string { + var n strings.Builder + + if fb.db != "" && fb.schema != "" { + n.WriteString(fmt.Sprintf(`"%v"."%v".`, fb.db, fb.schema)) + } + + if fb.db != "" && fb.schema == "" { + n.WriteString(fmt.Sprintf(`"%v"..`, fb.db)) + } + + if fb.db == "" && fb.schema != "" { + n.WriteString(fmt.Sprintf(`"%v".`, fb.schema)) + } + + n.WriteString(fmt.Sprintf(`"%v"`, fb.name)) + + return n.String() +} + +// QualifiedNameWithArgTypes appends all args' types to the qualified name. This is required to invoke 'DESC FUNCTION' and 'DROP FUNCTION' commands. +func (fb *ExternalFunctionBuilder) QualifiedNameWithArgTypes() string { + q := strings.Builder{} + q.WriteString(fmt.Sprintf(`%v (%s)`, fb.QualifiedName(), fb.argtypes)) + return q.String() +} + +// WithArgs sets the args on the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithArgs(args []map[string]string) *ExternalFunctionBuilder { + fb.args = args + return fb +} + +// WithArgTypes sets the args on the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithArgTypes(argtypes string) *ExternalFunctionBuilder { + argtypeslist := strings.ReplaceAll(argtypes, "-", ", ") + fb.argtypes = argtypeslist + return fb +} + +// WithNullInputBehavior adds a nullInputBehavior to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithNullInputBehavior(nullInputBehavior string) *ExternalFunctionBuilder { + fb.nullInputBehavior = nullInputBehavior + return fb +} + +// WithReturnType adds a returnType to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithReturnType(returnType string) *ExternalFunctionBuilder { + fb.returnType = returnType + return fb +} + +// WithReturnNullAllowed adds a returnNullAllowed to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithReturnNullAllowed(returnNullAllowed bool) *ExternalFunctionBuilder { + fb.returnNullAllowed = returnNullAllowed + return fb +} + +// WithReturnBehavior adds a returnBehavior to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithReturnBehavior(returnBehavior string) *ExternalFunctionBuilder { + fb.returnBehavior = returnBehavior + return fb +} + +// WithAPIIntegration adds a apiIntegration to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithAPIIntegration(apiIntegration string) *ExternalFunctionBuilder { + fb.apiIntegration = apiIntegration + return fb +} + +// WithHeaders sets the headers on the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithHeaders(headers []map[string]string) *ExternalFunctionBuilder { + fb.headers = headers + return fb +} + +// WithContextHeaders sets the context headers on the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithContextHeaders(contextHeaders []string) *ExternalFunctionBuilder { + fb.contextHeaders = contextHeaders + return fb +} + +// WithMaxBatchRows adds a maxBatchRows to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithMaxBatchRows(maxBatchRows int) *ExternalFunctionBuilder { + fb.maxBatchRows = maxBatchRows + return fb +} + +// WithCompression adds a compression to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithCompression(compression string) *ExternalFunctionBuilder { + fb.compression = compression + return fb +} + +// WithURLOfProxyAndResource adds a urlOfProxyAndResource to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithURLOfProxyAndResource(urlOfProxyAndResource string) *ExternalFunctionBuilder { + fb.urlOfProxyAndResource = urlOfProxyAndResource + return fb +} + +// WithComment adds a comment to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithComment(c string) *ExternalFunctionBuilder { + fb.comment = c + return fb +} + +// ExternalFunction returns a pointer to a Builder that abstracts the DDL operations for an external function. +// +// Supported DDL operations are: +// - CREATE EXTERNAL FUNCTION +// - ALTER EXTERNAL FUNCTION +// - DROP FUNCTION +// - SHOW EXTERNAL FUNCTIONS +// - DESCRIBE FUNCTION +// +// [Snowflake Reference](https://docs.snowflake.com/en/sql-reference/ddl-udf.html#external-function-management) +func ExternalFunction(name, db, schema string) *ExternalFunctionBuilder { + return &ExternalFunctionBuilder{ + name: name, + db: db, + schema: schema, + returnNullAllowed: true, + } +} + +// Create returns the SQL statement required to create an external function +func (fb *ExternalFunctionBuilder) Create() string { + q := strings.Builder{} + q.WriteString(fmt.Sprintf(`CREATE EXTERNAL FUNCTION %v`, fb.QualifiedName())) + + q.WriteString(` (`) + args := []string{} + for _, arg := range fb.args { + args = append(args, fmt.Sprintf(`%v %v`, EscapeString(arg["name"]), EscapeString(arg["type"]))) + } + q.WriteString(strings.Join(args, ", ")) + q.WriteString(`)`) + + q.WriteString(` RETURNS ` + EscapeString(fb.returnType)) + + if !fb.returnNullAllowed { + q.WriteString(` NOT`) + } + q.WriteString(` NULL`) + + if fb.nullInputBehavior != "" { + q.WriteString(fmt.Sprintf(` %v`, EscapeString(fb.nullInputBehavior))) + } + + q.WriteString(fmt.Sprintf(` %v`, EscapeString(fb.returnBehavior))) + + if fb.comment != "" { + q.WriteString(fmt.Sprintf(` COMMENT = '%v'`, EscapeString(fb.comment))) + } + + q.WriteString(fmt.Sprintf(` API_INTEGRATION = '%v'`, EscapeString(fb.apiIntegration))) + + if len(fb.headers) > 0 { + q.WriteString(` HEADERS = (`) + headers := []string{} + for _, header := range fb.headers { + headers = append(headers, fmt.Sprintf(`'%v' = '%v'`, EscapeString(header["name"]), EscapeString(header["value"]))) + } + q.WriteString(strings.Join(headers, ", ")) + q.WriteString(`)`) + } + + if len(fb.contextHeaders) > 0 { + q.WriteString(` CONTEXT_HEADERS = (`) + q.WriteString(EscapeString(strings.Join(fb.contextHeaders, ", "))) + q.WriteString(`)`) + } + + if fb.maxBatchRows > 0 { + q.WriteString(fmt.Sprintf(` MAX_BATCH_ROWS = %d`, fb.maxBatchRows)) + } + + if fb.compression != "" { + q.WriteString(fmt.Sprintf(` COMPRESSION = '%v'`, EscapeString(fb.compression))) + } + + q.WriteString(fmt.Sprintf(` AS '%v'`, EscapeString(fb.urlOfProxyAndResource))) + + return q.String() +} + +// Drop returns the SQL query that will drop an external function. +func (fb *ExternalFunctionBuilder) Drop() string { + return fmt.Sprintf(`DROP FUNCTION %v`, fb.QualifiedNameWithArgTypes()) +} + +// Show returns the SQL query that will show an external function. +func (fb *ExternalFunctionBuilder) Show() string { + return fmt.Sprintf(`SHOW EXTERNAL FUNCTIONS LIKE '%v' IN SCHEMA "%v"."%v"`, fb.name, fb.db, fb.schema) +} + +// Describe returns the SQL query that will describe an external function. +func (fb *ExternalFunctionBuilder) Describe() string { + return fmt.Sprintf(`DESCRIBE FUNCTION %s`, fb.QualifiedNameWithArgTypes()) +} + +type externalFunction struct { + CreatedOn sql.NullString `db:"created_on"` + ExternalFunctionName sql.NullString `db:"name"` + DatabaseName sql.NullString `db:"catalog_name"` + SchemaName sql.NullString `db:"schema_name"` + Comment sql.NullString `db:"description"` + IsExternalFunction sql.NullString `db:"is_external_function"` + Language sql.NullString `db:"language"` +} + +// ScanExternalFunction +func ScanExternalFunction(row *sqlx.Row) (*externalFunction, error) { + f := &externalFunction{} + e := row.StructScan(f) + return f, e +} + +type externalFunctionDescription struct { + Property sql.NullString `db:"property"` + Value sql.NullString `db:"value"` +} + +// ScanExternalFunctionDescription +func ScanExternalFunctionDescription(rows *sqlx.Rows) ([]externalFunctionDescription, error) { + efds := []externalFunctionDescription{} + for rows.Next() { + efd := externalFunctionDescription{} + err := rows.StructScan(&efd) + if err != nil { + return nil, err + } + efds = append(efds, efd) + } + return efds, rows.Err() +} diff --git a/pkg/snowflake/external_function_test.go b/pkg/snowflake/external_function_test.go new file mode 100644 index 0000000000..b2c9f581e8 --- /dev/null +++ b/pkg/snowflake/external_function_test.go @@ -0,0 +1,42 @@ +package snowflake + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExternalFunctionCreate(t *testing.T) { + r := require.New(t) + s := ExternalFunction("test_function", "test_db", "test_schema") + s.WithArgs([]map[string]string{{"name": "data", "type": "varchar"}}) + s.WithArgTypes("varchar") + s.WithReturnType("varchar") + s.WithNullInputBehavior("RETURNS NULL ON NULL INPUT") + s.WithReturnBehavior("IMMUTABLE") + s.WithAPIIntegration("test_api_integration_01") + s.WithURLOfProxyAndResource("https://123456.execute-api.us-west-2.amazonaws.com/prod/test_func") + + r.Equal(s.QualifiedName(), `"test_db"."test_schema"."test_function"`) + r.Equal(s.QualifiedNameWithArgTypes(), `"test_db"."test_schema"."test_function" (varchar)`) + + r.Equal(s.Create(), `CREATE EXTERNAL FUNCTION "test_db"."test_schema"."test_function" (data varchar) RETURNS varchar NULL RETURNS NULL ON NULL INPUT IMMUTABLE API_INTEGRATION = 'test_api_integration_01' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/test_func'`) +} + +func TestExternalFunctionDrop(t *testing.T) { + r := require.New(t) + + // Without arg + s := ExternalFunction("test_function", "test_db", "test_schema") + r.Equal(s.Drop(), `DROP FUNCTION "test_db"."test_schema"."test_function" ()`) + + // With arg + s = ExternalFunction("test_function", "test_db", "test_schema").WithArgTypes("varchar") + r.Equal(s.Drop(), `DROP FUNCTION "test_db"."test_schema"."test_function" (varchar)`) +} + +func TestExternalFunctionShow(t *testing.T) { + r := require.New(t) + s := ExternalFunction("test_function", "test_db", "test_schema") + r.Equal(s.Show(), `SHOW EXTERNAL FUNCTIONS LIKE 'test_function' IN SCHEMA "test_db"."test_schema"`) +} diff --git a/pkg/snowflake/generic.go b/pkg/snowflake/generic.go index 6b0bb3a533..513bf1d353 100644 --- a/pkg/snowflake/generic.go +++ b/pkg/snowflake/generic.go @@ -11,6 +11,7 @@ import ( type EntityType string const ( + ApiIntegrationType EntityType = "API INTEGRATION" DatabaseType EntityType = "DATABASE" ManagedAccountType EntityType = "MANAGED ACCOUNT" ResourceMonitorType EntityType = "RESOURCE MONITOR" @@ -49,6 +50,7 @@ type SettingBuilder interface { SetBool(string, bool) SetInt(string, int) SetFloat(string, float64) + SetRaw(string) } type AlterPropertiesBuilder struct { @@ -59,6 +61,7 @@ type AlterPropertiesBuilder struct { boolProperties map[string]bool intProperties map[string]int floatProperties map[string]float64 + rawStatement string } func (b *Builder) Alter() *AlterPropertiesBuilder { @@ -93,10 +96,18 @@ func (ab *AlterPropertiesBuilder) SetFloat(key string, value float64) { ab.floatProperties[key] = value } +func (ab *AlterPropertiesBuilder) SetRaw(rawStatement string) { + var sb strings.Builder + sb.WriteString(fmt.Sprintf(`%s %s`, ab.rawStatement, rawStatement)) + ab.rawStatement = sb.String() +} + func (ab *AlterPropertiesBuilder) Statement() string { var sb strings.Builder sb.WriteString(fmt.Sprintf(`ALTER %s "%s" SET`, ab.entityType, ab.name)) // TODO handle error + sb.WriteString(ab.rawStatement) + for k, v := range ab.stringProperties { sb.WriteString(fmt.Sprintf(" %s='%s'", strings.ToUpper(k), EscapeString(v))) } @@ -128,6 +139,7 @@ type CreateBuilder struct { boolProperties map[string]bool intProperties map[string]int floatProperties map[string]float64 + rawStatement string } func (b *Builder) Create() *CreateBuilder { @@ -162,10 +174,18 @@ func (b *CreateBuilder) SetFloat(key string, value float64) { b.floatProperties[key] = value } +func (b *CreateBuilder) SetRaw(rawStatement string) { + var sb strings.Builder + sb.WriteString(fmt.Sprintf(`%s %s`, b.rawStatement, rawStatement)) + b.rawStatement = sb.String() +} + func (b *CreateBuilder) Statement() string { var sb strings.Builder sb.WriteString(fmt.Sprintf(`CREATE %s "%s"`, b.entityType, b.name)) // TODO handle error + sb.WriteString(b.rawStatement) + sortedStringProperties := make([]string, 0) for k := range b.stringProperties { sortedStringProperties = append(sortedStringProperties, k)