From 87b96220a4c58174b0fe5cdabd1c459c8b7fd119 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Tue, 9 Mar 2021 09:52:56 +0100 Subject: [PATCH 01/23] Add support for api integration and external function --- VERSION | 2 +- go.mod | 2 +- go.sum | 4 +- pkg/provider/provider.go | 1 + pkg/resources/api_integration.go | 340 ++++++++++++++++++++++++++ pkg/resources/api_integration_test.go | 85 +++++++ pkg/resources/external_function.go | 29 +++ pkg/resources/helpers_test.go | 8 + pkg/snowflake/api_integration.go | 38 +++ pkg/snowflake/api_integration_test.go | 27 ++ pkg/snowflake/exec.go | 1 + pkg/snowflake/generic.go | 1 + 12 files changed, 534 insertions(+), 4 deletions(-) create mode 100644 pkg/resources/api_integration.go create mode 100644 pkg/resources/api_integration_test.go create mode 100644 pkg/resources/external_function.go create mode 100644 pkg/snowflake/api_integration.go create mode 100644 pkg/snowflake/api_integration_test.go diff --git a/VERSION b/VERSION index c86a09df3c..286d5b09c8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.23.2 \ No newline at end of file +0.24.0 \ No newline at end of file diff --git a/go.mod b/go.mod index b8cc16f55d..806065e250 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.9.1 github.com/posener/complete v1.2.1 // indirect - github.com/snowflakedb/gosnowflake v1.4.0 + github.com/snowflakedb/gosnowflake v1.4.1 github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 golang.org/x/tools v0.1.0 diff --git a/go.sum b/go.sum index f61062a597..d55ff7c6ec 100644 --- a/go.sum +++ b/go.sum @@ -649,8 +649,8 @@ github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/snowflakedb/gosnowflake v1.4.0 h1:tt3nMrv+qQ4DZtTjasornB6nD8siIU5VpTI+qEyK4sc= -github.com/snowflakedb/gosnowflake v1.4.0/go.mod h1:6nfka9aTXkUNha1p1cjeeyjDvcyh7jfjp0l8kGpDBok= +github.com/snowflakedb/gosnowflake v1.4.1 h1:5Yu1Pi0wh6gyebzxtwmngd63VtUIps1HvrmLwxtpAEI= +github.com/snowflakedb/gosnowflake v1.4.1/go.mod h1:6nfka9aTXkUNha1p1cjeeyjDvcyh7jfjp0l8kGpDBok= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 8bd3d1a6d2..12f993e387 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -106,6 +106,7 @@ func GetGrantResources() resources.TerraformGrantResources { func getResources() map[string]*schema.Resource { others := map[string]*schema.Resource{ + "snowflake_api_integration": resources.APIIntegration(), "snowflake_database": resources.Database(), "snowflake_managed_account": resources.ManagedAccount(), "snowflake_masking_policy": resources.MaskingPolicy(), diff --git a/pkg/resources/api_integration.go b/pkg/resources/api_integration.go new file mode 100644 index 0000000000..06b547310f --- /dev/null +++ b/pkg/resources/api_integration.go @@ -0,0 +1,340 @@ +package resources + +import ( + "database/sql" + "fmt" + "log" + "strings" + + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/snowflake" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" +) + +var apiIntegrationSchema = map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "Specifies the name of the api integration. This name follows the rules for Object Identifiers. The name should be unique among api integrations in your account.", + }, + "api_provider": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringInSlice([]string{"aws_api_gateway", "aws_private_api_gateway", "azure_api_management"}, false), + Description: "Specifies the HTTPS proxy service type.", + }, + "api_aws_role_arn": { + Type: schema.TypeString, + Optional: true, + Default: "", + Description: "ARN of a cloud platform role.", + }, + // Computed. Info you get by issuing a 'DESCRIBE INTEGRATION ' command (API_AWS_IAM_USER_ARN) + "api_aws_iam_user_arn": { + Type: schema.TypeString, + Computed: true, + Description: "The Snowflake user that will attempt to assume the AWS role.", + }, + // Computed. Info you get by issuing a 'DESCRIBE INTEGRATION ' command (API_AWS_EXTERNAL_ID) + "api_aws_external_id": { + Type: schema.TypeString, + Computed: true, + Description: "The external ID that Snowflake will use when assuming the AWS role.", + }, + "azure_tenant_id": { + Type: schema.TypeString, + Optional: true, + Default: "", + Description: "Specifies the ID for your Office 365 tenant that all Azure API Management instances belong to.", + }, + "azure_ad_application_id": { + Type: schema.TypeString, + Optional: true, + Default: "", + Description: "The 'Application (client) id' of the Azure AD app for your remote service.", + }, + // Computed. Info you get by issuing a 'DESCRIBE INTEGRATION ' command (AZURE_MULTI_TENANT_APP_NAME) + "azure_multi_tenant_app_name": { + Type: schema.TypeString, + Computed: true, + }, + // Computed. Info you get by issuing a 'DESCRIBE INTEGRATION ' command (AZURE_CONSENT_URL) + "azure_consent_url": { + Type: schema.TypeString, + Computed: true, + }, + "api_allowed_prefixes": { + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Required: true, + Description: "Explicitly limits external functions that use the integration to reference one or more HTTPS proxy service endpoints and resources within those proxies.", + MinItems: 1, + }, + "api_blocked_prefixes": { + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + Description: "Lists the endpoints and resources in the HTTPS proxy service that are not allowed to be called from Snowflake.", + }, + "enabled": { + Type: schema.TypeBool, + Optional: true, + Default: true, + Description: "Specifies whether this api integration is enabled or disabled. If the api integration is disabled, any external function that relies on it will not work.", + }, + "comment": { + Type: schema.TypeString, + Optional: true, + Description: "A description of the api integration.", + }, + "created_on": { + Type: schema.TypeString, + Computed: true, + Description: "Date and time when the api integration was created.", + }, +} + +// APIIntegration returns a pointer to the resource representing an api integration +func APIIntegration() *schema.Resource { + return &schema.Resource{ + Create: CreateAPIIntegration, + Read: ReadAPIIntegration, + Update: UpdateAPIIntegration, + Delete: DeleteAPIIntegration, + + Schema: apiIntegrationSchema, + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + } +} + +// CreateAPIIntegration implements schema.CreateFunc +func CreateAPIIntegration(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + name := d.Get("name").(string) + + stmt := snowflake.ApiIntegration(name).Create() + + // Set required fields + stmt.SetBool(`ENABLED`, d.Get("enabled").(bool)) + + stmt.SetStringList("API_ALLOWED_PREFIXES", expandStringList(d.Get("api_allowed_prefixes").([]interface{}))) + + // Set optional fields + if v, ok := d.GetOk("comment"); ok { + stmt.SetString(`COMMENT`, v.(string)) + } + + if _, ok := d.GetOk("api_blocked_prefixes"); ok { + stmt.SetStringList("API_BLOCKED_PREFIXES", expandStringList(d.Get("api_blocked_prefixes").([]interface{}))) + } + + // Now, set the API provider + err := setAPIProviderSettings(d, stmt) + if err != nil { + return err + } + + err = snowflake.Exec(db, stmt.Statement()) + if err != nil { + return fmt.Errorf("error creating api integration: %w", err) + } + + d.SetId(name) + + return ReadAPIIntegration(d, meta) +} + +// ReadAPIIntegration implements schema.ReadFunc +func ReadAPIIntegration(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + id := d.Id() + + stmt := snowflake.ApiIntegration(d.Id()).Show() + row := snowflake.QueryRow(db, stmt) + + // Some properties can come from the SHOW INTEGRATION call + + s, err := snowflake.ScanApiIntegration(row) + if err != nil { + return fmt.Errorf("Could not show api integration: %w", err) + } + + // Note: category must be API or something is broken + if c := s.Category.String; c != "API" { + return fmt.Errorf("Expected %v to be an api integration, got %v", id, c) + } + + if err := d.Set("name", s.Name.String); err != nil { + return err + } + + if err := d.Set("created_on", s.CreatedOn.String); err != nil { + return err + } + + if err := d.Set("enabled", s.Enabled.Bool); err != nil { + return err + } + + // Some properties come from the DESCRIBE INTEGRATION call + // We need to grab them in a loop + var k, pType string + var v, unused interface{} + stmt = snowflake.ApiIntegration(d.Id()).Describe() + rows, err := db.Query(stmt) + if err != nil { + return fmt.Errorf("Could not describe api integration: %w", err) + } + defer rows.Close() + for rows.Next() { + if err := rows.Scan(&k, &pType, &v, &unused); err != nil { + return err + } + switch k { + case "ENABLED": + // We set this using the SHOW INTEGRATION call so let's ignore it here + case "API_ALLOWED_PREFIXES": + if err = d.Set("api_allowed_prefixes", strings.Split(v.(string), ",")); err != nil { + return err + } + case "API_BLOCKED_PREFIXES": + if val := v.(string); val != "" { + if err = d.Set("api_blocked_prefixes", strings.Split(val, ",")); err != nil { + return err + } + } + case "API_AWS_IAM_USER_ARN": + if err = d.Set("api_aws_iam_user_arn", v.(string)); err != nil { + return err + } + case "API_AWS_ROLE_ARN": + if err = d.Set("api_aws_role_arn", v.(string)); err != nil { + return err + } + case "API_AWS_EXTERNAL_ID": + if err = d.Set("api_aws_external_id", v.(string)); err != nil { + return err + } + case "AZURE_CONSENT_URL": + if err = d.Set("azure_consent_url", v.(string)); err != nil { + return err + } + case "AZURE_MULTI_TENANT_APP_NAME": + if err = d.Set("azure_multi_tenant_app_name", v.(string)); err != nil { + return err + } + default: + log.Printf("[WARN] unexpected property %v returned from Snowflake", k) + } + } + + return err +} + +// UpdateAPIIntegration implements schema.UpdateFunc +func UpdateAPIIntegration(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + id := d.Id() + + stmt := snowflake.ApiIntegration(id).Alter() + + // This is required in case the only change is to UNSET API_ALLOWED_PREFIXES. + var runSetStatement bool + + if d.HasChange("comment") { + runSetStatement = true + stmt.SetString("COMMENT", d.Get("comment").(string)) + } + + if d.HasChange("enabled") { + runSetStatement = true + stmt.SetBool(`ENABLED`, d.Get("enabled").(bool)) + } + + if d.HasChange("api_allowed_prefixes") { + runSetStatement = true + stmt.SetStringList("API_ALLOWED_PREFIXES", expandStringList(d.Get("api_allowed_prefixes").([]interface{}))) + } + + // We need to UNSET this if we remove all api blocked prefixes. I don't think + // @TODO move the SQL back to the snowflake package + if d.HasChange("api_blocked_prefixes") { + v := d.Get("api_blocked_prefixes").([]interface{}) + if len(v) == 0 { + err := snowflake.Exec(db, fmt.Sprintf(`ALTER API INTEGRATION %v UNSET API_BLOCKED_PREFIXES`, d.Id())) + if err != nil { + return fmt.Errorf("error unsetting api_blocked_prefixes: %w", err) + } + } else { + runSetStatement = true + stmt.SetStringList("API_BLOCKED_PREFIXES", expandStringList(v)) + } + } + + if d.HasChange("api_provider") { + runSetStatement = true + err := setAPIProviderSettings(d, stmt) + if err != nil { + return err + } + } else { + if d.HasChange("api_aws_role_arn") { + runSetStatement = true + stmt.SetString("API_AWS_ROLE_ARN", d.Get("api_aws_role_arn").(string)) + } + if d.HasChange("azure_tenant_id") { + runSetStatement = true + stmt.SetString("AZURE_TENANT_ID", d.Get("azure_tenant_id").(string)) + } + if d.HasChange("azure_ad_application_id") { + runSetStatement = true + stmt.SetString("AZURE_AD_APPLICATION_ID", d.Get("azure_ad_application_id").(string)) + } + } + + if runSetStatement { + if err := snowflake.Exec(db, stmt.Statement()); err != nil { + return fmt.Errorf("error updating api integration: %w", err) + } + } + + return ReadAPIIntegration(d, meta) +} + +// DeleteAPIIntegration implements schema.DeleteFunc +func DeleteAPIIntegration(d *schema.ResourceData, meta interface{}) error { + return DeleteResource("", snowflake.ApiIntegration)(d, meta) +} + +func setAPIProviderSettings(data *schema.ResourceData, stmt snowflake.SettingBuilder) error { + apiProvider := data.Get("api_provider").(string) + stmt.SetString("API_PROVIDER", apiProvider) + + switch apiProvider { + case "aws_api_gateway", "aws_private_api_gateway": + v, ok := data.GetOk("api_aws_role_arn") + if !ok { + return fmt.Errorf("If you use AWS api provider you must specify an api_aws_role_arn") + } + stmt.SetString(`API_AWS_ROLE_ARN`, v.(string)) + case "azure_api_management": + v, ok := data.GetOk("azure_tenant_id") + if !ok { + return fmt.Errorf("If you use the Azure api provider you must specify an azure_tenant_id") + } + stmt.SetString(`AZURE_TENANT_ID`, v.(string)) + + v, ok = data.GetOk("azure_ad_application_id") + if !ok { + return fmt.Errorf("If you use the Azure api provider you must specify an azure_ad_application_id") + } + stmt.SetString(`AZURE_AD_APPLICATION_ID`, v.(string)) + default: + return fmt.Errorf("Unexpected provider %v", apiProvider) + } + + return nil +} diff --git a/pkg/resources/api_integration_test.go b/pkg/resources/api_integration_test.go new file mode 100644 index 0000000000..b10094b807 --- /dev/null +++ b/pkg/resources/api_integration_test.go @@ -0,0 +1,85 @@ +package resources_test + +import ( + "database/sql" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/provider" + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/resources" + . "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/testhelpers" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/require" +) + +func TestAPIIntegration(t *testing.T) { + r := require.New(t) + err := resources.APIIntegration().InternalValidate(provider.Provider().Schema, true) + r.NoError(err) +} + +func TestAPIIntegrationCreate(t *testing.T) { + r := require.New(t) + + in := map[string]interface{}{ + "name": "test_api_integration", + "comment": "great comment", + "api_allowed_prefixes": []interface{}{"https://123456.execute-api.us-west-2.amazonaws.com/prod/"}, + "api_provider": "aws_api_gateway", + "api_aws_role_arn": "we-should-probably-validate-this-string", + } + d := schema.TestResourceDataRaw(t, resources.APIIntegration().Schema, in) + r.NotNil(d) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `^CREATE API INTEGRATION "test_api_integration" API_AWS_ROLE_ARN='we-should-probably-validate-this-string' API_PROVIDER='aws_api_gateway' COMMENT='great comment' API_ALLOWED_PREFIXES=\('https://123456.execute-api.us-west-2.amazonaws.com/prod/'\) ENABLED=true$`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + expectReadAPIIntegration(mock) + + err := resources.CreateAPIIntegration(d, db) + r.NoError(err) + }) +} + +func TestAPIIntegrationRead(t *testing.T) { + r := require.New(t) + + d := apiIntegration(t, "test_api_integration", map[string]interface{}{"name": "test_api_integration"}) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + expectReadAPIIntegration(mock) + + err := resources.ReadAPIIntegration(d, db) + r.NoError(err) + }) +} + +func TestAPIIntegrationDelete(t *testing.T) { + r := require.New(t) + + d := apiIntegration(t, "drop_it", map[string]interface{}{"name": "drop_it"}) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec(`DROP API INTEGRATION "drop_it"`).WillReturnResult(sqlmock.NewResult(1, 1)) + err := resources.DeleteAPIIntegration(d, db) + r.NoError(err) + }) +} + +func expectReadAPIIntegration(mock sqlmock.Sqlmock) { + showRows := sqlmock.NewRows([]string{ + "name", "type", "category", "enabled", "created_on"}, + ).AddRow("test_api_integration", "EXTERNAL_API", "API", true, "now") + mock.ExpectQuery(`^SHOW API INTEGRATIONS LIKE 'test_api_integration'$`).WillReturnRows(showRows) + + descRows := sqlmock.NewRows([]string{ + "property", "property_type", "property_value", "property_default", + }).AddRow("ENABLED", "Boolean", true, false). + AddRow("API_ALLOWED_PREFIXES", "List", "https://123456.execute-api.us-west-2.amazonaws.com/prod/,https://123456.execute-api.us-west-2.amazonaws.com/staging/", nil). + AddRow("API_AWS_IAM_USER_ARN", "String", "arn:aws:iam::000000000000:/user/test", nil). + AddRow("API_AWS_ROLE_ARN", "String", "arn:aws:iam::000000000001:/role/test", nil). + AddRow("API_AWS_EXTERNAL_ID", "String", "AGreatExternalID", nil) + + mock.ExpectQuery(`DESCRIBE API INTEGRATION "test_api_integration"$`).WillReturnRows(descRows) +} diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go new file mode 100644 index 0000000000..8402ddfcf0 --- /dev/null +++ b/pkg/resources/external_function.go @@ -0,0 +1,29 @@ +package resources + +import "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + +var externalFunctionSchema = map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "Specifies the identifier for the function. The identifier can contain the schema name and database name, as well as the function name. The function's signature (name and argument data types) must be unique within the schema.", + }, + "schema": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The schema in which to create the table.", + }, + "database": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The database in which to create the table.", + }, + "comment": { + Type: schema.TypeString, + Optional: true, + Description: "A description of the external function.", + }, +} diff --git a/pkg/resources/helpers_test.go b/pkg/resources/helpers_test.go index 33cd2c59bf..9cd4003e2c 100644 --- a/pkg/resources/helpers_test.go +++ b/pkg/resources/helpers_test.go @@ -176,6 +176,14 @@ func roleGrants(t *testing.T, id string, params map[string]interface{}) *schema. return d } +func apiIntegration(t *testing.T, id string, params map[string]interface{}) *schema.ResourceData { + r := require.New(t) + d := schema.TestResourceDataRaw(t, resources.APIIntegration().Schema, params) + r.NotNil(d) + d.SetId(id) + return d +} + func storageIntegration(t *testing.T, id string, params map[string]interface{}) *schema.ResourceData { r := require.New(t) d := schema.TestResourceDataRaw(t, resources.StorageIntegration().Schema, params) diff --git a/pkg/snowflake/api_integration.go b/pkg/snowflake/api_integration.go new file mode 100644 index 0000000000..70ca8cd810 --- /dev/null +++ b/pkg/snowflake/api_integration.go @@ -0,0 +1,38 @@ +package snowflake + +import ( + "database/sql" + + "github.com/jmoiron/sqlx" +) + +// ApiIntegration returns a pointer to a Builder that abstracts the DDL operations for an api integration. +// +// Supported DDL operations are: +// - CREATE API INTEGRATION +// - ALTER API INTEGRATION +// - DROP INTEGRATION +// - SHOW INTEGRATIONS +// - DESCRIBE INTEGRATION +// +// [Snowflake Reference](https://docs.snowflake.com/en/sql-reference/ddl-user-security.html#api-integrations) +func ApiIntegration(name string) *Builder { + return &Builder{ + entityType: ApiIntegrationType, + name: name, + } +} + +type apiIntegration struct { + Name sql.NullString `db:"name"` + Category sql.NullString `db:"category"` + IntegrationType sql.NullString `db:"type"` + CreatedOn sql.NullString `db:"created_on"` + Enabled sql.NullBool `db:"enabled"` +} + +func ScanApiIntegration(row *sqlx.Row) (*apiIntegration, error) { + r := &apiIntegration{} + err := row.StructScan(r) + return r, err +} diff --git a/pkg/snowflake/api_integration_test.go b/pkg/snowflake/api_integration_test.go new file mode 100644 index 0000000000..c8dfa0932b --- /dev/null +++ b/pkg/snowflake/api_integration_test.go @@ -0,0 +1,27 @@ +package snowflake_test + +import ( + "testing" + + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/snowflake" + "github.com/stretchr/testify/require" +) + +func TestApiIntegration(t *testing.T) { + r := require.New(t) + builder := snowflake.ApiIntegration("aws_api") + r.NotNil(builder) + + q := builder.Show() + r.Equal("SHOW API INTEGRATIONS LIKE 'aws_api'", q) + + c := builder.Create() + + c.SetString(`api_provider`, `aws_private_api_gateway`) + c.SetString(`api_aws_role_arn`, "arn:aws:iam::xxxx:role/snowflake-execute-externalfunc-privendpoint-role") + c.SetStringList(`api_allowed_prefixes`, []string{"https://123456.execute-api.us-west-2.amazonaws.com/prod/", "https://123456.execute-api.us-west-2.amazonaws.com/test/"}) + c.SetBool(`enabled`, true) + q = c.Statement() + + r.Equal(`CREATE API INTEGRATION "aws_api" API_AWS_ROLE_ARN='arn:aws:iam::xxxx:role/snowflake-execute-externalfunc-privendpoint-role' API_PROVIDER='aws_private_api_gateway' API_ALLOWED_PREFIXES=('https://123456.execute-api.us-west-2.amazonaws.com/prod/', 'https://123456.execute-api.us-west-2.amazonaws.com/test/') ENABLED=true`, q) +} diff --git a/pkg/snowflake/exec.go b/pkg/snowflake/exec.go index 6f676366c9..fd5ae17ee7 100644 --- a/pkg/snowflake/exec.go +++ b/pkg/snowflake/exec.go @@ -44,6 +44,7 @@ func QueryRow(db *sql.DB, stmt string) *sqlx.Row { // [DB.Unsafe](https://godoc.org/github.com/jmoiron/sqlx#DB.Unsafe) so that we can scan to structs // without worrying about newly introduced columns func Query(db *sql.DB, stmt string) (*sqlx.Rows, error) { + log.Print("[DEBUG] query stmt ", stmt) sdb := sqlx.NewDb(db, "snowflake").Unsafe() return sdb.Queryx(stmt) } diff --git a/pkg/snowflake/generic.go b/pkg/snowflake/generic.go index 6b0bb3a533..f478437f9f 100644 --- a/pkg/snowflake/generic.go +++ b/pkg/snowflake/generic.go @@ -11,6 +11,7 @@ import ( type EntityType string const ( + ApiIntegrationType EntityType = "API INTEGRATION" DatabaseType EntityType = "DATABASE" ManagedAccountType EntityType = "MANAGED ACCOUNT" ResourceMonitorType EntityType = "RESOURCE MONITOR" From 51c3c4db5e1d693445de0c11ca9ef6b301a7d172 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Tue, 9 Mar 2021 09:56:34 +0100 Subject: [PATCH 02/23] Upd --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 777aad2c32..d55ff7c6ec 100644 --- a/go.sum +++ b/go.sum @@ -649,8 +649,6 @@ github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/snowflakedb/gosnowflake v1.4.0 h1:tt3nMrv+qQ4DZtTjasornB6nD8siIU5VpTI+qEyK4sc= -github.com/snowflakedb/gosnowflake v1.4.0/go.mod h1:6nfka9aTXkUNha1p1cjeeyjDvcyh7jfjp0l8kGpDBok= github.com/snowflakedb/gosnowflake v1.4.1 h1:5Yu1Pi0wh6gyebzxtwmngd63VtUIps1HvrmLwxtpAEI= github.com/snowflakedb/gosnowflake v1.4.1/go.mod h1:6nfka9aTXkUNha1p1cjeeyjDvcyh7jfjp0l8kGpDBok= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= From c66e9328644533c852c5c9c1bd4256b7c2a8f57f Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Tue, 9 Mar 2021 19:20:57 +0100 Subject: [PATCH 03/23] Add External Function --- go.sum | 1 + pkg/provider/provider.go | 1 + pkg/resources/api_integration.go | 8 +- pkg/resources/external_function.go | 332 +++++++++++++++++++++++- pkg/resources/external_function_test.go | 74 ++++++ pkg/resources/helpers_test.go | 8 + pkg/snowflake/external_function.go | 266 +++++++++++++++++++ pkg/snowflake/external_function_test.go | 42 +++ 8 files changed, 724 insertions(+), 8 deletions(-) create mode 100644 pkg/resources/external_function_test.go create mode 100644 pkg/snowflake/external_function.go create mode 100644 pkg/snowflake/external_function_test.go diff --git a/go.sum b/go.sum index d55ff7c6ec..5acd58f0e7 100644 --- a/go.sum +++ b/go.sum @@ -669,6 +669,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 12f993e387..fefbf2e482 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -108,6 +108,7 @@ func getResources() map[string]*schema.Resource { others := map[string]*schema.Resource{ "snowflake_api_integration": resources.APIIntegration(), "snowflake_database": resources.Database(), + "snowflake_external_function": resources.ExternalFunction(), "snowflake_managed_account": resources.ManagedAccount(), "snowflake_masking_policy": resources.MaskingPolicy(), "snowflake_materialized_view": resources.MaterializedView(), diff --git a/pkg/resources/api_integration.go b/pkg/resources/api_integration.go index 06b547310f..8f8ffb946f 100644 --- a/pkg/resources/api_integration.go +++ b/pkg/resources/api_integration.go @@ -16,7 +16,7 @@ var apiIntegrationSchema = map[string]*schema.Schema{ Type: schema.TypeString, Required: true, ForceNew: true, - Description: "Specifies the name of the api integration. This name follows the rules for Object Identifiers. The name should be unique among api integrations in your account.", + Description: "Specifies the name of the API integration. This name follows the rules for Object Identifiers. The name should be unique among api integrations in your account.", }, "api_provider": { Type: schema.TypeString, @@ -81,17 +81,17 @@ var apiIntegrationSchema = map[string]*schema.Schema{ Type: schema.TypeBool, Optional: true, Default: true, - Description: "Specifies whether this api integration is enabled or disabled. If the api integration is disabled, any external function that relies on it will not work.", + Description: "Specifies whether this API integration is enabled or disabled. If the API integration is disabled, any external function that relies on it will not work.", }, "comment": { Type: schema.TypeString, Optional: true, - Description: "A description of the api integration.", + Description: "A description of the API integration.", }, "created_on": { Type: schema.TypeString, Computed: true, - Description: "Date and time when the api integration was created.", + Description: "Date and time when the API integration was created.", }, } diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go index 8402ddfcf0..6509ef6d6d 100644 --- a/pkg/resources/external_function.go +++ b/pkg/resources/external_function.go @@ -1,29 +1,353 @@ package resources -import "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +import ( + "bytes" + "database/sql" + "encoding/csv" + "fmt" + "strings" + + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/snowflake" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" + "github.com/pkg/errors" +) + +const ( + externalFunctionIDDelimiter = '|' +) var externalFunctionSchema = map[string]*schema.Schema{ "name": { Type: schema.TypeString, Required: true, ForceNew: true, - Description: "Specifies the identifier for the function. The identifier can contain the schema name and database name, as well as the function name. The function's signature (name and argument data types) must be unique within the schema.", + Description: "Specifies the identifier for the external function. The identifier can contain the schema name and database name, as well as the function name. The function's signature (name and argument data types) must be unique within the schema.", }, "schema": { Type: schema.TypeString, Required: true, ForceNew: true, - Description: "The schema in which to create the table.", + Description: "The schema in which to create the external function.", }, "database": { Type: schema.TypeString, Required: true, ForceNew: true, - Description: "The database in which to create the table.", + Description: "The database in which to create the external function.", + }, + "args": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Description: "Specifies the arguments/inputs for the external function. These should correspond to the arguments that the remote service expects.", + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + Description: "Argument name", + }, + "type": { + Type: schema.TypeString, + Required: true, + Description: "Argument type, e.g. VARCHAR", + }, + }, + }, + }, + "null_input_behavior": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validation.StringInSlice([]string{"CALLED ON NULL INPUT", "RETURNS NULL ON NULL INPUT", "STRICT"}, false), + Description: "Specifies the behavior of the external function when called with null inputs.", + }, + "return_type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "Specifies the data type returned by the external function.", + }, + "return_null_allowed": { + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + Description: "Indicates whether the function can return NULL values or must return only NON-NULL values.", + }, + "return_behavior": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validation.StringInSlice([]string{"VOLATILE", "IMMUTABLE"}, false), + Description: "Specifies the behavior of the function when returning results", + }, + "api_integration": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The name of the API integration object that should be used to authenticate the call to the proxy service.", + }, + "headers": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Description: "Allows users to specify key-value metadata that is sent with every request as HTTP headers.", + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Required: true, + Description: "Header name", + }, + "value": { + Type: schema.TypeString, + Required: true, + Description: "Header value", + }, + }, + }, + }, + "context_headers": { + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + ForceNew: true, + Description: "Binds Snowflake context function results to HTTP headers.", + }, + "max_batch_rows": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + Description: "This specifies the maximum number of rows in each batch sent to the proxy service.", + }, + "compression": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validation.StringInSlice([]string{"NONE", "AUTO", "GZIP", "DEFLATE"}, false), + Description: "If specified, the JSON payload is compressed when sent from Snowflake to the proxy service, and when sent back from the proxy service to Snowflake.", + }, + "url_of_proxy_and_resource": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "This is the invocation URL of the proxy service and resource through which Snowflake calls the remote service.", }, "comment": { Type: schema.TypeString, Optional: true, + ForceNew: true, Description: "A description of the external function.", }, + "created_on": { + Type: schema.TypeString, + Computed: true, + Description: "Date and time when the external function was created.", + }, +} + +// ExternalFunction returns a pointer to the resource representing an external function +func ExternalFunction() *schema.Resource { + return &schema.Resource{ + Create: CreateExternalFunction, + Read: ReadExternalFunction, + Delete: DeleteExternalFunction, + + Schema: externalFunctionSchema, + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + } +} + +type externalFunctionID struct { + DatabaseName string + SchemaName string + ExternalFunctionName string +} + +func (si *externalFunctionID) String() (string, error) { + var buf bytes.Buffer + csvWriter := csv.NewWriter(&buf) + csvWriter.Comma = externalFunctionIDDelimiter + err := csvWriter.WriteAll([][]string{{si.DatabaseName, si.SchemaName, si.ExternalFunctionName}}) + if err != nil { + return "", err + } + + return strings.TrimSpace(buf.String()), nil +} + +func externalFunctionIDFromString(stringID string) (*externalFunctionID, error) { + reader := csv.NewReader(strings.NewReader(stringID)) + reader.Comma = externalFunctionIDDelimiter + lines, err := reader.ReadAll() + if err != nil { + return nil, fmt.Errorf("Not CSV compatible") + } + + if len(lines) != 1 { + return nil, fmt.Errorf("1 line at a time") + } + if len(lines[0]) != 3 { + return nil, fmt.Errorf("3 fields allowed") + } + + return &externalFunctionID{ + DatabaseName: lines[0][0], + SchemaName: lines[0][1], + ExternalFunctionName: lines[0][2], + }, nil +} + +// CreateExternalFunction implements schema.CreateFunc +func CreateExternalFunction(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + database := d.Get("database").(string) + dbSchema := d.Get("schema").(string) + name := d.Get("name").(string) + + builder := snowflake.ExternalFunction(name, database, dbSchema) + builder.WithReturnType(d.Get("return_type").(string)) + builder.WithReturnBehavior(d.Get("return_behavior").(string)) + builder.WithAPIIntegration(d.Get("api_integration").(string)) + builder.WithURLOfProxyAndResource(d.Get("url_of_proxy_and_resource").(string)) + + // Set optionals + if _, ok := d.GetOk("args"); ok { + args := []map[string]string{} + for _, arg := range d.Get("args").([]interface{}) { + argDef := map[string]string{} + for key, val := range arg.(map[string]interface{}) { + argDef[key] = val.(string) + } + args = append(args, argDef) + } + + builder.WithArgs(args) + } + + if v, ok := d.GetOk("return_null_allowed"); ok { + builder.WithReturnNullAllowed(v.(bool)) + } + + if v, ok := d.GetOk("null_input_behavior"); ok { + builder.WithNullInputBehavior(v.(string)) + } + + if v, ok := d.GetOk("comment"); ok { + builder.WithComment(v.(string)) + } + + if _, ok := d.GetOk("headers"); ok { + headers := []map[string]string{} + for _, header := range d.Get("headers").([]interface{}) { + headerDef := map[string]string{} + for key, val := range header.(map[string]interface{}) { + headerDef[key] = val.(string) + } + headers = append(headers, headerDef) + } + + builder.WithHeaders(headers) + } + + if v, ok := d.GetOk("context_headers"); ok { + contextHeaders := expandStringList(v.(*schema.Set).List()) + builder.WithContextHeaders(contextHeaders) + } + + if v, ok := d.GetOk("max_batch_rows"); ok { + builder.WithMaxBatchRows(v.(int)) + } + + if v, ok := d.GetOk("compression"); ok { + builder.WithNullInputBehavior(v.(string)) + } + + stmt := builder.Create() + err := snowflake.Exec(db, stmt) + if err != nil { + return errors.Wrapf(err, "error creating external function %v", name) + } + + externalFunctionID := &externalFunctionID{ + DatabaseName: database, + SchemaName: dbSchema, + ExternalFunctionName: name, + } + dataIDInput, err := externalFunctionID.String() + if err != nil { + return err + } + d.SetId(dataIDInput) + + return ReadExternalFunction(d, meta) +} + +// ReadExternalFunction implements schema.ReadFunc +func ReadExternalFunction(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + externalFunctionID, err := externalFunctionIDFromString(d.Id()) + if err != nil { + return err + } + + dbName := externalFunctionID.DatabaseName + dbSchema := externalFunctionID.SchemaName + name := externalFunctionID.ExternalFunctionName + + stmt := snowflake.ExternalFunction(name, dbName, dbSchema).Show() + row := snowflake.QueryRow(db, stmt) + externalFunction, err := snowflake.ScanExternalFunction(row) + if err != nil { + return err + } + + // Note: 'language' must be EXTERNAL and 'is_external_function' set to Y + if externalFunction.Language.String != "EXTERNAL" || externalFunction.IsExternalFunction.String != "Y" { + return fmt.Errorf("Expected %v to be an external function, got 'language=%v' and 'is_external_function=%v'", d.Id(), externalFunction.Language.String, externalFunction.IsExternalFunction.String) + } + + if err := d.Set("name", externalFunction.ExternalFunctionName.String); err != nil { + return err + } + + if err := d.Set("schema", externalFunction.SchemaName.String); err != nil { + return err + } + + if err := d.Set("database", externalFunction.DatabaseName.String); err != nil { + return err + } + + if err := d.Set("created_on", externalFunction.CreatedOn.String); err != nil { + return err + } + + return nil +} + +// DeleteExternalFunction implements schema.DeleteFunc +func DeleteExternalFunction(d *schema.ResourceData, meta interface{}) error { + db := meta.(*sql.DB) + externalFunctionID, err := externalFunctionIDFromString(d.Id()) + if err != nil { + return err + } + + dbName := externalFunctionID.DatabaseName + dbSchema := externalFunctionID.SchemaName + name := externalFunctionID.ExternalFunctionName + + q := snowflake.ExternalFunction(name, dbName, dbSchema).Drop() + + err = snowflake.Exec(db, q) + if err != nil { + return errors.Wrapf(err, "error deleting external function %v", d.Id()) + } + + d.SetId("") + return nil } diff --git a/pkg/resources/external_function_test.go b/pkg/resources/external_function_test.go new file mode 100644 index 0000000000..79fc3cb259 --- /dev/null +++ b/pkg/resources/external_function_test.go @@ -0,0 +1,74 @@ +package resources_test + +import ( + "database/sql" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/provider" + "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/resources" + . "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/testhelpers" + "github.com/stretchr/testify/require" +) + +func TestExternalFunction(t *testing.T) { + r := require.New(t) + err := resources.ExternalFunction().InternalValidate(provider.Provider().Schema, true) + r.NoError(err) +} + +func TestExternalFunctionCreate(t *testing.T) { + r := require.New(t) + + in := map[string]interface{}{ + "name": "my_test_function", + "database": "database_name", + "schema": "schema_name", + "return_type": "varchar", + "return_behavior": "IMMUTABLE", + "api_integration": "test_api_integration_01", + "url_of_proxy_and_resource": "https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function", + } + d := externalFunction(t, "database_name|schema_name|my_test_function", in) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec(`CREATE EXTERNAL FUNCTION "database_name"."schema_name"."my_test_function" \(\) RETURNS varchar NULL IMMUTABLE API_INTEGRATION = 'test_api_integration_01' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function'`).WillReturnResult(sqlmock.NewResult(1, 1)) + + expectExternalFunctionRead(mock) + err := resources.CreateExternalFunction(d, db) + r.NoError(err) + r.Equal("my_test_function", d.Get("name").(string)) + }) +} + +func expectExternalFunctionRead(mock sqlmock.Sqlmock) { + rows := sqlmock.NewRows([]string{"created_on", "name", "schema_name", "is_builtin", "is_aggregate", "is_ansi", "min_num_arguments", "max_num_arguments", "arguments", "description", "catalog_name", "is_table_function", "valid_for_clustering", "is_secure", "is_external_function", "language"}).AddRow("now", "my_test_function", "schema_name", "N", "N", "N", "0", "0", "NULL", "mock comment", "database_name", "N", "N", "N", "Y", "EXTERNAL") + mock.ExpectQuery(`SHOW EXTERNAL FUNCTIONS LIKE 'my_test_function' IN SCHEMA "database_name"."schema_name"`).WillReturnRows(rows) +} + +func TestExternalFunctionRead(t *testing.T) { + r := require.New(t) + + d := externalFunction(t, "database_name|schema_name|my_test_function", map[string]interface{}{"name": "my_test_function", "comment": "mock comment"}) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + expectExternalFunctionRead(mock) + + err := resources.ReadExternalFunction(d, db) + r.NoError(err) + r.Equal("my_test_function", d.Get("name").(string)) + r.Equal("mock comment", d.Get("comment").(string)) + }) +} + +func TestExternalFunctionDelete(t *testing.T) { + r := require.New(t) + + d := externalFunction(t, "database_name|schema_name|drop_it", map[string]interface{}{"name": "drop_it"}) + + WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec(`DROP FUNCTION "database_name"."schema_name"."drop_it"`).WillReturnResult(sqlmock.NewResult(1, 1)) + err := resources.DeleteExternalFunction(d, db) + r.NoError(err) + }) +} diff --git a/pkg/resources/helpers_test.go b/pkg/resources/helpers_test.go index 9cd4003e2c..fa5155b133 100644 --- a/pkg/resources/helpers_test.go +++ b/pkg/resources/helpers_test.go @@ -184,6 +184,14 @@ func apiIntegration(t *testing.T, id string, params map[string]interface{}) *sch return d } +func externalFunction(t *testing.T, id string, params map[string]interface{}) *schema.ResourceData { + r := require.New(t) + d := schema.TestResourceDataRaw(t, resources.ExternalFunction().Schema, params) + r.NotNil(d) + d.SetId(id) + return d +} + func storageIntegration(t *testing.T, id string, params map[string]interface{}) *schema.ResourceData { r := require.New(t) d := schema.TestResourceDataRaw(t, resources.StorageIntegration().Schema, params) diff --git a/pkg/snowflake/external_function.go b/pkg/snowflake/external_function.go new file mode 100644 index 0000000000..c143faf460 --- /dev/null +++ b/pkg/snowflake/external_function.go @@ -0,0 +1,266 @@ +package snowflake + +import ( + "database/sql" + "fmt" + "strings" + + "github.com/jmoiron/sqlx" +) + +// ExternalFunctionBuilder abstracts the creation of SQL queries for a Snowflake schema +type ExternalFunctionBuilder struct { + name string + db string + schema string + args []map[string]string + nullInputBehavior string + returnType string + returnNullAllowed bool + returnBehavior string + apiIntegration string + headers []map[string]string + contextHeaders []string + maxBatchRows int + compression string + urlOfProxyAndResource string + comment string +} + +// QualifiedName prepends the db and schema if set and escapes everything nicely +func (fb *ExternalFunctionBuilder) QualifiedName() string { + var n strings.Builder + + if fb.db != "" && fb.schema != "" { + n.WriteString(fmt.Sprintf(`"%v"."%v".`, fb.db, fb.schema)) + } + + if fb.db != "" && fb.schema == "" { + n.WriteString(fmt.Sprintf(`"%v"..`, fb.db)) + } + + if fb.db == "" && fb.schema != "" { + n.WriteString(fmt.Sprintf(`"%v".`, fb.schema)) + } + + n.WriteString(fmt.Sprintf(`"%v"`, fb.name)) + + return n.String() +} + +// QualifiedNameWithArgTypes appends all args' types to the qualified name. This is required to invoke 'DESC FUNCTION' and 'DROP FUNCTION' commands. +func (fb *ExternalFunctionBuilder) QualifiedNameWithArgTypes() string { + q := strings.Builder{} + + q.WriteString(fmt.Sprintf(`%v (`, fb.QualifiedName())) + argTypes := []string{} + for _, arg := range fb.args { + argTypes = append(argTypes, fmt.Sprintf(`%v`, EscapeString(arg["type"]))) + } + q.WriteString(strings.Join(argTypes, ", ")) + q.WriteString(fmt.Sprintf(`)`)) + + return q.String() +} + +// WithArgs sets the args on the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithArgs(args []map[string]string) *ExternalFunctionBuilder { + fb.args = args + return fb +} + +// WithNullInputBehavior adds a nullInputBehavior to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithNullInputBehavior(nullInputBehavior string) *ExternalFunctionBuilder { + fb.nullInputBehavior = nullInputBehavior + return fb +} + +// WithReturnType adds a returnType to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithReturnType(returnType string) *ExternalFunctionBuilder { + fb.returnType = returnType + return fb +} + +// WithReturnNullAllowed adds a returnNullAllowed to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithReturnNullAllowed(returnNullAllowed bool) *ExternalFunctionBuilder { + fb.returnNullAllowed = returnNullAllowed + return fb +} + +// WithReturnBehavior adds a returnBehavior to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithReturnBehavior(returnBehavior string) *ExternalFunctionBuilder { + fb.returnBehavior = returnBehavior + return fb +} + +// WithAPIIntegration adds a apiIntegration to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithAPIIntegration(apiIntegration string) *ExternalFunctionBuilder { + fb.apiIntegration = apiIntegration + return fb +} + +// WithHeaders sets the headers on the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithHeaders(headers []map[string]string) *ExternalFunctionBuilder { + fb.headers = headers + return fb +} + +// WithContextHeaders sets the context headers on the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithContextHeaders(contextHeaders []string) *ExternalFunctionBuilder { + fb.contextHeaders = contextHeaders + return fb +} + +// WithMaxBatchRows adds a maxBatchRows to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithMaxBatchRows(maxBatchRows int) *ExternalFunctionBuilder { + fb.maxBatchRows = maxBatchRows + return fb +} + +// WithCompression adds a compression to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithCompression(compression string) *ExternalFunctionBuilder { + fb.compression = compression + return fb +} + +// WithURLOfProxyAndResource adds a urlOfProxyAndResource to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithURLOfProxyAndResource(urlOfProxyAndResource string) *ExternalFunctionBuilder { + fb.urlOfProxyAndResource = urlOfProxyAndResource + return fb +} + +// WithComment adds a comment to the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithComment(c string) *ExternalFunctionBuilder { + fb.comment = c + return fb +} + +// ExternalFunction returns a pointer to a Builder that abstracts the DDL operations for an external function. +// +// Supported DDL operations are: +// - CREATE EXTERNAL FUNCTION +// - ALTER EXTERNAL FUNCTION +// - DROP FUNCTION +// - SHOW EXTERNAL FUNCTIONS +// - DESCRIBE FUNCTION +// +// [Snowflake Reference](https://docs.snowflake.com/en/sql-reference/ddl-udf.html#external-function-management) +func ExternalFunction(name, db, schema string) *ExternalFunctionBuilder { + return &ExternalFunctionBuilder{ + name: name, + db: db, + schema: schema, + returnNullAllowed: true, + } +} + +// Create returns the SQL statement required to create an external function +func (fb *ExternalFunctionBuilder) Create() string { + q := strings.Builder{} + q.WriteString(fmt.Sprintf(`CREATE EXTERNAL FUNCTION %v`, fb.QualifiedName())) + + q.WriteString(fmt.Sprintf(` (`)) + args := []string{} + for _, arg := range fb.args { + args = append(args, fmt.Sprintf(`%v %v`, EscapeString(arg["name"]), EscapeString(arg["type"]))) + } + q.WriteString(strings.Join(args, ", ")) + q.WriteString(fmt.Sprintf(`)`)) + + q.WriteString(` RETURNS ` + EscapeString(fb.returnType)) + + if !fb.returnNullAllowed { + q.WriteString(` NOT`) + } + q.WriteString(` NULL`) + + if fb.nullInputBehavior != "" { + q.WriteString(fmt.Sprintf(` %v`, EscapeString(fb.nullInputBehavior))) + } + + q.WriteString(fmt.Sprintf(` %v`, EscapeString(fb.returnBehavior))) + + if fb.comment != "" { + q.WriteString(fmt.Sprintf(` COMMENT = '%v'`, EscapeString(fb.comment))) + } + + q.WriteString(fmt.Sprintf(` API_INTEGRATION = '%v'`, EscapeString(fb.apiIntegration))) + + if len(fb.headers) > 0 { + q.WriteString(` HEADERS = (`) + headers := []string{} + for _, header := range fb.headers { + headers = append(headers, fmt.Sprintf(`'%v' = '%v'`, EscapeString(header["name"]), EscapeString(header["value"]))) + } + q.WriteString(strings.Join(headers, ", ")) + q.WriteString(fmt.Sprintf(`)`)) + } + + if len(fb.contextHeaders) > 0 { + q.WriteString(` CONTEXT_HEADERS = (`) + q.WriteString(EscapeString(strings.Join(fb.contextHeaders, ", "))) + q.WriteString(fmt.Sprintf(`)`)) + } + + if fb.maxBatchRows > 0 { + q.WriteString(fmt.Sprintf(` MAX_BATCH_ROWS = %d`, fb.maxBatchRows)) + } + + if fb.compression != "" { + q.WriteString(fmt.Sprintf(` COMPRESSION = '%v'`, EscapeString(fb.compression))) + } + + q.WriteString(fmt.Sprintf(` AS '%v'`, EscapeString(fb.urlOfProxyAndResource))) + + return q.String() +} + +// Drop returns the SQL query that will drop an external function. +func (fb *ExternalFunctionBuilder) Drop() string { + return fmt.Sprintf(`DROP FUNCTION %v`, fb.QualifiedNameWithArgTypes()) +} + +// Show returns the SQL query that will show an external function. +func (fb *ExternalFunctionBuilder) Show() string { + return fmt.Sprintf(`SHOW EXTERNAL FUNCTIONS LIKE '%v' IN SCHEMA "%v"."%v"`, fb.name, fb.db, fb.schema) +} + +// Describe returns the SQL query that will describe an external function. +func (fb *ExternalFunctionBuilder) Describe() string { + return fmt.Sprintf(`DESCRIBE FUNCTION %s`, fb.QualifiedNameWithArgTypes()) +} + +type externalFunction struct { + CreatedOn sql.NullString `db:"created_on"` + ExternalFunctionName sql.NullString `db:"name"` + DatabaseName sql.NullString `db:"catalog_name"` + SchemaName sql.NullString `db:"schema_name"` + IsExternalFunction sql.NullString `db:"is_external_function"` + Language sql.NullString `db:"language"` +} + +// ScanExternalFunction +func ScanExternalFunction(row *sqlx.Row) (*externalFunction, error) { + f := &externalFunction{} + e := row.StructScan(f) + return f, e +} + +type externalFunctionDescription struct { + Property sql.NullString `db:"property"` + Value sql.NullString `db:"value"` +} + +// ScanExternalFunctionDescription +func ScanExternalFunctionDescription(rows *sqlx.Rows) ([]externalFunctionDescription, error) { + efds := []externalFunctionDescription{} + for rows.Next() { + efd := externalFunctionDescription{} + err := rows.StructScan(&efd) + if err != nil { + return nil, err + } + efds = append(efds, efd) + } + return efds, rows.Err() +} diff --git a/pkg/snowflake/external_function_test.go b/pkg/snowflake/external_function_test.go new file mode 100644 index 0000000000..1ab86b30df --- /dev/null +++ b/pkg/snowflake/external_function_test.go @@ -0,0 +1,42 @@ +package snowflake + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExternalFunctionCreate(t *testing.T) { + r := require.New(t) + s := ExternalFunction("test_function", "test_db", "test_schema") + s.WithArgs([]map[string]string{{"name": "data", "type": "varchar"}}) + s.WithReturnType("varchar") + s.WithNullInputBehavior("RETURNS NULL ON NULL INPUT") + s.WithReturnBehavior("IMMUTABLE") + s.WithAPIIntegration("test_api_integration_01") + s.WithURLOfProxyAndResource("https://123456.execute-api.us-west-2.amazonaws.com/prod/test_func") + + r.Equal(s.QualifiedName(), `"test_db"."test_schema"."test_function"`) + r.Equal(s.QualifiedNameWithArgTypes(), `"test_db"."test_schema"."test_function" (varchar)`) + + r.Equal(s.Create(), `CREATE EXTERNAL FUNCTION "test_db"."test_schema"."test_function" (data varchar) RETURNS varchar NULL RETURNS NULL ON NULL INPUT IMMUTABLE API_INTEGRATION = 'test_api_integration_01' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/test_func'`) +} + +func TestExternalFunctionDrop(t *testing.T) { + r := require.New(t) + + // Without arg + s := ExternalFunction("test_function", "test_db", "test_schema") + r.Equal(s.Drop(), `DROP FUNCTION "test_db"."test_schema"."test_function" ()`) + + // With arg + s = ExternalFunction("test_function", "test_db", "test_schema") + s.WithArgs([]map[string]string{{"name": "data", "type": "varchar"}}) + r.Equal(s.Drop(), `DROP FUNCTION "test_db"."test_schema"."test_function" (varchar)`) +} + +func TestExternalFunctionShow(t *testing.T) { + r := require.New(t) + s := ExternalFunction("test_function", "test_db", "test_schema") + r.Equal(s.Show(), `SHOW EXTERNAL FUNCTIONS LIKE 'test_function' IN SCHEMA "test_db"."test_schema"`) +} From b0f9349d02e5539af5e3c0e70ecf82c259350d39 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Tue, 9 Mar 2021 19:33:20 +0100 Subject: [PATCH 04/23] Generate docs --- docs/resources/api_integration.md | 42 ++++++++++++++++++++ docs/resources/external_function.md | 61 +++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 docs/resources/api_integration.md create mode 100644 docs/resources/external_function.md diff --git a/docs/resources/api_integration.md b/docs/resources/api_integration.md new file mode 100644 index 0000000000..e5659649aa --- /dev/null +++ b/docs/resources/api_integration.md @@ -0,0 +1,42 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "snowflake_api_integration Resource - terraform-provider-snowflake" +subcategory: "" +description: |- + +--- + +# snowflake_api_integration (Resource) + + + + + + +## Schema + +### Required + +- **api_allowed_prefixes** (List of String) Explicitly limits external functions that use the integration to reference one or more HTTPS proxy service endpoints and resources within those proxies. +- **api_provider** (String) Specifies the HTTPS proxy service type. +- **name** (String) Specifies the name of the API integration. This name follows the rules for Object Identifiers. The name should be unique among api integrations in your account. + +### Optional + +- **api_aws_role_arn** (String) ARN of a cloud platform role. +- **api_blocked_prefixes** (List of String) Lists the endpoints and resources in the HTTPS proxy service that are not allowed to be called from Snowflake. +- **azure_ad_application_id** (String) The 'Application (client) id' of the Azure AD app for your remote service. +- **azure_tenant_id** (String) Specifies the ID for your Office 365 tenant that all Azure API Management instances belong to. +- **comment** (String) A description of the API integration. +- **enabled** (Boolean) Specifies whether this API integration is enabled or disabled. If the API integration is disabled, any external function that relies on it will not work. +- **id** (String) The ID of this resource. + +### Read-Only + +- **api_aws_external_id** (String) The external ID that Snowflake will use when assuming the AWS role. +- **api_aws_iam_user_arn** (String) The Snowflake user that will attempt to assume the AWS role. +- **azure_consent_url** (String) +- **azure_multi_tenant_app_name** (String) +- **created_on** (String) Date and time when the API integration was created. + + diff --git a/docs/resources/external_function.md b/docs/resources/external_function.md new file mode 100644 index 0000000000..368b6a8c22 --- /dev/null +++ b/docs/resources/external_function.md @@ -0,0 +1,61 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "snowflake_external_function Resource - terraform-provider-snowflake" +subcategory: "" +description: |- + +--- + +# snowflake_external_function (Resource) + + + + + + +## Schema + +### Required + +- **api_integration** (String) The name of the API integration object that should be used to authenticate the call to the proxy service. +- **database** (String) The database in which to create the external function. +- **name** (String) Specifies the identifier for the external function. The identifier can contain the schema name and database name, as well as the function name. The function's signature (name and argument data types) must be unique within the schema. +- **return_behavior** (String) Specifies the behavior of the function when returning results +- **return_type** (String) Specifies the data type returned by the external function. +- **schema** (String) The schema in which to create the external function. +- **url_of_proxy_and_resource** (String) This is the invocation URL of the proxy service and resource through which Snowflake calls the remote service. + +### Optional + +- **args** (Block List) Specifies the arguments/inputs for the external function. These should correspond to the arguments that the remote service expects. (see [below for nested schema](#nestedblock--args)) +- **comment** (String) A description of the external function. +- **compression** (String) If specified, the JSON payload is compressed when sent from Snowflake to the proxy service, and when sent back from the proxy service to Snowflake. +- **context_headers** (List of String) Binds Snowflake context function results to HTTP headers. +- **headers** (Block List) Allows users to specify key-value metadata that is sent with every request as HTTP headers. (see [below for nested schema](#nestedblock--headers)) +- **id** (String) The ID of this resource. +- **max_batch_rows** (Number) This specifies the maximum number of rows in each batch sent to the proxy service. +- **null_input_behavior** (String) Specifies the behavior of the external function when called with null inputs. +- **return_null_allowed** (Boolean) Indicates whether the function can return NULL values or must return only NON-NULL values. + +### Read-Only + +- **created_on** (String) Date and time when the external function was created. + + +### Nested Schema for `args` + +Required: + +- **name** (String) Argument name +- **type** (String) Argument type, e.g. VARCHAR + + + +### Nested Schema for `headers` + +Required: + +- **name** (String) Header name +- **value** (String) Header value + + From 1e82617d7aceca5009e3552bcb488543be23f740 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Tue, 9 Mar 2021 19:39:11 +0100 Subject: [PATCH 05/23] Linting --- pkg/snowflake/external_function.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/snowflake/external_function.go b/pkg/snowflake/external_function.go index c143faf460..f84b4b5764 100644 --- a/pkg/snowflake/external_function.go +++ b/pkg/snowflake/external_function.go @@ -58,7 +58,7 @@ func (fb *ExternalFunctionBuilder) QualifiedNameWithArgTypes() string { argTypes = append(argTypes, fmt.Sprintf(`%v`, EscapeString(arg["type"]))) } q.WriteString(strings.Join(argTypes, ", ")) - q.WriteString(fmt.Sprintf(`)`)) + q.WriteString(`)`) return q.String() } @@ -159,13 +159,13 @@ func (fb *ExternalFunctionBuilder) Create() string { q := strings.Builder{} q.WriteString(fmt.Sprintf(`CREATE EXTERNAL FUNCTION %v`, fb.QualifiedName())) - q.WriteString(fmt.Sprintf(` (`)) + q.WriteString(` (`) args := []string{} for _, arg := range fb.args { args = append(args, fmt.Sprintf(`%v %v`, EscapeString(arg["name"]), EscapeString(arg["type"]))) } q.WriteString(strings.Join(args, ", ")) - q.WriteString(fmt.Sprintf(`)`)) + q.WriteString(`)`) q.WriteString(` RETURNS ` + EscapeString(fb.returnType)) @@ -193,13 +193,13 @@ func (fb *ExternalFunctionBuilder) Create() string { headers = append(headers, fmt.Sprintf(`'%v' = '%v'`, EscapeString(header["name"]), EscapeString(header["value"]))) } q.WriteString(strings.Join(headers, ", ")) - q.WriteString(fmt.Sprintf(`)`)) + q.WriteString(`)`) } if len(fb.contextHeaders) > 0 { q.WriteString(` CONTEXT_HEADERS = (`) q.WriteString(EscapeString(strings.Join(fb.contextHeaders, ", "))) - q.WriteString(fmt.Sprintf(`)`)) + q.WriteString(`)`) } if fb.maxBatchRows > 0 { From 8ab525a336c45f17fcf6e2bcb78ae81b4ca51e2e Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Wed, 10 Mar 2021 10:34:10 +0100 Subject: [PATCH 06/23] Add acceptance tests --- .../api_integration_acceptance_test.go | 75 +++++++++++++++++++ .../external_function_acceptance_test.go | 72 ++++++++++++++++++ pkg/resources/external_function_test.go | 5 +- 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 pkg/resources/api_integration_acceptance_test.go create mode 100644 pkg/resources/external_function_acceptance_test.go diff --git a/pkg/resources/api_integration_acceptance_test.go b/pkg/resources/api_integration_acceptance_test.go new file mode 100644 index 0000000000..a33f9bfa85 --- /dev/null +++ b/pkg/resources/api_integration_acceptance_test.go @@ -0,0 +1,75 @@ +package resources_test + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestAccApiIntegration(t *testing.T) { + if _, ok := os.LookupEnv("SKIP_API_INTEGRATION_TESTS"); ok { + t.Skip("Skipping TestAccApiIntegration") + } + + apiIntName := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) + apiIntName2 := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) + + resource.Test(t, resource.TestCase{ + Providers: providers(), + Steps: []resource.TestStep{ + { + Config: apiIntegrationConfig_aws(apiIntName, []string{"https://123456.execute-api.us-west-2.amazonaws.com/prod/"}), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("snowflake_api_integration.test_aws_int", "name", apiIntName), + resource.TestCheckResourceAttr("snowflake_api_integration.test_aws_int", "api_provider", "aws_api_gateway"), + resource.TestCheckResourceAttr("snowflake_api_integration.test_aws_int", "comment", "Terraform acceptance test"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_aws_int", "created_on"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_aws_int", "api_aws_iam_user_arn"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_aws_int", "api_aws_external_id"), + ), + }, + { + Config: apiIntegrationConfig_azure(apiIntName2, []string{"https://apim-hello-world.azure-api.net/"}), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("snowflake_api_integration.test_azure_int", "name", apiIntName2), + resource.TestCheckResourceAttr("snowflake_api_integration.test_azure_int", "api_provider", "azure_api_management"), + resource.TestCheckResourceAttr("snowflake_api_integration.test_azure_int", "comment", "Terraform acceptance test"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_azure_int", "created_on"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_azure_int", "azure_multi_tenant_app_name"), + resource.TestCheckResourceAttrSet("snowflake_api_integration.test_azure_int", "azure_consent_url"), + ), + }, + }, + }) +} + +func apiIntegrationConfig_aws(name string, prefixes []string) string { + return fmt.Sprintf(` + resource "snowflake_api_integration" "test_aws_int" { + name = "%s" + api_provider = aws_api_gateway + api_aws_role_arn = 'arn:aws:iam::000000000001:/role/test' + api_allowed_prefixes = %q + enabled = true + comment = "Terraform acceptance test" + } + `, name, prefixes) +} + +func apiIntegrationConfig_azure(name string, prefixes []string) string { + return fmt.Sprintf(` + resource "snowflake_api_integration" "test_azure_int" { + name = "%s" + api_provider = azure_api_management + azure_tenant_id = '123456' + azure_ad_application_id = '7890' + api_allowed_prefixes = %q + enabled = true + comment = "Terraform acceptance test" + } + `, name, prefixes) +} diff --git a/pkg/resources/external_function_acceptance_test.go b/pkg/resources/external_function_acceptance_test.go new file mode 100644 index 0000000000..511efae30f --- /dev/null +++ b/pkg/resources/external_function_acceptance_test.go @@ -0,0 +1,72 @@ +package resources_test + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestAccExternalFunction(t *testing.T) { + if _, ok := os.LookupEnv("SKIP_EXTERNAL_FUNCTION_TESTS"); ok { + t.Skip("Skipping TestAccExternalFunction") + } + + accName := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha)) + + resource.Test(t, resource.TestCase{ + Providers: providers(), + Steps: []resource.TestStep{ + { + Config: externalFunctionConfig(accName, []string{"https://123456.execute-api.us-west-2.amazonaws.com/prod/"}, "https://123456.execute-api.us-west-2.amazonaws.com/prod/test_func"), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("snowflake_external_function.test_func", "name", accName), + resource.TestCheckResourceAttr("snowflake_external_function.test_func", "comment", "Terraform acceptance test"), + resource.TestCheckResourceAttrSet("snowflake_external_function.test_func", "created_on"), + ), + }, + }, + }) +} + +func externalFunctionConfig(name string, prefixes []string, url string) string { + return fmt.Sprintf(` + resource "snowflake_database" "test_database" { + name = "%s" + comment = "Terraform acceptance test" + } + + resource "snowflake_schema" "test_schema" { + name = "%s" + database = snowflake_database.test_database.name + comment = "Terraform acceptance test" + } + + resource "snowflake_api_integration" "test_api_int" { + name = "%s" + api_provider = aws_api_gateway + api_aws_role_arn = 'arn:aws:iam::000000000001:/role/test' + api_allowed_prefixes = %q + enabled = true + comment = "Terraform acceptance test" + } + + resource "snowflake_external_function" "test_func" { + name = "%s" + database = snowflake_database.test_database.name + schema = snowflake_schema.test_schema.name + args { + name = "data" + type = "varchar" + } + comment = "Terraform acceptance test" + return_type = "varchar" + return_behavior = "IMMUTABLE" + api_integration = snowflake_api_integration.test_api_int.name + url_of_proxy_and_resource = "%s" + } + `, name, name, name, prefixes, name, url) +} diff --git a/pkg/resources/external_function_test.go b/pkg/resources/external_function_test.go index 79fc3cb259..3dbf66cd43 100644 --- a/pkg/resources/external_function_test.go +++ b/pkg/resources/external_function_test.go @@ -24,6 +24,7 @@ func TestExternalFunctionCreate(t *testing.T) { "name": "my_test_function", "database": "database_name", "schema": "schema_name", + "args": []interface{}{map[string]interface{}{"name": "data", "type": "varchar"}}, "return_type": "varchar", "return_behavior": "IMMUTABLE", "api_integration": "test_api_integration_01", @@ -32,7 +33,7 @@ func TestExternalFunctionCreate(t *testing.T) { d := externalFunction(t, "database_name|schema_name|my_test_function", in) WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec(`CREATE EXTERNAL FUNCTION "database_name"."schema_name"."my_test_function" \(\) RETURNS varchar NULL IMMUTABLE API_INTEGRATION = 'test_api_integration_01' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function'`).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(`CREATE EXTERNAL FUNCTION "database_name"."schema_name"."my_test_function" \(data varchar\) RETURNS varchar NULL IMMUTABLE API_INTEGRATION = 'test_api_integration_01' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function'`).WillReturnResult(sqlmock.NewResult(1, 1)) expectExternalFunctionRead(mock) err := resources.CreateExternalFunction(d, db) @@ -42,7 +43,7 @@ func TestExternalFunctionCreate(t *testing.T) { } func expectExternalFunctionRead(mock sqlmock.Sqlmock) { - rows := sqlmock.NewRows([]string{"created_on", "name", "schema_name", "is_builtin", "is_aggregate", "is_ansi", "min_num_arguments", "max_num_arguments", "arguments", "description", "catalog_name", "is_table_function", "valid_for_clustering", "is_secure", "is_external_function", "language"}).AddRow("now", "my_test_function", "schema_name", "N", "N", "N", "0", "0", "NULL", "mock comment", "database_name", "N", "N", "N", "Y", "EXTERNAL") + rows := sqlmock.NewRows([]string{"created_on", "name", "schema_name", "is_builtin", "is_aggregate", "is_ansi", "min_num_arguments", "max_num_arguments", "arguments", "description", "catalog_name", "is_table_function", "valid_for_clustering", "is_secure", "is_external_function", "language"}).AddRow("now", "my_test_function", "schema_name", "N", "N", "N", "1", "1", "MY_TEST_FUNCTION(VARCHAR) RETURN VARCHAR", "mock comment", "database_name", "N", "N", "N", "Y", "EXTERNAL") mock.ExpectQuery(`SHOW EXTERNAL FUNCTIONS LIKE 'my_test_function' IN SCHEMA "database_name"."schema_name"`).WillReturnRows(rows) } From 06a6f6a880282953b8003a80cf7aa2ed24d688b3 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Wed, 10 Mar 2021 10:40:09 +0100 Subject: [PATCH 07/23] Add acceptance tests --- pkg/resources/api_integration_acceptance_test.go | 2 +- pkg/resources/external_function_acceptance_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/resources/api_integration_acceptance_test.go b/pkg/resources/api_integration_acceptance_test.go index a33f9bfa85..e7ec68d246 100644 --- a/pkg/resources/api_integration_acceptance_test.go +++ b/pkg/resources/api_integration_acceptance_test.go @@ -10,7 +10,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" ) -func TestAccApiIntegration(t *testing.T) { +func TestAcc_ApiIntegration(t *testing.T) { if _, ok := os.LookupEnv("SKIP_API_INTEGRATION_TESTS"); ok { t.Skip("Skipping TestAccApiIntegration") } diff --git a/pkg/resources/external_function_acceptance_test.go b/pkg/resources/external_function_acceptance_test.go index 511efae30f..04e06e97c7 100644 --- a/pkg/resources/external_function_acceptance_test.go +++ b/pkg/resources/external_function_acceptance_test.go @@ -10,7 +10,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" ) -func TestAccExternalFunction(t *testing.T) { +func TestAcc_ExternalFunction(t *testing.T) { if _, ok := os.LookupEnv("SKIP_EXTERNAL_FUNCTION_TESTS"); ok { t.Skip("Skipping TestAccExternalFunction") } From 486429a4496e734a16fd6cd180a7c48a8edeff2e Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Wed, 10 Mar 2021 10:52:43 +0100 Subject: [PATCH 08/23] Add acceptance tests --- pkg/resources/api_integration_acceptance_test.go | 6 +++--- pkg/resources/external_function_acceptance_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/resources/api_integration_acceptance_test.go b/pkg/resources/api_integration_acceptance_test.go index e7ec68d246..1dbc9f3c36 100644 --- a/pkg/resources/api_integration_acceptance_test.go +++ b/pkg/resources/api_integration_acceptance_test.go @@ -52,7 +52,7 @@ func apiIntegrationConfig_aws(name string, prefixes []string) string { resource "snowflake_api_integration" "test_aws_int" { name = "%s" api_provider = aws_api_gateway - api_aws_role_arn = 'arn:aws:iam::000000000001:/role/test' + api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" api_allowed_prefixes = %q enabled = true comment = "Terraform acceptance test" @@ -65,8 +65,8 @@ func apiIntegrationConfig_azure(name string, prefixes []string) string { resource "snowflake_api_integration" "test_azure_int" { name = "%s" api_provider = azure_api_management - azure_tenant_id = '123456' - azure_ad_application_id = '7890' + azure_tenant_id = "123456" + azure_ad_application_id = "7890" api_allowed_prefixes = %q enabled = true comment = "Terraform acceptance test" diff --git a/pkg/resources/external_function_acceptance_test.go b/pkg/resources/external_function_acceptance_test.go index 04e06e97c7..395d9b7d85 100644 --- a/pkg/resources/external_function_acceptance_test.go +++ b/pkg/resources/external_function_acceptance_test.go @@ -48,7 +48,7 @@ func externalFunctionConfig(name string, prefixes []string, url string) string { resource "snowflake_api_integration" "test_api_int" { name = "%s" api_provider = aws_api_gateway - api_aws_role_arn = 'arn:aws:iam::000000000001:/role/test' + api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" api_allowed_prefixes = %q enabled = true comment = "Terraform acceptance test" From 717d573f856447d958cd42ec76d53f343db3a282 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Wed, 10 Mar 2021 13:14:56 +0100 Subject: [PATCH 09/23] Add acceptance tests --- .../api_integration_acceptance_test.go | 4 +- pkg/resources/external_function.go | 41 +++++++++++++------ .../external_function_acceptance_test.go | 15 ++++++- pkg/resources/external_function_test.go | 8 ++-- pkg/snowflake/external_function.go | 18 ++++---- pkg/snowflake/external_function_test.go | 4 +- 6 files changed, 58 insertions(+), 32 deletions(-) diff --git a/pkg/resources/api_integration_acceptance_test.go b/pkg/resources/api_integration_acceptance_test.go index 1dbc9f3c36..1db1a0aa0c 100644 --- a/pkg/resources/api_integration_acceptance_test.go +++ b/pkg/resources/api_integration_acceptance_test.go @@ -51,7 +51,7 @@ func apiIntegrationConfig_aws(name string, prefixes []string) string { return fmt.Sprintf(` resource "snowflake_api_integration" "test_aws_int" { name = "%s" - api_provider = aws_api_gateway + api_provider = "aws_api_gateway" api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" api_allowed_prefixes = %q enabled = true @@ -64,7 +64,7 @@ func apiIntegrationConfig_azure(name string, prefixes []string) string { return fmt.Sprintf(` resource "snowflake_api_integration" "test_azure_int" { name = "%s" - api_provider = azure_api_management + api_provider = "azure_api_management" azure_tenant_id = "123456" azure_ad_application_id = "7890" api_allowed_prefixes = %q diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go index 6509ef6d6d..0de9ce93e8 100644 --- a/pkg/resources/external_function.go +++ b/pkg/resources/external_function.go @@ -162,16 +162,17 @@ func ExternalFunction() *schema.Resource { } type externalFunctionID struct { - DatabaseName string - SchemaName string - ExternalFunctionName string + DatabaseName string + SchemaName string + ExternalFunctionName string + ExternalFunctionArgTypes string } func (si *externalFunctionID) String() (string, error) { var buf bytes.Buffer csvWriter := csv.NewWriter(&buf) csvWriter.Comma = externalFunctionIDDelimiter - err := csvWriter.WriteAll([][]string{{si.DatabaseName, si.SchemaName, si.ExternalFunctionName}}) + err := csvWriter.WriteAll([][]string{{si.DatabaseName, si.SchemaName, si.ExternalFunctionName, si.ExternalFunctionArgTypes}}) if err != nil { return "", err } @@ -190,14 +191,15 @@ func externalFunctionIDFromString(stringID string) (*externalFunctionID, error) if len(lines) != 1 { return nil, fmt.Errorf("1 line at a time") } - if len(lines[0]) != 3 { - return nil, fmt.Errorf("3 fields allowed") + if len(lines[0]) != 4 { + return nil, fmt.Errorf("4 fields allowed") } return &externalFunctionID{ - DatabaseName: lines[0][0], - SchemaName: lines[0][1], - ExternalFunctionName: lines[0][2], + DatabaseName: lines[0][0], + SchemaName: lines[0][1], + ExternalFunctionName: lines[0][2], + ExternalFunctionArgTypes: lines[0][3], }, nil } @@ -207,6 +209,7 @@ func CreateExternalFunction(d *schema.ResourceData, meta interface{}) error { database := d.Get("database").(string) dbSchema := d.Get("schema").(string) name := d.Get("name").(string) + var argtypes string builder := snowflake.ExternalFunction(name, database, dbSchema) builder.WithReturnType(d.Get("return_type").(string)) @@ -216,16 +219,26 @@ func CreateExternalFunction(d *schema.ResourceData, meta interface{}) error { // Set optionals if _, ok := d.GetOk("args"); ok { + var types []string args := []map[string]string{} for _, arg := range d.Get("args").([]interface{}) { argDef := map[string]string{} for key, val := range arg.(map[string]interface{}) { argDef[key] = val.(string) + + if key == "type" { + // Also store arg types in distinct array as list of types is required for some Snowflake commands (DESC, DROP) + types = append(types, argDef[key]) + } } args = append(args, argDef) } + // Use '-' as a separator between arg types as the result will end in the Terraform resource id + argtypes = strings.Join(types, "-") + builder.WithArgs(args) + builder.WithArgTypes(argtypes) } if v, ok := d.GetOk("return_null_allowed"); ok { @@ -273,9 +286,10 @@ func CreateExternalFunction(d *schema.ResourceData, meta interface{}) error { } externalFunctionID := &externalFunctionID{ - DatabaseName: database, - SchemaName: dbSchema, - ExternalFunctionName: name, + DatabaseName: database, + SchemaName: dbSchema, + ExternalFunctionName: name, + ExternalFunctionArgTypes: argtypes, } dataIDInput, err := externalFunctionID.String() if err != nil { @@ -340,8 +354,9 @@ func DeleteExternalFunction(d *schema.ResourceData, meta interface{}) error { dbName := externalFunctionID.DatabaseName dbSchema := externalFunctionID.SchemaName name := externalFunctionID.ExternalFunctionName + argtypes := externalFunctionID.ExternalFunctionArgTypes - q := snowflake.ExternalFunction(name, dbName, dbSchema).Drop() + q := snowflake.ExternalFunction(name, dbName, dbSchema).WithArgTypes(argtypes).Drop() err = snowflake.Exec(db, q) if err != nil { diff --git a/pkg/resources/external_function_acceptance_test.go b/pkg/resources/external_function_acceptance_test.go index 395d9b7d85..005f970e9c 100644 --- a/pkg/resources/external_function_acceptance_test.go +++ b/pkg/resources/external_function_acceptance_test.go @@ -47,7 +47,7 @@ func externalFunctionConfig(name string, prefixes []string, url string) string { resource "snowflake_api_integration" "test_api_int" { name = "%s" - api_provider = aws_api_gateway + api_provider = "aws_api_gateway" api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" api_allowed_prefixes = %q enabled = true @@ -68,5 +68,16 @@ func externalFunctionConfig(name string, prefixes []string, url string) string { api_integration = snowflake_api_integration.test_api_int.name url_of_proxy_and_resource = "%s" } - `, name, name, name, prefixes, name, url) + + resource "snowflake_external_function" "test_func_2" { + name = "%s" + database = snowflake_database.test_database.name + schema = snowflake_schema.test_schema.name + comment = "Terraform acceptance test" + return_type = "varchar" + return_behavior = "IMMUTABLE" + api_integration = snowflake_api_integration.test_api_int.name + url_of_proxy_and_resource = "%s" + } + `, name, name, name, prefixes, name, url, name, url+"_2") } diff --git a/pkg/resources/external_function_test.go b/pkg/resources/external_function_test.go index 3dbf66cd43..4488a817f7 100644 --- a/pkg/resources/external_function_test.go +++ b/pkg/resources/external_function_test.go @@ -30,7 +30,7 @@ func TestExternalFunctionCreate(t *testing.T) { "api_integration": "test_api_integration_01", "url_of_proxy_and_resource": "https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function", } - d := externalFunction(t, "database_name|schema_name|my_test_function", in) + d := externalFunction(t, "database_name|schema_name|my_test_function|varchar", in) WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { mock.ExpectExec(`CREATE EXTERNAL FUNCTION "database_name"."schema_name"."my_test_function" \(data varchar\) RETURNS varchar NULL IMMUTABLE API_INTEGRATION = 'test_api_integration_01' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function'`).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -50,7 +50,7 @@ func expectExternalFunctionRead(mock sqlmock.Sqlmock) { func TestExternalFunctionRead(t *testing.T) { r := require.New(t) - d := externalFunction(t, "database_name|schema_name|my_test_function", map[string]interface{}{"name": "my_test_function", "comment": "mock comment"}) + d := externalFunction(t, "database_name|schema_name|my_test_function|", map[string]interface{}{"name": "my_test_function", "comment": "mock comment"}) WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { expectExternalFunctionRead(mock) @@ -65,10 +65,10 @@ func TestExternalFunctionRead(t *testing.T) { func TestExternalFunctionDelete(t *testing.T) { r := require.New(t) - d := externalFunction(t, "database_name|schema_name|drop_it", map[string]interface{}{"name": "drop_it"}) + d := externalFunction(t, "database_name|schema_name|drop_it|", map[string]interface{}{"name": "drop_it"}) WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec(`DROP FUNCTION "database_name"."schema_name"."drop_it"`).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(`DROP FUNCTION "database_name"."schema_name"."drop_it" ()`).WillReturnResult(sqlmock.NewResult(1, 1)) err := resources.DeleteExternalFunction(d, db) r.NoError(err) }) diff --git a/pkg/snowflake/external_function.go b/pkg/snowflake/external_function.go index f84b4b5764..a9a3adb1c2 100644 --- a/pkg/snowflake/external_function.go +++ b/pkg/snowflake/external_function.go @@ -14,6 +14,7 @@ type ExternalFunctionBuilder struct { db string schema string args []map[string]string + argtypes string // only used for 'DESC FUNCTION' & 'DROP FUNCTION' commands as of today (list of args types is required) nullInputBehavior string returnType string returnNullAllowed bool @@ -51,15 +52,7 @@ func (fb *ExternalFunctionBuilder) QualifiedName() string { // QualifiedNameWithArgTypes appends all args' types to the qualified name. This is required to invoke 'DESC FUNCTION' and 'DROP FUNCTION' commands. func (fb *ExternalFunctionBuilder) QualifiedNameWithArgTypes() string { q := strings.Builder{} - - q.WriteString(fmt.Sprintf(`%v (`, fb.QualifiedName())) - argTypes := []string{} - for _, arg := range fb.args { - argTypes = append(argTypes, fmt.Sprintf(`%v`, EscapeString(arg["type"]))) - } - q.WriteString(strings.Join(argTypes, ", ")) - q.WriteString(`)`) - + q.WriteString(fmt.Sprintf(`%v (%s)`, fb.QualifiedName(), fb.argtypes)) return q.String() } @@ -69,6 +62,13 @@ func (fb *ExternalFunctionBuilder) WithArgs(args []map[string]string) *ExternalF return fb } +// WithArgTypes sets the args on the ExternalFunctionBuilder +func (fb *ExternalFunctionBuilder) WithArgTypes(argtypes string) *ExternalFunctionBuilder { + argtypeslist := strings.ReplaceAll(argtypes, "-", ", ") + fb.argtypes = argtypeslist + return fb +} + // WithNullInputBehavior adds a nullInputBehavior to the ExternalFunctionBuilder func (fb *ExternalFunctionBuilder) WithNullInputBehavior(nullInputBehavior string) *ExternalFunctionBuilder { fb.nullInputBehavior = nullInputBehavior diff --git a/pkg/snowflake/external_function_test.go b/pkg/snowflake/external_function_test.go index 1ab86b30df..b2c9f581e8 100644 --- a/pkg/snowflake/external_function_test.go +++ b/pkg/snowflake/external_function_test.go @@ -10,6 +10,7 @@ func TestExternalFunctionCreate(t *testing.T) { r := require.New(t) s := ExternalFunction("test_function", "test_db", "test_schema") s.WithArgs([]map[string]string{{"name": "data", "type": "varchar"}}) + s.WithArgTypes("varchar") s.WithReturnType("varchar") s.WithNullInputBehavior("RETURNS NULL ON NULL INPUT") s.WithReturnBehavior("IMMUTABLE") @@ -30,8 +31,7 @@ func TestExternalFunctionDrop(t *testing.T) { r.Equal(s.Drop(), `DROP FUNCTION "test_db"."test_schema"."test_function" ()`) // With arg - s = ExternalFunction("test_function", "test_db", "test_schema") - s.WithArgs([]map[string]string{{"name": "data", "type": "varchar"}}) + s = ExternalFunction("test_function", "test_db", "test_schema").WithArgTypes("varchar") r.Equal(s.Drop(), `DROP FUNCTION "test_db"."test_schema"."test_function" (varchar)`) } From 402188136d4b71d5571c9840fb86572ac4ef96cd Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Wed, 10 Mar 2021 13:57:09 +0100 Subject: [PATCH 10/23] Upd examples & docs --- docs/resources/api_integration.md | 18 ++++++++++++- docs/resources/external_function.md | 25 ++++++++++++++++++- .../snowflake_api_integration/import.sh | 1 + .../snowflake_api_integration/resource.tf | 7 ++++++ .../snowflake_external_function/import.sh | 2 ++ .../snowflake_external_function/resource.tf | 13 ++++++++++ 6 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 examples/resources/snowflake_api_integration/import.sh create mode 100644 examples/resources/snowflake_api_integration/resource.tf create mode 100644 examples/resources/snowflake_external_function/import.sh create mode 100644 examples/resources/snowflake_external_function/resource.tf diff --git a/docs/resources/api_integration.md b/docs/resources/api_integration.md index e5659649aa..aab2e05533 100644 --- a/docs/resources/api_integration.md +++ b/docs/resources/api_integration.md @@ -10,7 +10,17 @@ description: |- - +## Example Usage + +```terraform +resource "snowflake_api_integration" "api_integration" { + name = "aws_integration" + api_provider = "aws_api_gateway" + api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" + api_allowed_prefixes = "https://123456.execute-api.us-west-2.amazonaws.com/prod/" + enabled = true +} +``` ## Schema @@ -39,4 +49,10 @@ description: |- - **azure_multi_tenant_app_name** (String) - **created_on** (String) Date and time when the API integration was created. +## Import + +Import is supported using the following syntax: +```shell +terraform import snowflake_api_integration.example name +``` diff --git a/docs/resources/external_function.md b/docs/resources/external_function.md index 368b6a8c22..01a1133e4b 100644 --- a/docs/resources/external_function.md +++ b/docs/resources/external_function.md @@ -10,7 +10,23 @@ description: |- - +## Example Usage + +```terraform +resource "snowflake_external_function" "test_ext_func" { + name = "my_function" + database = "my_test_db" + schema = "my_test_schema" + args { + name = "data" + type = "varchar" + } + return_type = "varchar" + return_behavior = "IMMUTABLE" + api_integration = "api_integration_name" + url_of_proxy_and_resource = "https://123456.execute-api.us-west-2.amazonaws.com/prod/test_func" +} +``` ## Schema @@ -58,4 +74,11 @@ Required: - **name** (String) Header name - **value** (String) Header value +## Import + +Import is supported using the following syntax: +```shell +# format is database name | schema name | external function name | +terraform import snowflake_external_function.example 'dbName|schemaName|externalFunctionName|varchar-varchar-varchar' +``` diff --git a/examples/resources/snowflake_api_integration/import.sh b/examples/resources/snowflake_api_integration/import.sh new file mode 100644 index 0000000000..44c1faf658 --- /dev/null +++ b/examples/resources/snowflake_api_integration/import.sh @@ -0,0 +1 @@ +terraform import snowflake_api_integration.example name diff --git a/examples/resources/snowflake_api_integration/resource.tf b/examples/resources/snowflake_api_integration/resource.tf new file mode 100644 index 0000000000..b93efae34e --- /dev/null +++ b/examples/resources/snowflake_api_integration/resource.tf @@ -0,0 +1,7 @@ +resource "snowflake_api_integration" "api_integration" { + name = "aws_integration" + api_provider = "aws_api_gateway" + api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" + api_allowed_prefixes = "https://123456.execute-api.us-west-2.amazonaws.com/prod/" + enabled = true +} \ No newline at end of file diff --git a/examples/resources/snowflake_external_function/import.sh b/examples/resources/snowflake_external_function/import.sh new file mode 100644 index 0000000000..5210458738 --- /dev/null +++ b/examples/resources/snowflake_external_function/import.sh @@ -0,0 +1,2 @@ +# format is database name | schema name | external function name | +terraform import snowflake_external_function.example 'dbName|schemaName|externalFunctionName|varchar-varchar-varchar' diff --git a/examples/resources/snowflake_external_function/resource.tf b/examples/resources/snowflake_external_function/resource.tf new file mode 100644 index 0000000000..a03da1f06c --- /dev/null +++ b/examples/resources/snowflake_external_function/resource.tf @@ -0,0 +1,13 @@ +resource "snowflake_external_function" "test_ext_func" { + name = "my_function" + database = "my_test_db" + schema = "my_test_schema" + args { + name = "data" + type = "varchar" + } + return_type = "varchar" + return_behavior = "IMMUTABLE" + api_integration = "api_integration_name" + url_of_proxy_and_resource = "https://123456.execute-api.us-west-2.amazonaws.com/prod/test_func" +} \ No newline at end of file From a68a71c6c6d4257edd5c6b4ce1f4580c2fa7e4a8 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Thu, 11 Mar 2021 11:13:07 +0100 Subject: [PATCH 11/23] Improve external function read & unit test --- VERSION | 2 +- pkg/resources/api_integration.go | 2 +- pkg/resources/external_function.go | 80 +++++++++++++++++++++++++ pkg/resources/external_function_test.go | 20 ++++++- pkg/snowflake/external_function.go | 1 + 5 files changed, 102 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 286d5b09c8..c86a09df3c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.24.0 \ No newline at end of file +0.23.2 \ No newline at end of file diff --git a/pkg/resources/api_integration.go b/pkg/resources/api_integration.go index 8f8ffb946f..900f81daa7 100644 --- a/pkg/resources/api_integration.go +++ b/pkg/resources/api_integration.go @@ -227,7 +227,7 @@ func ReadAPIIntegration(d *schema.ResourceData, meta interface{}) error { return err } default: - log.Printf("[WARN] unexpected property %v returned from Snowflake", k) + log.Printf("[WARN] unexpected api integration property %v returned from Snowflake", k) } } diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go index 0de9ce93e8..280925b580 100644 --- a/pkg/resources/external_function.go +++ b/pkg/resources/external_function.go @@ -5,6 +5,8 @@ import ( "database/sql" "encoding/csv" "fmt" + "log" + "regexp" "strings" "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/snowflake" @@ -311,7 +313,9 @@ func ReadExternalFunction(d *schema.ResourceData, meta interface{}) error { dbName := externalFunctionID.DatabaseName dbSchema := externalFunctionID.SchemaName name := externalFunctionID.ExternalFunctionName + argtypes := externalFunctionID.ExternalFunctionArgTypes + // Some properties can come from the SHOW EXTERNAL FUNCTION call stmt := snowflake.ExternalFunction(name, dbName, dbSchema).Show() row := snowflake.QueryRow(db, stmt) externalFunction, err := snowflake.ScanExternalFunction(row) @@ -336,10 +340,86 @@ func ReadExternalFunction(d *schema.ResourceData, meta interface{}) error { return err } + if err := d.Set("comment", externalFunction.Comment.String); err != nil { + return err + } + if err := d.Set("created_on", externalFunction.CreatedOn.String); err != nil { return err } + // Some properties come from the DESCRIBE FUNCTION call + stmt = snowflake.ExternalFunction(name, dbName, dbSchema).WithArgTypes(argtypes).Describe() + externalFunctionDescriptionRows, err := snowflake.Query(db, stmt) + if err != nil { + return err + } + + externalFunctionDescription, err := snowflake.ScanExternalFunctionDescription(externalFunctionDescriptionRows) + if err != nil { + return err + } + + for _, desc := range externalFunctionDescription { + switch desc.Property.String { + case "signature": + // Format in Snowflake DB is: (argName argType, argName argType, ...) + args := strings.ReplaceAll(strings.ReplaceAll(desc.Value.String, "(", ""), ")", "") + argPairs := strings.Split(args, ", ") + flattenedArgs := []interface{}{} + + for _, argPair := range argPairs { + arg := strings.Split(argPair, " ") + + flatArg := map[string]interface{}{} + flatArg["name"] = arg[0] + flatArg["type"] = arg[1] + flattenedArgs = append(flattenedArgs, flatArg) + } + + if err = d.Set("args", flattenedArgs); err != nil { + return err + } + case "returns": + // Format in Snowflake DB is returnType() + re := regexp.MustCompile(`^(.*)\([0-9]*\)$`) + match := re.FindStringSubmatch(desc.Value.String) + if err = d.Set("return_type", match[1]); err != nil { + return err + } + case "null handling": + if err = d.Set("null_input_behavior", desc.Value.String); err != nil { + return err + } + case "volatility": + if err = d.Set("return_behavior", desc.Value.String); err != nil { + return err + } + case "headers": + //TODO - Format in Snowflake DB is: {"head1":"val1","head2":"val2"} + case "context_headers": + //TODO - Format in Snowflake DB is: ["context_function_1","context_function_2"] + case "max_batch_rows": + if desc.Value.String != "not set" { + if err = d.Set("max_batch_rows", desc.Value.String); err != nil { + return err + } + } + case "compression": + if err = d.Set("compression", desc.Value.String); err != nil { + return err + } + case "body": + if err = d.Set("url_of_proxy_and_resource", desc.Value.String); err != nil { + return err + } + case "language": + // To ignore + default: + log.Printf("[WARN] unexpected external function property %v returned from Snowflake", desc.Property.String) + } + } + return nil } diff --git a/pkg/resources/external_function_test.go b/pkg/resources/external_function_test.go index 4488a817f7..22399590ac 100644 --- a/pkg/resources/external_function_test.go +++ b/pkg/resources/external_function_test.go @@ -45,12 +45,22 @@ func TestExternalFunctionCreate(t *testing.T) { func expectExternalFunctionRead(mock sqlmock.Sqlmock) { rows := sqlmock.NewRows([]string{"created_on", "name", "schema_name", "is_builtin", "is_aggregate", "is_ansi", "min_num_arguments", "max_num_arguments", "arguments", "description", "catalog_name", "is_table_function", "valid_for_clustering", "is_secure", "is_external_function", "language"}).AddRow("now", "my_test_function", "schema_name", "N", "N", "N", "1", "1", "MY_TEST_FUNCTION(VARCHAR) RETURN VARCHAR", "mock comment", "database_name", "N", "N", "N", "Y", "EXTERNAL") mock.ExpectQuery(`SHOW EXTERNAL FUNCTIONS LIKE 'my_test_function' IN SCHEMA "database_name"."schema_name"`).WillReturnRows(rows) + + describeRows := sqlmock.NewRows([]string{"property", "value"}). + AddRow("returns", "VARCHAR(123456789)"). // This is how return type is stored in Snowflake DB + AddRow("null handling", "CALLED ON NULL INPUT"). + AddRow("volatility", "IMMUTABLE"). + AddRow("body", "https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function"). + AddRow("max_batch_rows", "not set"). + AddRow("compression", "AUTO") + + mock.ExpectQuery(`DESCRIBE FUNCTION "database_name"."schema_name"."my_test_function" \(varchar\)`).WillReturnRows(describeRows) } func TestExternalFunctionRead(t *testing.T) { r := require.New(t) - d := externalFunction(t, "database_name|schema_name|my_test_function|", map[string]interface{}{"name": "my_test_function", "comment": "mock comment"}) + d := externalFunction(t, "database_name|schema_name|my_test_function|varchar", map[string]interface{}{"name": "my_test_function", "args": []interface{}{map[string]interface{}{"name": "data", "type": "varchar"}}, "return_type": "varchar", "comment": "mock comment"}) WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { expectExternalFunctionRead(mock) @@ -59,6 +69,14 @@ func TestExternalFunctionRead(t *testing.T) { r.NoError(err) r.Equal("my_test_function", d.Get("name").(string)) r.Equal("mock comment", d.Get("comment").(string)) + r.Equal("VARCHAR", d.Get("return_type").(string)) + + args := d.Get("args").([]interface{}) + r.Len(args, 1) + test_func_args := args[0].(map[string]interface{}) + r.Len(test_func_args, 2) + r.Equal("data", test_func_args["name"].(string)) + r.Equal("varchar", test_func_args["type"].(string)) }) } diff --git a/pkg/snowflake/external_function.go b/pkg/snowflake/external_function.go index a9a3adb1c2..60f7bc0e43 100644 --- a/pkg/snowflake/external_function.go +++ b/pkg/snowflake/external_function.go @@ -235,6 +235,7 @@ type externalFunction struct { ExternalFunctionName sql.NullString `db:"name"` DatabaseName sql.NullString `db:"catalog_name"` SchemaName sql.NullString `db:"schema_name"` + Comment sql.NullString `db:"description"` IsExternalFunction sql.NullString `db:"is_external_function"` Language sql.NullString `db:"language"` } From cf08e1374ac117ae0e1f8c26c50912e657daa943 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Thu, 11 Mar 2021 11:22:56 +0100 Subject: [PATCH 12/23] Improve external function read & unit test --- pkg/resources/external_function.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go index 280925b580..87e8860ecf 100644 --- a/pkg/resources/external_function.go +++ b/pkg/resources/external_function.go @@ -365,20 +365,23 @@ func ReadExternalFunction(d *schema.ResourceData, meta interface{}) error { case "signature": // Format in Snowflake DB is: (argName argType, argName argType, ...) args := strings.ReplaceAll(strings.ReplaceAll(desc.Value.String, "(", ""), ")", "") - argPairs := strings.Split(args, ", ") - flattenedArgs := []interface{}{} - for _, argPair := range argPairs { - arg := strings.Split(argPair, " ") + if args != "" { // Do nothing for functions without arguments + argPairs := strings.Split(args, ", ") + args := []interface{}{} - flatArg := map[string]interface{}{} - flatArg["name"] = arg[0] - flatArg["type"] = arg[1] - flattenedArgs = append(flattenedArgs, flatArg) - } + for _, argPair := range argPairs { + argItem := strings.Split(argPair, " ") - if err = d.Set("args", flattenedArgs); err != nil { - return err + arg := map[string]interface{}{} + arg["name"] = argItem[0] + arg["type"] = argItem[1] + args = append(args, arg) + } + + if err = d.Set("args", args); err != nil { + return err + } } case "returns": // Format in Snowflake DB is returnType() From e557bcdbcceb991914f0518618ccc3af78b7357b Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Thu, 11 Mar 2021 12:14:29 +0100 Subject: [PATCH 13/23] Fixes --- pkg/resources/external_function.go | 53 ++++++++++++++++++++----- pkg/resources/external_function_test.go | 2 +- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go index 87e8860ecf..a11a42cf6c 100644 --- a/pkg/resources/external_function.go +++ b/pkg/resources/external_function.go @@ -21,9 +21,16 @@ const ( var externalFunctionSchema = map[string]*schema.Schema{ "name": { - Type: schema.TypeString, - Required: true, - ForceNew: true, + Type: schema.TypeString, + Required: true, + ForceNew: true, + // Suppress the diff shown if the base_image name are equal when both compared in lower case. + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + if strings.ToLower(old) == strings.ToLower(new) { + return true + } + return false + }, Description: "Specifies the identifier for the external function. The identifier can contain the schema name and database name, as well as the function name. The function's signature (name and argument data types) must be unique within the schema.", }, "schema": { @@ -46,13 +53,27 @@ var externalFunctionSchema = map[string]*schema.Schema{ Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "name": { - Type: schema.TypeString, - Required: true, + Type: schema.TypeString, + Required: true, + // Suppress the diff shown if the base_image name are equal when both compared in lower case. + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + if strings.ToLower(old) == strings.ToLower(new) { + return true + } + return false + }, Description: "Argument name", }, "type": { - Type: schema.TypeString, - Required: true, + Type: schema.TypeString, + Required: true, + // Suppress the diff shown if the base_image name are equal when both compared in lower case. + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + if strings.ToLower(old) == strings.ToLower(new) { + return true + } + return false + }, Description: "Argument type, e.g. VARCHAR", }, }, @@ -61,14 +82,22 @@ var externalFunctionSchema = map[string]*schema.Schema{ "null_input_behavior": { Type: schema.TypeString, Optional: true, + Default: "CALLED ON NULL INPUT", ForceNew: true, ValidateFunc: validation.StringInSlice([]string{"CALLED ON NULL INPUT", "RETURNS NULL ON NULL INPUT", "STRICT"}, false), Description: "Specifies the behavior of the external function when called with null inputs.", }, "return_type": { - Type: schema.TypeString, - Required: true, - ForceNew: true, + Type: schema.TypeString, + Required: true, + ForceNew: true, + // Suppress the diff shown if the base_image name are equal when both compared in lower case. + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + if strings.ToLower(old) == strings.ToLower(new) { + return true + } + return false + }, Description: "Specifies the data type returned by the external function.", }, "return_null_allowed": { @@ -126,6 +155,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ "compression": { Type: schema.TypeString, Optional: true, + Default: "AUTO", ForceNew: true, ValidateFunc: validation.StringInSlice([]string{"NONE", "AUTO", "GZIP", "DEFLATE"}, false), Description: "If specified, the JSON payload is compressed when sent from Snowflake to the proxy service, and when sent back from the proxy service to Snowflake.", @@ -139,6 +169,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ "comment": { Type: schema.TypeString, Optional: true, + Default: "user-defined function", ForceNew: true, Description: "A description of the external function.", }, @@ -278,7 +309,7 @@ func CreateExternalFunction(d *schema.ResourceData, meta interface{}) error { } if v, ok := d.GetOk("compression"); ok { - builder.WithNullInputBehavior(v.(string)) + builder.WithCompression(v.(string)) } stmt := builder.Create() diff --git a/pkg/resources/external_function_test.go b/pkg/resources/external_function_test.go index 22399590ac..0411b9620d 100644 --- a/pkg/resources/external_function_test.go +++ b/pkg/resources/external_function_test.go @@ -33,7 +33,7 @@ func TestExternalFunctionCreate(t *testing.T) { d := externalFunction(t, "database_name|schema_name|my_test_function|varchar", in) WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec(`CREATE EXTERNAL FUNCTION "database_name"."schema_name"."my_test_function" \(data varchar\) RETURNS varchar NULL IMMUTABLE API_INTEGRATION = 'test_api_integration_01' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function'`).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(`CREATE EXTERNAL FUNCTION "database_name"."schema_name"."my_test_function" \(data varchar\) RETURNS varchar NULL CALLED ON NULL INPUT IMMUTABLE COMMENT = 'user-defined function' API_INTEGRATION = 'test_api_integration_01' COMPRESSION = 'AUTO' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function'`).WillReturnResult(sqlmock.NewResult(1, 1)) expectExternalFunctionRead(mock) err := resources.CreateExternalFunction(d, db) From 6afc128511221f641e60a7b4e10dcbf654675a81 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Thu, 11 Mar 2021 16:42:11 +0100 Subject: [PATCH 14/23] Fix doc example on api integration --- docs/resources/api_integration.md | 2 +- examples/resources/snowflake_api_integration/resource.tf | 2 +- pkg/resources/external_function.go | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/resources/api_integration.md b/docs/resources/api_integration.md index aab2e05533..20c596faac 100644 --- a/docs/resources/api_integration.md +++ b/docs/resources/api_integration.md @@ -17,7 +17,7 @@ resource "snowflake_api_integration" "api_integration" { name = "aws_integration" api_provider = "aws_api_gateway" api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" - api_allowed_prefixes = "https://123456.execute-api.us-west-2.amazonaws.com/prod/" + api_allowed_prefixes = ["https://123456.execute-api.us-west-2.amazonaws.com/prod/"] enabled = true } ``` diff --git a/examples/resources/snowflake_api_integration/resource.tf b/examples/resources/snowflake_api_integration/resource.tf index b93efae34e..98bbf7a974 100644 --- a/examples/resources/snowflake_api_integration/resource.tf +++ b/examples/resources/snowflake_api_integration/resource.tf @@ -2,6 +2,6 @@ resource "snowflake_api_integration" "api_integration" { name = "aws_integration" api_provider = "aws_api_gateway" api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" - api_allowed_prefixes = "https://123456.execute-api.us-west-2.amazonaws.com/prod/" + api_allowed_prefixes = ["https://123456.execute-api.us-west-2.amazonaws.com/prod/"] enabled = true } \ No newline at end of file diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go index a11a42cf6c..151f805a10 100644 --- a/pkg/resources/external_function.go +++ b/pkg/resources/external_function.go @@ -24,7 +24,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ Type: schema.TypeString, Required: true, ForceNew: true, - // Suppress the diff shown if the base_image name are equal when both compared in lower case. + // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { if strings.ToLower(old) == strings.ToLower(new) { return true @@ -55,7 +55,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ "name": { Type: schema.TypeString, Required: true, - // Suppress the diff shown if the base_image name are equal when both compared in lower case. + // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { if strings.ToLower(old) == strings.ToLower(new) { return true @@ -67,7 +67,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ "type": { Type: schema.TypeString, Required: true, - // Suppress the diff shown if the base_image name are equal when both compared in lower case. + // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { if strings.ToLower(old) == strings.ToLower(new) { return true @@ -91,7 +91,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ Type: schema.TypeString, Required: true, ForceNew: true, - // Suppress the diff shown if the base_image name are equal when both compared in lower case. + // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { if strings.ToLower(old) == strings.ToLower(new) { return true From fa34c7eb1215fda99dbd017be9c7a22e0fd52846 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Thu, 18 Mar 2021 09:41:43 +0100 Subject: [PATCH 15/23] Consistent use of id --- pkg/resources/api_integration.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/resources/api_integration.go b/pkg/resources/api_integration.go index 900f81daa7..6cf690411d 100644 --- a/pkg/resources/api_integration.go +++ b/pkg/resources/api_integration.go @@ -152,7 +152,7 @@ func ReadAPIIntegration(d *schema.ResourceData, meta interface{}) error { db := meta.(*sql.DB) id := d.Id() - stmt := snowflake.ApiIntegration(d.Id()).Show() + stmt := snowflake.ApiIntegration(id).Show() row := snowflake.QueryRow(db, stmt) // Some properties can come from the SHOW INTEGRATION call @@ -183,7 +183,7 @@ func ReadAPIIntegration(d *schema.ResourceData, meta interface{}) error { // We need to grab them in a loop var k, pType string var v, unused interface{} - stmt = snowflake.ApiIntegration(d.Id()).Describe() + stmt = snowflake.ApiIntegration(id).Describe() rows, err := db.Query(stmt) if err != nil { return fmt.Errorf("Could not describe api integration: %w", err) @@ -264,7 +264,7 @@ func UpdateAPIIntegration(d *schema.ResourceData, meta interface{}) error { if d.HasChange("api_blocked_prefixes") { v := d.Get("api_blocked_prefixes").([]interface{}) if len(v) == 0 { - err := snowflake.Exec(db, fmt.Sprintf(`ALTER API INTEGRATION %v UNSET API_BLOCKED_PREFIXES`, d.Id())) + err := snowflake.Exec(db, fmt.Sprintf(`ALTER API INTEGRATION %v UNSET API_BLOCKED_PREFIXES`, id)) if err != nil { return fmt.Errorf("error unsetting api_blocked_prefixes: %w", err) } From 46803dc026c5a6cb811ddb4a114d3672d805505f Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Thu, 18 Mar 2021 11:05:37 +0100 Subject: [PATCH 16/23] Remove 'comment' from api integration as not persisted in Snowflake --- docs/resources/api_integration.md | 1 - pkg/resources/api_integration.go | 14 -------------- pkg/resources/api_integration_acceptance_test.go | 4 ---- pkg/resources/api_integration_test.go | 5 ++--- 4 files changed, 2 insertions(+), 22 deletions(-) diff --git a/docs/resources/api_integration.md b/docs/resources/api_integration.md index 20c596faac..f672da9219 100644 --- a/docs/resources/api_integration.md +++ b/docs/resources/api_integration.md @@ -37,7 +37,6 @@ resource "snowflake_api_integration" "api_integration" { - **api_blocked_prefixes** (List of String) Lists the endpoints and resources in the HTTPS proxy service that are not allowed to be called from Snowflake. - **azure_ad_application_id** (String) The 'Application (client) id' of the Azure AD app for your remote service. - **azure_tenant_id** (String) Specifies the ID for your Office 365 tenant that all Azure API Management instances belong to. -- **comment** (String) A description of the API integration. - **enabled** (Boolean) Specifies whether this API integration is enabled or disabled. If the API integration is disabled, any external function that relies on it will not work. - **id** (String) The ID of this resource. diff --git a/pkg/resources/api_integration.go b/pkg/resources/api_integration.go index 6cf690411d..ec7a849046 100644 --- a/pkg/resources/api_integration.go +++ b/pkg/resources/api_integration.go @@ -83,11 +83,6 @@ var apiIntegrationSchema = map[string]*schema.Schema{ Default: true, Description: "Specifies whether this API integration is enabled or disabled. If the API integration is disabled, any external function that relies on it will not work.", }, - "comment": { - Type: schema.TypeString, - Optional: true, - Description: "A description of the API integration.", - }, "created_on": { Type: schema.TypeString, Computed: true, @@ -123,10 +118,6 @@ func CreateAPIIntegration(d *schema.ResourceData, meta interface{}) error { stmt.SetStringList("API_ALLOWED_PREFIXES", expandStringList(d.Get("api_allowed_prefixes").([]interface{}))) // Set optional fields - if v, ok := d.GetOk("comment"); ok { - stmt.SetString(`COMMENT`, v.(string)) - } - if _, ok := d.GetOk("api_blocked_prefixes"); ok { stmt.SetStringList("API_BLOCKED_PREFIXES", expandStringList(d.Get("api_blocked_prefixes").([]interface{}))) } @@ -244,11 +235,6 @@ func UpdateAPIIntegration(d *schema.ResourceData, meta interface{}) error { // This is required in case the only change is to UNSET API_ALLOWED_PREFIXES. var runSetStatement bool - if d.HasChange("comment") { - runSetStatement = true - stmt.SetString("COMMENT", d.Get("comment").(string)) - } - if d.HasChange("enabled") { runSetStatement = true stmt.SetBool(`ENABLED`, d.Get("enabled").(bool)) diff --git a/pkg/resources/api_integration_acceptance_test.go b/pkg/resources/api_integration_acceptance_test.go index 1db1a0aa0c..cafd087110 100644 --- a/pkg/resources/api_integration_acceptance_test.go +++ b/pkg/resources/api_integration_acceptance_test.go @@ -26,7 +26,6 @@ func TestAcc_ApiIntegration(t *testing.T) { Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("snowflake_api_integration.test_aws_int", "name", apiIntName), resource.TestCheckResourceAttr("snowflake_api_integration.test_aws_int", "api_provider", "aws_api_gateway"), - resource.TestCheckResourceAttr("snowflake_api_integration.test_aws_int", "comment", "Terraform acceptance test"), resource.TestCheckResourceAttrSet("snowflake_api_integration.test_aws_int", "created_on"), resource.TestCheckResourceAttrSet("snowflake_api_integration.test_aws_int", "api_aws_iam_user_arn"), resource.TestCheckResourceAttrSet("snowflake_api_integration.test_aws_int", "api_aws_external_id"), @@ -37,7 +36,6 @@ func TestAcc_ApiIntegration(t *testing.T) { Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("snowflake_api_integration.test_azure_int", "name", apiIntName2), resource.TestCheckResourceAttr("snowflake_api_integration.test_azure_int", "api_provider", "azure_api_management"), - resource.TestCheckResourceAttr("snowflake_api_integration.test_azure_int", "comment", "Terraform acceptance test"), resource.TestCheckResourceAttrSet("snowflake_api_integration.test_azure_int", "created_on"), resource.TestCheckResourceAttrSet("snowflake_api_integration.test_azure_int", "azure_multi_tenant_app_name"), resource.TestCheckResourceAttrSet("snowflake_api_integration.test_azure_int", "azure_consent_url"), @@ -55,7 +53,6 @@ func apiIntegrationConfig_aws(name string, prefixes []string) string { api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" api_allowed_prefixes = %q enabled = true - comment = "Terraform acceptance test" } `, name, prefixes) } @@ -69,7 +66,6 @@ func apiIntegrationConfig_azure(name string, prefixes []string) string { azure_ad_application_id = "7890" api_allowed_prefixes = %q enabled = true - comment = "Terraform acceptance test" } `, name, prefixes) } diff --git a/pkg/resources/api_integration_test.go b/pkg/resources/api_integration_test.go index b10094b807..5b203eb93f 100644 --- a/pkg/resources/api_integration_test.go +++ b/pkg/resources/api_integration_test.go @@ -23,17 +23,16 @@ func TestAPIIntegrationCreate(t *testing.T) { in := map[string]interface{}{ "name": "test_api_integration", - "comment": "great comment", "api_allowed_prefixes": []interface{}{"https://123456.execute-api.us-west-2.amazonaws.com/prod/"}, "api_provider": "aws_api_gateway", - "api_aws_role_arn": "we-should-probably-validate-this-string", + "api_aws_role_arn": "arn:aws:iam::000000000001:/role/test", } d := schema.TestResourceDataRaw(t, resources.APIIntegration().Schema, in) r.NotNil(d) WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { mock.ExpectExec( - `^CREATE API INTEGRATION "test_api_integration" API_AWS_ROLE_ARN='we-should-probably-validate-this-string' API_PROVIDER='aws_api_gateway' COMMENT='great comment' API_ALLOWED_PREFIXES=\('https://123456.execute-api.us-west-2.amazonaws.com/prod/'\) ENABLED=true$`, + `^CREATE API INTEGRATION "test_api_integration" API_AWS_ROLE_ARN='arn:aws:iam::000000000001:/role/test' API_PROVIDER='aws_api_gateway' API_ALLOWED_PREFIXES=\('https://123456.execute-api.us-west-2.amazonaws.com/prod/'\) ENABLED=true$`, ).WillReturnResult(sqlmock.NewResult(1, 1)) expectReadAPIIntegration(mock) From 1daa3e38323e31ae30e35d8cb7baa6609da5c0be Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Thu, 18 Mar 2021 11:19:49 +0100 Subject: [PATCH 17/23] Remove useless comments --- pkg/resources/api_integration.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/resources/api_integration.go b/pkg/resources/api_integration.go index ec7a849046..c6c908a9a0 100644 --- a/pkg/resources/api_integration.go +++ b/pkg/resources/api_integration.go @@ -232,7 +232,6 @@ func UpdateAPIIntegration(d *schema.ResourceData, meta interface{}) error { stmt := snowflake.ApiIntegration(id).Alter() - // This is required in case the only change is to UNSET API_ALLOWED_PREFIXES. var runSetStatement bool if d.HasChange("enabled") { @@ -245,8 +244,7 @@ func UpdateAPIIntegration(d *schema.ResourceData, meta interface{}) error { stmt.SetStringList("API_ALLOWED_PREFIXES", expandStringList(d.Get("api_allowed_prefixes").([]interface{}))) } - // We need to UNSET this if we remove all api blocked prefixes. I don't think - // @TODO move the SQL back to the snowflake package + // We need to UNSET this if we remove all api blocked prefixes. if d.HasChange("api_blocked_prefixes") { v := d.Get("api_blocked_prefixes").([]interface{}) if len(v) == 0 { From 7e124a9819ad3fe0b34027f57d517fbd49472ea4 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Thu, 18 Mar 2021 13:37:09 +0100 Subject: [PATCH 18/23] Remove 'comment' from api integration as not persisted in Snowflake --- pkg/resources/external_function_acceptance_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/resources/external_function_acceptance_test.go b/pkg/resources/external_function_acceptance_test.go index 005f970e9c..e4eb0cac9d 100644 --- a/pkg/resources/external_function_acceptance_test.go +++ b/pkg/resources/external_function_acceptance_test.go @@ -51,7 +51,6 @@ func externalFunctionConfig(name string, prefixes []string, url string) string { api_aws_role_arn = "arn:aws:iam::000000000001:/role/test" api_allowed_prefixes = %q enabled = true - comment = "Terraform acceptance test" } resource "snowflake_external_function" "test_func" { From 175fec8ac560b6284cd98e6ecbfd4e0f036ad4b4 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Sun, 21 Mar 2021 17:14:17 +0100 Subject: [PATCH 19/23] Add SetRaw method, 'API_PROVIDER' value now unquoted --- pkg/resources/api_integration.go | 2 +- pkg/resources/api_integration_test.go | 2 +- pkg/snowflake/api_integration_test.go | 4 ++-- pkg/snowflake/generic.go | 19 +++++++++++++++++++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/pkg/resources/api_integration.go b/pkg/resources/api_integration.go index c6c908a9a0..b0c461c821 100644 --- a/pkg/resources/api_integration.go +++ b/pkg/resources/api_integration.go @@ -295,7 +295,7 @@ func DeleteAPIIntegration(d *schema.ResourceData, meta interface{}) error { func setAPIProviderSettings(data *schema.ResourceData, stmt snowflake.SettingBuilder) error { apiProvider := data.Get("api_provider").(string) - stmt.SetString("API_PROVIDER", apiProvider) + stmt.SetRaw("API_PROVIDER=" + apiProvider) switch apiProvider { case "aws_api_gateway", "aws_private_api_gateway": diff --git a/pkg/resources/api_integration_test.go b/pkg/resources/api_integration_test.go index 5b203eb93f..ab91ff31cd 100644 --- a/pkg/resources/api_integration_test.go +++ b/pkg/resources/api_integration_test.go @@ -32,7 +32,7 @@ func TestAPIIntegrationCreate(t *testing.T) { WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { mock.ExpectExec( - `^CREATE API INTEGRATION "test_api_integration" API_AWS_ROLE_ARN='arn:aws:iam::000000000001:/role/test' API_PROVIDER='aws_api_gateway' API_ALLOWED_PREFIXES=\('https://123456.execute-api.us-west-2.amazonaws.com/prod/'\) ENABLED=true$`, + `^CREATE API INTEGRATION "test_api_integration" API_PROVIDER=aws_api_gateway API_AWS_ROLE_ARN='arn:aws:iam::000000000001:/role/test' API_ALLOWED_PREFIXES=\('https://123456.execute-api.us-west-2.amazonaws.com/prod/'\) ENABLED=true$`, ).WillReturnResult(sqlmock.NewResult(1, 1)) expectReadAPIIntegration(mock) diff --git a/pkg/snowflake/api_integration_test.go b/pkg/snowflake/api_integration_test.go index c8dfa0932b..d8399bcdcc 100644 --- a/pkg/snowflake/api_integration_test.go +++ b/pkg/snowflake/api_integration_test.go @@ -17,11 +17,11 @@ func TestApiIntegration(t *testing.T) { c := builder.Create() - c.SetString(`api_provider`, `aws_private_api_gateway`) + c.SetRaw(`API_PROVIDER=aws_private_api_gateway`) c.SetString(`api_aws_role_arn`, "arn:aws:iam::xxxx:role/snowflake-execute-externalfunc-privendpoint-role") c.SetStringList(`api_allowed_prefixes`, []string{"https://123456.execute-api.us-west-2.amazonaws.com/prod/", "https://123456.execute-api.us-west-2.amazonaws.com/test/"}) c.SetBool(`enabled`, true) q = c.Statement() - r.Equal(`CREATE API INTEGRATION "aws_api" API_AWS_ROLE_ARN='arn:aws:iam::xxxx:role/snowflake-execute-externalfunc-privendpoint-role' API_PROVIDER='aws_private_api_gateway' API_ALLOWED_PREFIXES=('https://123456.execute-api.us-west-2.amazonaws.com/prod/', 'https://123456.execute-api.us-west-2.amazonaws.com/test/') ENABLED=true`, q) + r.Equal(`CREATE API INTEGRATION "aws_api" API_PROVIDER=aws_private_api_gateway API_AWS_ROLE_ARN='arn:aws:iam::xxxx:role/snowflake-execute-externalfunc-privendpoint-role' API_ALLOWED_PREFIXES=('https://123456.execute-api.us-west-2.amazonaws.com/prod/', 'https://123456.execute-api.us-west-2.amazonaws.com/test/') ENABLED=true`, q) } diff --git a/pkg/snowflake/generic.go b/pkg/snowflake/generic.go index f478437f9f..79d5cb924f 100644 --- a/pkg/snowflake/generic.go +++ b/pkg/snowflake/generic.go @@ -50,6 +50,7 @@ type SettingBuilder interface { SetBool(string, bool) SetInt(string, int) SetFloat(string, float64) + SetRaw(string) } type AlterPropertiesBuilder struct { @@ -60,6 +61,7 @@ type AlterPropertiesBuilder struct { boolProperties map[string]bool intProperties map[string]int floatProperties map[string]float64 + rawStatement string } func (b *Builder) Alter() *AlterPropertiesBuilder { @@ -94,10 +96,18 @@ func (ab *AlterPropertiesBuilder) SetFloat(key string, value float64) { ab.floatProperties[key] = value } +func (ab *AlterPropertiesBuilder) SetRaw(rawStatement string) { + var sb strings.Builder + sb.WriteString(fmt.Sprintf(`%s %s`, ab.rawStatement, rawStatement)) + ab.rawStatement = sb.String() +} + func (ab *AlterPropertiesBuilder) Statement() string { var sb strings.Builder sb.WriteString(fmt.Sprintf(`ALTER %s "%s" SET`, ab.entityType, ab.name)) // TODO handle error + sb.WriteString(fmt.Sprintf(`%s`, ab.rawStatement)) + for k, v := range ab.stringProperties { sb.WriteString(fmt.Sprintf(" %s='%s'", strings.ToUpper(k), EscapeString(v))) } @@ -129,6 +139,7 @@ type CreateBuilder struct { boolProperties map[string]bool intProperties map[string]int floatProperties map[string]float64 + rawStatement string } func (b *Builder) Create() *CreateBuilder { @@ -163,10 +174,18 @@ func (b *CreateBuilder) SetFloat(key string, value float64) { b.floatProperties[key] = value } +func (b *CreateBuilder) SetRaw(rawStatement string) { + var sb strings.Builder + sb.WriteString(fmt.Sprintf(`%s %s`, b.rawStatement, rawStatement)) + b.rawStatement = sb.String() +} + func (b *CreateBuilder) Statement() string { var sb strings.Builder sb.WriteString(fmt.Sprintf(`CREATE %s "%s"`, b.entityType, b.name)) // TODO handle error + sb.WriteString(fmt.Sprintf(`%s`, b.rawStatement)) + sortedStringProperties := make([]string, 0) for k := range b.stringProperties { sortedStringProperties = append(sortedStringProperties, k) From c577c1b8a53a439e130261d0ba9f370351160e52 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Sun, 21 Mar 2021 17:24:30 +0100 Subject: [PATCH 20/23] Lint --- pkg/resources/external_function.go | 20 ++++---------------- pkg/snowflake/generic.go | 4 ++-- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go index 151f805a10..8e9a1a8fc9 100644 --- a/pkg/resources/external_function.go +++ b/pkg/resources/external_function.go @@ -26,10 +26,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ ForceNew: true, // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { - if strings.ToLower(old) == strings.ToLower(new) { - return true - } - return false + return strings.EqualFold(old, new) }, Description: "Specifies the identifier for the external function. The identifier can contain the schema name and database name, as well as the function name. The function's signature (name and argument data types) must be unique within the schema.", }, @@ -57,10 +54,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ Required: true, // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { - if strings.ToLower(old) == strings.ToLower(new) { - return true - } - return false + return strings.EqualFold(old, new) }, Description: "Argument name", }, @@ -69,10 +63,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ Required: true, // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { - if strings.ToLower(old) == strings.ToLower(new) { - return true - } - return false + return strings.EqualFold(old, new) }, Description: "Argument type, e.g. VARCHAR", }, @@ -93,10 +84,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ ForceNew: true, // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { - if strings.ToLower(old) == strings.ToLower(new) { - return true - } - return false + return strings.EqualFold(old, new) }, Description: "Specifies the data type returned by the external function.", }, diff --git a/pkg/snowflake/generic.go b/pkg/snowflake/generic.go index 79d5cb924f..513bf1d353 100644 --- a/pkg/snowflake/generic.go +++ b/pkg/snowflake/generic.go @@ -106,7 +106,7 @@ func (ab *AlterPropertiesBuilder) Statement() string { var sb strings.Builder sb.WriteString(fmt.Sprintf(`ALTER %s "%s" SET`, ab.entityType, ab.name)) // TODO handle error - sb.WriteString(fmt.Sprintf(`%s`, ab.rawStatement)) + sb.WriteString(ab.rawStatement) for k, v := range ab.stringProperties { sb.WriteString(fmt.Sprintf(" %s='%s'", strings.ToUpper(k), EscapeString(v))) @@ -184,7 +184,7 @@ func (b *CreateBuilder) Statement() string { var sb strings.Builder sb.WriteString(fmt.Sprintf(`CREATE %s "%s"`, b.entityType, b.name)) // TODO handle error - sb.WriteString(fmt.Sprintf(`%s`, b.rawStatement)) + sb.WriteString(b.rawStatement) sortedStringProperties := make([]string, 0) for k := range b.stringProperties { From 9d05487cded173bf295aeb699e2003b449d7aacf Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Sun, 21 Mar 2021 18:12:45 +0100 Subject: [PATCH 21/23] Fix --- pkg/resources/external_function.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go index 8e9a1a8fc9..6cd3784f9d 100644 --- a/pkg/resources/external_function.go +++ b/pkg/resources/external_function.go @@ -21,13 +21,9 @@ const ( var externalFunctionSchema = map[string]*schema.Schema{ "name": { - Type: schema.TypeString, - Required: true, - ForceNew: true, - // Suppress the diff shown if the values are equal when both compared in lower case. - DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { - return strings.EqualFold(old, new) - }, + Type: schema.TypeString, + Required: true, + ForceNew: true, Description: "Specifies the identifier for the external function. The identifier can contain the schema name and database name, as well as the function name. The function's signature (name and argument data types) must be unique within the schema.", }, "schema": { @@ -54,7 +50,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ Required: true, // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { - return strings.EqualFold(old, new) + return strings.EqualFold(strings.ToLower(old), strings.ToLower(new)) }, Description: "Argument name", }, @@ -63,7 +59,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ Required: true, // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { - return strings.EqualFold(old, new) + return strings.EqualFold(strings.ToLower(old), strings.ToLower(new)) }, Description: "Argument type, e.g. VARCHAR", }, @@ -84,7 +80,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ ForceNew: true, // Suppress the diff shown if the values are equal when both compared in lower case. DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { - return strings.EqualFold(old, new) + return strings.EqualFold(strings.ToLower(old), strings.ToLower(new)) }, Description: "Specifies the data type returned by the external function.", }, From e9b3c10b4bda58c84edad5ed63b20aec8ba5ee41 Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Mon, 22 Mar 2021 18:15:17 +0100 Subject: [PATCH 22/23] Fix convert issue on max_batch_rows --- pkg/resources/external_function.go | 8 +++++++- pkg/resources/external_function_acceptance_test.go | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go index 6cd3784f9d..f420f45c23 100644 --- a/pkg/resources/external_function.go +++ b/pkg/resources/external_function.go @@ -7,6 +7,7 @@ import ( "fmt" "log" "regexp" + "strconv" "strings" "github.com/chanzuckerberg/terraform-provider-snowflake/pkg/snowflake" @@ -419,7 +420,12 @@ func ReadExternalFunction(d *schema.ResourceData, meta interface{}) error { //TODO - Format in Snowflake DB is: ["context_function_1","context_function_2"] case "max_batch_rows": if desc.Value.String != "not set" { - if err = d.Set("max_batch_rows", desc.Value.String); err != nil { + i, err := strconv.ParseInt(desc.Value.String, 10, 64) + if err != nil { + return err + } + + if err = d.Set("max_batch_rows", i); err != nil { return err } } diff --git a/pkg/resources/external_function_acceptance_test.go b/pkg/resources/external_function_acceptance_test.go index e4eb0cac9d..1767ee192f 100644 --- a/pkg/resources/external_function_acceptance_test.go +++ b/pkg/resources/external_function_acceptance_test.go @@ -76,6 +76,7 @@ func externalFunctionConfig(name string, prefixes []string, url string) string { return_type = "varchar" return_behavior = "IMMUTABLE" api_integration = snowflake_api_integration.test_api_int.name + max_batch_rows = 500 url_of_proxy_and_resource = "%s" } `, name, name, name, prefixes, name, url, name, url+"_2") From ba8097ddd9a2a046fc0e3736d51c6e8aa416becd Mon Sep 17 00:00:00 2001 From: Alain Saint-Sever Date: Wed, 24 Mar 2021 11:45:40 +0100 Subject: [PATCH 23/23] Complete handling of headers and context_headers --- docs/resources/external_function.md | 20 ++++--- .../snowflake_external_function/resource.tf | 8 ++- pkg/resources/external_function.go | 56 ++++++++++++++----- .../external_function_acceptance_test.go | 12 +++- pkg/resources/external_function_test.go | 36 ++++++++++-- 5 files changed, 102 insertions(+), 30 deletions(-) diff --git a/docs/resources/external_function.md b/docs/resources/external_function.md index 01a1133e4b..362ddb093a 100644 --- a/docs/resources/external_function.md +++ b/docs/resources/external_function.md @@ -17,8 +17,12 @@ resource "snowflake_external_function" "test_ext_func" { name = "my_function" database = "my_test_db" schema = "my_test_schema" - args { - name = "data" + arg { + name = "arg1" + type = "varchar" + } + arg { + name = "arg2" type = "varchar" } return_type = "varchar" @@ -43,11 +47,11 @@ resource "snowflake_external_function" "test_ext_func" { ### Optional -- **args** (Block List) Specifies the arguments/inputs for the external function. These should correspond to the arguments that the remote service expects. (see [below for nested schema](#nestedblock--args)) +- **arg** (Block List) Specifies the arguments/inputs for the external function. These should correspond to the arguments that the remote service expects. (see [below for nested schema](#nestedblock--arg)) - **comment** (String) A description of the external function. - **compression** (String) If specified, the JSON payload is compressed when sent from Snowflake to the proxy service, and when sent back from the proxy service to Snowflake. - **context_headers** (List of String) Binds Snowflake context function results to HTTP headers. -- **headers** (Block List) Allows users to specify key-value metadata that is sent with every request as HTTP headers. (see [below for nested schema](#nestedblock--headers)) +- **header** (Block List) Allows users to specify key-value metadata that is sent with every request as HTTP headers. (see [below for nested schema](#nestedblock--header)) - **id** (String) The ID of this resource. - **max_batch_rows** (Number) This specifies the maximum number of rows in each batch sent to the proxy service. - **null_input_behavior** (String) Specifies the behavior of the external function when called with null inputs. @@ -57,8 +61,8 @@ resource "snowflake_external_function" "test_ext_func" { - **created_on** (String) Date and time when the external function was created. - -### Nested Schema for `args` + +### Nested Schema for `arg` Required: @@ -66,8 +70,8 @@ Required: - **type** (String) Argument type, e.g. VARCHAR - -### Nested Schema for `headers` + +### Nested Schema for `header` Required: diff --git a/examples/resources/snowflake_external_function/resource.tf b/examples/resources/snowflake_external_function/resource.tf index a03da1f06c..7c4bfafdda 100644 --- a/examples/resources/snowflake_external_function/resource.tf +++ b/examples/resources/snowflake_external_function/resource.tf @@ -2,8 +2,12 @@ resource "snowflake_external_function" "test_ext_func" { name = "my_function" database = "my_test_db" schema = "my_test_schema" - args { - name = "data" + arg { + name = "arg1" + type = "varchar" + } + arg { + name = "arg2" type = "varchar" } return_type = "varchar" diff --git a/pkg/resources/external_function.go b/pkg/resources/external_function.go index f420f45c23..a76bcb709d 100644 --- a/pkg/resources/external_function.go +++ b/pkg/resources/external_function.go @@ -39,7 +39,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ ForceNew: true, Description: "The database in which to create the external function.", }, - "args": { + "arg": { Type: schema.TypeList, Optional: true, ForceNew: true, @@ -104,7 +104,7 @@ var externalFunctionSchema = map[string]*schema.Schema{ ForceNew: true, Description: "The name of the API integration object that should be used to authenticate the call to the proxy service.", }, - "headers": { + "header": { Type: schema.TypeList, Optional: true, ForceNew: true, @@ -125,10 +125,14 @@ var externalFunctionSchema = map[string]*schema.Schema{ }, }, "context_headers": { - Type: schema.TypeList, - Elem: &schema.Schema{Type: schema.TypeString}, - Optional: true, - ForceNew: true, + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + ForceNew: true, + // Suppress the diff shown if the values are equal when both compared in lower case. + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + return strings.EqualFold(strings.ToLower(old), strings.ToLower(new)) + }, Description: "Binds Snowflake context function results to HTTP headers.", }, "max_batch_rows": { @@ -236,10 +240,10 @@ func CreateExternalFunction(d *schema.ResourceData, meta interface{}) error { builder.WithURLOfProxyAndResource(d.Get("url_of_proxy_and_resource").(string)) // Set optionals - if _, ok := d.GetOk("args"); ok { + if _, ok := d.GetOk("arg"); ok { var types []string args := []map[string]string{} - for _, arg := range d.Get("args").([]interface{}) { + for _, arg := range d.Get("arg").([]interface{}) { argDef := map[string]string{} for key, val := range arg.(map[string]interface{}) { argDef[key] = val.(string) @@ -271,9 +275,9 @@ func CreateExternalFunction(d *schema.ResourceData, meta interface{}) error { builder.WithComment(v.(string)) } - if _, ok := d.GetOk("headers"); ok { + if _, ok := d.GetOk("header"); ok { headers := []map[string]string{} - for _, header := range d.Get("headers").([]interface{}) { + for _, header := range d.Get("header").([]interface{}) { headerDef := map[string]string{} for key, val := range header.(map[string]interface{}) { headerDef[key] = val.(string) @@ -285,7 +289,7 @@ func CreateExternalFunction(d *schema.ResourceData, meta interface{}) error { } if v, ok := d.GetOk("context_headers"); ok { - contextHeaders := expandStringList(v.(*schema.Set).List()) + contextHeaders := expandStringList(v.([]interface{})) builder.WithContextHeaders(contextHeaders) } @@ -395,7 +399,7 @@ func ReadExternalFunction(d *schema.ResourceData, meta interface{}) error { args = append(args, arg) } - if err = d.Set("args", args); err != nil { + if err = d.Set("arg", args); err != nil { return err } } @@ -415,9 +419,33 @@ func ReadExternalFunction(d *schema.ResourceData, meta interface{}) error { return err } case "headers": - //TODO - Format in Snowflake DB is: {"head1":"val1","head2":"val2"} + if desc.Value.Valid && desc.Value.String != "null" { + // Format in Snowflake DB is: {"head1":"val1","head2":"val2"} + headerPairs := strings.Split(strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(desc.Value.String, "{", ""), "}", ""), "\"", ""), ",") + headers := []interface{}{} + + for _, headerPair := range headerPairs { + headerItem := strings.Split(headerPair, ":") + + header := map[string]interface{}{} + header["name"] = headerItem[0] + header["value"] = headerItem[1] + headers = append(headers, header) + } + + if err = d.Set("header", headers); err != nil { + return err + } + } case "context_headers": - //TODO - Format in Snowflake DB is: ["context_function_1","context_function_2"] + if desc.Value.Valid && desc.Value.String != "null" { + // Format in Snowflake DB is: ["CONTEXT_FUNCTION_1","CONTEXT_FUNCTION_2"] + contextHeaders := strings.Split(strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(desc.Value.String, "[", ""), "]", ""), "\"", ""), ",") + + if err = d.Set("context_headers", contextHeaders); err != nil { + return err + } + } case "max_batch_rows": if desc.Value.String != "not set" { i, err := strconv.ParseInt(desc.Value.String, 10, 64) diff --git a/pkg/resources/external_function_acceptance_test.go b/pkg/resources/external_function_acceptance_test.go index 1767ee192f..b091a4a432 100644 --- a/pkg/resources/external_function_acceptance_test.go +++ b/pkg/resources/external_function_acceptance_test.go @@ -57,8 +57,12 @@ func externalFunctionConfig(name string, prefixes []string, url string) string { name = "%s" database = snowflake_database.test_database.name schema = snowflake_schema.test_schema.name - args { - name = "data" + arg { + name = "arg1" + type = "varchar" + } + arg { + name = "arg2" type = "varchar" } comment = "Terraform acceptance test" @@ -76,6 +80,10 @@ func externalFunctionConfig(name string, prefixes []string, url string) string { return_type = "varchar" return_behavior = "IMMUTABLE" api_integration = snowflake_api_integration.test_api_int.name + header { + name = "x-custom-header" + value = "snowflake" + } max_batch_rows = 500 url_of_proxy_and_resource = "%s" } diff --git a/pkg/resources/external_function_test.go b/pkg/resources/external_function_test.go index 0411b9620d..8a51aad0d8 100644 --- a/pkg/resources/external_function_test.go +++ b/pkg/resources/external_function_test.go @@ -24,16 +24,18 @@ func TestExternalFunctionCreate(t *testing.T) { "name": "my_test_function", "database": "database_name", "schema": "schema_name", - "args": []interface{}{map[string]interface{}{"name": "data", "type": "varchar"}}, + "arg": []interface{}{map[string]interface{}{"name": "data", "type": "varchar"}}, "return_type": "varchar", "return_behavior": "IMMUTABLE", "api_integration": "test_api_integration_01", + "header": []interface{}{map[string]interface{}{"name": "x-custom-header", "value": "snowflake"}}, + "context_headers": []interface{}{"current_timestamp"}, "url_of_proxy_and_resource": "https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function", } d := externalFunction(t, "database_name|schema_name|my_test_function|varchar", in) WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec(`CREATE EXTERNAL FUNCTION "database_name"."schema_name"."my_test_function" \(data varchar\) RETURNS varchar NULL CALLED ON NULL INPUT IMMUTABLE COMMENT = 'user-defined function' API_INTEGRATION = 'test_api_integration_01' COMPRESSION = 'AUTO' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function'`).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(`CREATE EXTERNAL FUNCTION "database_name"."schema_name"."my_test_function" \(data varchar\) RETURNS varchar NULL CALLED ON NULL INPUT IMMUTABLE COMMENT = 'user-defined function' API_INTEGRATION = 'test_api_integration_01' HEADERS = \('x-custom-header' = 'snowflake'\) CONTEXT_HEADERS = \(current_timestamp\) COMPRESSION = 'AUTO' AS 'https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function'`).WillReturnResult(sqlmock.NewResult(1, 1)) expectExternalFunctionRead(mock) err := resources.CreateExternalFunction(d, db) @@ -51,6 +53,8 @@ func expectExternalFunctionRead(mock sqlmock.Sqlmock) { AddRow("null handling", "CALLED ON NULL INPUT"). AddRow("volatility", "IMMUTABLE"). AddRow("body", "https://123456.execute-api.us-west-2.amazonaws.com/prod/my_test_function"). + AddRow("headers", "{\"x-custom-header\":\"snowflake\""). + AddRow("context_headers", "[\"CURRENT_TIMESTAMP\"]"). AddRow("max_batch_rows", "not set"). AddRow("compression", "AUTO") @@ -60,7 +64,7 @@ func expectExternalFunctionRead(mock sqlmock.Sqlmock) { func TestExternalFunctionRead(t *testing.T) { r := require.New(t) - d := externalFunction(t, "database_name|schema_name|my_test_function|varchar", map[string]interface{}{"name": "my_test_function", "args": []interface{}{map[string]interface{}{"name": "data", "type": "varchar"}}, "return_type": "varchar", "comment": "mock comment"}) + d := externalFunction(t, "database_name|schema_name|my_test_function|varchar", map[string]interface{}{"name": "my_test_function", "arg": []interface{}{map[string]interface{}{"name": "data", "type": "varchar"}}, "return_type": "varchar", "comment": "mock comment"}) WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { expectExternalFunctionRead(mock) @@ -71,12 +75,25 @@ func TestExternalFunctionRead(t *testing.T) { r.Equal("mock comment", d.Get("comment").(string)) r.Equal("VARCHAR", d.Get("return_type").(string)) - args := d.Get("args").([]interface{}) + args := d.Get("arg").([]interface{}) r.Len(args, 1) test_func_args := args[0].(map[string]interface{}) r.Len(test_func_args, 2) r.Equal("data", test_func_args["name"].(string)) r.Equal("varchar", test_func_args["type"].(string)) + + headers := d.Get("header").([]interface{}) + r.Len(headers, 1) + test_func_headers := headers[0].(map[string]interface{}) + r.Len(test_func_headers, 2) + r.Equal("x-custom-header", test_func_headers["name"].(string)) + r.Equal("snowflake", test_func_headers["value"].(string)) + + context_headers := d.Get("context_headers").([]interface{}) + r.Len(context_headers, 1) + test_func_context_headers := expandStringList(context_headers) + r.Len(test_func_context_headers, 1) + r.Equal("CURRENT_TIMESTAMP", test_func_context_headers[0]) }) } @@ -91,3 +108,14 @@ func TestExternalFunctionDelete(t *testing.T) { r.NoError(err) }) } + +func expandStringList(configured []interface{}) []string { + vs := make([]string, 0, len(configured)) + for _, v := range configured { + val, ok := v.(string) + if ok && val != "" { + vs = append(vs, val) + } + } + return vs +}