diff --git a/.github/ISSUE_TEMPLATE/provider-issue.md b/.github/ISSUE_TEMPLATE/provider-issue.md index daf65b5dfb..123b86ba7b 100644 --- a/.github/ISSUE_TEMPLATE/provider-issue.md +++ b/.github/ISSUE_TEMPLATE/provider-issue.md @@ -32,6 +32,12 @@ To get relevant environment variable _names_ please copypaste the output of the ### Debug Output Please add turn on logging, e.g. `TF_LOG=DEBUG terraform apply` and run command again, paste it to gist & provide the link to gist. If you're still willing to paste in log output, make sure you provide only relevant log lines with requests. +It would make it more readable, if you pipe the log through `| grep databricks | sed -E 's/^.* plugin[^:]+: (.*)$/\1/'`, e.g.: + +``` +TF_LOG=DEBUG terraform plan 2>&1 | grep databricks | sed -E 's/^.* plugin[^:]+: (.*)$/\1/' +``` + ### Panic Output If Terraform produced a panic, please provide a link to a GitHub Gist containing the output of the `crash.log`. diff --git a/CHANGELOG.md b/CHANGELOG.md index a308b18284..250da6efa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,19 @@ ## 0.2.8 -* Added [Azure Key Vault support](https://github.com/databrickslabs/terraform-provider-databricks/pull/381) for databricks_secret_scope for Azure CLI authenticated users -* Added support for pinning clusters (issue #113) -* Internal: API for retrieval of the cluster events +* Added [databricks_mws_log_delivery](https://github.com/databrickslabs/terraform-provider-databricks/pull/343) resource for billable usage & audit logs consumption. +* Added [databricks_node_type](https://github.com/databrickslabs/terraform-provider-databricks/pull/376) data source for simpler selection of node types across AWS & Azure. +* Added [Azure Key Vault support](https://github.com/databrickslabs/terraform-provider-databricks/pull/381) for databricks_secret_scope for Azure CLI authenticated users. +* Added [is_pinned](https://github.com/databrickslabs/terraform-provider-databricks/pull/348) support for `databricks_cluster` resource. +* Internal: API for retrieval of the cluster events. + +Updated dependency versions: + +* github.com/Azure/go-autorest/autorest v0.11.9 +* github.com/Azure/go-autorest/autorest/adal v0.9.5 +* github.com/Azure/go-autorest/autorest/azure/auth v0.5.3 +* github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 +* gopkg.in/ini.v1 1.62.0 ## 0.2.7 diff --git a/README.md b/README.md index 21cc5648c6..28028bd293 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,11 @@ End-to-end workspace creation on [AWS](scripts/awsmt-integration) or [Azure](scr | [databricks_job](docs/resources/job.md) | [databricks_mws_credentials](docs/resources/mws_credentials.md) | [databricks_mws_customer_managed_keys](docs/resources/mws_customer_managed_keys.md) +| [databricks_mws_log_delivery](docs/resources/mws_log_delivery.md) | [databricks_mws_networks](docs/resources/mws_networks.md) | [databricks_mws_storage_configurations](docs/resources/mws_storage_configurations.md) | [databricks_mws_workspaces](docs/resources/mws_workspaces.md) +| [databricks_node_type](docs/data-sources/node_type.md) data | [databricks_notebook](docs/resources/notebook.md) | [databricks_notebook](docs/data-sources/notebook.md) data | [databricks_notebook_paths](docs/data-sources/notebook_paths.md) data diff --git a/access/data_aws_policies.go b/access/data_aws_policies.go index 0147ae3fb3..0456cded93 100644 --- a/access/data_aws_policies.go +++ b/access/data_aws_policies.go @@ -153,7 +153,7 @@ func DataAwsAssumeRolePolicy() *schema.Resource { return &schema.Resource{ Read: func(d *schema.ResourceData, m interface{}) error { externalID := d.Get("external_id").(string) - policyJSON, err := json.MarshalIndent(awsIamPolicy{ + policy := awsIamPolicy{ Version: "2008-10-17", Statements: []*awsIamPolicyStatement{ { @@ -169,7 +169,15 @@ func DataAwsAssumeRolePolicy() *schema.Resource { }, }, }, - }, "", " ") + } + if v, ok := d.GetOk("for_log_delivery"); ok { + if v.(bool) { + // this is production UsageDelivery IAM role, that is considered a constant + logDeliveryARN := "arn:aws:iam::414351767826:role/SaasUsageDeliveryRole-prod-IAMRole-3PLHICCRR1TK" + policy.Statements[0].Principal["AWS"] = logDeliveryARN + } + } + policyJSON, err := json.MarshalIndent(policy, "", " ") if err != nil { return err } @@ -182,6 +190,12 @@ func DataAwsAssumeRolePolicy() *schema.Resource { Default: "414351767826", Optional: true, }, + "for_log_delivery": { + Type: schema.TypeBool, + Description: "Grant AssumeRole to Databricks SaasUsageDeliveryRole instead of root account", + Optional: true, + Default: false, + }, "external_id": { Type: schema.TypeString, Required: true, diff --git a/docs/data-sources/aws_assume_role_policy.md b/docs/data-sources/aws_assume_role_policy.md index 4548e48655..a1512dc827 100644 --- a/docs/data-sources/aws_assume_role_policy.md +++ b/docs/data-sources/aws_assume_role_policy.md @@ -46,6 +46,7 @@ resource "databricks_mws_credentials" "this" { ## Argument Reference * `external_id` (Required) (String) External ID that can be found at http://accounts.cloud.databricks.com/#aws +* `for_log_delivery` (Optional) Either or not this assume role policy should be created for usage log delivery. Defaults to false. ## Attribute Reference diff --git a/docs/resources/mws_log_delivery.md b/docs/resources/mws_log_delivery.md new file mode 100644 index 0000000000..6e48d61706 --- /dev/null +++ b/docs/resources/mws_log_delivery.md @@ -0,0 +1,140 @@ +# databricks_mws_log_delivery Resource + +-> **Note** This resource has an evolving API, which may change in future versions of the provider. + +This resource configures the delivery of the two supported log types from Databricks workspaces: [billable usage logs](https://docs.databricks.com/administration-guide/account-settings/billable-usage-delivery.html) and [audit logs](https://docs.databricks.com/administration-guide/account-settings/audit-logs.html). You cannot delete a log delivery configuration, but you can disable it when you no longer need it. This fact is important because there is a limit to the number of enabled log delivery configurations that you can create for an account. You can create a maximum of two enabled using the account level *(without workspace filter)* and two that use the workspace filter. There is an additional uniqueness constraint that two enabled configurations cannot share all their fields (not including the `config_name`). Re-enabling may fail when there's a violation of limit or uniqueness constraints. + +## Example Usage + +End-to-end example of usage and audit log delivery: + +```hcl +resource "aws_s3_bucket" "logdelivery" { + bucket = "${var.prefix}-logdelivery" + acl = "private" + versioning { + enabled = false + } + force_destroy = true + tags = merge(var.tags, { + Name = "${var.prefix}-logdelivery" + }) +} + +resource "aws_s3_bucket_public_access_block" "logdelivery" { + bucket = aws_s3_bucket.logdelivery.id + ignore_public_acls = true +} + +data "databricks_aws_assume_role_policy" "logdelivery" { + external_id = var.account_id + for_log_delivery = true +} + +resource "aws_iam_role" "logdelivery" { + name = "${var.prefix}-logdelivery" + description = "(${var.prefix}) UsageDelivery role" + assume_role_policy = data.databricks_aws_assume_role_policy.logdelivery.json + tags = var.tags +} + +data "databricks_aws_bucket_policy" "logdelivery" { + full_access_role = aws_iam_role.logdelivery.arn + bucket = aws_s3_bucket.logdelivery.bucket +} + +resource "aws_s3_bucket_policy" "logdelivery" { + bucket = aws_s3_bucket.logdelivery.id + policy = data.databricks_aws_bucket_policy.logdelivery.json +} + +resource "databricks_mws_credentials" "log_writer" { + account_id = var.account_id + credentials_name = "Usage Delivery" + role_arn = aws_iam_role.logdelivery.arn +} + +resource "databricks_mws_storage_configurations" "log_bucket" { + account_id = var.account_id + storage_configuration_name = "Usage Logs" + bucket_name = aws_s3_bucket.logdelivery.bucket +} + +resource "databricks_mws_log_delivery" "usage_logs" { + account_id = var.account_id + credentials_id = databricks_mws_credentials.log_writer.credentials_id + storage_configuration_id = databricks_mws_storage_configurations.log_bucket.storage_configuration_id + delivery_path_prefix = "billable-usage" + config_name = "Usage Logs" + log_type = "BILLABLE_USAGE" + output_format = "CSV" +} + +resource "databricks_mws_log_delivery" "audit_logs" { + account_id = var.account_id + credentials_id = databricks_mws_credentials.log_writer.credentials_id + storage_configuration_id = databricks_mws_storage_configurations.log_bucket.storage_configuration_id + delivery_path_prefix = "audit-logs" + config_name = "Audit Logs" + log_type = "AUDIT_LOGS" + output_format = "JSON" +} +``` + +## Billable Usage + +CSV files with [static schema](https://docs.databricks.com/administration-guide/account-settings/usage.html) are delivered to `/billable-usage/csv/`. Files are named `workspaceId=-usageMonth=.csv`, which are delivered daily by overwriting the month's CSV file for each workspace. + +```hcl +resource "databricks_mws_log_delivery" "usage_logs" { + account_id = var.account_id + credentials_id = databricks_mws_credentials.log_writer.credentials_id + storage_configuration_id = databricks_mws_storage_configurations.log_bucket.storage_configuration_id + delivery_path_prefix = "billable-usage" + config_name = "Usage Logs" + log_type = "BILLABLE_USAGE" + output_format = "CSV" +} +``` + +## Audit Logs + +JSON files with [static schema](https://docs.databricks.com/administration-guide/account-settings/audit-logs.html#audit-log-schema) are delivered to `/workspaceId=/date=/auditlogs_.json`. Logs are available within 15 minutes of activation for audit logs. New JSON files are delivered every few minutes, potentially overwriting existing files for each workspace. Sometimes data may arrive later than 15 minutes. Databricks can overwrite the delivered log files in your bucket at any time. If a file is overwritten, the existing content remains, but there may be additional lines for more auditable events. Overwriting ensures exactly-once semantics without requiring read or delete access to your account. + +```hcl +resource "databricks_mws_log_delivery" "audit_logs" { + account_id = var.account_id + credentials_id = databricks_mws_credentials.log_writer.credentials_id + storage_configuration_id = databricks_mws_storage_configurations.log_bucket.storage_configuration_id + delivery_path_prefix = "audit-logs" + config_name = "Audit Logs" + log_type = "AUDIT_LOGS" + output_format = "JSON" +} +``` + +## Argument reference + +* `account_id` - The Databricks account ID that hosts the log delivery configuration. +* `config_name` - The optional human-readable name of the log delivery configuration. Defaults to empty. +* `log_type` - The type of log delivery. `BILLABLE_USAGE` and `AUDIT_LOGS` are supported. +* `output_format` - The file type of log delivery. Currently `CSV` (for `BILLABLE_USAGE`) and `JSON` (for `AUDIT_LOGS`) are supported. +* `credentials_id` - The ID for a Databricks [credential configuration](mws_credentials.md) that represents the AWS IAM role [with policy](../data-sources/aws_assume_role_policy.md) and [trust relationship](../data-sources/aws_assume_role_policy.md) as described in the main billable usage documentation page. +* `storage_configuration_id` - The ID for a Databricks [storage configuration](mws_storage_configurations.md) that represents the S3 bucket with [bucket policy](../data-sources/aws_bucket_policy.md) as described in the main billable usage documentation page. +* `workspace_ids_filter` - (Optional) By default, this log configuration applies to all workspaces associated with your account ID. If your account is on the E2 version of the platform or on a select custom plan that allows multiple workspaces per account, you may have multiple workspaces associated with your account ID. You can optionally set the field as mentioned earlier to an array of workspace IDs. If you plan to use different log delivery configurations for several workspaces, set this explicitly rather than leaving it blank. If you leave this blank and your account ID gets additional workspaces in the future, this configuration will also apply to the new workspaces. +* `delivery_path_prefix` - (Optional) Defaults to empty, which means that logs delivered to the root of the bucket. The value must be a valid S3 object key. It must not start or end with a slash character. +* `delivery_start_time` - (Optional) The optional start month and year for delivery, specified in YYYY-MM format. Defaults to current year and month. Usage is not available before 2019-03. + +## Attribute reference + +Resource exports the following attributes: + +* `config_id` - Databricks log delivery configuration ID. + +## Import + +This resource can be imported by specifying a combination of an account id and log config id separated by `|`: + +```bash +$ terraform import databricks_mws_log_delivery.usage "|" +``` \ No newline at end of file diff --git a/mws/acceptance/log_delivery_test.go b/mws/acceptance/log_delivery_test.go new file mode 100644 index 0000000000..3c9a5d2f75 --- /dev/null +++ b/mws/acceptance/log_delivery_test.go @@ -0,0 +1,60 @@ +package acceptance + +import ( + "os" + "testing" + + "github.com/databrickslabs/databricks-terraform/internal/acceptance" + "github.com/databrickslabs/databricks-terraform/internal/qa" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestMwsAccLogDelivery(t *testing.T) { + if os.Getenv("CLOUD_ENV") != "MWS" { + t.Skip("Cannot run test on non-MWS environment") + } + acceptance.AccTest(t, resource.TestCase{ + Steps: []resource.TestStep{ + { + Config: qa.EnvironmentTemplate(t, ` + provider "databricks" { + host = "{env.DATABRICKS_HOST}" + username = "{env.DATABRICKS_USERNAME}" + password = "{env.DATABRICKS_PASSWORD}" + } + + resource "databricks_mws_credentials" "ld" { + account_id = "{env.DATABRICKS_ACCOUNT_ID}" + credentials_name = "tf-acceptance-logdelivery-{var.RANDOM}" + role_arn = "{env.TEST_LOGDELIVERY_ARN}" + } + + resource "databricks_mws_storage_configurations" "ld" { + account_id = "{env.DATABRICKS_ACCOUNT_ID}" + storage_configuration_name = "tf-acceptance-logdelivery-{var.RANDOM}" + bucket_name = "{env.TEST_LOGDELIVERY_BUCKET}" + } + + resource "databricks_mws_log_delivery" "usage_logs" { + account_id = "{env.DATABRICKS_ACCOUNT_ID}" + credentials_id = databricks_mws_credentials.ld.credentials_id + storage_configuration_id = databricks_mws_storage_configurations.ld.storage_configuration_id + delivery_path_prefix = "tf-{var.RANDOM}/billable-usage" + config_name = "Usage {var.RANDOM}" + log_type = "BILLABLE_USAGE" + output_format = "CSV" + } + + resource "databricks_mws_log_delivery" "audit_logs" { + account_id = "{env.DATABRICKS_ACCOUNT_ID}" + credentials_id = databricks_mws_credentials.ld.credentials_id + storage_configuration_id = databricks_mws_storage_configurations.ld.storage_configuration_id + delivery_path_prefix = "tf-{var.RANDOM}/audit-logs" + config_name = "Audit {var.RANDOM}" + log_type = "AUDIT_LOGS" + output_format = "JSON" + }`), + }, + }, + }) +} diff --git a/mws/resource_log_delivery.go b/mws/resource_log_delivery.go new file mode 100644 index 0000000000..59f97634c5 --- /dev/null +++ b/mws/resource_log_delivery.go @@ -0,0 +1,149 @@ +package mws + +import ( + "context" + "fmt" + "log" + + "github.com/databrickslabs/databricks-terraform/common" + "github.com/databrickslabs/databricks-terraform/internal" + "github.com/databrickslabs/databricks-terraform/internal/util" + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" +) + +// LogDelivery wrapper +type LogDelivery struct { + LogDeliveryConfiguration LogDeliveryConfiguration `json:"log_delivery_configuration"` +} + +// LogDeliveryConfiguration describes log delivery +type LogDeliveryConfiguration struct { + AccountID string `json:"account_id"` + ConfigID string `json:"config_id,omitempty" tf:"computed"` + CredentialsID string `json:"credentials_id"` + StorageConfigurationID string `json:"storage_configuration_id"` + WorkspaceIdsFilter []string `json:"workspace_ids_filter,omitempty"` + ConfigName string `json:"config_name,omitempty"` + Status string `json:"status,omitempty" tf:"computed"` + LogType string `json:"log_type"` + OutputFormat string `json:"output_format"` + DeliveryPathPrefix string `json:"delivery_path_prefix,omitempty"` + DeliveryStartTime string `json:"delivery_start_time,omitempty" tf:"computed"` +} + +// LogDeliveryAPI ... +type LogDeliveryAPI struct { + client *common.DatabricksClient +} + +// NewLogDeliveryAPI ... +func NewLogDeliveryAPI(m interface{}) LogDeliveryAPI { + return LogDeliveryAPI{client: m.(*common.DatabricksClient)} +} + +// Read reads log delivery configuration +func (a LogDeliveryAPI) Read(accountID, configID string) (LogDeliveryConfiguration, error) { + var ld LogDelivery + err := a.client.Get(fmt.Sprintf("/accounts/%s/log-delivery/%s", accountID, configID), nil, &ld) + return ld.LogDeliveryConfiguration, err +} + +// Create new log delivery configuration +func (a LogDeliveryAPI) Create(ldc LogDeliveryConfiguration) (string, error) { + var ld LogDelivery + err := a.client.Post(fmt.Sprintf("/accounts/%s/log-delivery", ldc.AccountID), LogDelivery{ + LogDeliveryConfiguration: ldc, + }, &ld) + // todo: verify with empty response - structs should have empty default strings + return ld.LogDeliveryConfiguration.ConfigID, err +} + +// Disable log delivery configuration - e.g. delete it +func (a LogDeliveryAPI) Disable(accountID, configID string) error { + return a.client.Patch(fmt.Sprintf("/accounts/%s/log-delivery/%s", accountID, configID), map[string]string{ + "status": "DISABLED", + }) +} + +// ResourceLogDelivery .. +func ResourceLogDelivery() *schema.Resource { + p := util.NewPairID("account_id", "config_id") + s := internal.StructToSchema(LogDeliveryConfiguration{}, + func(s map[string]*schema.Schema) map[string]*schema.Schema { + // nolint + s["config_name"].ValidateFunc = validation.StringLenBetween(0, 255) + for k, v := range s { + if v.Computed { + continue + } + s[k].ForceNew = true + } + s["delivery_start_time"].DiffSuppressFunc = func( + k, old, new string, d *schema.ResourceData) bool { + return false + } + return s + }) + readContext := func(ctx context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics { + accountID, configID, err := p.Unpack(d) + if err != nil { + return diag.FromErr(err) + } + ldc, err := NewLogDeliveryAPI(m).Read(accountID, configID) + if e, ok := err.(common.APIError); ok && e.IsMissing() { + log.Printf("[DEBUG] Log delivery configuration %s was not found. Removing from state.", configID) + d.SetId("") + return nil + } + if err != nil { + return diag.FromErr(err) + } + if ldc.Status == "DISABLED" { + log.Printf("[DEBUG] Log delivery configuration %s was disabled. Removing from state.", configID) + d.SetId("") + return nil + } + err = internal.StructToData(ldc, s, d) + if err != nil { + return diag.FromErr(err) + } + return nil + } + return &schema.Resource{ + Schema: s, + ReadContext: readContext, + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + CreateContext: func(ctx context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics { + var ldc LogDeliveryConfiguration + err := internal.DataToStructPointer(d, s, &ldc) + if err != nil { + return diag.FromErr(err) + } + configID, err := NewLogDeliveryAPI(m).Create(ldc) + if err != nil { + return diag.FromErr(err) + } + err = d.Set("config_id", configID) + if err != nil { + return diag.FromErr(err) + } + p.Pack(d) + return readContext(ctx, d, m) + }, + DeleteContext: func(ctx context.Context, d *schema.ResourceData, m interface{}) diag.Diagnostics { + accountID, configID, err := p.Unpack(d) + if err != nil { + return diag.FromErr(err) + } + err = NewLogDeliveryAPI(m).Disable(accountID, configID) + if err != nil { + return diag.FromErr(err) + } + return nil + }, + } +} diff --git a/mws/resource_log_delivery_test.go b/mws/resource_log_delivery_test.go new file mode 100644 index 0000000000..45b6fcd579 --- /dev/null +++ b/mws/resource_log_delivery_test.go @@ -0,0 +1,261 @@ +package mws + +import ( + "testing" + + "github.com/databrickslabs/databricks-terraform/common" + "github.com/databrickslabs/databricks-terraform/internal/qa" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMwsAccLogDelivery(t *testing.T) { + acctID := qa.GetEnvOrSkipTest(t, "DATABRICKS_ACCOUNT_ID") + roleARN := qa.GetEnvOrSkipTest(t, "TEST_LOGDELIVERY_ARN") + bucket := qa.GetEnvOrSkipTest(t, "TEST_LOGDELIVERY_BUCKET") + client := common.CommonEnvironmentClient() + randomName := qa.RandomName("tf-logdelivery-") + + logDeliveryAPI := NewLogDeliveryAPI(client) + credentialsAPI := NewCredentialsAPI(client) + storageConfigurationsAPI := NewStorageConfigurationsAPI(client) + + creds, err := credentialsAPI.Create(acctID, randomName, roleARN) + require.NoError(t, err) + defer func() { + assert.NoError(t, credentialsAPI.Delete(acctID, creds.CredentialsID)) + }() + + storageConfig, err := storageConfigurationsAPI.Create(acctID, randomName, bucket) + require.NoError(t, err) + defer func() { + assert.NoError(t, storageConfigurationsAPI.Delete(acctID, storageConfig.StorageConfigurationID)) + }() + + configID, err := logDeliveryAPI.Create(LogDeliveryConfiguration{ + AccountID: acctID, + CredentialsID: creds.CredentialsID, + StorageConfigurationID: storageConfig.StorageConfigurationID, + ConfigName: randomName, + DeliveryPathPrefix: randomName, + LogType: "AUDIT_LOGS", + OutputFormat: "JSON", + }) + require.NoError(t, err) + defer func() { + assert.NoError(t, logDeliveryAPI.Disable(acctID, configID)) + }() + + ldc, err := logDeliveryAPI.Read(acctID, configID) + require.NoError(t, err) + assert.Equal(t, "ENABLED", ldc.Status) +} + +func TestResourceLogDeliveryCreate(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "POST", + Resource: "/api/2.0/accounts/abc/log-delivery", + ExpectedRequest: LogDelivery{ + LogDeliveryConfiguration: LogDeliveryConfiguration{ + AccountID: "abc", + ConfigName: "Audit logs", + CredentialsID: "bcd", + DeliveryPathPrefix: "/a/b", + LogType: "AUDIT_LOGS", + OutputFormat: "JSON", + StorageConfigurationID: "def", + DeliveryStartTime: "2020-10", + WorkspaceIdsFilter: []string{"e", "f"}, + }, + }, + Response: LogDelivery{ + LogDeliveryConfiguration: LogDeliveryConfiguration{ + ConfigID: "nid", + }, + }, + }, + { + Method: "GET", + Resource: "/api/2.0/accounts/abc/log-delivery/nid", + Response: LogDelivery{ + LogDeliveryConfiguration: LogDeliveryConfiguration{ + ConfigID: "nid", + AccountID: "abc", + ConfigName: "Audit logs", + CredentialsID: "bcd", + DeliveryPathPrefix: "/a/b", + LogType: "AUDIT_LOGS", + OutputFormat: "JSON", + StorageConfigurationID: "def", + DeliveryStartTime: "2020-10", + WorkspaceIdsFilter: []string{"e", "f"}, + }, + }, + }, + }, + Resource: ResourceLogDelivery(), + HCL: ` + account_id = "abc" + credentials_id = "bcd" + storage_configuration_id = "def" + config_name = "Audit logs" + log_type = "AUDIT_LOGS" + output_format = "JSON" + delivery_path_prefix = "/a/b" + workspace_ids_filter = ["e", "f"] + delivery_start_time = "2020-10"`, + Create: true, + }.Apply(t) + assert.NoError(t, err, err) + assert.Equal(t, "abc|nid", d.Id()) + assert.Equal(t, "nid", d.Get("config_id")) +} + +func TestResourceLogDeliveryCreate_Error(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "POST", + Resource: "/api/2.0/accounts/abc/log-delivery", + Response: common.APIErrorBody{ + ErrorCode: "INVALID_REQUEST", + Message: "Internal error happened", + }, + Status: 400, + }, + }, + Resource: ResourceLogDelivery(), + HCL: ` + account_id = "abc" + credentials_id = "bcd" + storage_configuration_id = "def" + config_name = "Audit logs" + log_type = "AUDIT_LOGS" + output_format = "JSON" + delivery_path_prefix = "/a/b" + workspace_ids_filter = ["e", "f"] + delivery_start_time = "2020-10"`, + Create: true, + }.Apply(t) + qa.AssertErrorStartsWith(t, err, "Internal error happened") + assert.Equal(t, "", d.Id(), "Id should be empty for error creates") +} + +func TestResourceLogDeliveryRead(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.0/accounts/abc/log-delivery/nid", + Response: LogDelivery{ + LogDeliveryConfiguration: LogDeliveryConfiguration{ + ConfigID: "nid", + Status: "ENABLED", + AccountID: "abc", + ConfigName: "Audit logs", + CredentialsID: "bcd", + DeliveryPathPrefix: "/a/b", + LogType: "AUDIT_LOGS", + OutputFormat: "JSON", + StorageConfigurationID: "def", + DeliveryStartTime: "2020-10", + WorkspaceIdsFilter: []string{"e", "f"}, + }, + }, + }, + }, + Resource: ResourceLogDelivery(), + Read: true, + New: true, + ID: "abc|nid", + }.Apply(t) + assert.NoError(t, err, err) + assert.Equal(t, "abc|nid", d.Id(), "Id should not be empty") + assert.Equal(t, "bcd", d.Get("credentials_id")) + assert.Equal(t, "def", d.Get("storage_configuration_id")) +} + +func TestResourceLogDeliveryRead_NotFound(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.0/accounts/abc/log-delivery/nid", + Response: LogDelivery{ + LogDeliveryConfiguration: LogDeliveryConfiguration{ + Status: "DISABLED", + }, + }, + }, + }, + Resource: ResourceLogDelivery(), + Read: true, + ID: "abc|nid", + }.Apply(t) + assert.NoError(t, err, err) + assert.Equal(t, "", d.Id(), "Id should be empty for missing resources") +} + +func TestResourceLogDeliveryRead_Error(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.0/accounts/abc/log-delivery/nid", + Response: common.APIErrorBody{ + ErrorCode: "INVALID_REQUEST", + Message: "Internal error happened", + }, + Status: 400, + }, + }, + Resource: ResourceLogDelivery(), + Read: true, + ID: "abc|nid", + }.Apply(t) + qa.AssertErrorStartsWith(t, err, "Internal error happened") + assert.Equal(t, "abc|nid", d.Id(), "Id should not be empty for error reads") +} + +func TestResourceLogDeliveryDelete(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "PATCH", + Resource: "/api/2.0/accounts/abc/log-delivery/nid", + ExpectedRequest: map[string]string{ + "status": "DISABLED", + }, + }, + }, + Resource: ResourceLogDelivery(), + Delete: true, + ID: "abc|nid", + }.Apply(t) + assert.NoError(t, err, err) + assert.Equal(t, "abc|nid", d.Id()) +} + +func TestResourceLogDeliveryDelete_Error(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "PATCH", + Resource: "/api/2.0/accounts/abc/log-delivery/nid", + Response: common.APIErrorBody{ + ErrorCode: "INVALID_REQUEST", + Message: "Internal error happened", + }, + Status: 400, + }, + }, + Resource: ResourceLogDelivery(), + Delete: true, + ID: "abc|nid", + }.Apply(t) + qa.AssertErrorStartsWith(t, err, "Internal error happened") + assert.Equal(t, "abc|nid", d.Id()) +} diff --git a/provider/provider.go b/provider/provider.go index 86b3bd9f43..301d9fb07e 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -56,11 +56,12 @@ func DatabricksProvider() *schema.Provider { "databricks_token": identity.ResourceToken(), "databricks_user": identity.ResourceUser(), + "databricks_mws_customer_managed_keys": mws.ResourceCustomerManagedKey(), "databricks_mws_credentials": mws.ResourceCredentials(), - "databricks_mws_storage_configurations": mws.ResourceStorageConfiguration(), + "databricks_mws_log_delivery": mws.ResourceLogDelivery(), "databricks_mws_networks": mws.ResourceNetwork(), + "databricks_mws_storage_configurations": mws.ResourceStorageConfiguration(), "databricks_mws_workspaces": mws.ResourceWorkspace(), - "databricks_mws_customer_managed_keys": mws.ResourceCustomerManagedKey(), "databricks_aws_s3_mount": storage.ResourceAWSS3Mount(), "databricks_azure_adls_gen1_mount": storage.ResourceAzureAdlsGen1Mount(), diff --git a/scripts/mws-integration/main.tf b/scripts/mws-integration/main.tf index 025d572e04..6f6235c030 100644 --- a/scripts/mws-integration/main.tf +++ b/scripts/mws-integration/main.tf @@ -55,6 +55,53 @@ module "aws_common" { tags = local.tags } +resource "aws_s3_bucket" "logdelivery" { + bucket = "${local.prefix}-logdelivery" + acl = "private" + versioning { + enabled = false + } + force_destroy = true + tags = merge(local.tags, { + Name = "${local.prefix}-logdelivery" + }) +} + +output "test_logdelivery_bucket" { + value = aws_s3_bucket.logdelivery.bucket +} + +resource "aws_s3_bucket_public_access_block" "logdelivery" { + bucket = aws_s3_bucket.logdelivery.id + ignore_public_acls = true +} + +data "databricks_aws_assume_role_policy" "logdelivery" { + external_id = data.external.env.result.DATABRICKS_ACCOUNT_ID + for_log_delivery = true +} + +resource "aws_iam_role" "logdelivery" { + name = "${local.prefix}-logdelivery" + description = "(${local.prefix}) UsageDelivery role" + assume_role_policy = data.databricks_aws_assume_role_policy.logdelivery.json + tags = local.tags +} + +output "test_logdelivery_arn" { + value = aws_iam_role.logdelivery.arn +} + +data "databricks_aws_bucket_policy" "logdelivery" { + full_access_role = aws_iam_role.logdelivery.arn + bucket = aws_s3_bucket.logdelivery.bucket +} + +resource "aws_s3_bucket_policy" "logdelivery" { + bucket = aws_s3_bucket.logdelivery.id + policy = data.databricks_aws_bucket_policy.logdelivery.json +} + output "cloud_env" { // needed to distinguish between azure, aws & mws tests value = "MWS"