From eb8b5e5257bffc60319402dc6feab259ba8c236b Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Wed, 10 Apr 2024 11:58:16 -0700 Subject: [PATCH 01/18] Replaces `ComposeTestCheckFunc` with `ComposeAggregateTestCheckFunc` --- internal/service/dms/replication_task_test.go | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index 4ab75048c784..2df160ac2e62 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -44,7 +44,7 @@ func TestAccDMSReplicationTask_basic(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccReplicationTaskConfig_basic(rName, tags), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), ), @@ -62,7 +62,7 @@ func TestAccDMSReplicationTask_basic(t *testing.T) { }, { Config: testAccReplicationTaskConfig_basic(rName, updatedTags), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), ), }, @@ -83,7 +83,7 @@ func TestAccDMSReplicationTask_update(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "ZedsDead"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), @@ -93,7 +93,7 @@ func TestAccDMSReplicationTask_update(t *testing.T) { }, { Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "EMBRZ"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1024")), @@ -107,7 +107,7 @@ func TestAccDMSReplicationTask_update(t *testing.T) { }, { Config: testAccReplicationTaskConfig_update(rName, "full-load", 1248, "ZedsDead"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), @@ -117,7 +117,7 @@ func TestAccDMSReplicationTask_update(t *testing.T) { }, { Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "ZedsDead"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1024")), @@ -131,7 +131,7 @@ func TestAccDMSReplicationTask_update(t *testing.T) { }, { Config: testAccReplicationTaskConfig_update(rName, "full-load", 1248, "ZedsDead"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), @@ -141,7 +141,7 @@ func TestAccDMSReplicationTask_update(t *testing.T) { }, { Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "EMBRZ"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1024")), @@ -170,7 +170,7 @@ func TestAccDMSReplicationTask_cdcStartPosition(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccReplicationTaskConfig_cdcStartPosition(rName, "mysql-bin-changelog.000024:373"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "cdc_start_position", "mysql-bin-changelog.000024:373"), ), @@ -230,7 +230,7 @@ func TestAccDMSReplicationTask_startReplicationTask(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccReplicationTaskConfig_start(rName, true, "testrule"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "status", "running"), ), @@ -243,14 +243,14 @@ func TestAccDMSReplicationTask_startReplicationTask(t *testing.T) { }, { Config: testAccReplicationTaskConfig_start(rName, true, "changedtestrule"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "status", "running"), ), }, { Config: testAccReplicationTaskConfig_start(rName, false, "changedtestrule"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "status", "stopped"), ), @@ -274,7 +274,7 @@ func TestAccDMSReplicationTask_s3ToRDS(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccReplicationTaskConfig_s3ToRDS(rName), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), ), @@ -305,7 +305,7 @@ func TestAccDMSReplicationTask_disappears(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccReplicationTaskConfig_basic(rName, tags), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), acctest.CheckResourceDisappears(ctx, acctest.Provider, tfdms.ResourceReplicationTask(), resourceName), ), @@ -332,7 +332,7 @@ func TestAccDMSReplicationTask_cdcStartTime_rfc3339_date(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccReplicationTaskConfig_cdcStartTime(rName, rfc3339Time), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "cdc_start_position", awsDmsExpectedOutput), ), @@ -365,7 +365,7 @@ func TestAccDMSReplicationTask_cdcStartTime_unix_timestamp(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccReplicationTaskConfig_cdcStartTime(rName, unixDateTime), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "cdc_start_position", awsDmsExpectedOutput), ), @@ -394,7 +394,7 @@ func TestAccDMSReplicationTask_move(t *testing.T) { Steps: []resource.TestStep{ { Config: testAccReplicationTaskConfig_move(rName, "aws_dms_replication_instance.test.replication_instance_arn"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationInstanceExists(ctx, resourceName), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), resource.TestCheckResourceAttrPair(resourceName, "replication_instance_arn", instanceOne, "replication_instance_arn"), @@ -402,7 +402,7 @@ func TestAccDMSReplicationTask_move(t *testing.T) { }, { Config: testAccReplicationTaskConfig_move(rName, "aws_dms_replication_instance.test2.replication_instance_arn"), - Check: resource.ComposeTestCheckFunc( + Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationInstanceExists(ctx, resourceName), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), resource.TestCheckResourceAttrPair(resourceName, "replication_instance_arn", instanceTwo, "replication_instance_arn"), From 67d3d5d4ed3fa9d03d73919b406f44b65c5f3ad6 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Wed, 10 Apr 2024 12:52:39 -0700 Subject: [PATCH 02/18] Updates ARN checks --- internal/service/dms/replication_task_test.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index 2df160ac2e62..5c7e7cb1a9bd 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -46,14 +46,9 @@ func TestAccDMSReplicationTask_basic(t *testing.T) { Config: testAccReplicationTaskConfig_basic(rName, tags), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), + acctest.MatchResourceAttrRegionalARN(resourceName, "replication_task_arn", "dms", regexache.MustCompile(`task:[A-Z0-9]{26}`)), ), }, - { - Config: testAccReplicationTaskConfig_basic(rName, tags), - PlanOnly: true, - ExpectNonEmptyPlan: false, - }, { ResourceName: resourceName, ImportState: true, @@ -85,7 +80,6 @@ func TestAccDMSReplicationTask_update(t *testing.T) { Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1024")), resource.TestMatchResourceAttr(resourceName, "table_mappings", regexache.MustCompile("rule-name\":\"ZedsDead")), @@ -109,7 +103,6 @@ func TestAccDMSReplicationTask_update(t *testing.T) { Config: testAccReplicationTaskConfig_update(rName, "full-load", 1248, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1248")), resource.TestMatchResourceAttr(resourceName, "table_mappings", regexache.MustCompile("rule-name\":\"ZedsDead")), @@ -133,7 +126,6 @@ func TestAccDMSReplicationTask_update(t *testing.T) { Config: testAccReplicationTaskConfig_update(rName, "full-load", 1248, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1248")), resource.TestMatchResourceAttr(resourceName, "table_mappings", regexache.MustCompile("rule-name\":\"ZedsDead")), From 71c756540155a0387e20eea5b85ac5affee6d177 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Wed, 10 Apr 2024 14:51:16 -0700 Subject: [PATCH 03/18] Uses `CheckResourceAttrJMES` instead of regex for JSON attributes --- internal/service/dms/replication_task_test.go | 99 +++++++++++-------- 1 file changed, 60 insertions(+), 39 deletions(-) diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index 5c7e7cb1a9bd..54238efb0910 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -65,7 +65,7 @@ func TestAccDMSReplicationTask_basic(t *testing.T) { }) } -func TestAccDMSReplicationTask_update(t *testing.T) { +func TestAccDMSReplicationTask_updateSettingsAndMappings(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" @@ -77,73 +77,94 @@ func TestAccDMSReplicationTask_update(t *testing.T) { CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), Steps: []resource.TestStep{ { - Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "ZedsDead"), + Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1024, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), - resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1024")), - resource.TestMatchResourceAttr(resourceName, "table_mappings", regexache.MustCompile("rule-name\":\"ZedsDead")), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1024"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "ZedsDead"), ), }, { - Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "EMBRZ"), + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication_task"}, + }, + { + Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1024, "EMBRZ"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), - resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1024")), - resource.TestMatchResourceAttr(resourceName, "table_mappings", regexache.MustCompile("rule-name\":\"EMBRZ")), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1024"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "EMBRZ"), ), }, { - Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "EMBRZ"), - PlanOnly: true, - ExpectNonEmptyPlan: false, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication_task"}, }, { - Config: testAccReplicationTaskConfig_update(rName, "full-load", 1248, "ZedsDead"), + Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1248, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), - resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1248")), - resource.TestMatchResourceAttr(resourceName, "table_mappings", regexache.MustCompile("rule-name\":\"ZedsDead")), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1248"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "ZedsDead"), ), }, { - Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "ZedsDead"), + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication_task"}, + }, + { + Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1024, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), - resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1024")), - resource.TestMatchResourceAttr(resourceName, "table_mappings", regexache.MustCompile("rule-name\":\"ZedsDead")), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1024"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "ZedsDead"), ), }, { - Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "ZedsDead"), - PlanOnly: true, - ExpectNonEmptyPlan: false, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication_task"}, }, { - Config: testAccReplicationTaskConfig_update(rName, "full-load", 1248, "ZedsDead"), + Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1248, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), - resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1248")), - resource.TestMatchResourceAttr(resourceName, "table_mappings", regexache.MustCompile("rule-name\":\"ZedsDead")), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1248"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "ZedsDead"), ), }, { - Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "EMBRZ"), + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication_task"}, + }, + { + Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1024, "EMBRZ"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), - resource.TestMatchResourceAttr(resourceName, "replication_task_settings", regexache.MustCompile("MemoryLimitTotal\":1024")), - resource.TestMatchResourceAttr(resourceName, "table_mappings", regexache.MustCompile("rule-name\":\"EMBRZ")), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1024"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "EMBRZ"), ), }, { - Config: testAccReplicationTaskConfig_update(rName, "full-load", 1024, "EMBRZ"), - PlanOnly: true, - ExpectNonEmptyPlan: false, + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication_task"}, }, }, }) @@ -526,15 +547,15 @@ resource "aws_dms_replication_task" "test" { `, rName)) } -func testAccReplicationTaskConfig_update(rName, migType string, memLimitTotal int, ruleName string) string { +func testAccReplicationTaskConfig_updateSettingsAndMappings(rName string, memLimitTotal int, ruleName string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { - migration_type = %[2]q + migration_type = "full-load" replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn replication_task_id = %[1]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":%[3]d,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" + replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":%[2]d,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"%[4]s\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" + table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"%[3]s\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" tags = { Name = %[1]q @@ -542,7 +563,7 @@ resource "aws_dms_replication_task" "test" { target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn } -`, rName, migType, memLimitTotal, ruleName)) +`, rName, memLimitTotal, ruleName)) } func testAccReplicationTaskConfig_cdcStartPosition(rName, cdcStartPosition string) string { From 23f8578efb80eaa636dbdc6c27f7d7b032d7b32a Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Wed, 10 Apr 2024 17:17:46 -0700 Subject: [PATCH 04/18] Makes `TestAccDMSReplicationTask_basic` check defaults --- internal/service/dms/replication_task.go | 1 + internal/service/dms/replication_task_test.go | 231 +++++++++++++++--- 2 files changed, 198 insertions(+), 34 deletions(-) diff --git a/internal/service/dms/replication_task.go b/internal/service/dms/replication_task.go index e7513f12788b..3e7d10527c69 100644 --- a/internal/service/dms/replication_task.go +++ b/internal/service/dms/replication_task.go @@ -76,6 +76,7 @@ func ResourceReplicationTask() *schema.Resource { "replication_task_settings": { Type: schema.TypeString, Optional: true, + Computed: true, ValidateFunc: validation.StringIsJSON, DiffSuppressFunc: suppressEquivalentTaskSettings, DiffSuppressOnRefresh: true, diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index 54238efb0910..fbe547decfd3 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -27,15 +27,6 @@ func TestAccDMSReplicationTask_basic(t *testing.T) { rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" - tags := ` - Update = "to-update" - Remove = "to-remove" -` - updatedTags := ` - Update = "updated" - Add = "added" -` - resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), @@ -43,10 +34,23 @@ func TestAccDMSReplicationTask_basic(t *testing.T) { CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), Steps: []resource.TestStep{ { - Config: testAccReplicationTaskConfig_basic(rName, tags), + Config: testAccReplicationTaskConfig_basic(rName), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), + resource.TestCheckResourceAttr(resourceName, "replication_task_id", rName), acctest.MatchResourceAttrRegionalARN(resourceName, "replication_task_arn", "dms", regexache.MustCompile(`task:[A-Z0-9]{26}`)), + resource.TestCheckResourceAttr(resourceName, "cdc_start_position", ""), + resource.TestCheckNoResourceAttr(resourceName, "cdc_start_time"), + resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), + resource.TestCheckResourceAttrPair(resourceName, "replication_instance_arn", "aws_dms_replication_instance.test", "replication_instance_arn"), + acctest.CheckResourceAttrEquivalentJSON(resourceName, "replication_task_settings", defaultReplicationTaskConfig), + resource.TestCheckResourceAttrPair(resourceName, "source_endpoint_arn", "aws_dms_endpoint.source", "endpoint_arn"), + resource.TestCheckResourceAttr(resourceName, "start_replication_task", "false"), + resource.TestCheckResourceAttr(resourceName, "status", "ready"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), + resource.TestCheckResourceAttrPair(resourceName, "target_endpoint_arn", "aws_dms_endpoint.target", "endpoint_arn"), + resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), + resource.TestCheckResourceAttr(resourceName, "tags_all.%", "0"), ), }, { @@ -55,12 +59,6 @@ func TestAccDMSReplicationTask_basic(t *testing.T) { ImportStateVerify: true, ImportStateVerifyIgnore: []string{"start_replication_task"}, }, - { - Config: testAccReplicationTaskConfig_basic(rName, updatedTags), - Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), - ), - }, }, }) } @@ -306,10 +304,6 @@ func TestAccDMSReplicationTask_disappears(t *testing.T) { rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" - tags := ` - Test = "test" -` - resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), @@ -317,7 +311,7 @@ func TestAccDMSReplicationTask_disappears(t *testing.T) { CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), Steps: []resource.TestStep{ { - Config: testAccReplicationTaskConfig_basic(rName, tags), + Config: testAccReplicationTaskConfig_basic(rName), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), acctest.CheckResourceDisappears(ctx, acctest.Provider, tfdms.ResourceReplicationTask(), resourceName), @@ -511,24 +505,18 @@ resource "aws_dms_replication_instance" "test" { `, rName)) } -func testAccReplicationTaskConfig_basic(rName, tags string) string { +func testAccReplicationTaskConfig_basic(rName string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { - migration_type = "full-load" - replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn - replication_task_id = %[1]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" - source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - - tags = { - Name = %[1]q -%[2]s - } + migration_type = "full-load" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + replication_task_id = %[1]q + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn } -`, rName, tags)) +`, rName)) } func testAccReplicationTaskConfig_resourceIdentifier(rName string) string { @@ -815,3 +803,178 @@ resource "aws_dms_replication_task" "test" { } `, rName, arn)) } + +const defaultReplicationTaskConfig = `{ + "Logging": { + "EnableLogging": false, + "EnableLogContext": false, + "LogComponents": [ + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "DATA_STRUCTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMUNICATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "IO" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMON" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_FACTORY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_TRANSFER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "REST_SERVER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "ADDONS" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_LOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_APPLY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_UNLOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_CAPTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TRANSFORMATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SORTER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TASK_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TABLES_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "METADATA_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "PERFORMANCE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "VALIDATOR_EXT" + } + ], + "CloudWatchLogGroup": null, + "CloudWatchLogStream": null + }, + "StreamBufferSettings": { + "StreamBufferCount": 3, + "CtrlStreamBufferSizeInMB": 5, + "StreamBufferSizeInMB": 8 + }, + "ErrorBehavior": { + "FailOnNoTablesCaptured": true, + "ApplyErrorUpdatePolicy": "LOG_ERROR", + "FailOnTransactionConsistencyBreached": false, + "RecoverableErrorThrottlingMax": 1800, + "DataErrorEscalationPolicy": "SUSPEND_TABLE", + "ApplyErrorEscalationCount": 0, + "RecoverableErrorStopRetryAfterThrottlingMax": true, + "RecoverableErrorThrottling": true, + "ApplyErrorFailOnTruncationDdl": false, + "DataTruncationErrorPolicy": "LOG_ERROR", + "ApplyErrorInsertPolicy": "LOG_ERROR", + "EventErrorPolicy": "IGNORE", + "ApplyErrorEscalationPolicy": "LOG_ERROR", + "RecoverableErrorCount": -1, + "DataErrorEscalationCount": 0, + "TableErrorEscalationPolicy": "STOP_TASK", + "RecoverableErrorInterval": 5, + "ApplyErrorDeletePolicy": "IGNORE_RECORD", + "TableErrorEscalationCount": 0, + "FullLoadIgnoreConflicts": true, + "DataErrorPolicy": "LOG_ERROR", + "TableErrorPolicy": "SUSPEND_TABLE" + }, + "TTSettings": null, + "FullLoadSettings": { + "CommitRate": 10000, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CreatePkAfterFullLoad": false, + "TargetTablePrepMode": "DROP_AND_CREATE" + }, + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 32, + "BatchApplyEnabled": false, + "FullLobMode": false, + "LimitedSizeLobMode": true, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "BeforeImageSettings": null, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": false, + "HistoryTableEnabled": false, + "ControlSchema": "", + "FullLoadExceptionTableEnabled": false + }, + "LoopbackPreventionSettings": null, + "CharacterSetSettings": null, + "FailTaskWhenCleanTaskResourceFailed": false, + "ChangeProcessingTuning": { + "StatementCacheSize": 50, + "CommitTimeout": 1, + "BatchApplyPreserveTransaction": true, + "BatchApplyTimeoutMin": 1, + "BatchSplitSize": 0, + "BatchApplyTimeoutMax": 30, + "MinTransactionSize": 1000, + "MemoryKeepTime": 60, + "BatchApplyMemoryLimit": 500, + "MemoryLimitTotal": 1024 + }, + "ChangeProcessingDdlHandlingPolicy": { + "HandleSourceTableDropped": true, + "HandleSourceTableTruncated": true, + "HandleSourceTableAltered": true + }, + "PostProcessingRules": null +}` From f2f768f3565fd239605febda5b7bd1ae4dbf4ae8 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Thu, 11 Apr 2024 09:24:17 -0700 Subject: [PATCH 05/18] Replaces massive JSON documents on one line in replication task tests with `jsonencode` --- internal/service/dms/replication_task_test.go | 262 ++++++++++++------ 1 file changed, 171 insertions(+), 91 deletions(-) diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index fbe547decfd3..4e53c48ad192 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -184,6 +184,7 @@ func TestAccDMSReplicationTask_cdcStartPosition(t *testing.T) { Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "cdc_start_position", "mysql-bin-changelog.000024:373"), + resource.TestCheckNoResourceAttr(resourceName, "cdc_start_time"), ), }, { @@ -342,6 +343,7 @@ func TestAccDMSReplicationTask_cdcStartTime_rfc3339_date(t *testing.T) { Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "cdc_start_position", awsDmsExpectedOutput), + resource.TestCheckResourceAttr(resourceName, "cdc_start_time", rfc3339Time), ), }, { @@ -375,6 +377,7 @@ func TestAccDMSReplicationTask_cdcStartTime_unix_timestamp(t *testing.T) { Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttr(resourceName, "cdc_start_position", awsDmsExpectedOutput), + resource.TestCheckResourceAttr(resourceName, "cdc_start_time", unixDateTime), ), }, { @@ -508,13 +511,27 @@ resource "aws_dms_replication_instance" "test" { func testAccReplicationTaskConfig_basic(rName string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { + replication_task_id = %[1]q migration_type = "full-load" replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn - replication_task_id = %[1]q source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - - target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) } `, rName)) } @@ -522,15 +539,28 @@ resource "aws_dms_replication_task" "test" { func testAccReplicationTaskConfig_resourceIdentifier(rName string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { - migration_type = "full-load" - replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn - replication_task_id = %[1]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" - source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn - resource_identifier = "identifier" - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - - target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + replication_task_id = %[1]q + resource_identifier = "identifier" + migration_type = "full-load" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) } `, rName)) } @@ -538,18 +568,15 @@ resource "aws_dms_replication_task" "test" { func testAccReplicationTaskConfig_updateSettingsAndMappings(rName string, memLimitTotal int, ruleName string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { - migration_type = "full-load" - replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn - replication_task_id = %[1]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":%[2]d,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" - source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + replication_task_id = %[1]q + migration_type = "full-load" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + # terrafmt can't handle this using jsonencode or a heredoc + replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":%[2]d,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" # terrafmt can't handle this using jsonencode or a heredoc + # terrafmt can't handle this using jsonencode or a heredoc table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"%[3]s\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - - tags = { - Name = %[1]q - } - - target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn } `, rName, memLimitTotal, ruleName)) } @@ -557,20 +584,61 @@ resource "aws_dms_replication_task" "test" { func testAccReplicationTaskConfig_cdcStartPosition(rName, cdcStartPosition string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { - cdc_start_position = %[1]q - migration_type = "cdc" - replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn - replication_task_id = %[2]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" - source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + replication_task_id = %[2]q + migration_type = "cdc" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) + + cdc_start_position = %[1]q } `, cdcStartPosition, rName)) } func testAccReplicationTaskConfig_start(rName string, startTask bool, ruleName string) string { return acctest.ConfigCompose(testAccReplicationConfigConfig_base(rName), fmt.Sprintf(` +resource "aws_dms_replication_task" "test" { + replication_task_id = %[1]q + migration_type = "full-load-and-cdc" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) + + start_replication_task = %[2]t +} + resource "aws_dms_replication_instance" "test" { allocated_storage = 5 auto_minor_version_upgrade = true @@ -581,30 +649,35 @@ resource "aws_dms_replication_instance" "test" { replication_subnet_group_id = aws_dms_replication_subnet_group.test.replication_subnet_group_id vpc_security_group_ids = [aws_security_group.test.id] } - -resource "aws_dms_replication_task" "test" { - migration_type = "full-load-and-cdc" - replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn - replication_task_id = %[1]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" - source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"%[3]s\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - - start_replication_task = %[2]t - - tags = { - Name = %[1]q - } - - target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn - - depends_on = [aws_rds_cluster_instance.source, aws_rds_cluster_instance.target] -} `, rName, startTask, ruleName)) } func testAccReplicationTaskConfig_s3ToRDS(rName string) string { return acctest.ConfigCompose(acctest.ConfigVPCWithSubnets(rName, 2), testAccS3EndpointConfig_base(rName), fmt.Sprintf(` +resource "aws_dms_replication_task" "test" { + replication_task_id = %[1]q + migration_type = "full-load-and-cdc" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_s3_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) +} + resource "aws_security_group" "test" { name = %[1]q vpc_id = aws_vpc.test.id @@ -740,43 +813,65 @@ resource "aws_dms_replication_instance" "test" { replication_subnet_group_id = aws_dms_replication_subnet_group.test.replication_subnet_group_id vpc_security_group_ids = [aws_security_group.test.id] } - -resource "aws_dms_replication_task" "test" { - migration_type = "full-load-and-cdc" - replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn - replication_task_id = %[1]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" - source_endpoint_arn = aws_dms_s3_endpoint.source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - - tags = { - Name = %[1]q - } - - target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn - - depends_on = [aws_rds_cluster_instance.test] -} `, rName)) } func testAccReplicationTaskConfig_cdcStartTime(rName, cdcStartPosition string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { - cdc_start_time = %[1]q - migration_type = "cdc" - replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn - replication_task_id = %[2]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" - source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + replication_task_id = %[2]q + migration_type = "cdc" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) + + cdc_start_time = %[1]q } `, cdcStartPosition, rName)) } func testAccReplicationTaskConfig_move(rName, arn string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` +resource "aws_dms_replication_task" "test" { + replication_task_id = %[1]q + migration_type = "full-load" + replication_instance_arn = %[2]s + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) +} + resource "aws_dms_replication_instance" "test2" { allocated_storage = 5 auto_minor_version_upgrade = true @@ -786,21 +881,6 @@ resource "aws_dms_replication_instance" "test2" { publicly_accessible = false replication_subnet_group_id = aws_dms_replication_subnet_group.test.replication_subnet_group_id } - -resource "aws_dms_replication_task" "test" { - migration_type = "full-load" - replication_instance_arn = %[2]s - replication_task_id = %[1]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" - source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - - tags = { - Name = %[1]q - } - - target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn -} `, rName, arn)) } From 07581b03856ebcd99aaddae419dc2c1e9f8d85da Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Thu, 11 Apr 2024 10:51:43 -0700 Subject: [PATCH 06/18] Makes `TestAccDMSReplicationConfig_basic` check defaults --- .../service/dms/replication_config_test.go | 193 +++++++++++++++++- internal/service/dms/replication_task_test.go | 4 +- 2 files changed, 191 insertions(+), 6 deletions(-) diff --git a/internal/service/dms/replication_config_test.go b/internal/service/dms/replication_config_test.go index b6b297dcdd08..c330954af989 100644 --- a/internal/service/dms/replication_config_test.go +++ b/internal/service/dms/replication_config_test.go @@ -8,6 +8,7 @@ import ( "fmt" "testing" + "github.com/YakDriver/regexache" sdkacctest "github.com/hashicorp/terraform-plugin-testing/helper/acctest" "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-plugin-testing/terraform" @@ -34,6 +35,7 @@ func TestAccDMSReplicationConfig_basic(t *testing.T) { Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationConfigExists(ctx, resourceName), resource.TestCheckResourceAttrSet(resourceName, "arn"), + acctest.MatchResourceAttrRegionalARN(resourceName, "arn", "dms", regexache.MustCompile(`replication-config:[A-Z0-9]{26}`)), resource.TestCheckResourceAttr(resourceName, "compute_config.#", "1"), resource.TestCheckResourceAttr(resourceName, "compute_config.0.availability_zone", ""), resource.TestCheckResourceAttr(resourceName, "compute_config.0.dns_name_servers", ""), @@ -46,14 +48,16 @@ func TestAccDMSReplicationConfig_basic(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "compute_config.0.vpc_security_group_ids.#", "0"), resource.TestCheckResourceAttr(resourceName, "replication_config_identifier", rName), resource.TestCheckResourceAttrSet(resourceName, "replication_settings"), + acctest.CheckResourceAttrEquivalentJSON(resourceName, "replication_settings", defaultReplicationConfigSettings), resource.TestCheckResourceAttr(resourceName, "replication_type", "cdc"), resource.TestCheckNoResourceAttr(resourceName, "resource_identifier"), - resource.TestCheckResourceAttrSet(resourceName, "source_endpoint_arn"), + resource.TestCheckResourceAttrPair(resourceName, "source_endpoint_arn", "aws_dms_endpoint.source", "endpoint_arn"), resource.TestCheckResourceAttr(resourceName, "start_replication", "false"), resource.TestCheckResourceAttr(resourceName, "supplemental_settings", ""), - resource.TestCheckResourceAttrSet(resourceName, "table_mappings"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), + resource.TestCheckResourceAttrPair(resourceName, "target_endpoint_arn", "aws_dms_endpoint.target", "endpoint_arn"), resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), - resource.TestCheckResourceAttrSet(resourceName, "target_endpoint_arn"), + resource.TestCheckResourceAttr(resourceName, "tags_all.%", "0"), ), }, { @@ -439,7 +443,6 @@ resource "aws_dms_replication_config" "test" { source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - replication_settings = "{\"BeforeImageSettings\":null,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"CommitPositionTableEnabled\":false,\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FailTaskWhenCleanTaskResourceFailed\":false,\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TTSettings\":{\"EnableTT\":false,\"FailTaskOnTTFailure\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false}}" compute_config { replication_subnet_group_id = aws_dms_replication_subnet_group.test.replication_subnet_group_id @@ -559,3 +562,185 @@ resource "aws_dms_replication_config" "test" { } `, rName, tagKey1, tagValue1, tagKey2, tagValue2)) } + +const defaultReplicationConfigSettings = `{ + "Logging": { + "EnableLogging": false, + "EnableLogContext": false, + "LogComponents": [ + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "DATA_STRUCTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMUNICATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "IO" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMON" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_FACTORY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_TRANSFER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "REST_SERVER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "ADDONS" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_LOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_APPLY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_UNLOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_CAPTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TRANSFORMATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SORTER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TASK_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TABLES_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "METADATA_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "PERFORMANCE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "VALIDATOR_EXT" + } + ], + "CloudWatchLogGroup": null, + "CloudWatchLogStream": null + }, + "StreamBufferSettings": { + "StreamBufferCount": 3, + "CtrlStreamBufferSizeInMB": 5, + "StreamBufferSizeInMB": 8 + }, + "ErrorBehavior": { + "FailOnNoTablesCaptured": true, + "ApplyErrorUpdatePolicy": "LOG_ERROR", + "FailOnTransactionConsistencyBreached": false, + "RecoverableErrorThrottlingMax": 1800, + "DataErrorEscalationPolicy": "SUSPEND_TABLE", + "ApplyErrorEscalationCount": 0, + "RecoverableErrorStopRetryAfterThrottlingMax": true, + "RecoverableErrorThrottling": true, + "ApplyErrorFailOnTruncationDdl": false, + "DataMaskingErrorPolicy": "STOP_TASK", + "DataTruncationErrorPolicy": "LOG_ERROR", + "ApplyErrorInsertPolicy": "LOG_ERROR", + "EventErrorPolicy": "IGNORE", + "ApplyErrorEscalationPolicy": "LOG_ERROR", + "RecoverableErrorCount": -1, + "DataErrorEscalationCount": 0, + "TableErrorEscalationPolicy": "STOP_TASK", + "RecoverableErrorInterval": 5, + "ApplyErrorDeletePolicy": "IGNORE_RECORD", + "TableErrorEscalationCount": 0, + "FullLoadIgnoreConflicts": true, + "DataErrorPolicy": "LOG_ERROR", + "TableErrorPolicy": "SUSPEND_TABLE" + }, + "TTSettings": { + "TTS3Settings": null, + "TTRecordSettings": null, + "FailTaskOnTTFailure": false, + "EnableTT": false + }, + "FullLoadSettings": { + "CommitRate": 10000, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CreatePkAfterFullLoad": false, + "TargetTablePrepMode": "DO_NOTHING" + }, + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 32, + "BatchApplyEnabled": false, + "FullLobMode": false, + "LimitedSizeLobMode": true, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "BeforeImageSettings": null, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "CommitPositionTableEnabled": false, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": false, + "HistoryTableEnabled": false, + "ControlSchema": "", + "FullLoadExceptionTableEnabled": false + }, + "LoopbackPreventionSettings": null, + "CharacterSetSettings": null, + "FailTaskWhenCleanTaskResourceFailed": false, + "ChangeProcessingTuning": { + "StatementCacheSize": 50, + "CommitTimeout": 1, + "BatchApplyPreserveTransaction": true, + "BatchApplyTimeoutMin": 1, + "BatchSplitSize": 0, + "BatchApplyTimeoutMax": 30, + "MinTransactionSize": 1000, + "MemoryKeepTime": 60, + "BatchApplyMemoryLimit": 500, + "MemoryLimitTotal": 1024 + }, + "ChangeProcessingDdlHandlingPolicy": { + "HandleSourceTableDropped": true, + "HandleSourceTableTruncated": true, + "HandleSourceTableAltered": true + }, + "PostProcessingRules": null +}` diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index 4e53c48ad192..1e914f62b966 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -43,7 +43,7 @@ func TestAccDMSReplicationTask_basic(t *testing.T) { resource.TestCheckNoResourceAttr(resourceName, "cdc_start_time"), resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), resource.TestCheckResourceAttrPair(resourceName, "replication_instance_arn", "aws_dms_replication_instance.test", "replication_instance_arn"), - acctest.CheckResourceAttrEquivalentJSON(resourceName, "replication_task_settings", defaultReplicationTaskConfig), + acctest.CheckResourceAttrEquivalentJSON(resourceName, "replication_task_settings", defaultReplicationTaskSettings), resource.TestCheckResourceAttrPair(resourceName, "source_endpoint_arn", "aws_dms_endpoint.source", "endpoint_arn"), resource.TestCheckResourceAttr(resourceName, "start_replication_task", "false"), resource.TestCheckResourceAttr(resourceName, "status", "ready"), @@ -884,7 +884,7 @@ resource "aws_dms_replication_instance" "test2" { `, rName, arn)) } -const defaultReplicationTaskConfig = `{ +const defaultReplicationTaskSettings = `{ "Logging": { "EnableLogging": false, "EnableLogContext": false, From 694e7ba1585ed201245b0460a8b2332847cb572e Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Fri, 12 Apr 2024 11:57:48 -0700 Subject: [PATCH 07/18] Uses dummy databases for Replication Config acceptance tests that don't start replication This saves approximately six minutes for setup and teardown --- internal/service/dms/endpoint_test.go | 112 +++++++- .../service/dms/replication_config_test.go | 250 ++++++------------ internal/service/dms/replication_task_test.go | 15 +- 3 files changed, 207 insertions(+), 170 deletions(-) diff --git a/internal/service/dms/endpoint_test.go b/internal/service/dms/endpoint_test.go index 43cdd86e1242..906050da4650 100644 --- a/internal/service/dms/endpoint_test.go +++ b/internal/service/dms/endpoint_test.go @@ -4680,7 +4680,7 @@ resource "aws_kms_key" "test" { } func testAccEndpointConfig_pauseReplicationTasks(rName string, pause bool) string { - return acctest.ConfigCompose(testAccRDSClustersConfig_base(rName), fmt.Sprintf(` + return acctest.ConfigCompose(testAccEndpointConfig_rdsClusterBase(rName), fmt.Sprintf(` resource "aws_dms_endpoint" "source" { database_name = "tftest" endpoint_id = "%[1]s-source" @@ -4742,3 +4742,113 @@ resource "aws_dms_replication_task" "test" { } `, rName, pause)) } + +// testAccEndpointConfig_rdsClusterBase configures a pair of Aurora RDS clusters (and instances) ready for replication. +func testAccEndpointConfig_rdsClusterBase(rName string) string { + return acctest.ConfigCompose(acctest.ConfigVPCWithSubnets(rName, 2), fmt.Sprintf(` +resource "aws_db_subnet_group" "test" { + name = %[1]q + subnet_ids = aws_subnet.test[*].id +} + +resource "aws_security_group" "test" { + name = %[1]q + vpc_id = aws_vpc.test.id + + ingress { + from_port = 0 + to_port = 0 + protocol = -1 + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = %[1]q + } +} + +data "aws_rds_engine_version" "default" { + engine = "aurora-mysql" +} + +data "aws_rds_orderable_db_instance" "test" { + engine = data.aws_rds_engine_version.default.engine + engine_version = data.aws_rds_engine_version.default.version + preferred_instance_classes = ["db.t3.small", "db.t3.medium", "db.t3.large"] +} + +resource "aws_rds_cluster_parameter_group" "test" { + name = "%[1]s-pg-cluster" + family = data.aws_rds_engine_version.default.parameter_group_family + description = "DMS cluster parameter group" + + parameter { + name = "binlog_format" + value = "ROW" + apply_method = "pending-reboot" + } + + parameter { + name = "binlog_row_image" + value = "Full" + apply_method = "pending-reboot" + } + + parameter { + name = "binlog_checksum" + value = "NONE" + apply_method = "pending-reboot" + } +} + +resource "aws_rds_cluster" "source" { + cluster_identifier = "%[1]s-aurora-cluster-source" + engine = data.aws_rds_orderable_db_instance.test.engine + engine_version = data.aws_rds_orderable_db_instance.test.engine_version + database_name = "tftest" + master_username = "tftest" + master_password = "mustbeeightcharaters" + skip_final_snapshot = true + vpc_security_group_ids = [aws_security_group.test.id] + db_subnet_group_name = aws_db_subnet_group.test.name + db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.test.name +} + +resource "aws_rds_cluster_instance" "source" { + identifier = "%[1]s-source-primary" + cluster_identifier = aws_rds_cluster.source.id + engine = data.aws_rds_orderable_db_instance.test.engine + engine_version = data.aws_rds_orderable_db_instance.test.engine_version + instance_class = data.aws_rds_orderable_db_instance.test.instance_class + db_subnet_group_name = aws_db_subnet_group.test.name +} + +resource "aws_rds_cluster" "target" { + cluster_identifier = "%[1]s-aurora-cluster-target" + engine = data.aws_rds_orderable_db_instance.test.engine + engine_version = data.aws_rds_orderable_db_instance.test.engine_version + database_name = "tftest" + master_username = "tftest" + master_password = "mustbeeightcharaters" + skip_final_snapshot = true + vpc_security_group_ids = [aws_security_group.test.id] + db_subnet_group_name = aws_db_subnet_group.test.name +} + +resource "aws_rds_cluster_instance" "target" { + identifier = "%[1]s-target-primary" + cluster_identifier = aws_rds_cluster.target.id + engine = data.aws_rds_orderable_db_instance.test.engine + engine_version = data.aws_rds_orderable_db_instance.test.engine_version + instance_class = data.aws_rds_orderable_db_instance.test.instance_class + db_subnet_group_name = aws_db_subnet_group.test.name +} +`, rName)) +} diff --git a/internal/service/dms/replication_config_test.go b/internal/service/dms/replication_config_test.go index c330954af989..316f13d65341 100644 --- a/internal/service/dms/replication_config_test.go +++ b/internal/service/dms/replication_config_test.go @@ -70,53 +70,53 @@ func TestAccDMSReplicationConfig_basic(t *testing.T) { }) } -func TestAccDMSReplicationConfig_noChangeOnDefault(t *testing.T) { - ctx := acctest.Context(t) - rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) - resourceName := "aws_dms_replication_config.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(ctx, t) }, - ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckReplicationConfigDestroy(ctx), - Steps: []resource.TestStep{ - { - Config: testAccReplicationConfigConfig_noChangeOnDefault(rName), - Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), - resource.TestCheckResourceAttrSet(resourceName, "arn"), - resource.TestCheckResourceAttr(resourceName, "compute_config.#", "1"), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.availability_zone", ""), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.dns_name_servers", ""), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.kms_key_id", ""), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.max_capacity_units", "128"), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.min_capacity_units", "2"), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.multi_az", "false"), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.preferred_maintenance_window", "sun:23:45-mon:00:30"), - resource.TestCheckResourceAttrSet(resourceName, "compute_config.0.replication_subnet_group_id"), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.vpc_security_group_ids.#", "0"), - resource.TestCheckResourceAttr(resourceName, "replication_config_identifier", rName), - resource.TestCheckResourceAttrSet(resourceName, "replication_settings"), - resource.TestCheckResourceAttr(resourceName, "replication_type", "cdc"), - resource.TestCheckNoResourceAttr(resourceName, "resource_identifier"), - resource.TestCheckResourceAttrSet(resourceName, "source_endpoint_arn"), - resource.TestCheckResourceAttr(resourceName, "start_replication", "false"), - resource.TestCheckResourceAttr(resourceName, "supplemental_settings", ""), - resource.TestCheckResourceAttrSet(resourceName, "table_mappings"), - resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), - resource.TestCheckResourceAttrSet(resourceName, "target_endpoint_arn"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"start_replication", "resource_identifier"}, - }, - }, - }) -} +// func TestAccDMSReplicationConfig_noChangeOnDefault(t *testing.T) { +// ctx := acctest.Context(t) +// rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) +// resourceName := "aws_dms_replication_config.test" + +// resource.ParallelTest(t, resource.TestCase{ +// PreCheck: func() { acctest.PreCheck(ctx, t) }, +// ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), +// ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, +// CheckDestroy: testAccCheckReplicationConfigDestroy(ctx), +// Steps: []resource.TestStep{ +// { +// Config: testAccReplicationConfigConfig_noChangeOnDefault(rName), +// Check: resource.ComposeAggregateTestCheckFunc( +// testAccCheckReplicationConfigExists(ctx, resourceName), +// resource.TestCheckResourceAttrSet(resourceName, "arn"), +// resource.TestCheckResourceAttr(resourceName, "compute_config.#", "1"), +// resource.TestCheckResourceAttr(resourceName, "compute_config.0.availability_zone", ""), +// resource.TestCheckResourceAttr(resourceName, "compute_config.0.dns_name_servers", ""), +// resource.TestCheckResourceAttr(resourceName, "compute_config.0.kms_key_id", ""), +// resource.TestCheckResourceAttr(resourceName, "compute_config.0.max_capacity_units", "128"), +// resource.TestCheckResourceAttr(resourceName, "compute_config.0.min_capacity_units", "2"), +// resource.TestCheckResourceAttr(resourceName, "compute_config.0.multi_az", "false"), +// resource.TestCheckResourceAttr(resourceName, "compute_config.0.preferred_maintenance_window", "sun:23:45-mon:00:30"), +// resource.TestCheckResourceAttrSet(resourceName, "compute_config.0.replication_subnet_group_id"), +// resource.TestCheckResourceAttr(resourceName, "compute_config.0.vpc_security_group_ids.#", "0"), +// resource.TestCheckResourceAttr(resourceName, "replication_config_identifier", rName), +// resource.TestCheckResourceAttrSet(resourceName, "replication_settings"), +// resource.TestCheckResourceAttr(resourceName, "replication_type", "cdc"), +// resource.TestCheckNoResourceAttr(resourceName, "resource_identifier"), +// resource.TestCheckResourceAttrSet(resourceName, "source_endpoint_arn"), +// resource.TestCheckResourceAttr(resourceName, "start_replication", "false"), +// resource.TestCheckResourceAttr(resourceName, "supplemental_settings", ""), +// resource.TestCheckResourceAttrSet(resourceName, "table_mappings"), +// resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), +// resource.TestCheckResourceAttrSet(resourceName, "target_endpoint_arn"), +// ), +// }, +// { +// ResourceName: resourceName, +// ImportState: true, +// ImportStateVerify: true, +// ImportStateVerifyIgnore: []string{"start_replication", "resource_identifier"}, +// }, +// }, +// }) +// } func TestAccDMSReplicationConfig_disappears(t *testing.T) { ctx := acctest.Context(t) @@ -293,118 +293,11 @@ func testAccCheckReplicationConfigDestroy(ctx context.Context) resource.TestChec } } -// testAccRDSClustersConfig_base configures a pair of Aurora RDS clusters (and instances) ready for replication. -func testAccRDSClustersConfig_base(rName string) string { - return acctest.ConfigCompose(acctest.ConfigVPCWithSubnets(rName, 2), fmt.Sprintf(` -resource "aws_db_subnet_group" "test" { - name = %[1]q - subnet_ids = aws_subnet.test[*].id -} - -resource "aws_security_group" "test" { - name = %[1]q - vpc_id = aws_vpc.test.id - - ingress { - from_port = 0 - to_port = 0 - protocol = -1 - cidr_blocks = ["0.0.0.0/0"] - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - - tags = { - Name = %[1]q - } -} - -data "aws_rds_engine_version" "default" { - engine = "aurora-mysql" -} - -data "aws_rds_orderable_db_instance" "test" { - engine = data.aws_rds_engine_version.default.engine - engine_version = data.aws_rds_engine_version.default.version - preferred_instance_classes = ["db.t3.small", "db.t3.medium", "db.t3.large"] -} - -resource "aws_rds_cluster_parameter_group" "test" { - name = "%[1]s-pg-cluster" - family = data.aws_rds_engine_version.default.parameter_group_family - description = "DMS cluster parameter group" - - parameter { - name = "binlog_format" - value = "ROW" - apply_method = "pending-reboot" - } - - parameter { - name = "binlog_row_image" - value = "Full" - apply_method = "pending-reboot" - } - - parameter { - name = "binlog_checksum" - value = "NONE" - apply_method = "pending-reboot" - } -} - -resource "aws_rds_cluster" "source" { - cluster_identifier = "%[1]s-aurora-cluster-source" - engine = data.aws_rds_orderable_db_instance.test.engine - engine_version = data.aws_rds_orderable_db_instance.test.engine_version - database_name = "tftest" - master_username = "tftest" - master_password = "mustbeeightcharaters" - skip_final_snapshot = true - vpc_security_group_ids = [aws_security_group.test.id] - db_subnet_group_name = aws_db_subnet_group.test.name - db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.test.name -} - -resource "aws_rds_cluster_instance" "source" { - identifier = "%[1]s-source-primary" - cluster_identifier = aws_rds_cluster.source.id - engine = data.aws_rds_orderable_db_instance.test.engine - engine_version = data.aws_rds_orderable_db_instance.test.engine_version - instance_class = data.aws_rds_orderable_db_instance.test.instance_class - db_subnet_group_name = aws_db_subnet_group.test.name -} - -resource "aws_rds_cluster" "target" { - cluster_identifier = "%[1]s-aurora-cluster-target" - engine = data.aws_rds_orderable_db_instance.test.engine - engine_version = data.aws_rds_orderable_db_instance.test.engine_version - database_name = "tftest" - master_username = "tftest" - master_password = "mustbeeightcharaters" - skip_final_snapshot = true - vpc_security_group_ids = [aws_security_group.test.id] - db_subnet_group_name = aws_db_subnet_group.test.name -} - -resource "aws_rds_cluster_instance" "target" { - identifier = "%[1]s-target-primary" - cluster_identifier = aws_rds_cluster.target.id - engine = data.aws_rds_orderable_db_instance.test.engine - engine_version = data.aws_rds_orderable_db_instance.test.engine_version - instance_class = data.aws_rds_orderable_db_instance.test.instance_class - db_subnet_group_name = aws_db_subnet_group.test.name -} -`, rName)) -} - -func testAccReplicationConfigConfig_base(rName string) string { - return acctest.ConfigCompose(testAccRDSClustersConfig_base(rName), fmt.Sprintf(` +// testAccReplicationConfigConfig_base_DummyDatabase creates Replication Endpoints referencing valid databases. +// This should only be used in cases where actual replication is started, since it requires approcimately +// six more minutes for setup and teardown. +func testAccReplicationConfigConfig_base_ValidDatabase(rName string) string { + return acctest.ConfigCompose(testAccEndpointConfig_rdsClusterBase(rName), fmt.Sprintf(` resource "aws_dms_replication_subnet_group" "test" { replication_subnet_group_id = %[1]q replication_subnet_group_description = "terraform test" @@ -435,8 +328,25 @@ resource "aws_dms_endpoint" "source" { `, rName)) } +// testAccReplicationConfigConfig_base_DummyDatabase creates Replication Endpoints referencing dummy databases. +// This should be used in all cases where actual replication is not started, since it shaves approcimately +// six minutes off setup and teardown time. +func testAccReplicationConfigConfig_base_DummyDatabase(rName string) string { + return acctest.ConfigCompose( + testAccReplicationEndpointConfig_dummyDatabase(rName), + fmt.Sprintf(` +resource "aws_dms_replication_subnet_group" "test" { + replication_subnet_group_id = %[1]q + replication_subnet_group_description = "terraform test" + subnet_ids = aws_subnet.test[*].id +} +`, rName)) +} + func testAccReplicationConfigConfig_basic(rName string) string { - return acctest.ConfigCompose(testAccReplicationConfigConfig_base(rName), fmt.Sprintf(` + return acctest.ConfigCompose( + testAccReplicationConfigConfig_base_DummyDatabase(rName), + fmt.Sprintf(` resource "aws_dms_replication_config" "test" { replication_config_identifier = %[1]q replication_type = "cdc" @@ -455,7 +365,9 @@ resource "aws_dms_replication_config" "test" { } func testAccReplicationConfigConfig_noChangeOnDefault(rName string) string { - return acctest.ConfigCompose(testAccReplicationConfigConfig_base(rName), fmt.Sprintf(` + return acctest.ConfigCompose( + testAccReplicationConfigConfig_base_DummyDatabase(rName), + fmt.Sprintf(` resource "aws_dms_replication_config" "test" { replication_config_identifier = %[1]q replication_type = "cdc" @@ -475,7 +387,9 @@ resource "aws_dms_replication_config" "test" { } func testAccReplicationConfigConfig_update(rName, replicationType string, minCapacity, maxCapacity int) string { - return acctest.ConfigCompose(testAccReplicationConfigConfig_base(rName), fmt.Sprintf(` + return acctest.ConfigCompose( + testAccReplicationConfigConfig_base_DummyDatabase(rName), + fmt.Sprintf(` resource "aws_dms_replication_config" "test" { replication_config_identifier = %[1]q resource_identifier = %[1]q @@ -495,7 +409,9 @@ resource "aws_dms_replication_config" "test" { } func testAccReplicationConfigConfig_startReplication(rName string, start bool) string { - return acctest.ConfigCompose(testAccReplicationConfigConfig_base(rName), fmt.Sprintf(` + return acctest.ConfigCompose( + testAccReplicationConfigConfig_base_ValidDatabase(rName), + fmt.Sprintf(` resource "aws_dms_replication_config" "test" { replication_config_identifier = %[1]q resource_identifier = %[1]q @@ -517,7 +433,9 @@ resource "aws_dms_replication_config" "test" { } func testAccReplicationConfigConfig_tags1(rName, tagKey1, tagValue1 string) string { - return acctest.ConfigCompose(testAccReplicationConfigConfig_base(rName), fmt.Sprintf(` + return acctest.ConfigCompose( + testAccReplicationConfigConfig_base_DummyDatabase(rName), + fmt.Sprintf(` resource "aws_dms_replication_config" "test" { replication_config_identifier = %[1]q replication_type = "cdc" @@ -540,7 +458,9 @@ resource "aws_dms_replication_config" "test" { } func testAccReplicationConfigConfig_tags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string { - return acctest.ConfigCompose(testAccReplicationConfigConfig_base(rName), fmt.Sprintf(` + return acctest.ConfigCompose( + testAccReplicationConfigConfig_base_DummyDatabase(rName), + fmt.Sprintf(` resource "aws_dms_replication_config" "test" { replication_config_identifier = %[1]q replication_type = "cdc" diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index 1e914f62b966..9864cb9feab6 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -463,7 +463,7 @@ func testAccCheckReplicationTaskDestroy(ctx context.Context) resource.TestCheckF } } -func testAccReplicationTaskConfig_base(rName string) string { +func testAccReplicationEndpointConfig_dummyDatabase(rName string) string { return acctest.ConfigCompose(acctest.ConfigVPCWithSubnets(rName, 2), fmt.Sprintf(` data "aws_partition" "current" {} data "aws_region" "current" {} @@ -489,7 +489,14 @@ resource "aws_dms_endpoint" "target" { username = "tftest" password = "tftest" } +`, rName)) +} +func testAccReplicationTaskConfig_base(rName string) string { + return acctest.ConfigCompose( + acctest.ConfigVPCWithSubnets(rName, 2), + testAccReplicationEndpointConfig_dummyDatabase(rName), + fmt.Sprintf(` resource "aws_dms_replication_subnet_group" "test" { replication_subnet_group_id = %[1]q replication_subnet_group_description = "terraform test for replication subnet group" @@ -574,9 +581,9 @@ resource "aws_dms_replication_task" "test" { source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn # terrafmt can't handle this using jsonencode or a heredoc - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":%[2]d,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" # terrafmt can't handle this using jsonencode or a heredoc + replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":%[2]d,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" # terrafmt can't handle this using jsonencode or a heredoc - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"%[3]s\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" + table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":%[3]q,\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" } `, rName, memLimitTotal, ruleName)) } @@ -612,7 +619,7 @@ resource "aws_dms_replication_task" "test" { } func testAccReplicationTaskConfig_start(rName string, startTask bool, ruleName string) string { - return acctest.ConfigCompose(testAccReplicationConfigConfig_base(rName), fmt.Sprintf(` + return acctest.ConfigCompose(testAccReplicationConfigConfig_base_ValidDatabase(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { replication_task_id = %[1]q migration_type = "full-load-and-cdc" From 1bcc2854627513adc936607bc86b20ee5d4f0582 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Fri, 12 Apr 2024 12:09:39 -0700 Subject: [PATCH 08/18] Cleanup --- internal/service/dms/endpoint_test.go | 26 ++++++++++++---- .../service/dms/replication_config_test.go | 1 - .../dms/replication_task_data_source_test.go | 31 +++++++++++++------ 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/internal/service/dms/endpoint_test.go b/internal/service/dms/endpoint_test.go index 906050da4650..7add882264c7 100644 --- a/internal/service/dms/endpoint_test.go +++ b/internal/service/dms/endpoint_test.go @@ -4723,12 +4723,26 @@ resource "aws_dms_replication_instance" "test" { } resource "aws_dms_replication_task" "test" { - migration_type = "full-load-and-cdc" - replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn - replication_task_id = %[1]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" - source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"testrule\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" + migration_type = "full-load-and-cdc" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + replication_task_id = %[1]q + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "testrule", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) start_replication_task = true diff --git a/internal/service/dms/replication_config_test.go b/internal/service/dms/replication_config_test.go index 316f13d65341..40456ff42c86 100644 --- a/internal/service/dms/replication_config_test.go +++ b/internal/service/dms/replication_config_test.go @@ -47,7 +47,6 @@ func TestAccDMSReplicationConfig_basic(t *testing.T) { resource.TestCheckResourceAttrSet(resourceName, "compute_config.0.replication_subnet_group_id"), resource.TestCheckResourceAttr(resourceName, "compute_config.0.vpc_security_group_ids.#", "0"), resource.TestCheckResourceAttr(resourceName, "replication_config_identifier", rName), - resource.TestCheckResourceAttrSet(resourceName, "replication_settings"), acctest.CheckResourceAttrEquivalentJSON(resourceName, "replication_settings", defaultReplicationConfigSettings), resource.TestCheckResourceAttr(resourceName, "replication_type", "cdc"), resource.TestCheckNoResourceAttr(resourceName, "resource_identifier"), diff --git a/internal/service/dms/replication_task_data_source_test.go b/internal/service/dms/replication_task_data_source_test.go index 7451424f34c7..6a7368abc8dc 100644 --- a/internal/service/dms/replication_task_data_source_test.go +++ b/internal/service/dms/replication_task_data_source_test.go @@ -28,7 +28,7 @@ func TestAccDMSReplicationTaskDataSource_basic(t *testing.T) { { Config: testAccReplicationTaskDataSourceConfig_basic(rName), Check: resource.ComposeTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, dataSourceName), + testAccCheckReplicationTaskExists(ctx, resourceName), resource.TestCheckResourceAttrPair(dataSourceName, "replication_task_id", resourceName, "replication_task_id"), ), }, @@ -39,14 +39,27 @@ func TestAccDMSReplicationTaskDataSource_basic(t *testing.T) { func testAccReplicationTaskDataSourceConfig_basic(rName string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { - migration_type = "full-load" - replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn - replication_task_id = %[1]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" - source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - - target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + replication_task_id = %[1]q + migration_type = "full-load" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) } data "aws_dms_replication_task" "test" { From 5fd4a192a51fef5383fafc1d1482c89897e76040 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Fri, 12 Apr 2024 13:28:51 -0700 Subject: [PATCH 09/18] Adds tests for replication settings defaults for each migration type --- .../service/dms/replication_config_test.go | 307 +++++------------- internal/service/dms/replication_task_test.go | 283 ++++------------ .../replication_config/defaults/cdc.json | 181 +++++++++++ .../defaults/full-load-and-cdc.json | 181 +++++++++++ .../defaults/full-load.json | 176 ++++++++++ .../replication_task/defaults/cdc.json | 178 ++++++++++ .../defaults/full-load-and-cdc.json | 178 ++++++++++ .../replication_task/defaults/full-load.json | 174 ++++++++++ 8 files changed, 1209 insertions(+), 449 deletions(-) create mode 100644 internal/service/dms/testdata/replication_config/defaults/cdc.json create mode 100644 internal/service/dms/testdata/replication_config/defaults/full-load-and-cdc.json create mode 100644 internal/service/dms/testdata/replication_config/defaults/full-load.json create mode 100644 internal/service/dms/testdata/replication_task/defaults/cdc.json create mode 100644 internal/service/dms/testdata/replication_task/defaults/full-load-and-cdc.json create mode 100644 internal/service/dms/testdata/replication_task/defaults/full-load.json diff --git a/internal/service/dms/replication_config_test.go b/internal/service/dms/replication_config_test.go index 40456ff42c86..309ff7a06f89 100644 --- a/internal/service/dms/replication_config_test.go +++ b/internal/service/dms/replication_config_test.go @@ -5,10 +5,12 @@ package dms_test import ( "context" + _ "embed" "fmt" "testing" "github.com/YakDriver/regexache" + dms "github.com/aws/aws-sdk-go/service/databasemigrationservice" sdkacctest "github.com/hashicorp/terraform-plugin-testing/helper/acctest" "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-plugin-testing/terraform" @@ -20,53 +22,59 @@ import ( ) func TestAccDMSReplicationConfig_basic(t *testing.T) { - ctx := acctest.Context(t) - rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) - resourceName := "aws_dms_replication_config.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(ctx, t) }, - ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckReplicationConfigDestroy(ctx), - Steps: []resource.TestStep{ - { - Config: testAccReplicationConfigConfig_basic(rName), - Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), - resource.TestCheckResourceAttrSet(resourceName, "arn"), - acctest.MatchResourceAttrRegionalARN(resourceName, "arn", "dms", regexache.MustCompile(`replication-config:[A-Z0-9]{26}`)), - resource.TestCheckResourceAttr(resourceName, "compute_config.#", "1"), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.availability_zone", ""), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.dns_name_servers", ""), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.kms_key_id", ""), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.max_capacity_units", "128"), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.min_capacity_units", "2"), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.multi_az", "false"), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.preferred_maintenance_window", "sun:23:45-mon:00:30"), - resource.TestCheckResourceAttrSet(resourceName, "compute_config.0.replication_subnet_group_id"), - resource.TestCheckResourceAttr(resourceName, "compute_config.0.vpc_security_group_ids.#", "0"), - resource.TestCheckResourceAttr(resourceName, "replication_config_identifier", rName), - acctest.CheckResourceAttrEquivalentJSON(resourceName, "replication_settings", defaultReplicationConfigSettings), - resource.TestCheckResourceAttr(resourceName, "replication_type", "cdc"), - resource.TestCheckNoResourceAttr(resourceName, "resource_identifier"), - resource.TestCheckResourceAttrPair(resourceName, "source_endpoint_arn", "aws_dms_endpoint.source", "endpoint_arn"), - resource.TestCheckResourceAttr(resourceName, "start_replication", "false"), - resource.TestCheckResourceAttr(resourceName, "supplemental_settings", ""), - acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), - resource.TestCheckResourceAttrPair(resourceName, "target_endpoint_arn", "aws_dms_endpoint.target", "endpoint_arn"), - resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), - resource.TestCheckResourceAttr(resourceName, "tags_all.%", "0"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"start_replication", "resource_identifier"}, - }, - }, - }) + t.Parallel() + + for _, migrationType := range dms.MigrationTypeValue_Values() { + t.Run(migrationType, func(t *testing.T) { + ctx := acctest.Context(t) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_dms_replication_config.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckReplicationConfigDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccReplicationConfigConfig_basic(rName, migrationType), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationConfigExists(ctx, resourceName), + resource.TestCheckResourceAttrSet(resourceName, "arn"), + acctest.MatchResourceAttrRegionalARN(resourceName, "arn", "dms", regexache.MustCompile(`replication-config:[A-Z0-9]{26}`)), + resource.TestCheckResourceAttr(resourceName, "compute_config.#", "1"), + resource.TestCheckResourceAttr(resourceName, "compute_config.0.availability_zone", ""), + resource.TestCheckResourceAttr(resourceName, "compute_config.0.dns_name_servers", ""), + resource.TestCheckResourceAttr(resourceName, "compute_config.0.kms_key_id", ""), + resource.TestCheckResourceAttr(resourceName, "compute_config.0.max_capacity_units", "128"), + resource.TestCheckResourceAttr(resourceName, "compute_config.0.min_capacity_units", "2"), + resource.TestCheckResourceAttr(resourceName, "compute_config.0.multi_az", "false"), + resource.TestCheckResourceAttr(resourceName, "compute_config.0.preferred_maintenance_window", "sun:23:45-mon:00:30"), + resource.TestCheckResourceAttrSet(resourceName, "compute_config.0.replication_subnet_group_id"), + resource.TestCheckResourceAttr(resourceName, "compute_config.0.vpc_security_group_ids.#", "0"), + resource.TestCheckResourceAttr(resourceName, "replication_config_identifier", rName), + acctest.CheckResourceAttrEquivalentJSON(resourceName, "replication_settings", defaultReplicationConfigSettings[migrationType]), + resource.TestCheckResourceAttr(resourceName, "replication_type", migrationType), + resource.TestCheckNoResourceAttr(resourceName, "resource_identifier"), + resource.TestCheckResourceAttrPair(resourceName, "source_endpoint_arn", "aws_dms_endpoint.source", "endpoint_arn"), + resource.TestCheckResourceAttr(resourceName, "start_replication", "false"), + resource.TestCheckResourceAttr(resourceName, "supplemental_settings", ""), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), + resource.TestCheckResourceAttrPair(resourceName, "target_endpoint_arn", "aws_dms_endpoint.target", "endpoint_arn"), + resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), + resource.TestCheckResourceAttr(resourceName, "tags_all.%", "0"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication", "resource_identifier"}, + }, + }, + }) + }) + } } // func TestAccDMSReplicationConfig_noChangeOnDefault(t *testing.T) { @@ -129,7 +137,7 @@ func TestAccDMSReplicationConfig_disappears(t *testing.T) { CheckDestroy: testAccCheckReplicationConfigDestroy(ctx), Steps: []resource.TestStep{ { - Config: testAccReplicationConfigConfig_basic(rName), + Config: testAccReplicationConfigConfig_basic(rName, "cdc"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationConfigExists(ctx, resourceName), acctest.CheckResourceDisappears(ctx, acctest.Provider, tfdms.ResourceReplicationConfig(), resourceName), @@ -342,13 +350,13 @@ resource "aws_dms_replication_subnet_group" "test" { `, rName)) } -func testAccReplicationConfigConfig_basic(rName string) string { +func testAccReplicationConfigConfig_basic(rName, migrationType string) string { return acctest.ConfigCompose( testAccReplicationConfigConfig_base_DummyDatabase(rName), fmt.Sprintf(` resource "aws_dms_replication_config" "test" { replication_config_identifier = %[1]q - replication_type = "cdc" + replication_type = %[2]q source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" @@ -360,7 +368,7 @@ resource "aws_dms_replication_config" "test" { preferred_maintenance_window = "sun:23:45-mon:00:30" } } -`, rName)) +`, rName, migrationType)) } func testAccReplicationConfigConfig_noChangeOnDefault(rName string) string { @@ -482,184 +490,19 @@ resource "aws_dms_replication_config" "test" { `, rName, tagKey1, tagValue1, tagKey2, tagValue2)) } -const defaultReplicationConfigSettings = `{ - "Logging": { - "EnableLogging": false, - "EnableLogContext": false, - "LogComponents": [ - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "DATA_STRUCTURE" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "COMMUNICATION" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "IO" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "COMMON" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "FILE_FACTORY" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "FILE_TRANSFER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "REST_SERVER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "ADDONS" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TARGET_LOAD" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TARGET_APPLY" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "SOURCE_UNLOAD" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "SOURCE_CAPTURE" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TRANSFORMATION" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "SORTER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TASK_MANAGER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TABLES_MANAGER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "METADATA_MANAGER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "PERFORMANCE" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "VALIDATOR_EXT" - } - ], - "CloudWatchLogGroup": null, - "CloudWatchLogStream": null - }, - "StreamBufferSettings": { - "StreamBufferCount": 3, - "CtrlStreamBufferSizeInMB": 5, - "StreamBufferSizeInMB": 8 - }, - "ErrorBehavior": { - "FailOnNoTablesCaptured": true, - "ApplyErrorUpdatePolicy": "LOG_ERROR", - "FailOnTransactionConsistencyBreached": false, - "RecoverableErrorThrottlingMax": 1800, - "DataErrorEscalationPolicy": "SUSPEND_TABLE", - "ApplyErrorEscalationCount": 0, - "RecoverableErrorStopRetryAfterThrottlingMax": true, - "RecoverableErrorThrottling": true, - "ApplyErrorFailOnTruncationDdl": false, - "DataMaskingErrorPolicy": "STOP_TASK", - "DataTruncationErrorPolicy": "LOG_ERROR", - "ApplyErrorInsertPolicy": "LOG_ERROR", - "EventErrorPolicy": "IGNORE", - "ApplyErrorEscalationPolicy": "LOG_ERROR", - "RecoverableErrorCount": -1, - "DataErrorEscalationCount": 0, - "TableErrorEscalationPolicy": "STOP_TASK", - "RecoverableErrorInterval": 5, - "ApplyErrorDeletePolicy": "IGNORE_RECORD", - "TableErrorEscalationCount": 0, - "FullLoadIgnoreConflicts": true, - "DataErrorPolicy": "LOG_ERROR", - "TableErrorPolicy": "SUSPEND_TABLE" - }, - "TTSettings": { - "TTS3Settings": null, - "TTRecordSettings": null, - "FailTaskOnTTFailure": false, - "EnableTT": false - }, - "FullLoadSettings": { - "CommitRate": 10000, - "StopTaskCachedChangesApplied": false, - "StopTaskCachedChangesNotApplied": false, - "MaxFullLoadSubTasks": 8, - "TransactionConsistencyTimeout": 600, - "CreatePkAfterFullLoad": false, - "TargetTablePrepMode": "DO_NOTHING" - }, - "TargetMetadata": { - "ParallelApplyBufferSize": 0, - "ParallelApplyQueuesPerThread": 0, - "ParallelApplyThreads": 0, - "TargetSchema": "", - "InlineLobMaxSize": 0, - "ParallelLoadQueuesPerThread": 0, - "SupportLobs": true, - "LobChunkSize": 64, - "TaskRecoveryTableEnabled": false, - "ParallelLoadThreads": 0, - "LobMaxSize": 32, - "BatchApplyEnabled": false, - "FullLobMode": false, - "LimitedSizeLobMode": true, - "LoadMaxFileSize": 0, - "ParallelLoadBufferSize": 0 - }, - "BeforeImageSettings": null, - "ControlTablesSettings": { - "historyTimeslotInMinutes": 5, - "CommitPositionTableEnabled": false, - "HistoryTimeslotInMinutes": 5, - "StatusTableEnabled": false, - "SuspendedTablesTableEnabled": false, - "HistoryTableEnabled": false, - "ControlSchema": "", - "FullLoadExceptionTableEnabled": false - }, - "LoopbackPreventionSettings": null, - "CharacterSetSettings": null, - "FailTaskWhenCleanTaskResourceFailed": false, - "ChangeProcessingTuning": { - "StatementCacheSize": 50, - "CommitTimeout": 1, - "BatchApplyPreserveTransaction": true, - "BatchApplyTimeoutMin": 1, - "BatchSplitSize": 0, - "BatchApplyTimeoutMax": 30, - "MinTransactionSize": 1000, - "MemoryKeepTime": 60, - "BatchApplyMemoryLimit": 500, - "MemoryLimitTotal": 1024 - }, - "ChangeProcessingDdlHandlingPolicy": { - "HandleSourceTableDropped": true, - "HandleSourceTableTruncated": true, - "HandleSourceTableAltered": true - }, - "PostProcessingRules": null -}` +var ( + defaultReplicationConfigSettings = map[string]string{ + "cdc": defaultReplicationConfigCdcSettings, + "full-load": defaultReplicationConfigFullLoadSettings, + "full-load-and-cdc": defaultReplicationConfigFullLoadAndCdcSettings, + } + + //go:embed testdata/replication_config/defaults/cdc.json + defaultReplicationConfigCdcSettings string + + //go:embed testdata/replication_config/defaults/full-load.json + defaultReplicationConfigFullLoadSettings string + + //go:embed testdata/replication_config/defaults/full-load-and-cdc.json + defaultReplicationConfigFullLoadAndCdcSettings string +) diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index 9864cb9feab6..d918d7cf33cd 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -5,6 +5,7 @@ package dms_test import ( "context" + _ "embed" "fmt" "strconv" "strings" @@ -12,6 +13,7 @@ import ( "time" "github.com/YakDriver/regexache" + dms "github.com/aws/aws-sdk-go/service/databasemigrationservice" sdkacctest "github.com/hashicorp/terraform-plugin-testing/helper/acctest" "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-plugin-testing/terraform" @@ -23,44 +25,50 @@ import ( ) func TestAccDMSReplicationTask_basic(t *testing.T) { - ctx := acctest.Context(t) - rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) - resourceName := "aws_dms_replication_task.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(ctx, t) }, - ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), - Steps: []resource.TestStep{ - { - Config: testAccReplicationTaskConfig_basic(rName), - Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), - resource.TestCheckResourceAttr(resourceName, "replication_task_id", rName), - acctest.MatchResourceAttrRegionalARN(resourceName, "replication_task_arn", "dms", regexache.MustCompile(`task:[A-Z0-9]{26}`)), - resource.TestCheckResourceAttr(resourceName, "cdc_start_position", ""), - resource.TestCheckNoResourceAttr(resourceName, "cdc_start_time"), - resource.TestCheckResourceAttr(resourceName, "migration_type", "full-load"), - resource.TestCheckResourceAttrPair(resourceName, "replication_instance_arn", "aws_dms_replication_instance.test", "replication_instance_arn"), - acctest.CheckResourceAttrEquivalentJSON(resourceName, "replication_task_settings", defaultReplicationTaskSettings), - resource.TestCheckResourceAttrPair(resourceName, "source_endpoint_arn", "aws_dms_endpoint.source", "endpoint_arn"), - resource.TestCheckResourceAttr(resourceName, "start_replication_task", "false"), - resource.TestCheckResourceAttr(resourceName, "status", "ready"), - acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), - resource.TestCheckResourceAttrPair(resourceName, "target_endpoint_arn", "aws_dms_endpoint.target", "endpoint_arn"), - resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), - resource.TestCheckResourceAttr(resourceName, "tags_all.%", "0"), - ), - }, - { - ResourceName: resourceName, - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"start_replication_task"}, - }, - }, - }) + t.Parallel() + + for _, migrationType := range dms.MigrationTypeValue_Values() { + t.Run(migrationType, func(t *testing.T) { + ctx := acctest.Context(t) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_dms_replication_task.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccReplicationTaskConfig_basic(rName, migrationType), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationTaskExists(ctx, resourceName), + resource.TestCheckResourceAttr(resourceName, "replication_task_id", rName), + acctest.MatchResourceAttrRegionalARN(resourceName, "replication_task_arn", "dms", regexache.MustCompile(`task:[A-Z0-9]{26}`)), + resource.TestCheckResourceAttr(resourceName, "cdc_start_position", ""), + resource.TestCheckNoResourceAttr(resourceName, "cdc_start_time"), + resource.TestCheckResourceAttr(resourceName, "migration_type", migrationType), + resource.TestCheckResourceAttrPair(resourceName, "replication_instance_arn", "aws_dms_replication_instance.test", "replication_instance_arn"), + acctest.CheckResourceAttrEquivalentJSON(resourceName, "replication_task_settings", defaultReplicationTaskSettings[migrationType]), + resource.TestCheckResourceAttrPair(resourceName, "source_endpoint_arn", "aws_dms_endpoint.source", "endpoint_arn"), + resource.TestCheckResourceAttr(resourceName, "start_replication_task", "false"), + resource.TestCheckResourceAttr(resourceName, "status", "ready"), + acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), + resource.TestCheckResourceAttrPair(resourceName, "target_endpoint_arn", "aws_dms_endpoint.target", "endpoint_arn"), + resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), + resource.TestCheckResourceAttr(resourceName, "tags_all.%", "0"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication_task"}, + }, + }, + }) + }) + } } func TestAccDMSReplicationTask_updateSettingsAndMappings(t *testing.T) { @@ -312,7 +320,7 @@ func TestAccDMSReplicationTask_disappears(t *testing.T) { CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), Steps: []resource.TestStep{ { - Config: testAccReplicationTaskConfig_basic(rName), + Config: testAccReplicationTaskConfig_basic(rName, "full-load"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName), acctest.CheckResourceDisappears(ctx, acctest.Provider, tfdms.ResourceReplicationTask(), resourceName), @@ -494,7 +502,6 @@ resource "aws_dms_endpoint" "target" { func testAccReplicationTaskConfig_base(rName string) string { return acctest.ConfigCompose( - acctest.ConfigVPCWithSubnets(rName, 2), testAccReplicationEndpointConfig_dummyDatabase(rName), fmt.Sprintf(` resource "aws_dms_replication_subnet_group" "test" { @@ -515,11 +522,11 @@ resource "aws_dms_replication_instance" "test" { `, rName)) } -func testAccReplicationTaskConfig_basic(rName string) string { +func testAccReplicationTaskConfig_basic(rName, migrationType string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { replication_task_id = %[1]q - migration_type = "full-load" + migration_type = %[2]q replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn @@ -540,7 +547,7 @@ resource "aws_dms_replication_task" "test" { } ) } -`, rName)) +`, rName, migrationType)) } func testAccReplicationTaskConfig_resourceIdentifier(rName string) string { @@ -891,177 +898,19 @@ resource "aws_dms_replication_instance" "test2" { `, rName, arn)) } -const defaultReplicationTaskSettings = `{ - "Logging": { - "EnableLogging": false, - "EnableLogContext": false, - "LogComponents": [ - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "DATA_STRUCTURE" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "COMMUNICATION" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "IO" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "COMMON" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "FILE_FACTORY" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "FILE_TRANSFER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "REST_SERVER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "ADDONS" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TARGET_LOAD" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TARGET_APPLY" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "SOURCE_UNLOAD" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "SOURCE_CAPTURE" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TRANSFORMATION" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "SORTER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TASK_MANAGER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TABLES_MANAGER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "METADATA_MANAGER" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "PERFORMANCE" - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "VALIDATOR_EXT" - } - ], - "CloudWatchLogGroup": null, - "CloudWatchLogStream": null - }, - "StreamBufferSettings": { - "StreamBufferCount": 3, - "CtrlStreamBufferSizeInMB": 5, - "StreamBufferSizeInMB": 8 - }, - "ErrorBehavior": { - "FailOnNoTablesCaptured": true, - "ApplyErrorUpdatePolicy": "LOG_ERROR", - "FailOnTransactionConsistencyBreached": false, - "RecoverableErrorThrottlingMax": 1800, - "DataErrorEscalationPolicy": "SUSPEND_TABLE", - "ApplyErrorEscalationCount": 0, - "RecoverableErrorStopRetryAfterThrottlingMax": true, - "RecoverableErrorThrottling": true, - "ApplyErrorFailOnTruncationDdl": false, - "DataTruncationErrorPolicy": "LOG_ERROR", - "ApplyErrorInsertPolicy": "LOG_ERROR", - "EventErrorPolicy": "IGNORE", - "ApplyErrorEscalationPolicy": "LOG_ERROR", - "RecoverableErrorCount": -1, - "DataErrorEscalationCount": 0, - "TableErrorEscalationPolicy": "STOP_TASK", - "RecoverableErrorInterval": 5, - "ApplyErrorDeletePolicy": "IGNORE_RECORD", - "TableErrorEscalationCount": 0, - "FullLoadIgnoreConflicts": true, - "DataErrorPolicy": "LOG_ERROR", - "TableErrorPolicy": "SUSPEND_TABLE" - }, - "TTSettings": null, - "FullLoadSettings": { - "CommitRate": 10000, - "StopTaskCachedChangesApplied": false, - "StopTaskCachedChangesNotApplied": false, - "MaxFullLoadSubTasks": 8, - "TransactionConsistencyTimeout": 600, - "CreatePkAfterFullLoad": false, - "TargetTablePrepMode": "DROP_AND_CREATE" - }, - "TargetMetadata": { - "ParallelApplyBufferSize": 0, - "ParallelApplyQueuesPerThread": 0, - "ParallelApplyThreads": 0, - "TargetSchema": "", - "InlineLobMaxSize": 0, - "ParallelLoadQueuesPerThread": 0, - "SupportLobs": true, - "LobChunkSize": 64, - "TaskRecoveryTableEnabled": false, - "ParallelLoadThreads": 0, - "LobMaxSize": 32, - "BatchApplyEnabled": false, - "FullLobMode": false, - "LimitedSizeLobMode": true, - "LoadMaxFileSize": 0, - "ParallelLoadBufferSize": 0 - }, - "BeforeImageSettings": null, - "ControlTablesSettings": { - "historyTimeslotInMinutes": 5, - "HistoryTimeslotInMinutes": 5, - "StatusTableEnabled": false, - "SuspendedTablesTableEnabled": false, - "HistoryTableEnabled": false, - "ControlSchema": "", - "FullLoadExceptionTableEnabled": false - }, - "LoopbackPreventionSettings": null, - "CharacterSetSettings": null, - "FailTaskWhenCleanTaskResourceFailed": false, - "ChangeProcessingTuning": { - "StatementCacheSize": 50, - "CommitTimeout": 1, - "BatchApplyPreserveTransaction": true, - "BatchApplyTimeoutMin": 1, - "BatchSplitSize": 0, - "BatchApplyTimeoutMax": 30, - "MinTransactionSize": 1000, - "MemoryKeepTime": 60, - "BatchApplyMemoryLimit": 500, - "MemoryLimitTotal": 1024 - }, - "ChangeProcessingDdlHandlingPolicy": { - "HandleSourceTableDropped": true, - "HandleSourceTableTruncated": true, - "HandleSourceTableAltered": true - }, - "PostProcessingRules": null -}` +var ( + defaultReplicationTaskSettings = map[string]string{ + "cdc": defaultReplicationTaskCdcSettings, + "full-load": defaultReplicationTaskFullLoadSettings, + "full-load-and-cdc": defaultReplicationTaskFullLoadAndCdcSettings, + } + + //go:embed testdata/replication_task/defaults/cdc.json + defaultReplicationTaskCdcSettings string + + //go:embed testdata/replication_task/defaults/full-load.json + defaultReplicationTaskFullLoadSettings string + + //go:embed testdata/replication_task/defaults/full-load-and-cdc.json + defaultReplicationTaskFullLoadAndCdcSettings string +) diff --git a/internal/service/dms/testdata/replication_config/defaults/cdc.json b/internal/service/dms/testdata/replication_config/defaults/cdc.json new file mode 100644 index 000000000000..3a0248855af2 --- /dev/null +++ b/internal/service/dms/testdata/replication_config/defaults/cdc.json @@ -0,0 +1,181 @@ +{ + "Logging": { + "EnableLogging": false, + "EnableLogContext": false, + "LogComponents": [ + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "DATA_STRUCTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMUNICATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "IO" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMON" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_FACTORY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_TRANSFER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "REST_SERVER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "ADDONS" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_LOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_APPLY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_UNLOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_CAPTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TRANSFORMATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SORTER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TASK_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TABLES_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "METADATA_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "PERFORMANCE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "VALIDATOR_EXT" + } + ], + "CloudWatchLogGroup": null, + "CloudWatchLogStream": null + }, + "StreamBufferSettings": { + "StreamBufferCount": 3, + "CtrlStreamBufferSizeInMB": 5, + "StreamBufferSizeInMB": 8 + }, + "ErrorBehavior": { + "FailOnNoTablesCaptured": true, + "ApplyErrorUpdatePolicy": "LOG_ERROR", + "FailOnTransactionConsistencyBreached": false, + "RecoverableErrorThrottlingMax": 1800, + "DataErrorEscalationPolicy": "SUSPEND_TABLE", + "ApplyErrorEscalationCount": 0, + "RecoverableErrorStopRetryAfterThrottlingMax": true, + "RecoverableErrorThrottling": true, + "ApplyErrorFailOnTruncationDdl": false, + "DataMaskingErrorPolicy": "STOP_TASK", + "DataTruncationErrorPolicy": "LOG_ERROR", + "ApplyErrorInsertPolicy": "LOG_ERROR", + "EventErrorPolicy": "IGNORE", + "ApplyErrorEscalationPolicy": "LOG_ERROR", + "RecoverableErrorCount": -1, + "DataErrorEscalationCount": 0, + "TableErrorEscalationPolicy": "STOP_TASK", + "RecoverableErrorInterval": 5, + "ApplyErrorDeletePolicy": "IGNORE_RECORD", + "TableErrorEscalationCount": 0, + "FullLoadIgnoreConflicts": true, + "DataErrorPolicy": "LOG_ERROR", + "TableErrorPolicy": "SUSPEND_TABLE" + }, + "TTSettings": { + "TTS3Settings": null, + "TTRecordSettings": null, + "FailTaskOnTTFailure": false, + "EnableTT": false + }, + "FullLoadSettings": { + "CommitRate": 10000, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CreatePkAfterFullLoad": false, + "TargetTablePrepMode": "DO_NOTHING" + }, + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 32, + "BatchApplyEnabled": false, + "FullLobMode": false, + "LimitedSizeLobMode": true, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "BeforeImageSettings": null, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "CommitPositionTableEnabled": false, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": false, + "HistoryTableEnabled": false, + "ControlSchema": "", + "FullLoadExceptionTableEnabled": false + }, + "LoopbackPreventionSettings": null, + "CharacterSetSettings": null, + "FailTaskWhenCleanTaskResourceFailed": false, + "ChangeProcessingTuning": { + "StatementCacheSize": 50, + "CommitTimeout": 1, + "BatchApplyPreserveTransaction": true, + "BatchApplyTimeoutMin": 1, + "BatchSplitSize": 0, + "BatchApplyTimeoutMax": 30, + "MinTransactionSize": 1000, + "MemoryKeepTime": 60, + "BatchApplyMemoryLimit": 500, + "MemoryLimitTotal": 1024 + }, + "ChangeProcessingDdlHandlingPolicy": { + "HandleSourceTableDropped": true, + "HandleSourceTableTruncated": true, + "HandleSourceTableAltered": true + }, + "PostProcessingRules": null +} diff --git a/internal/service/dms/testdata/replication_config/defaults/full-load-and-cdc.json b/internal/service/dms/testdata/replication_config/defaults/full-load-and-cdc.json new file mode 100644 index 000000000000..9de78a459848 --- /dev/null +++ b/internal/service/dms/testdata/replication_config/defaults/full-load-and-cdc.json @@ -0,0 +1,181 @@ +{ + "Logging": { + "EnableLogging": false, + "EnableLogContext": false, + "LogComponents": [ + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "DATA_STRUCTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMUNICATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "IO" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMON" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_FACTORY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_TRANSFER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "REST_SERVER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "ADDONS" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_LOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_APPLY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_UNLOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_CAPTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TRANSFORMATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SORTER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TASK_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TABLES_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "METADATA_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "PERFORMANCE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "VALIDATOR_EXT" + } + ], + "CloudWatchLogGroup": null, + "CloudWatchLogStream": null + }, + "StreamBufferSettings": { + "StreamBufferCount": 3, + "CtrlStreamBufferSizeInMB": 5, + "StreamBufferSizeInMB": 8 + }, + "ErrorBehavior": { + "FailOnNoTablesCaptured": true, + "ApplyErrorUpdatePolicy": "LOG_ERROR", + "FailOnTransactionConsistencyBreached": false, + "RecoverableErrorThrottlingMax": 1800, + "DataErrorEscalationPolicy": "SUSPEND_TABLE", + "ApplyErrorEscalationCount": 0, + "RecoverableErrorStopRetryAfterThrottlingMax": true, + "RecoverableErrorThrottling": true, + "ApplyErrorFailOnTruncationDdl": false, + "DataMaskingErrorPolicy": "STOP_TASK", + "DataTruncationErrorPolicy": "LOG_ERROR", + "ApplyErrorInsertPolicy": "LOG_ERROR", + "EventErrorPolicy": "IGNORE", + "ApplyErrorEscalationPolicy": "LOG_ERROR", + "RecoverableErrorCount": -1, + "DataErrorEscalationCount": 0, + "TableErrorEscalationPolicy": "STOP_TASK", + "RecoverableErrorInterval": 5, + "ApplyErrorDeletePolicy": "IGNORE_RECORD", + "TableErrorEscalationCount": 0, + "FullLoadIgnoreConflicts": true, + "DataErrorPolicy": "LOG_ERROR", + "TableErrorPolicy": "SUSPEND_TABLE" + }, + "TTSettings": { + "TTS3Settings": null, + "TTRecordSettings": null, + "FailTaskOnTTFailure": false, + "EnableTT": false + }, + "FullLoadSettings": { + "CommitRate": 10000, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CreatePkAfterFullLoad": false, + "TargetTablePrepMode": "DROP_AND_CREATE" + }, + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 32, + "BatchApplyEnabled": false, + "FullLobMode": false, + "LimitedSizeLobMode": true, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "BeforeImageSettings": null, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "CommitPositionTableEnabled": false, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": false, + "HistoryTableEnabled": false, + "ControlSchema": "", + "FullLoadExceptionTableEnabled": false + }, + "LoopbackPreventionSettings": null, + "CharacterSetSettings": null, + "FailTaskWhenCleanTaskResourceFailed": false, + "ChangeProcessingTuning": { + "StatementCacheSize": 50, + "CommitTimeout": 1, + "BatchApplyPreserveTransaction": true, + "BatchApplyTimeoutMin": 1, + "BatchSplitSize": 0, + "BatchApplyTimeoutMax": 30, + "MinTransactionSize": 1000, + "MemoryKeepTime": 60, + "BatchApplyMemoryLimit": 500, + "MemoryLimitTotal": 1024 + }, + "ChangeProcessingDdlHandlingPolicy": { + "HandleSourceTableDropped": true, + "HandleSourceTableTruncated": true, + "HandleSourceTableAltered": true + }, + "PostProcessingRules": null +} diff --git a/internal/service/dms/testdata/replication_config/defaults/full-load.json b/internal/service/dms/testdata/replication_config/defaults/full-load.json new file mode 100644 index 000000000000..40e5e4375a7d --- /dev/null +++ b/internal/service/dms/testdata/replication_config/defaults/full-load.json @@ -0,0 +1,176 @@ +{ + "Logging": { + "EnableLogging": false, + "EnableLogContext": false, + "LogComponents": [ + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "DATA_STRUCTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMUNICATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "IO" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMON" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_FACTORY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_TRANSFER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "REST_SERVER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "ADDONS" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_LOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_APPLY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_UNLOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_CAPTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TRANSFORMATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SORTER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TASK_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TABLES_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "METADATA_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "PERFORMANCE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "VALIDATOR_EXT" + } + ], + "CloudWatchLogGroup": null, + "CloudWatchLogStream": null + }, + "StreamBufferSettings": { + "StreamBufferCount": 3, + "CtrlStreamBufferSizeInMB": 5, + "StreamBufferSizeInMB": 8 + }, + "ErrorBehavior": { + "FailOnNoTablesCaptured": true, + "ApplyErrorUpdatePolicy": "LOG_ERROR", + "FailOnTransactionConsistencyBreached": false, + "RecoverableErrorThrottlingMax": 1800, + "DataErrorEscalationPolicy": "SUSPEND_TABLE", + "ApplyErrorEscalationCount": 0, + "RecoverableErrorStopRetryAfterThrottlingMax": true, + "RecoverableErrorThrottling": true, + "ApplyErrorFailOnTruncationDdl": false, + "DataMaskingErrorPolicy": "STOP_TASK", + "DataTruncationErrorPolicy": "LOG_ERROR", + "ApplyErrorInsertPolicy": "LOG_ERROR", + "EventErrorPolicy": "IGNORE", + "ApplyErrorEscalationPolicy": "LOG_ERROR", + "RecoverableErrorCount": -1, + "DataErrorEscalationCount": 0, + "TableErrorEscalationPolicy": "STOP_TASK", + "RecoverableErrorInterval": 5, + "ApplyErrorDeletePolicy": "IGNORE_RECORD", + "TableErrorEscalationCount": 0, + "FullLoadIgnoreConflicts": true, + "DataErrorPolicy": "LOG_ERROR", + "TableErrorPolicy": "SUSPEND_TABLE" + }, + "TTSettings": null, + "FullLoadSettings": { + "CommitRate": 10000, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CreatePkAfterFullLoad": false, + "TargetTablePrepMode": "DROP_AND_CREATE" + }, + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 32, + "BatchApplyEnabled": false, + "FullLobMode": false, + "LimitedSizeLobMode": true, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "BeforeImageSettings": null, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "CommitPositionTableEnabled": false, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": false, + "HistoryTableEnabled": false, + "ControlSchema": "", + "FullLoadExceptionTableEnabled": false + }, + "LoopbackPreventionSettings": null, + "CharacterSetSettings": null, + "FailTaskWhenCleanTaskResourceFailed": false, + "ChangeProcessingTuning": { + "StatementCacheSize": 50, + "CommitTimeout": 1, + "BatchApplyPreserveTransaction": true, + "BatchApplyTimeoutMin": 1, + "BatchSplitSize": 0, + "BatchApplyTimeoutMax": 30, + "MinTransactionSize": 1000, + "MemoryKeepTime": 60, + "BatchApplyMemoryLimit": 500, + "MemoryLimitTotal": 1024 + }, + "ChangeProcessingDdlHandlingPolicy": { + "HandleSourceTableDropped": true, + "HandleSourceTableTruncated": true, + "HandleSourceTableAltered": true + }, + "PostProcessingRules": null +} diff --git a/internal/service/dms/testdata/replication_task/defaults/cdc.json b/internal/service/dms/testdata/replication_task/defaults/cdc.json new file mode 100644 index 000000000000..3a1101f8674f --- /dev/null +++ b/internal/service/dms/testdata/replication_task/defaults/cdc.json @@ -0,0 +1,178 @@ +{ + "Logging": { + "EnableLogging": false, + "EnableLogContext": false, + "LogComponents": [ + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "DATA_STRUCTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMUNICATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "IO" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMON" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_FACTORY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_TRANSFER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "REST_SERVER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "ADDONS" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_LOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_APPLY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_UNLOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_CAPTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TRANSFORMATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SORTER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TASK_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TABLES_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "METADATA_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "PERFORMANCE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "VALIDATOR_EXT" + } + ], + "CloudWatchLogGroup": null, + "CloudWatchLogStream": null + }, + "StreamBufferSettings": { + "StreamBufferCount": 3, + "CtrlStreamBufferSizeInMB": 5, + "StreamBufferSizeInMB": 8 + }, + "ErrorBehavior": { + "FailOnNoTablesCaptured": true, + "ApplyErrorUpdatePolicy": "LOG_ERROR", + "FailOnTransactionConsistencyBreached": false, + "RecoverableErrorThrottlingMax": 1800, + "DataErrorEscalationPolicy": "SUSPEND_TABLE", + "ApplyErrorEscalationCount": 0, + "RecoverableErrorStopRetryAfterThrottlingMax": true, + "RecoverableErrorThrottling": true, + "ApplyErrorFailOnTruncationDdl": false, + "DataTruncationErrorPolicy": "LOG_ERROR", + "ApplyErrorInsertPolicy": "LOG_ERROR", + "EventErrorPolicy": "IGNORE", + "ApplyErrorEscalationPolicy": "LOG_ERROR", + "RecoverableErrorCount": -1, + "DataErrorEscalationCount": 0, + "TableErrorEscalationPolicy": "STOP_TASK", + "RecoverableErrorInterval": 5, + "ApplyErrorDeletePolicy": "IGNORE_RECORD", + "TableErrorEscalationCount": 0, + "FullLoadIgnoreConflicts": true, + "DataErrorPolicy": "LOG_ERROR", + "TableErrorPolicy": "SUSPEND_TABLE" + }, + "TTSettings": { + "TTS3Settings": null, + "TTRecordSettings": null, + "EnableTT": false + }, + "FullLoadSettings": { + "CommitRate": 10000, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CreatePkAfterFullLoad": false, + "TargetTablePrepMode": "DO_NOTHING" + }, + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 32, + "BatchApplyEnabled": false, + "FullLobMode": false, + "LimitedSizeLobMode": true, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "BeforeImageSettings": null, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": false, + "HistoryTableEnabled": false, + "ControlSchema": "", + "FullLoadExceptionTableEnabled": false + }, + "LoopbackPreventionSettings": null, + "CharacterSetSettings": null, + "FailTaskWhenCleanTaskResourceFailed": false, + "ChangeProcessingTuning": { + "StatementCacheSize": 50, + "CommitTimeout": 1, + "BatchApplyPreserveTransaction": true, + "BatchApplyTimeoutMin": 1, + "BatchSplitSize": 0, + "BatchApplyTimeoutMax": 30, + "MinTransactionSize": 1000, + "MemoryKeepTime": 60, + "BatchApplyMemoryLimit": 500, + "MemoryLimitTotal": 1024 + }, + "ChangeProcessingDdlHandlingPolicy": { + "HandleSourceTableDropped": true, + "HandleSourceTableTruncated": true, + "HandleSourceTableAltered": true + }, + "PostProcessingRules": null +} diff --git a/internal/service/dms/testdata/replication_task/defaults/full-load-and-cdc.json b/internal/service/dms/testdata/replication_task/defaults/full-load-and-cdc.json new file mode 100644 index 000000000000..2d102ba0c638 --- /dev/null +++ b/internal/service/dms/testdata/replication_task/defaults/full-load-and-cdc.json @@ -0,0 +1,178 @@ +{ + "Logging": { + "EnableLogging": false, + "EnableLogContext": false, + "LogComponents": [ + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "DATA_STRUCTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMUNICATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "IO" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMON" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_FACTORY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_TRANSFER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "REST_SERVER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "ADDONS" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_LOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_APPLY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_UNLOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_CAPTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TRANSFORMATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SORTER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TASK_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TABLES_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "METADATA_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "PERFORMANCE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "VALIDATOR_EXT" + } + ], + "CloudWatchLogGroup": null, + "CloudWatchLogStream": null + }, + "StreamBufferSettings": { + "StreamBufferCount": 3, + "CtrlStreamBufferSizeInMB": 5, + "StreamBufferSizeInMB": 8 + }, + "ErrorBehavior": { + "FailOnNoTablesCaptured": true, + "ApplyErrorUpdatePolicy": "LOG_ERROR", + "FailOnTransactionConsistencyBreached": false, + "RecoverableErrorThrottlingMax": 1800, + "DataErrorEscalationPolicy": "SUSPEND_TABLE", + "ApplyErrorEscalationCount": 0, + "RecoverableErrorStopRetryAfterThrottlingMax": true, + "RecoverableErrorThrottling": true, + "ApplyErrorFailOnTruncationDdl": false, + "DataTruncationErrorPolicy": "LOG_ERROR", + "ApplyErrorInsertPolicy": "LOG_ERROR", + "EventErrorPolicy": "IGNORE", + "ApplyErrorEscalationPolicy": "LOG_ERROR", + "RecoverableErrorCount": -1, + "DataErrorEscalationCount": 0, + "TableErrorEscalationPolicy": "STOP_TASK", + "RecoverableErrorInterval": 5, + "ApplyErrorDeletePolicy": "IGNORE_RECORD", + "TableErrorEscalationCount": 0, + "FullLoadIgnoreConflicts": true, + "DataErrorPolicy": "LOG_ERROR", + "TableErrorPolicy": "SUSPEND_TABLE" + }, + "TTSettings": { + "TTS3Settings": null, + "TTRecordSettings": null, + "EnableTT": false + }, + "FullLoadSettings": { + "CommitRate": 10000, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CreatePkAfterFullLoad": false, + "TargetTablePrepMode": "DROP_AND_CREATE" + }, + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 32, + "BatchApplyEnabled": false, + "FullLobMode": false, + "LimitedSizeLobMode": true, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "BeforeImageSettings": null, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": false, + "HistoryTableEnabled": false, + "ControlSchema": "", + "FullLoadExceptionTableEnabled": false + }, + "LoopbackPreventionSettings": null, + "CharacterSetSettings": null, + "FailTaskWhenCleanTaskResourceFailed": false, + "ChangeProcessingTuning": { + "StatementCacheSize": 50, + "CommitTimeout": 1, + "BatchApplyPreserveTransaction": true, + "BatchApplyTimeoutMin": 1, + "BatchSplitSize": 0, + "BatchApplyTimeoutMax": 30, + "MinTransactionSize": 1000, + "MemoryKeepTime": 60, + "BatchApplyMemoryLimit": 500, + "MemoryLimitTotal": 1024 + }, + "ChangeProcessingDdlHandlingPolicy": { + "HandleSourceTableDropped": true, + "HandleSourceTableTruncated": true, + "HandleSourceTableAltered": true + }, + "PostProcessingRules": null +} diff --git a/internal/service/dms/testdata/replication_task/defaults/full-load.json b/internal/service/dms/testdata/replication_task/defaults/full-load.json new file mode 100644 index 000000000000..01c0240acf5c --- /dev/null +++ b/internal/service/dms/testdata/replication_task/defaults/full-load.json @@ -0,0 +1,174 @@ +{ + "Logging": { + "EnableLogging": false, + "EnableLogContext": false, + "LogComponents": [ + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "DATA_STRUCTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMUNICATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "IO" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMON" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_FACTORY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_TRANSFER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "REST_SERVER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "ADDONS" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_LOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_APPLY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_UNLOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_CAPTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TRANSFORMATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SORTER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TASK_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TABLES_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "METADATA_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "PERFORMANCE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "VALIDATOR_EXT" + } + ], + "CloudWatchLogGroup": null, + "CloudWatchLogStream": null + }, + "StreamBufferSettings": { + "StreamBufferCount": 3, + "CtrlStreamBufferSizeInMB": 5, + "StreamBufferSizeInMB": 8 + }, + "ErrorBehavior": { + "FailOnNoTablesCaptured": true, + "ApplyErrorUpdatePolicy": "LOG_ERROR", + "FailOnTransactionConsistencyBreached": false, + "RecoverableErrorThrottlingMax": 1800, + "DataErrorEscalationPolicy": "SUSPEND_TABLE", + "ApplyErrorEscalationCount": 0, + "RecoverableErrorStopRetryAfterThrottlingMax": true, + "RecoverableErrorThrottling": true, + "ApplyErrorFailOnTruncationDdl": false, + "DataTruncationErrorPolicy": "LOG_ERROR", + "ApplyErrorInsertPolicy": "LOG_ERROR", + "EventErrorPolicy": "IGNORE", + "ApplyErrorEscalationPolicy": "LOG_ERROR", + "RecoverableErrorCount": -1, + "DataErrorEscalationCount": 0, + "TableErrorEscalationPolicy": "STOP_TASK", + "RecoverableErrorInterval": 5, + "ApplyErrorDeletePolicy": "IGNORE_RECORD", + "TableErrorEscalationCount": 0, + "FullLoadIgnoreConflicts": true, + "DataErrorPolicy": "LOG_ERROR", + "TableErrorPolicy": "SUSPEND_TABLE" + }, + "TTSettings": null, + "FullLoadSettings": { + "CommitRate": 10000, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CreatePkAfterFullLoad": false, + "TargetTablePrepMode": "DROP_AND_CREATE" + }, + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 32, + "BatchApplyEnabled": false, + "FullLobMode": false, + "LimitedSizeLobMode": true, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "BeforeImageSettings": null, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": false, + "HistoryTableEnabled": false, + "ControlSchema": "", + "FullLoadExceptionTableEnabled": false + }, + "LoopbackPreventionSettings": null, + "CharacterSetSettings": null, + "FailTaskWhenCleanTaskResourceFailed": false, + "ChangeProcessingTuning": { + "StatementCacheSize": 50, + "CommitTimeout": 1, + "BatchApplyPreserveTransaction": true, + "BatchApplyTimeoutMin": 1, + "BatchSplitSize": 0, + "BatchApplyTimeoutMax": 30, + "MinTransactionSize": 1000, + "MemoryKeepTime": 60, + "BatchApplyMemoryLimit": 500, + "MemoryLimitTotal": 1024 + }, + "ChangeProcessingDdlHandlingPolicy": { + "HandleSourceTableDropped": true, + "HandleSourceTableTruncated": true, + "HandleSourceTableAltered": true + }, + "PostProcessingRules": null +} From c5ecee4a4b2a04d303b6bd853fa928a371f2f9c3 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Fri, 12 Apr 2024 13:48:35 -0700 Subject: [PATCH 10/18] Adds return parameter to `Exists` tests --- internal/service/dms/endpoint_test.go | 6 ++- .../service/dms/replication_config_test.go | 35 +++++++----- .../dms/replication_task_data_source_test.go | 4 +- internal/service/dms/replication_task_test.go | 53 ++++++++++++------- 4 files changed, 65 insertions(+), 33 deletions(-) diff --git a/internal/service/dms/endpoint_test.go b/internal/service/dms/endpoint_test.go index 7add882264c7..02891bc7627f 100644 --- a/internal/service/dms/endpoint_test.go +++ b/internal/service/dms/endpoint_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/YakDriver/regexache" + dms "github.com/aws/aws-sdk-go/service/databasemigrationservice" sdkacctest "github.com/hashicorp/terraform-plugin-testing/helper/acctest" "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-plugin-testing/terraform" @@ -2211,6 +2212,7 @@ func TestAccDMSEndpoint_pauseReplicationTasks(t *testing.T) { endpointNameSource := "aws_dms_endpoint.source" endpointNameTarget := "aws_dms_endpoint.target" replicationTaskName := "aws_dms_replication_task.test" + var task dms.ReplicationTask resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -2223,7 +2225,7 @@ func TestAccDMSEndpoint_pauseReplicationTasks(t *testing.T) { Check: resource.ComposeTestCheckFunc( testAccCheckEndpointExists(ctx, endpointNameSource), testAccCheckEndpointExists(ctx, endpointNameTarget), - testAccCheckReplicationTaskExists(ctx, replicationTaskName), + testAccCheckReplicationTaskExists(ctx, replicationTaskName, &task), resource.TestCheckResourceAttr(replicationTaskName, "status", "running"), ), }, @@ -2232,7 +2234,7 @@ func TestAccDMSEndpoint_pauseReplicationTasks(t *testing.T) { Check: resource.ComposeTestCheckFunc( testAccCheckEndpointExists(ctx, endpointNameSource), testAccCheckEndpointExists(ctx, endpointNameTarget), - testAccCheckReplicationTaskExists(ctx, replicationTaskName), + testAccCheckReplicationTaskExists(ctx, replicationTaskName, &task), resource.TestCheckResourceAttr(replicationTaskName, "status", "running"), ), }, diff --git a/internal/service/dms/replication_config_test.go b/internal/service/dms/replication_config_test.go index 309ff7a06f89..c7cb9b03acdb 100644 --- a/internal/service/dms/replication_config_test.go +++ b/internal/service/dms/replication_config_test.go @@ -29,6 +29,7 @@ func TestAccDMSReplicationConfig_basic(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_config.test" + var v dms.ReplicationConfig resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -39,7 +40,7 @@ func TestAccDMSReplicationConfig_basic(t *testing.T) { { Config: testAccReplicationConfigConfig_basic(rName, migrationType), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), + testAccCheckReplicationConfigExists(ctx, resourceName, &v), resource.TestCheckResourceAttrSet(resourceName, "arn"), acctest.MatchResourceAttrRegionalARN(resourceName, "arn", "dms", regexache.MustCompile(`replication-config:[A-Z0-9]{26}`)), resource.TestCheckResourceAttr(resourceName, "compute_config.#", "1"), @@ -129,6 +130,7 @@ func TestAccDMSReplicationConfig_disappears(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_config.test" + var v dms.ReplicationConfig resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -139,7 +141,7 @@ func TestAccDMSReplicationConfig_disappears(t *testing.T) { { Config: testAccReplicationConfigConfig_basic(rName, "cdc"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), + testAccCheckReplicationConfigExists(ctx, resourceName, &v), acctest.CheckResourceDisappears(ctx, acctest.Provider, tfdms.ResourceReplicationConfig(), resourceName), ), ExpectNonEmptyPlan: true, @@ -152,6 +154,7 @@ func TestAccDMSReplicationConfig_tags(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_config.test" + var v dms.ReplicationConfig resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -162,7 +165,7 @@ func TestAccDMSReplicationConfig_tags(t *testing.T) { { Config: testAccReplicationConfigConfig_tags1(rName, "key1", "value1"), Check: resource.ComposeTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), + testAccCheckReplicationConfigExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "tags.%", "1"), resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1"), ), @@ -170,7 +173,7 @@ func TestAccDMSReplicationConfig_tags(t *testing.T) { { Config: testAccReplicationConfigConfig_tags2(rName, "key1", "value1updated", "key2", "value2"), Check: resource.ComposeTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), + testAccCheckReplicationConfigExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "tags.%", "2"), resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1updated"), resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"), @@ -179,7 +182,7 @@ func TestAccDMSReplicationConfig_tags(t *testing.T) { { Config: testAccReplicationConfigConfig_tags1(rName, "key2", "value2"), Check: resource.ComposeTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), + testAccCheckReplicationConfigExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "tags.%", "1"), resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"), ), @@ -192,6 +195,7 @@ func TestAccDMSReplicationConfig_update(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_config.test" + var v dms.ReplicationConfig resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -202,7 +206,7 @@ func TestAccDMSReplicationConfig_update(t *testing.T) { { Config: testAccReplicationConfigConfig_update(rName, "cdc", 2, 16), Check: resource.ComposeTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), + testAccCheckReplicationConfigExists(ctx, resourceName, &v), resource.TestCheckResourceAttrSet(resourceName, "arn"), resource.TestCheckResourceAttr(resourceName, "replication_type", "cdc"), resource.TestCheckResourceAttr(resourceName, "compute_config.0.max_capacity_units", "16"), @@ -212,7 +216,7 @@ func TestAccDMSReplicationConfig_update(t *testing.T) { { Config: testAccReplicationConfigConfig_update(rName, "cdc", 4, 32), Check: resource.ComposeTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), + testAccCheckReplicationConfigExists(ctx, resourceName, &v), resource.TestCheckResourceAttrSet(resourceName, "arn"), resource.TestCheckResourceAttr(resourceName, "replication_type", "cdc"), resource.TestCheckResourceAttr(resourceName, "compute_config.0.max_capacity_units", "32"), @@ -228,6 +232,7 @@ func TestAccDMSReplicationConfig_startReplication(t *testing.T) { rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_config.test" + var v dms.ReplicationConfig resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -238,7 +243,7 @@ func TestAccDMSReplicationConfig_startReplication(t *testing.T) { { Config: testAccReplicationConfigConfig_startReplication(rName, true), Check: resource.ComposeTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), + testAccCheckReplicationConfigExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "start_replication", "true"), ), }, @@ -251,7 +256,7 @@ func TestAccDMSReplicationConfig_startReplication(t *testing.T) { { Config: testAccReplicationConfigConfig_startReplication(rName, false), Check: resource.ComposeTestCheckFunc( - testAccCheckReplicationConfigExists(ctx, resourceName), + testAccCheckReplicationConfigExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "start_replication", "false"), ), }, @@ -259,7 +264,7 @@ func TestAccDMSReplicationConfig_startReplication(t *testing.T) { }) } -func testAccCheckReplicationConfigExists(ctx context.Context, n string) resource.TestCheckFunc { +func testAccCheckReplicationConfigExists(ctx context.Context, n string, v *dms.ReplicationConfig) resource.TestCheckFunc { return func(s *terraform.State) error { rs, ok := s.RootModule().Resources[n] if !ok { @@ -268,9 +273,15 @@ func testAccCheckReplicationConfigExists(ctx context.Context, n string) resource conn := acctest.Provider.Meta().(*conns.AWSClient).DMSConn(ctx) - _, err := tfdms.FindReplicationConfigByARN(ctx, conn, rs.Primary.ID) + output, err := tfdms.FindReplicationConfigByARN(ctx, conn, rs.Primary.ID) - return err + if err != nil { + return err + } + + *v = *output + + return nil } } diff --git a/internal/service/dms/replication_task_data_source_test.go b/internal/service/dms/replication_task_data_source_test.go index 6a7368abc8dc..06ca0f53db22 100644 --- a/internal/service/dms/replication_task_data_source_test.go +++ b/internal/service/dms/replication_task_data_source_test.go @@ -7,6 +7,7 @@ import ( "fmt" "testing" + dms "github.com/aws/aws-sdk-go/service/databasemigrationservice" sdkacctest "github.com/hashicorp/terraform-plugin-testing/helper/acctest" "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/hashicorp/terraform-provider-aws/internal/acctest" @@ -18,6 +19,7 @@ func TestAccDMSReplicationTaskDataSource_basic(t *testing.T) { rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" dataSourceName := "data.aws_dms_replication_task.test" + var v dms.ReplicationTask resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -28,7 +30,7 @@ func TestAccDMSReplicationTaskDataSource_basic(t *testing.T) { { Config: testAccReplicationTaskDataSourceConfig_basic(rName), Check: resource.ComposeTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttrPair(dataSourceName, "replication_task_id", resourceName, "replication_task_id"), ), }, diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index d918d7cf33cd..343350507d0f 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -32,6 +32,7 @@ func TestAccDMSReplicationTask_basic(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -42,7 +43,7 @@ func TestAccDMSReplicationTask_basic(t *testing.T) { { Config: testAccReplicationTaskConfig_basic(rName, migrationType), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "replication_task_id", rName), acctest.MatchResourceAttrRegionalARN(resourceName, "replication_task_arn", "dms", regexache.MustCompile(`task:[A-Z0-9]{26}`)), resource.TestCheckResourceAttr(resourceName, "cdc_start_position", ""), @@ -75,6 +76,7 @@ func TestAccDMSReplicationTask_updateSettingsAndMappings(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -85,7 +87,7 @@ func TestAccDMSReplicationTask_updateSettingsAndMappings(t *testing.T) { { Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1024, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1024"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "ZedsDead"), @@ -100,7 +102,7 @@ func TestAccDMSReplicationTask_updateSettingsAndMappings(t *testing.T) { { Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1024, "EMBRZ"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1024"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "EMBRZ"), @@ -115,7 +117,7 @@ func TestAccDMSReplicationTask_updateSettingsAndMappings(t *testing.T) { { Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1248, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1248"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "ZedsDead"), @@ -130,7 +132,7 @@ func TestAccDMSReplicationTask_updateSettingsAndMappings(t *testing.T) { { Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1024, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1024"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "ZedsDead"), @@ -145,7 +147,7 @@ func TestAccDMSReplicationTask_updateSettingsAndMappings(t *testing.T) { { Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1248, "ZedsDead"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1248"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "ZedsDead"), @@ -160,7 +162,7 @@ func TestAccDMSReplicationTask_updateSettingsAndMappings(t *testing.T) { { Config: testAccReplicationTaskConfig_updateSettingsAndMappings(rName, 1024, "EMBRZ"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "ChangeProcessingTuning.MemoryLimitTotal", "1024"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", "length(rules)", "1"), acctest.CheckResourceAttrJMES(resourceName, "table_mappings", `rules[0]."rule-name"`, "EMBRZ"), @@ -180,6 +182,7 @@ func TestAccDMSReplicationTask_cdcStartPosition(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -190,7 +193,7 @@ func TestAccDMSReplicationTask_cdcStartPosition(t *testing.T) { { Config: testAccReplicationTaskConfig_cdcStartPosition(rName, "mysql-bin-changelog.000024:373"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "cdc_start_position", "mysql-bin-changelog.000024:373"), resource.TestCheckNoResourceAttr(resourceName, "cdc_start_time"), ), @@ -241,6 +244,7 @@ func TestAccDMSReplicationTask_startReplicationTask(t *testing.T) { rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -251,7 +255,7 @@ func TestAccDMSReplicationTask_startReplicationTask(t *testing.T) { { Config: testAccReplicationTaskConfig_start(rName, true, "testrule"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "status", "running"), ), }, @@ -264,14 +268,14 @@ func TestAccDMSReplicationTask_startReplicationTask(t *testing.T) { { Config: testAccReplicationTaskConfig_start(rName, true, "changedtestrule"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "status", "running"), ), }, { Config: testAccReplicationTaskConfig_start(rName, false, "changedtestrule"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "status", "stopped"), ), }, @@ -283,6 +287,7 @@ func TestAccDMSReplicationTask_s3ToRDS(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask //https://github.com/hashicorp/terraform-provider-aws/issues/28277 @@ -295,7 +300,7 @@ func TestAccDMSReplicationTask_s3ToRDS(t *testing.T) { { Config: testAccReplicationTaskConfig_s3ToRDS(rName), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), ), }, @@ -312,6 +317,7 @@ func TestAccDMSReplicationTask_disappears(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -322,7 +328,7 @@ func TestAccDMSReplicationTask_disappears(t *testing.T) { { Config: testAccReplicationTaskConfig_basic(rName, "full-load"), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), acctest.CheckResourceDisappears(ctx, acctest.Provider, tfdms.ResourceReplicationTask(), resourceName), ), ExpectNonEmptyPlan: true, @@ -335,6 +341,7 @@ func TestAccDMSReplicationTask_cdcStartTime_rfc3339_date(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask currentTime := time.Now().UTC() rfc3339Time := currentTime.Format(time.RFC3339) @@ -349,7 +356,7 @@ func TestAccDMSReplicationTask_cdcStartTime_rfc3339_date(t *testing.T) { { Config: testAccReplicationTaskConfig_cdcStartTime(rName, rfc3339Time), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "cdc_start_position", awsDmsExpectedOutput), resource.TestCheckResourceAttr(resourceName, "cdc_start_time", rfc3339Time), ), @@ -367,6 +374,7 @@ func TestAccDMSReplicationTask_cdcStartTime_unix_timestamp(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask currentTime := time.Now().UTC() rfc3339Time := currentTime.Format(time.RFC3339) @@ -383,7 +391,7 @@ func TestAccDMSReplicationTask_cdcStartTime_unix_timestamp(t *testing.T) { { Config: testAccReplicationTaskConfig_cdcStartTime(rName, unixDateTime), Check: resource.ComposeAggregateTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "cdc_start_position", awsDmsExpectedOutput), resource.TestCheckResourceAttr(resourceName, "cdc_start_time", unixDateTime), ), @@ -403,6 +411,7 @@ func TestAccDMSReplicationTask_move(t *testing.T) { resourceName := "aws_dms_replication_task.test" instanceOne := "aws_dms_replication_instance.test" instanceTwo := "aws_dms_replication_instance.test2" + var v dms.ReplicationTask resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -414,6 +423,7 @@ func TestAccDMSReplicationTask_move(t *testing.T) { Config: testAccReplicationTaskConfig_move(rName, "aws_dms_replication_instance.test.replication_instance_arn"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationInstanceExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), resource.TestCheckResourceAttrPair(resourceName, "replication_instance_arn", instanceOne, "replication_instance_arn"), ), @@ -422,6 +432,7 @@ func TestAccDMSReplicationTask_move(t *testing.T) { Config: testAccReplicationTaskConfig_move(rName, "aws_dms_replication_instance.test2.replication_instance_arn"), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationInstanceExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), resource.TestCheckResourceAttrPair(resourceName, "replication_instance_arn", instanceTwo, "replication_instance_arn"), ), @@ -430,7 +441,7 @@ func TestAccDMSReplicationTask_move(t *testing.T) { }) } -func testAccCheckReplicationTaskExists(ctx context.Context, n string) resource.TestCheckFunc { +func testAccCheckReplicationTaskExists(ctx context.Context, n string, v *dms.ReplicationTask) resource.TestCheckFunc { return func(s *terraform.State) error { rs, ok := s.RootModule().Resources[n] if !ok { @@ -439,9 +450,15 @@ func testAccCheckReplicationTaskExists(ctx context.Context, n string) resource.T conn := acctest.Provider.Meta().(*conns.AWSClient).DMSConn(ctx) - _, err := tfdms.FindReplicationTaskByID(ctx, conn, rs.Primary.ID) + output, err := tfdms.FindReplicationTaskByID(ctx, conn, rs.Primary.ID) - return err + if err != nil { + return err + } + + *v = *output + + return nil } } From f37d6d5db049e103ef149588ec03437176e0d0b2 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Tue, 16 Apr 2024 10:21:13 -0700 Subject: [PATCH 11/18] Updates validation for `replication_settings` and `replication_task_settings` --- internal/service/dms/replication_config.go | 13 +- .../service/dms/replication_config_test.go | 296 ++++++++++++--- internal/service/dms/replication_task.go | 13 +- internal/service/dms/replication_task_test.go | 300 ++++++++++++++- internal/service/dms/task_settings_json.go | 353 +++++------------- .../service/dms/task_settings_json_test.go | 111 ++++++ 6 files changed, 768 insertions(+), 318 deletions(-) create mode 100644 internal/service/dms/task_settings_json_test.go diff --git a/internal/service/dms/replication_config.go b/internal/service/dms/replication_config.go index cd04271011b6..d986aa535b81 100644 --- a/internal/service/dms/replication_config.go +++ b/internal/service/dms/replication_config.go @@ -113,11 +113,16 @@ func ResourceReplicationConfig() *schema.Resource { Required: true, ForceNew: true, }, + // "replication_settings" is equivalent to "replication_task_settings" on "aws_dms_replication_task" + // All changes to this field and supporting tests should be mirrored in "aws_dms_replication_task" "replication_settings": { - Type: schema.TypeString, - Optional: true, - Computed: true, - ValidateFunc: validation.StringIsJSON, + Type: schema.TypeString, + Optional: true, + Computed: true, + ValidateDiagFunc: validation.AllDiag( + validation.ToDiagFunc(validation.StringIsJSON), + validateReplicationSettings, + ), DiffSuppressFunc: suppressEquivalentTaskSettings, DiffSuppressOnRefresh: true, }, diff --git a/internal/service/dms/replication_config_test.go b/internal/service/dms/replication_config_test.go index c7cb9b03acdb..e6a8ecaae84a 100644 --- a/internal/service/dms/replication_config_test.go +++ b/internal/service/dms/replication_config_test.go @@ -78,54 +78,6 @@ func TestAccDMSReplicationConfig_basic(t *testing.T) { } } -// func TestAccDMSReplicationConfig_noChangeOnDefault(t *testing.T) { -// ctx := acctest.Context(t) -// rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) -// resourceName := "aws_dms_replication_config.test" - -// resource.ParallelTest(t, resource.TestCase{ -// PreCheck: func() { acctest.PreCheck(ctx, t) }, -// ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), -// ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, -// CheckDestroy: testAccCheckReplicationConfigDestroy(ctx), -// Steps: []resource.TestStep{ -// { -// Config: testAccReplicationConfigConfig_noChangeOnDefault(rName), -// Check: resource.ComposeAggregateTestCheckFunc( -// testAccCheckReplicationConfigExists(ctx, resourceName), -// resource.TestCheckResourceAttrSet(resourceName, "arn"), -// resource.TestCheckResourceAttr(resourceName, "compute_config.#", "1"), -// resource.TestCheckResourceAttr(resourceName, "compute_config.0.availability_zone", ""), -// resource.TestCheckResourceAttr(resourceName, "compute_config.0.dns_name_servers", ""), -// resource.TestCheckResourceAttr(resourceName, "compute_config.0.kms_key_id", ""), -// resource.TestCheckResourceAttr(resourceName, "compute_config.0.max_capacity_units", "128"), -// resource.TestCheckResourceAttr(resourceName, "compute_config.0.min_capacity_units", "2"), -// resource.TestCheckResourceAttr(resourceName, "compute_config.0.multi_az", "false"), -// resource.TestCheckResourceAttr(resourceName, "compute_config.0.preferred_maintenance_window", "sun:23:45-mon:00:30"), -// resource.TestCheckResourceAttrSet(resourceName, "compute_config.0.replication_subnet_group_id"), -// resource.TestCheckResourceAttr(resourceName, "compute_config.0.vpc_security_group_ids.#", "0"), -// resource.TestCheckResourceAttr(resourceName, "replication_config_identifier", rName), -// resource.TestCheckResourceAttrSet(resourceName, "replication_settings"), -// resource.TestCheckResourceAttr(resourceName, "replication_type", "cdc"), -// resource.TestCheckNoResourceAttr(resourceName, "resource_identifier"), -// resource.TestCheckResourceAttrSet(resourceName, "source_endpoint_arn"), -// resource.TestCheckResourceAttr(resourceName, "start_replication", "false"), -// resource.TestCheckResourceAttr(resourceName, "supplemental_settings", ""), -// resource.TestCheckResourceAttrSet(resourceName, "table_mappings"), -// resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), -// resource.TestCheckResourceAttrSet(resourceName, "target_endpoint_arn"), -// ), -// }, -// { -// ResourceName: resourceName, -// ImportState: true, -// ImportStateVerify: true, -// ImportStateVerifyIgnore: []string{"start_replication", "resource_identifier"}, -// }, -// }, -// }) -// } - func TestAccDMSReplicationConfig_disappears(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) @@ -150,6 +102,146 @@ func TestAccDMSReplicationConfig_disappears(t *testing.T) { }) } +func TestAccDMSReplicationConfig_settings_EnableLogging(t *testing.T) { + ctx := acctest.Context(t) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_dms_replication_config.test" + var v dms.ReplicationConfig + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccReplicationConfigConfig_settings_EnableLogging(rName, true), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationConfigExists(ctx, resourceName, &v), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.EnableLogging", "true"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.EnableLogContext", "false"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.LogComponents[?Id=='DATA_STRUCTURE'].Severity | [0]", "LOGGER_SEVERITY_DEFAULT"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "type(Logging.CloudWatchLogGroup)", "null"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "type(Logging.CloudWatchLogStream)", "null"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication"}, + }, + { + Config: testAccReplicationConfigConfig_settings_EnableLogging(rName, false), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationConfigExists(ctx, resourceName, &v), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.EnableLogging", "false"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.EnableLogContext", "false"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.LogComponents[?Id=='DATA_STRUCTURE'].Severity | [0]", "LOGGER_SEVERITY_DEFAULT"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "type(Logging.CloudWatchLogGroup)", "null"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "type(Logging.CloudWatchLogStream)", "null"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication"}, + }, + }, + }) +} + +func TestAccDMSReplicationConfig_settings_LoggingValidation(t *testing.T) { + ctx := acctest.Context(t) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccReplicationConfigConfig_settings_EnableLogContext(rName, true), + ExpectError: regexache.MustCompile(`The parameter Logging.EnableLogContext is not allowed when\s+Logging.EnableLogging is not set to true.`), + }, + { + Config: testAccReplicationConfigConfig_settings_LoggingReadOnly(rName, "CloudWatchLogGroup"), + ExpectError: regexache.MustCompile(`The parameter Logging.CloudWatchLogGroup is read-only and cannot be set.`), + }, + { + Config: testAccReplicationConfigConfig_settings_LoggingReadOnly(rName, "CloudWatchLogStream"), + ExpectError: regexache.MustCompile(`The parameter Logging.CloudWatchLogStream is read-only and cannot be set.`), + }, + }, + }) +} + +func TestAccDMSReplicationConfig_settings_LogComponents(t *testing.T) { + ctx := acctest.Context(t) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_dms_replication_config.test" + var v dms.ReplicationConfig + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccReplicationConfigConfig_settings_LogComponents(rName), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationConfigExists(ctx, resourceName, &v), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.EnableLogging", "true"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.EnableLogContext", "false"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.LogComponents[?Id=='DATA_STRUCTURE'].Severity | [0]", "LOGGER_SEVERITY_WARNING"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "type(Logging.CloudWatchLogGroup)", "null"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "type(Logging.CloudWatchLogStream)", "null"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication"}, + }, + }, + }) +} + +func TestAccDMSReplicationConfig_settings_StreamBuffer(t *testing.T) { + ctx := acctest.Context(t) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_dms_replication_config.test" + var v dms.ReplicationConfig + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccReplicationConfigConfig_settings_StreamBuffer(rName, 4, 16), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationConfigExists(ctx, resourceName, &v), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "StreamBufferSettings.StreamBufferCount", "4"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "StreamBufferSettings.StreamBufferSizeInMB", "16"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "StreamBufferSettings.CtrlStreamBufferSizeInMB", "5"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication"}, + }, + }, + }) +} + func TestAccDMSReplicationConfig_tags(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) @@ -382,28 +474,130 @@ resource "aws_dms_replication_config" "test" { `, rName, migrationType)) } -func testAccReplicationConfigConfig_noChangeOnDefault(rName string) string { +func testAccReplicationConfigConfig_settings_EnableLogging(rName string, enabled bool) string { return acctest.ConfigCompose( testAccReplicationConfigConfig_base_DummyDatabase(rName), fmt.Sprintf(` resource "aws_dms_replication_config" "test" { replication_config_identifier = %[1]q - replication_type = "cdc" + replication_type = "full-load" source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - replication_settings = "{\"Logging\":{\"EnableLogging\":true}}" + compute_config { + replication_subnet_group_id = aws_dms_replication_subnet_group.test.replication_subnet_group_id + max_capacity_units = "128" + min_capacity_units = "2" + preferred_maintenance_window = "sun:23:45-mon:00:30" + } + + # terrafmt can't handle this using jsonencode or a heredoc + replication_settings = "{\"Logging\":{\"EnableLogging\":%[2]t}}" +} +`, rName, enabled)) +} +func testAccReplicationConfigConfig_settings_EnableLogContext(rName string, enabled bool) string { + return acctest.ConfigCompose( + testAccReplicationConfigConfig_base_DummyDatabase(rName), + fmt.Sprintf(` +resource "aws_dms_replication_config" "test" { + replication_config_identifier = %[1]q + replication_type = "full-load" + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" compute_config { replication_subnet_group_id = aws_dms_replication_subnet_group.test.replication_subnet_group_id max_capacity_units = "128" min_capacity_units = "2" preferred_maintenance_window = "sun:23:45-mon:00:30" } + + # terrafmt can't handle this using jsonencode or a heredoc + replication_settings = "{\"Logging\":{\"EnableLogContext\":%[2]t}}" +} +`, rName, enabled)) +} + +func testAccReplicationConfigConfig_settings_LoggingReadOnly(rName, field string) string { + return acctest.ConfigCompose( + testAccReplicationConfigConfig_base_DummyDatabase(rName), + fmt.Sprintf(` +resource "aws_dms_replication_config" "test" { + replication_config_identifier = %[1]q + replication_type = "full-load" + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" + compute_config { + replication_subnet_group_id = aws_dms_replication_subnet_group.test.replication_subnet_group_id + max_capacity_units = "128" + min_capacity_units = "2" + preferred_maintenance_window = "sun:23:45-mon:00:30" + } + + # terrafmt can't handle this using jsonencode or a heredoc + replication_settings = "{\"Logging\":{\"EnableLogging\":true, \"%[2]s\":\"value\"}}" +} +`, rName, field)) +} + +func testAccReplicationConfigConfig_settings_LogComponents(rName string) string { + return acctest.ConfigCompose( + testAccReplicationConfigConfig_base_DummyDatabase(rName), + fmt.Sprintf(` +resource "aws_dms_replication_config" "test" { + replication_config_identifier = %[1]q + replication_type = "full-load" + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" + compute_config { + replication_subnet_group_id = aws_dms_replication_subnet_group.test.replication_subnet_group_id + max_capacity_units = "128" + min_capacity_units = "2" + preferred_maintenance_window = "sun:23:45-mon:00:30" + } + + replication_settings = jsonencode( + { + Logging = { + EnableLogging = true, + LogComponents = [{ + Id = "DATA_STRUCTURE", + Severity = "LOGGER_SEVERITY_WARNING" + }] + } + } + ) } `, rName)) } +func testAccReplicationConfigConfig_settings_StreamBuffer(rName string, bufferCount, bufferSize int) string { + return acctest.ConfigCompose( + testAccReplicationConfigConfig_base_DummyDatabase(rName), + fmt.Sprintf(` +resource "aws_dms_replication_config" "test" { + replication_config_identifier = %[1]q + replication_type = "full-load" + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" + compute_config { + replication_subnet_group_id = aws_dms_replication_subnet_group.test.replication_subnet_group_id + max_capacity_units = "128" + min_capacity_units = "2" + preferred_maintenance_window = "sun:23:45-mon:00:30" + } + + # terrafmt can't handle this using jsonencode or a heredoc + replication_settings = "{\"StreamBufferSettings\":{\"StreamBufferCount\":%[2]d,\"StreamBufferSizeInMB\":%[3]d}}" +} +`, rName, bufferCount, bufferSize)) +} + func testAccReplicationConfigConfig_update(rName, replicationType string, minCapacity, maxCapacity int) string { return acctest.ConfigCompose( testAccReplicationConfigConfig_base_DummyDatabase(rName), diff --git a/internal/service/dms/replication_task.go b/internal/service/dms/replication_task.go index 3e7d10527c69..3bce90109a1b 100644 --- a/internal/service/dms/replication_task.go +++ b/internal/service/dms/replication_task.go @@ -73,11 +73,16 @@ func ResourceReplicationTask() *schema.Resource { ForceNew: true, ValidateFunc: validReplicationTaskID, }, + // "replication_task_settings" is equivalent to "replication_settings" on "aws_dms_replication_config" + // All changes to this field and supporting tests should be mirrored in "aws_dms_replication_config" "replication_task_settings": { - Type: schema.TypeString, - Optional: true, - Computed: true, - ValidateFunc: validation.StringIsJSON, + Type: schema.TypeString, + Optional: true, + Computed: true, + ValidateDiagFunc: validation.AllDiag( + validation.ToDiagFunc(validation.StringIsJSON), + validateReplicationSettings, + ), DiffSuppressFunc: suppressEquivalentTaskSettings, DiffSuppressOnRefresh: true, }, diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index 343350507d0f..f02f2c7acca1 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -13,6 +13,8 @@ import ( "time" "github.com/YakDriver/regexache" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/arn" dms "github.com/aws/aws-sdk-go/service/databasemigrationservice" sdkacctest "github.com/hashicorp/terraform-plugin-testing/helper/acctest" "github.com/hashicorp/terraform-plugin-testing/helper/resource" @@ -178,6 +180,133 @@ func TestAccDMSReplicationTask_updateSettingsAndMappings(t *testing.T) { }) } +func TestAccDMSReplicationTask_settings_EnableLogging(t *testing.T) { + ctx := acctest.Context(t) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccReplicationTaskConfig_settings_EnableLogging(rName, true), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationTaskExists(ctx, resourceName, &v), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogging", "true"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogContext", "false"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.CloudWatchLogGroup", fmt.Sprintf("dms-tasks-%s", rName)), + func(s *terraform.State) error { + arn, err := arn.Parse(aws.StringValue(v.ReplicationTaskArn)) + if err != nil { + return err + } + l := strings.Split(arn.Resource, ":") + if len(l) != 2 { + return fmt.Errorf("expected 2 parts in %s", arn.Resource) + } + id := l[1] + return acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.CloudWatchLogStream", fmt.Sprintf("dms-task-%s", id))(s) + }, + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication_task"}, + }, + { + Config: testAccReplicationTaskConfig_settings_EnableLogging(rName, false), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationTaskExists(ctx, resourceName, &v), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogging", "false"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogContext", "false"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.CloudWatchLogGroup", fmt.Sprintf("dms-tasks-%s", rName)), + func(s *terraform.State) error { + arn, err := arn.Parse(aws.StringValue(v.ReplicationTaskArn)) + if err != nil { + return err + } + l := strings.Split(arn.Resource, ":") + if len(l) != 2 { + return fmt.Errorf("expected 2 parts in %s", arn.Resource) + } + id := l[1] + return acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.CloudWatchLogStream", fmt.Sprintf("dms-task-%s", id))(s) + }, + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication_task"}, + }, + }, + }) +} + +func TestAccDMSReplicationTask_settings_LoggingValidation(t *testing.T) { + ctx := acctest.Context(t) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccReplicationTaskConfig_settings_EnableLogContext(rName, true), + ExpectError: regexache.MustCompile(`The parameter Logging.EnableLogContext is not allowed when\s+Logging.EnableLogging is not set to true.`), + }, + { + Config: testAccReplicationTaskConfig_settings_LoggingReadOnly(rName, "CloudWatchLogGroup"), + ExpectError: regexache.MustCompile(`The parameter Logging.CloudWatchLogGroup is read-only and cannot be set.`), + }, + { + Config: testAccReplicationTaskConfig_settings_LoggingReadOnly(rName, "CloudWatchLogStream"), + ExpectError: regexache.MustCompile(`The parameter Logging.CloudWatchLogStream is read-only and cannot be set.`), + }, + }, + }) +} + +func TestAccDMSReplicationTask_settings_StreamBuffer(t *testing.T) { + ctx := acctest.Context(t) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccReplicationTaskConfig_settings_StreamBuffer(rName, 4, 16), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationTaskExists(ctx, resourceName, &v), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "StreamBufferSettings.StreamBufferCount", "4"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "StreamBufferSettings.StreamBufferSizeInMB", "16"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "StreamBufferSettings.CtrlStreamBufferSizeInMB", "5"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication"}, + }, + }, + }) +} + func TestAccDMSReplicationTask_cdcStartPosition(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) @@ -530,7 +659,7 @@ resource "aws_dms_replication_subnet_group" "test" { resource "aws_dms_replication_instance" "test" { allocated_storage = 5 auto_minor_version_upgrade = true - replication_instance_class = "dms.c4.large" + replication_instance_class = "dms.t3.medium" replication_instance_id = %[1]q preferred_maintenance_window = "sun:00:30-sun:02:30" publicly_accessible = false @@ -607,11 +736,172 @@ resource "aws_dms_replication_task" "test" { # terrafmt can't handle this using jsonencode or a heredoc replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":%[2]d,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"EventErrorPolicy\":\"IGNORE\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false},\"TTSettings\":{\"EnableTT\":false,\"TTRecordSettings\":null,\"TTS3Settings\":null}}" # terrafmt can't handle this using jsonencode or a heredoc - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":%[3]q,\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" + table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"%[3]s\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" } `, rName, memLimitTotal, ruleName)) } +func testAccReplicationTaskConfig_settings_EnableLogging(rName string, enabled bool) string { + return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` +resource "aws_dms_replication_task" "test" { + replication_task_id = %[1]q + migration_type = "full-load" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) + # terrafmt can't handle this using jsonencode or a heredoc + replication_task_settings = "{\"Logging\":{\"EnableLogging\":%[2]t}}" +} +`, rName, enabled)) +} + +func testAccReplicationTaskConfig_settings_EnableLogContext(rName string, enabled bool) string { + return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` +resource "aws_dms_replication_task" "test" { + replication_task_id = %[1]q + migration_type = "full-load" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) + # terrafmt can't handle this using jsonencode or a heredoc + replication_task_settings = "{\"Logging\":{\"EnableLogContext\":%[2]t}}" +} +`, rName, enabled)) +} + +func testAccReplicationTaskConfig_settings_LoggingReadOnly(rName, field string) string { + return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` +resource "aws_dms_replication_task" "test" { + replication_task_id = %[1]q + migration_type = "full-load" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) + # terrafmt can't handle this using jsonencode or a heredoc + replication_task_settings = "{\"Logging\":{\"EnableLogging\":true, \"%[2]s\":\"value\"}}" +} +`, rName, field)) +} + +func testAccReplicationTaskConfig_settings_LogComponents(rName string) string { + return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` +resource "aws_dms_replication_task" "test" { + replication_task_id = %[1]q + migration_type = "full-load" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) + + replication_task_settings = jsonencode( + { + Logging = { + EnableLogging = true, + LogComponents = [{ + Id = "DATA_STRUCTURE", + Severity = "LOGGER_SEVERITY_WARNING" + }] + } + } + ) +} +`, rName)) +} + +func testAccReplicationTaskConfig_settings_StreamBuffer(rName string, bufferCount, bufferSize int) string { + return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` +resource "aws_dms_replication_task" "test" { + replication_task_id = %[1]q + migration_type = "full-load" + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn + table_mappings = jsonencode( + { + "rules" = [ + { + "rule-type" = "selection", + "rule-id" = "1", + "rule-name" = "1", + "object-locator" = { + "schema-name" = "%%", + "table-name" = "%%" + }, + "rule-action" = "include" + } + ] + } + ) + + # terrafmt can't handle this using jsonencode or a heredoc + replication_task_settings = "{\"StreamBufferSettings\":{\"StreamBufferCount\":%[2]d,\"StreamBufferSizeInMB\":%[3]d}}" +} + `, rName, bufferCount, bufferSize)) +} + func testAccReplicationTaskConfig_cdcStartPosition(rName, cdcStartPosition string) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { @@ -673,7 +963,7 @@ resource "aws_dms_replication_task" "test" { resource "aws_dms_replication_instance" "test" { allocated_storage = 5 auto_minor_version_upgrade = true - replication_instance_class = "dms.c4.large" + replication_instance_class = "dms.t3.medium" replication_instance_id = %[1]q preferred_maintenance_window = "sun:00:30-sun:02:30" publicly_accessible = false @@ -837,7 +1127,7 @@ resource "aws_dms_endpoint" "target" { resource "aws_dms_replication_instance" "test" { allocated_storage = 5 auto_minor_version_upgrade = true - replication_instance_class = "dms.c4.large" + replication_instance_class = "dms.t3.medium" replication_instance_id = %[1]q preferred_maintenance_window = "sun:00:30-sun:02:30" publicly_accessible = false @@ -906,7 +1196,7 @@ resource "aws_dms_replication_task" "test" { resource "aws_dms_replication_instance" "test2" { allocated_storage = 5 auto_minor_version_upgrade = true - replication_instance_class = "dms.c4.large" + replication_instance_class = "dms.t3.medium" replication_instance_id = "%[1]s-2" preferred_maintenance_window = "sun:00:30-sun:02:30" publicly_accessible = false diff --git a/internal/service/dms/task_settings_json.go b/internal/service/dms/task_settings_json.go index 4164403dad57..b0097f9a35e1 100644 --- a/internal/service/dms/task_settings_json.go +++ b/internal/service/dms/task_settings_json.go @@ -4,283 +4,128 @@ package dms import ( - "cmp" "encoding/json" "log" - "reflect" - "slices" + "github.com/hashicorp/go-cty/cty" + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" - tfjson "github.com/hashicorp/terraform-provider-aws/internal/json" - "github.com/hashicorp/terraform-provider-aws/internal/verify" + "github.com/hashicorp/terraform-provider-aws/internal/errs" ) -// https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Tasks.CustomizingTasks.TaskSettings.html#CHAP_Tasks.CustomizingTasks.TaskSettings.Example -// https://mholt.github.io/json-to-go/ +func suppressEquivalentTaskSettings(k, state, proposed string, d *schema.ResourceData) bool { + if !json.Valid([]byte(state)) || !json.Valid([]byte(proposed)) { + return state == proposed + } + + var stateMap, proposedMap map[string]any + + if err := json.Unmarshal([]byte(state), &stateMap); err != nil { + log.Printf("[ERROR] failed to unmarshal task settings JSON: %v", err) + return false + } + + if s, ok := stateMap["Logging"]; ok { + stateLogging := s.(map[string]any) + if stateLogging != nil { + delete(stateLogging, "CloudWatchLogGroup") + delete(stateLogging, "CloudWatchLogStream") + } + normalizeLogComponents(stateLogging) + } -// normalizeTaskSettings returns a normalized DMS task settings JSON string. -// Read-only (non-configurable) fields are removed by using the published "schema". -// Empty fields are then removed. -func normalizeTaskSettings(apiObject string) string { - defaultValues := map[string]interface{}{ - "ChangeProcessingTuning": map[string]interface{}{ - "BatchApplyMemoryLimit": 500, - "BatchApplyTimeoutMax": 30, - "BatchApplyTimeoutMin": 1, - "BatchSplitSize": 0, - "CommitTimeout": 1, - "MemoryKeepTime": 60, - "MemoryLimitTotal": 1024, - "MinTransactionSize": 1000, - "StatementCacheSize": 50, - "BatchApplyPreserveTransaction": true, - }, - "ControlTablesSettings": map[string]interface{}{ - "historyTimeslotInMinutes": 5, - "CommitPositionTableEnabled": false, - "HistoryTimeslotInMinutes": 5, - "StatusTableEnabled": false, - "SuspendedTablesTableEnabled": false, - "HistoryTableEnabled": false, - "ControlSchema": "", - "FullLoadExceptionTableEnabled": false, - }, - "BeforeImageSettings": nil, - "FailTaskWhenCleanTaskResourceFailed": false, - "ErrorBehavior": map[string]interface{}{ - "DataErrorPolicy": "LOG_ERROR", - "DataTruncationErrorPolicy": "LOG_ERROR", - "DataErrorEscalationPolicy": "SUSPEND_TABLE", - "EventErrorPolicy": "IGNORE", - "FailOnNoTablesCaptured": true, - "TableErrorPolicy": "SUSPEND_TABLE", - "TableErrorEscalationPolicy": "STOP_TASK", - "RecoverableErrorCount": -1, - "RecoverableErrorInterval": 5, - "RecoverableErrorThrottling": true, - "RecoverableErrorThrottlingMax": 1800, - "RecoverableErrorStopRetryAfterThrottlingMax": true, - "ApplyErrorDeletePolicy": "IGNORE_RECORD", - "ApplyErrorInsertPolicy": "LOG_ERROR", - "ApplyErrorUpdatePolicy": "LOG_ERROR", - "ApplyErrorEscalationPolicy": "LOG_ERROR", - "FullLoadIgnoreConflicts": true, - "ApplyErrorEscalationCount": 0, - "ApplyErrorFailOnTruncationDdl": false, - "DataErrorEscalationCount": 0, - "FailOnTransactionConsistencyBreached": false, - "TableErrorEscalationCount": 0, - }, - "TTSettings": map[string]interface{}{ - "TTS3Settings": nil, - "TTRecordSettings": nil, - "FailTaskOnTTFailure": false, - "EnableTT": false, - }, - "FullLoadSettings": map[string]interface{}{ - "CommitRate": 10000, - "StopTaskCachedChangesApplied": false, - "StopTaskCachedChangesNotApplied": false, - "MaxFullLoadSubTasks": 8, - "TransactionConsistencyTimeout": 600, - "CreatePkAfterFullLoad": false, - "TargetTablePrepMode": "DO_NOTHING", - }, - "Logging": map[string]interface{}{ - "EnableLogging": true, - "CloudWatchLogGroup": nil, - "CloudWatchLogStream": nil, - "EnableLogContext": false, - "LogComponents": []map[string]string{ - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TRANSFORMATION", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "SOURCE_UNLOAD", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "IO", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TARGET_LOAD", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "PERFORMANCE", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "SOURCE_CAPTURE", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "SORTER", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "REST_SERVER", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "VALIDATOR_EXT", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TARGET_APPLY", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TASK_MANAGER", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "TABLES_MANAGER", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "METADATA_MANAGER", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "FILE_FACTORY", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "COMMON", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "ADDONS", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "DATA_STRUCTURE", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "COMMUNICATION", - }, - { - "Severity": "LOGGER_SEVERITY_DEFAULT", - "Id": "FILE_TRANSFER", - }, - }, - }, - "StreamBufferSettings": map[string]interface{}{ - "CtrlStreamBufferSizeInMB": 5, - "StreamBufferCount": 3, - "StreamBufferSizeInMB": 8, - }, - "TargetMetadata": map[string]interface{}{ - "ParallelApplyBufferSize": 0, - "ParallelApplyQueuesPerThread": 0, - "ParallelApplyThreads": 0, - "TargetSchema": "", - "InlineLobMaxSize": 0, - "ParallelLoadQueuesPerThread": 0, - "SupportLobs": true, - "LobChunkSize": 64, - "TaskRecoveryTableEnabled": false, - "ParallelLoadThreads": 0, - "LobMaxSize": 32, - "BatchApplyEnabled": false, - "FullLobMode": false, - "LimitedSizeLobMode": true, - "LoadMaxFileSize": 0, - "ParallelLoadBufferSize": 0, - }, - "ChangeProcessingDdlHandlingPolicy": map[string]interface{}{ - "HandleSourceTableDropped": true, - "HandleSourceTableTruncated": true, - "HandleSourceTableAltered": true, - }, + if err := json.Unmarshal([]byte(proposed), &proposedMap); err != nil { + log.Printf("[ERROR] failed to unmarshal task settings JSON: %v", err) + return false } - var jsonMap map[string]interface{} + if p, ok := proposedMap["Logging"]; ok { + proposedLogging := p.(map[string]any) - if err := json.Unmarshal([]byte(apiObject), &jsonMap); err != nil { - log.Printf("[DEBUG] failed to unmarshal task settings JSON: %v", err) - return apiObject + normalizeLogComponents(proposedLogging) } - jsonMap = checkdefaultvalues(defaultValues, jsonMap) - if b, err := json.Marshal(&jsonMap); err != nil { - log.Printf("[DEBUG] failed to marshal task settings JSON: %v", err) - return apiObject - } else { - return string(tfjson.RemoveEmptyFields(b)) + return taskSettingsEqual(stateMap, proposedMap) +} + +func taskSettingsEqual(state, proposed any) bool { + if proposed == nil { + return true + } + + switch x := state.(type) { + case bool: + p := proposed.(bool) + return x == p + + case float64: + p := proposed.(float64) + return x == p + + case string: + p := proposed.(string) + return x == p + + case map[string]any: + proposedMap := proposed.(map[string]any) + for k, v := range x { + if !taskSettingsEqual(v, proposedMap[k]) { + return false + } + delete(proposedMap, k) + } + return len(proposedMap) == 0 } + return false } -// suppressEquivalentTaskSettings provides custom difference suppression for task settings. -func suppressEquivalentTaskSettings(k, old, new string, d *schema.ResourceData) bool { - if !json.Valid([]byte(old)) || !json.Valid([]byte(new)) { - return old == new +func normalizeLogComponents(m map[string]any) { + if m == nil { + return } - old, new = normalizeTaskSettings(old), normalizeTaskSettings(new) - return verify.JSONStringsEqual(old, new) + components, ok := m["LogComponents"] + if !ok { + return + } + + newComponents := make(map[string]any, len(components.([]any))) + for _, c := range components.([]any) { + component := c.(map[string]any) + id := component["Id"].(string) + newComponents[id] = component["Severity"].(string) + } + m["LogComponents"] = newComponents } -func checkdefaultvalues(defaultMap, oldMap map[string]interface{}) map[string]interface{} { - for k, v := range oldMap { - if value, ok := defaultMap[k]; ok && v != nil { - // Check the type of the value - switch t := reflect.TypeOf(value); t.Kind() { - // Check top level settings - case reflect.Bool, reflect.String, reflect.Float64, reflect.Int: - if reflect.DeepEqual(value, v) { - delete(oldMap, k) - } - case reflect.Map: - // Map of defaults - kMap := value.(map[string]interface{}) - // Map of inner map (from user) - vMap := v.(map[string]interface{}) +func validateReplicationSettings(i any, path cty.Path) diag.Diagnostics { + var diags diag.Diagnostics - for kInner, vInner := range vMap { - if kMap[kInner] != nil || vInner != nil { - if reflect.TypeOf(vInner).Kind() == reflect.Float64 { - if kMap[kInner] != nil { - kMap[kInner] = float64(kMap[kInner].(int)) - } - } - if reflect.TypeOf(vInner).Kind() == reflect.Slice { - temp := make([]map[string]string, 0) - for _, v := range vInner.([]interface{}) { - innerTemp := make(map[string]string) - for k, v := range v.(map[string]interface{}) { - innerTemp[k] = v.(string) - } - temp = append(temp, innerTemp) - } - // We are assuming the types; we know the type at the point of this code - slices.SortFunc(temp, func(i, j map[string]string) int { - return cmp.Compare(i["Id"], j["Id"]) - }) - vInner = temp + v, ok := i.(string) + if !ok { + return append(diags, errs.NewIncorrectValueTypeAttributeError(path, "string")) + } - slices.SortFunc(kMap[kInner].([]map[string]string), func(i, j map[string]string) int { - return cmp.Compare(i["Id"], j["Id"]) - }) - } - if reflect.DeepEqual(kMap[kInner], vInner) { - delete(vMap, kInner) - } - } - } - if len(vMap) == 0 { - delete(oldMap, k) - } + var m map[string]any - default: - return oldMap + if err := json.Unmarshal([]byte(v), &m); err != nil { + return append(diags, errs.NewInvalidValueAttributeError(path, "Unable to parse as JSON")) + } + + if l, ok := m["Logging"].(map[string]any); ok { + if _, ok := l["EnableLogContext"]; ok { + if enabled, ok := l["EnableLogging"]; !ok || !enabled.(bool) { + diags = append(diags, errs.NewInvalidValueAttributeError(path, "The parameter Logging.EnableLogContext is not allowed when Logging.EnableLogging is not set to true.")) } } + + if _, ok := l["CloudWatchLogGroup"]; ok { + diags = append(diags, errs.NewInvalidValueAttributeError(path, "The parameter Logging.CloudWatchLogGroup is read-only and cannot be set.")) + } + if _, ok := l["CloudWatchLogStream"]; ok { + diags = append(diags, errs.NewInvalidValueAttributeError(path, "The parameter Logging.CloudWatchLogStream is read-only and cannot be set.")) + } } - return oldMap + + return diags } diff --git a/internal/service/dms/task_settings_json_test.go b/internal/service/dms/task_settings_json_test.go new file mode 100644 index 000000000000..8354f493a49d --- /dev/null +++ b/internal/service/dms/task_settings_json_test.go @@ -0,0 +1,111 @@ +package dms + +import ( + "testing" +) + +func TestTaskSettingsEqual(t *testing.T) { + tests := map[string]map[string]struct { + a, b any + expected bool + }{ + "bool": { + "both true": { + a: true, + b: true, + expected: true, + }, + "not equal": { + a: true, + b: false, + expected: false, + }, + "both null": { + a: nil, + b: nil, + expected: true, + }, + "true proposed null": { + a: true, + b: nil, + expected: true, + }, + "false proposed null": { + a: false, + b: nil, + expected: true, + }, + "null proposed true": { + a: nil, + b: true, + expected: false, + }, + "null proposed false": { + a: nil, + b: false, + expected: false, + }, + }, + "float64": { + "equal": { + a: float64(1), + b: float64(1), + expected: true, + }, + "not equal": { + a: float64(1), + b: float64(2), + expected: false, + }, + "proposed null": { + a: float64(1), + b: nil, + expected: true, + }, + "null proposed value": { + a: nil, + b: float64(1), + expected: false, + }, + }, + "string": { + "equal": { + a: "value", + b: "value", + expected: true, + }, + "not equal": { + a: "value1", + b: "value2", + expected: false, + }, + "both null": { + a: nil, + b: nil, + expected: true, + }, + "proposed null": { + a: "value", + b: nil, + expected: true, + }, + "null proposed value": { + a: nil, + b: "value", + expected: false, + }, + }, + } + + for name, typeTest := range tests { + t.Run(name, func(t *testing.T) { + for name, test := range typeTest { + t.Run(name, func(t *testing.T) { + if taskSettingsEqual(test.a, test.b) != test.expected { + t.Fatalf("expected %v, got %v", test.expected, !test.expected) + } + }) + } + }) + } +} From 62b81cbf0ca78dc137804abe37917ad858299cf2 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Tue, 16 Apr 2024 10:38:28 -0700 Subject: [PATCH 12/18] Skips errors for serverless DMS in GovCloud --- internal/service/dms/dms_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 internal/service/dms/dms_test.go diff --git a/internal/service/dms/dms_test.go b/internal/service/dms/dms_test.go new file mode 100644 index 000000000000..51432b875e12 --- /dev/null +++ b/internal/service/dms/dms_test.go @@ -0,0 +1,21 @@ +package dms_test + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-provider-aws/internal/acctest" + "github.com/hashicorp/terraform-provider-aws/names" +) + +func init() { + acctest.RegisterServiceErrorCheckFunc(names.DMSServiceID, testAccErrorCheckSkip) +} + +// testAccErrorCheckSkip skips DMS tests that have error messages indicating unsupported features +func testAccErrorCheckSkip(t *testing.T) resource.ErrorCheckFunc { + return acctest.ErrorCheckSkipMessagesContaining(t, + // Serverless DMS in GovCloud + "SERVERLESS feature is not available", + ) +} From d4d0d197b6c34390c198b5cc233ec4ed9c675afa Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Tue, 16 Apr 2024 14:41:13 -0700 Subject: [PATCH 13/18] Handles automatically turning off `Logging.EnableLogContext` on Update if `Logging.EnableLogging` is `false` --- .../service/dms/replication_config_test.go | 27 ++++++++-- internal/service/dms/replication_task.go | 6 ++- internal/service/dms/replication_task_test.go | 51 +++++++++++++++++-- internal/service/dms/task_settings_json.go | 30 +++++++++++ 4 files changed, 105 insertions(+), 9 deletions(-) diff --git a/internal/service/dms/replication_config_test.go b/internal/service/dms/replication_config_test.go index e6a8ecaae84a..dcc57a3ee2e2 100644 --- a/internal/service/dms/replication_config_test.go +++ b/internal/service/dms/replication_config_test.go @@ -131,6 +131,23 @@ func TestAccDMSReplicationConfig_settings_EnableLogging(t *testing.T) { ImportStateVerify: true, ImportStateVerifyIgnore: []string{"start_replication"}, }, + { + Config: testAccReplicationConfigConfig_settings_EnableLogContext(rName, true, true), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationConfigExists(ctx, resourceName, &v), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.EnableLogging", "true"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.EnableLogContext", "true"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "Logging.LogComponents[?Id=='DATA_STRUCTURE'].Severity | [0]", "LOGGER_SEVERITY_DEFAULT"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "type(Logging.CloudWatchLogGroup)", "null"), + acctest.CheckResourceAttrJMES(resourceName, "replication_settings", "type(Logging.CloudWatchLogStream)", "null"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication"}, + }, { Config: testAccReplicationConfigConfig_settings_EnableLogging(rName, false), Check: resource.ComposeAggregateTestCheckFunc( @@ -163,7 +180,7 @@ func TestAccDMSReplicationConfig_settings_LoggingValidation(t *testing.T) { CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), Steps: []resource.TestStep{ { - Config: testAccReplicationConfigConfig_settings_EnableLogContext(rName, true), + Config: testAccReplicationConfigConfig_settings_EnableLogContext(rName, false, true), ExpectError: regexache.MustCompile(`The parameter Logging.EnableLogContext is not allowed when\s+Logging.EnableLogging is not set to true.`), }, { @@ -497,7 +514,7 @@ resource "aws_dms_replication_config" "test" { `, rName, enabled)) } -func testAccReplicationConfigConfig_settings_EnableLogContext(rName string, enabled bool) string { +func testAccReplicationConfigConfig_settings_EnableLogContext(rName string, enableLogging, enableLogContext bool) string { return acctest.ConfigCompose( testAccReplicationConfigConfig_base_DummyDatabase(rName), fmt.Sprintf(` @@ -515,9 +532,9 @@ resource "aws_dms_replication_config" "test" { } # terrafmt can't handle this using jsonencode or a heredoc - replication_settings = "{\"Logging\":{\"EnableLogContext\":%[2]t}}" + replication_settings = "{\"Logging\":{\"EnableLogging\":%[2]t,\"EnableLogContext\":%[3]t}}" } -`, rName, enabled)) +`, rName, enableLogging, enableLogContext)) } func testAccReplicationConfigConfig_settings_LoggingReadOnly(rName, field string) string { @@ -640,6 +657,8 @@ resource "aws_dms_replication_config" "test" { min_capacity_units = "2" preferred_maintenance_window = "sun:23:45-mon:00:30" } + + depends_on = [aws_rds_cluster_instance.source, aws_rds_cluster_instance.target] } `, rName, start)) } diff --git a/internal/service/dms/replication_task.go b/internal/service/dms/replication_task.go index 3bce90109a1b..0b917fb07c20 100644 --- a/internal/service/dms/replication_task.go +++ b/internal/service/dms/replication_task.go @@ -251,7 +251,11 @@ func resourceReplicationTaskUpdate(ctx context.Context, d *schema.ResourceData, if d.HasChange("replication_task_settings") { if v, ok := d.GetOk("replication_task_settings"); ok { - input.ReplicationTaskSettings = aws.String(v.(string)) + s, err := normalizeReplicationSettings(v.(string)) + if err != nil { + return sdkdiag.AppendErrorf(diags, "updating DMS Replication Task (%s): %s", d.Id(), err) + } + input.ReplicationTaskSettings = aws.String(s) } } diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index f02f2c7acca1..2104e4240a9e 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -18,6 +18,7 @@ import ( dms "github.com/aws/aws-sdk-go/service/databasemigrationservice" sdkacctest "github.com/hashicorp/terraform-plugin-testing/helper/acctest" "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-plugin-testing/plancheck" "github.com/hashicorp/terraform-plugin-testing/terraform" "github.com/hashicorp/terraform-provider-aws/internal/acctest" "github.com/hashicorp/terraform-provider-aws/internal/conns" @@ -198,6 +199,7 @@ func TestAccDMSReplicationTask_settings_EnableLogging(t *testing.T) { testAccCheckReplicationTaskExists(ctx, resourceName, &v), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogging", "true"), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogContext", "false"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.LogComponents[?Id=='DATA_STRUCTURE'].Severity | [0]", "LOGGER_SEVERITY_DEFAULT"), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.CloudWatchLogGroup", fmt.Sprintf("dms-tasks-%s", rName)), func(s *terraform.State) error { arn, err := arn.Parse(aws.StringValue(v.ReplicationTaskArn)) @@ -219,12 +221,46 @@ func TestAccDMSReplicationTask_settings_EnableLogging(t *testing.T) { ImportStateVerify: true, ImportStateVerifyIgnore: []string{"start_replication_task"}, }, + { + Config: testAccReplicationTaskConfig_settings_EnableLogContext(rName, true, true), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationTaskExists(ctx, resourceName, &v), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogging", "true"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogContext", "true"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.LogComponents[?Id=='DATA_STRUCTURE'].Severity | [0]", "LOGGER_SEVERITY_DEFAULT"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.CloudWatchLogGroup", fmt.Sprintf("dms-tasks-%s", rName)), + func(s *terraform.State) error { + arn, err := arn.Parse(aws.StringValue(v.ReplicationTaskArn)) + if err != nil { + return err + } + l := strings.Split(arn.Resource, ":") + if len(l) != 2 { + return fmt.Errorf("expected 2 parts in %s", arn.Resource) + } + id := l[1] + return acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.CloudWatchLogStream", fmt.Sprintf("dms-task-%s", id))(s) + }, + ), + ConfigPlanChecks: resource.ConfigPlanChecks{ + PreApply: []plancheck.PlanCheck{ + plancheck.ExpectResourceAction(resourceName, plancheck.ResourceActionUpdate), + }, + }, + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication_task"}, + }, { Config: testAccReplicationTaskConfig_settings_EnableLogging(rName, false), Check: resource.ComposeAggregateTestCheckFunc( testAccCheckReplicationTaskExists(ctx, resourceName, &v), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogging", "false"), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogContext", "false"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.LogComponents[?Id=='DATA_STRUCTURE'].Severity | [0]", "LOGGER_SEVERITY_DEFAULT"), acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.CloudWatchLogGroup", fmt.Sprintf("dms-tasks-%s", rName)), func(s *terraform.State) error { arn, err := arn.Parse(aws.StringValue(v.ReplicationTaskArn)) @@ -239,6 +275,11 @@ func TestAccDMSReplicationTask_settings_EnableLogging(t *testing.T) { return acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.CloudWatchLogStream", fmt.Sprintf("dms-task-%s", id))(s) }, ), + ConfigPlanChecks: resource.ConfigPlanChecks{ + PreApply: []plancheck.PlanCheck{ + plancheck.ExpectResourceAction(resourceName, plancheck.ResourceActionUpdate), + }, + }, }, { ResourceName: resourceName, @@ -261,7 +302,7 @@ func TestAccDMSReplicationTask_settings_LoggingValidation(t *testing.T) { CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), Steps: []resource.TestStep{ { - Config: testAccReplicationTaskConfig_settings_EnableLogContext(rName, true), + Config: testAccReplicationTaskConfig_settings_EnableLogContext(rName, false, true), ExpectError: regexache.MustCompile(`The parameter Logging.EnableLogContext is not allowed when\s+Logging.EnableLogging is not set to true.`), }, { @@ -771,7 +812,7 @@ resource "aws_dms_replication_task" "test" { `, rName, enabled)) } -func testAccReplicationTaskConfig_settings_EnableLogContext(rName string, enabled bool) string { +func testAccReplicationTaskConfig_settings_EnableLogContext(rName string, enableLogging, enableLogContext bool) string { return acctest.ConfigCompose(testAccReplicationTaskConfig_base(rName), fmt.Sprintf(` resource "aws_dms_replication_task" "test" { replication_task_id = %[1]q @@ -796,9 +837,9 @@ resource "aws_dms_replication_task" "test" { } ) # terrafmt can't handle this using jsonencode or a heredoc - replication_task_settings = "{\"Logging\":{\"EnableLogContext\":%[2]t}}" + replication_task_settings = "{\"Logging\":{\"EnableLogging\":%[2]t,\"EnableLogContext\":%[3]t}}" } -`, rName, enabled)) +`, rName, enableLogging, enableLogContext)) } func testAccReplicationTaskConfig_settings_LoggingReadOnly(rName, field string) string { @@ -958,6 +999,8 @@ resource "aws_dms_replication_task" "test" { ) start_replication_task = %[2]t + + depends_on = [aws_rds_cluster_instance.source, aws_rds_cluster_instance.target] } resource "aws_dms_replication_instance" "test" { diff --git a/internal/service/dms/task_settings_json.go b/internal/service/dms/task_settings_json.go index b0097f9a35e1..bb2f4a0b6abe 100644 --- a/internal/service/dms/task_settings_json.go +++ b/internal/service/dms/task_settings_json.go @@ -129,3 +129,33 @@ func validateReplicationSettings(i any, path cty.Path) diag.Diagnostics { return diags } + +func normalizeReplicationSettings(s string) (string, error) { + if s == "" { + return "", nil + } + + var m map[string]any + + if err := json.Unmarshal([]byte(s), &m); err != nil { + return s, err + } + + // If EnableLogging is false, set EnableLogContext to false unless it is explicitly set. + // Normally, if EnableLogContext is not set, it uses the existing value. + if l, ok := m["Logging"].(map[string]any); ok { + if enabled, ok := l["EnableLogging"]; ok && !enabled.(bool) { + delete(l, "EnableLogContext") + if _, ok := l["EnableLogContext"]; !ok { + l["EnableLogContext"] = false + b, err := json.Marshal(m) + if err != nil { + return s, err + } + s = string(b) + } + } + } + + return s, nil +} From ad3140466476f912adc1202368a73d45dc040641 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Tue, 16 Apr 2024 15:47:27 -0700 Subject: [PATCH 14/18] Fix --- internal/service/dms/replication_task_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index 2104e4240a9e..98cdec83fe26 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -382,6 +382,7 @@ func TestAccDMSReplicationTask_resourceIdentifier(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acctest.PreCheck(ctx, t) }, @@ -392,7 +393,7 @@ func TestAccDMSReplicationTask_resourceIdentifier(t *testing.T) { { Config: testAccReplicationTaskConfig_resourceIdentifier(rName), Check: resource.ComposeTestCheckFunc( - testAccCheckReplicationTaskExists(ctx, resourceName), + testAccCheckReplicationTaskExists(ctx, resourceName, &v), resource.TestCheckResourceAttr(resourceName, "resource_identifier", "identifier"), ), }, From 4702a865d4773e00d1b6efadc332acabec163854 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Wed, 17 Apr 2024 09:22:33 -0700 Subject: [PATCH 15/18] `copywrite headers` --- internal/service/dms/dms_test.go | 3 +++ internal/service/dms/task_settings_json_test.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/internal/service/dms/dms_test.go b/internal/service/dms/dms_test.go index 51432b875e12..732c98cf1dff 100644 --- a/internal/service/dms/dms_test.go +++ b/internal/service/dms/dms_test.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + package dms_test import ( diff --git a/internal/service/dms/task_settings_json_test.go b/internal/service/dms/task_settings_json_test.go index 8354f493a49d..2c834367c35a 100644 --- a/internal/service/dms/task_settings_json_test.go +++ b/internal/service/dms/task_settings_json_test.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + package dms import ( From a86c7c84277faca48b6444eca3db907c51febfdb Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Wed, 17 Apr 2024 09:23:17 -0700 Subject: [PATCH 16/18] Adds missing test --- internal/service/dms/replication_task_test.go | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index 98cdec83fe26..bd758582c129 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -317,6 +317,39 @@ func TestAccDMSReplicationTask_settings_LoggingValidation(t *testing.T) { }) } +func TestAccDMSReplicationTask_settings_LogComponents(t *testing.T) { + ctx := acctest.Context(t) + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_dms_replication_task.test" + var v dms.ReplicationTask + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.DMSServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckReplicationTaskDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccReplicationTaskConfig_settings_LogComponents(rName), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckReplicationTaskExists(ctx, resourceName, &v), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogging", "true"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.EnableLogContext", "false"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "Logging.LogComponents[?Id=='DATA_STRUCTURE'].Severity | [0]", "LOGGER_SEVERITY_WARNING"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "type(Logging.CloudWatchLogGroup)", "null"), + acctest.CheckResourceAttrJMES(resourceName, "replication_task_settings", "type(Logging.CloudWatchLogStream)", "null"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"start_replication"}, + }, + }, + }) +} + func TestAccDMSReplicationTask_settings_StreamBuffer(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) From 9dfba556e8962d9f6f2a9c93de455e7f41263118 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Wed, 17 Apr 2024 09:23:27 -0700 Subject: [PATCH 17/18] Parallel tests --- internal/service/dms/replication_config_test.go | 2 +- internal/service/dms/replication_task_test.go | 2 +- internal/service/dms/task_settings_json_test.go | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/service/dms/replication_config_test.go b/internal/service/dms/replication_config_test.go index dcc57a3ee2e2..660d71097888 100644 --- a/internal/service/dms/replication_config_test.go +++ b/internal/service/dms/replication_config_test.go @@ -24,7 +24,7 @@ import ( func TestAccDMSReplicationConfig_basic(t *testing.T) { t.Parallel() - for _, migrationType := range dms.MigrationTypeValue_Values() { + for _, migrationType := range dms.MigrationTypeValue_Values() { //nolint:paralleltest // false positive t.Run(migrationType, func(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) diff --git a/internal/service/dms/replication_task_test.go b/internal/service/dms/replication_task_test.go index bd758582c129..c2f07cfb3483 100644 --- a/internal/service/dms/replication_task_test.go +++ b/internal/service/dms/replication_task_test.go @@ -30,7 +30,7 @@ import ( func TestAccDMSReplicationTask_basic(t *testing.T) { t.Parallel() - for _, migrationType := range dms.MigrationTypeValue_Values() { + for _, migrationType := range dms.MigrationTypeValue_Values() { //nolint:paralleltest // false positive t.Run(migrationType, func(t *testing.T) { ctx := acctest.Context(t) rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) diff --git a/internal/service/dms/task_settings_json_test.go b/internal/service/dms/task_settings_json_test.go index 2c834367c35a..e6cfbaafe1fe 100644 --- a/internal/service/dms/task_settings_json_test.go +++ b/internal/service/dms/task_settings_json_test.go @@ -8,6 +8,8 @@ import ( ) func TestTaskSettingsEqual(t *testing.T) { + t.Parallel() + tests := map[string]map[string]struct { a, b any expected bool @@ -101,9 +103,15 @@ func TestTaskSettingsEqual(t *testing.T) { } for name, typeTest := range tests { + name, typeTest := name, typeTest t.Run(name, func(t *testing.T) { + t.Parallel() + for name, test := range typeTest { + name, test := name, test t.Run(name, func(t *testing.T) { + t.Parallel() + if taskSettingsEqual(test.a, test.b) != test.expected { t.Fatalf("expected %v, got %v", test.expected, !test.expected) } From d91e5269232c3ccb1c128b72496b4d84772ed192 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Wed, 17 Apr 2024 09:29:45 -0700 Subject: [PATCH 18/18] Adds CHANGELOG entry --- .changelog/36936.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .changelog/36936.txt diff --git a/.changelog/36936.txt b/.changelog/36936.txt new file mode 100644 index 000000000000..ad32a452ff80 --- /dev/null +++ b/.changelog/36936.txt @@ -0,0 +1,19 @@ +```release-note:bug +resource/aws_dms_replication_task: Allows leaving `replication_task_settings` unset to use default settings. +``` + +```release-note:bug +resource/aws_dms_replication_task: Suppresses differences in partial `replication_task_settings` JSON documents. +``` + +```release-note:bug +resource/aws_dms_replication_task: Adds validation to `replication_task_settings` to disallow `Logging.CloudWatchLogGroup` and `Logging.CloudWatchLogStream`. +``` + +```release-note:bug +resource/aws_dms_replication_config: Suppresses differences in partial `replication_settings` JSON documents. +``` + +```release-note:bug +resource/aws_dms_replication_config: Adds validation to `replication_settings` to disallow `Logging.CloudWatchLogGroup` and `Logging.CloudWatchLogStream`. +```