diff --git a/terraform/environments/electronic-monitoring-data/dms_main.tf b/terraform/environments/electronic-monitoring-data/dms_main.tf index 0795941c80a..caa39df91e6 100644 --- a/terraform/environments/electronic-monitoring-data/dms_main.tf +++ b/terraform/environments/electronic-monitoring-data/dms_main.tf @@ -5,13 +5,19 @@ module "dms_task" { database_name = each.key - # dms_vpc_id = data.aws_vpc.shared.id + # DMS Source Endpoint Inputs rds_db_security_group_id = aws_security_group.db.id rds_db_server_name = split(":", aws_db_instance.database_2022.endpoint)[0] rds_db_instance_port = aws_db_instance.database_2022.port rds_db_username = aws_db_instance.database_2022.username rds_db_instance_pasword = aws_db_instance.database_2022.password - + + # DMS Target Endpoint Inputs target_s3_bucket_name = data.aws_s3_bucket.existing_dms_bucket.id ep_service_access_role_arn = aws_iam_role.dms-endpoint-role.arn + + # DMS Migration Task Inputs + dms_replication_instance_arn = aws_dms_replication_instance.t3micro_dms_replication_instance.replication_instance_arn + rep_task_settings_filepath = "${trimspace(file("${path.module}/dms_replication_task_settings.json"))}" + rep_task_table_mapping_filepath = "${trimspace(file("${path.module}/dms_rep_task_table_mappings.json"))}" } \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/dms_rep_task_table_mappings.json b/terraform/environments/electronic-monitoring-data/dms_rep_task_table_mappings.json new file mode 100644 index 00000000000..770b67c9471 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/dms_rep_task_table_mappings.json @@ -0,0 +1,17 @@ +{ + "rules": [ + { + "rule-type": "selection", + "rule-id": "054476697", + "rule-name": "054476697", + "object-locator": { + "schema-name": "dbo", + "table-name": "%" + }, + "rule-action": "include", + "filters": [], + "parallel-load": null, + "isAutoSegmentationChecked": false + } + ] +} \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/dms_replication_instance.tf b/terraform/environments/electronic-monitoring-data/dms_replication_instance.tf new file mode 100644 index 00000000000..a7653fe1a18 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/dms_replication_instance.tf @@ -0,0 +1,52 @@ +# Create a new replication subnet group +resource "aws_dms_replication_subnet_group" "dms_replication_subnet_group" { + replication_subnet_group_description = "RDS subnet group" + replication_subnet_group_id = "rds-replication-subnet-group-tf" + + subnet_ids = tolist(aws_db_subnet_group.db.subnet_ids) + + tags = merge( + local.tags, + { + Resource_Type = "DMS Replication Subnet Group", + } + ) + + # explicit depends_on is needed since this resource doesn't reference the role or policy attachment + depends_on = [aws_iam_role_policy_attachment.dms-vpc-role-AmazonDMSVPCManagementRole] +} + +# ========================================================================== + +# Create a new replication instance +resource "aws_dms_replication_instance" "t3micro_dms_replication_instance" { + allocated_storage = 20 + apply_immediately = true + auto_minor_version_upgrade = true + availability_zone = "eu-west-2b" +# engine_version = "3.5.1" +# kms_key_arn = "arn:aws:kms:eu-west-2:800964199911:key/b7f54acb-16a3-4958-9340-3bdf5f5842d8" + multi_az = false +# preferred_maintenance_window = "sun:10:30-sun:14:30" + publicly_accessible = false + replication_instance_class = "dms.t3.micro" + replication_instance_id = "t3micro-dms-replication-instance-tf" + replication_subnet_group_id = aws_dms_replication_subnet_group.dms_replication_subnet_group.id + + tags = merge( + local.tags, + { + Resource_Type = "DMS Replication Instance", + } + ) + + vpc_security_group_ids = [ + aws_security_group.dms_ri_security_group.id, + ] + + depends_on = [ + aws_iam_role_policy_attachment.dms-endpoint-role, + aws_iam_role_policy_attachment.dms-cloudwatch-logs-role-AmazonDMSCloudWatchLogsRole, + aws_iam_role_policy_attachment.dms-vpc-role-AmazonDMSVPCManagementRole + ] +} \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/dms_replication_task_settings.json b/terraform/environments/electronic-monitoring-data/dms_replication_task_settings.json new file mode 100644 index 00000000000..ff3b9ba00d2 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/dms_replication_task_settings.json @@ -0,0 +1,174 @@ +{ + "Logging": { + "EnableLogging": false, + "EnableLogContext": false, + "LogComponents": [ + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TRANSFORMATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_UNLOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "IO" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_LOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "PERFORMANCE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_CAPTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SORTER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "REST_SERVER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "VALIDATOR_EXT" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_APPLY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TASK_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TABLES_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "METADATA_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_FACTORY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMON" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "ADDONS" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "DATA_STRUCTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMUNICATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_TRANSFER" + } + ], + "CloudWatchLogGroup": null, + "CloudWatchLogStream": null + }, + "StreamBufferSettings": { + "StreamBufferCount": 3, + "CtrlStreamBufferSizeInMB": 5, + "StreamBufferSizeInMB": 8 + }, + "ErrorBehavior": { + "FailOnNoTablesCaptured": true, + "ApplyErrorUpdatePolicy": "LOG_ERROR", + "FailOnTransactionConsistencyBreached": false, + "RecoverableErrorThrottlingMax": 1800, + "DataErrorEscalationPolicy": "SUSPEND_TABLE", + "ApplyErrorEscalationCount": 0, + "RecoverableErrorStopRetryAfterThrottlingMax": true, + "RecoverableErrorThrottling": true, + "ApplyErrorFailOnTruncationDdl": false, + "DataTruncationErrorPolicy": "LOG_ERROR", + "ApplyErrorInsertPolicy": "LOG_ERROR", + "EventErrorPolicy": "IGNORE", + "ApplyErrorEscalationPolicy": "LOG_ERROR", + "RecoverableErrorCount": -1, + "DataErrorEscalationCount": 0, + "TableErrorEscalationPolicy": "STOP_TASK", + "RecoverableErrorInterval": 5, + "ApplyErrorDeletePolicy": "IGNORE_RECORD", + "TableErrorEscalationCount": 0, + "FullLoadIgnoreConflicts": true, + "DataErrorPolicy": "LOG_ERROR", + "TableErrorPolicy": "SUSPEND_TABLE" + }, + "TTSettings": null, + "FullLoadSettings": { + "CommitRate": 10000, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CreatePkAfterFullLoad": false, + "TargetTablePrepMode": "DO_NOTHING" + }, + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 0, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 0, + "LobMaxSize": 32, + "BatchApplyEnabled": false, + "FullLobMode": false, + "LimitedSizeLobMode": true, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 0 + }, + "BeforeImageSettings": null, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": false, + "HistoryTableEnabled": false, + "ControlSchema": "", + "FullLoadExceptionTableEnabled": false + }, + "LoopbackPreventionSettings": null, + "CharacterSetSettings": null, + "FailTaskWhenCleanTaskResourceFailed": false, + "ChangeProcessingTuning": { + "StatementCacheSize": 50, + "CommitTimeout": 1, + "BatchApplyPreserveTransaction": true, + "BatchApplyTimeoutMin": 1, + "BatchSplitSize": 0, + "BatchApplyTimeoutMax": 30, + "MinTransactionSize": 1000, + "MemoryKeepTime": 60, + "BatchApplyMemoryLimit": 500, + "MemoryLimitTotal": 1024 + }, + "ChangeProcessingDdlHandlingPolicy": { + "HandleSourceTableDropped": true, + "HandleSourceTableTruncated": true, + "HandleSourceTableAltered": true + }, + "PostProcessingRules": null +} \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/modules/dms/dms_db_migration_task.tf b/terraform/environments/electronic-monitoring-data/modules/dms/dms_db_migration_task.tf new file mode 100644 index 00000000000..ca066eff1ea --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/dms/dms_db_migration_task.tf @@ -0,0 +1,18 @@ +# Create a new replication task +resource "aws_dms_replication_task" "t3micro-dms-replication-task" { + # cdc_start_time = "1993-05-21T05:50:00Z" + migration_type = "full-load" + replication_instance_arn = var.dms_replication_instance_arn + replication_task_id = "${var.database_name}-db-migration-task-tf" + replication_task_settings = var.rep_task_settings_filepath + source_endpoint_arn = aws_dms_endpoint.dms-rds-source.endpoint_arn + table_mappings = var.rep_task_table_mapping_filepath + target_endpoint_arn = aws_dms_s3_endpoint.dms-s3-csv-target.endpoint_arn + +# tags = merge( +# local.tags, +# { +# Resource_Type = "DMS Replication Task", +# } +# ) +} \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/modules/dms/dms_endpoints_rds_s3.tf b/terraform/environments/electronic-monitoring-data/modules/dms/dms_endpoints_rds_s3.tf index 92921045270..f4e4569640f 100644 --- a/terraform/environments/electronic-monitoring-data/modules/dms/dms_endpoints_rds_s3.tf +++ b/terraform/environments/electronic-monitoring-data/modules/dms/dms_endpoints_rds_s3.tf @@ -1,5 +1,5 @@ # Create DMS Endpoint to RDS Source -resource "aws_dms_endpoint" "dms-rds-conn-tf" { +resource "aws_dms_endpoint" "dms-rds-source" { # certificate_arn = "" database_name = var.database_name @@ -25,7 +25,7 @@ resource "aws_dms_endpoint" "dms-rds-conn-tf" { # ========================================================================== # Create DMS Endpoint to S3 Target -resource "aws_dms_s3_endpoint" "dms-s3-csv-tf" { +resource "aws_dms_s3_endpoint" "dms-s3-csv-target" { # Minimal Config: endpoint_id = "s3-${var.database_name}-tf" diff --git a/terraform/environments/electronic-monitoring-data/modules/dms/dms_variables.tf b/terraform/environments/electronic-monitoring-data/modules/dms/dms_variables.tf index 708d3b0f0f8..2d6262d2788 100644 --- a/terraform/environments/electronic-monitoring-data/modules/dms/dms_variables.tf +++ b/terraform/environments/electronic-monitoring-data/modules/dms/dms_variables.tf @@ -65,4 +65,19 @@ variable "ep_service_access_role_arn" { variable "target_s3_bucket_name" { description = "DMS S3 Target Endpoint Bucket Name" type = string +} + +variable "rep_task_settings_filepath" { + description = "JSON file with DMS relevant migration task settings" + type = string +} + +variable "rep_task_table_mapping_filepath" { + description = "JSON file with DMS table mappings" + type = string +} + +variable "dms_replication_instance_arn" { + description = "Assign the Replication Instance-ARN to be used" + type = string } \ No newline at end of file