Merge pull request #9196 from ministryofjustice/TM-606-migrate-tst-data

TM-606 migrate data

terraform/environments/edw/application_variables.json

@@ -4,7 +4,7 @@
       "edw_AppName": "EDW",
       "edw_environment": "dev",
       "edw_region": "eu-west-2",
-      "edw_ec2_ami_id": "ami-012aa1576af4948d1",
+      "edw_ec2_ami_id": "ami-0935a2693d3e9f859",
       "edw_ec2_instance_type": "m4.xlarge",
       "edw_ssh_key_name": "development-general",
       "edw_OrahomeVolumeSize": "150",
@@ -60,11 +60,11 @@
       "database_ec2_name": "edw-db-instance",
       "edw_ec2_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCTLaLYlRp9nXnprXkvSd9pxHxJq4KrZENz0PQ4fy8xe93x54TRfqzMEJdG4tl9Yb5Uv0HEmvi9kMSKZ8uorM67v9oFI5/xtnFoxO9M/cDLXbUmIQ5QFP6+BlDAPWPTV8hh9jAhY1VhkFJTGF0vMBCxdqfIwKH4l6wTciP/lrPYqp4uKddicg0T9mfk6y1EvaZhVTgpMdtj374xC+t57JN6TEuYxwcxsdUA4tB9y4B51LE9WEG7w1up8SeTduCAyGNpUu0lKVnLgpkcrJ3hD/pavUzdAmFESYmMjJkvtKHIuEaAVlxaWQDhHFAJ0BdMlJV4jAARaqgtqgLpsjp3NPnM4FOu3wXRY6+gw3cwDyg7TFlwksnayCHG3ZrSs3fF2cVGF/VQ9L0QZyayPNxnh1g+B7Nnuly3eXLOwD0KQ8XtNDvAX0kV7e2QGiDO0//n+LQHqDMFMTvVWo5g2Ct2HjP97Wf+QgYZ2wbAxb/JSnlHMxbrjeAo0DejscoYzp9f3AM= fani.foteva@MJ004598",
       "edw_pagerduty_integration_key_name": "laa_edw_nonprod_alarms",
-      "oradata_snapshot_id": "snap-095a18300a7758e86",
-      "orahome_snapshot_id": "snap-0d993d189ad092cac",
-      "oraarch_snapshot_id": "snap-0cf815da06fcc1115",
-      "software_snapshot_id": "snap-05c64a90c08debf2a",
-      "oraredo_snapshot_id": "snap-09bdeb4217c0bdab0",
+      "oradata_snapshot_id": "snap-0009509aa2f260b9e",
+      "orahome_snapshot_id": "snap-0a3414c321a9ee084",
+      "oraarch_snapshot_id": "snap-014baa633e2684d43",
+      "software_snapshot_id": "snap-0c5d946e0a884325d",
+      "oraredo_snapshot_id": "snap-09fcb4cc0c27bbf31",
       "old_mail_server_url": "mail.aws.dev.legalservices.gov.uk",
       "old_domain_name": "dev.legalservices.gov.uk",
       "laa_mail_relay_url": "laa-mail.laa-development.modernisation-platform.service.justice.gov.uk"

terraform/environments/edw/ec2.tf

@@ -38,7 +38,7 @@ fi
 
 # Check if the chrony.conf file exists and is properly configured
 if ! grep -q "server 169.254.169.123" /etc/chrony.conf; then
-  sudo bash -c 'cat << EOC9 > /etc/chrony.conf
+  sudo bash -c 'cat << EOC1 > /etc/chrony.conf
 server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4
 # Record the rate at which the system clock gains/losses time.
 driftfile /var/lib/chrony/drift
@@ -50,7 +50,7 @@ makestep 1.0 3
 logdir /var/log/chrony
 # Select which information is logged
 log measurements statistics tracking
-EOC9'
+EOC1'
 fi
 
 # Start chronyd service
@@ -94,17 +94,17 @@ sudo ln -s /bin/mail /bin/mailx
 # Set up log files
 echo "---creating /etc/awslogs/awscli.conf"
 mkdir -p /etc/awslogs
-cat > /etc/awslogs/awscli.conf <<-EOC1
+cat > /etc/awslogs/awscli.conf <<-EOC2
 [plugins]
 cwlogs = cwlogs
 [default]
 region = $REGION
-EOC1
+EOC2
 
 echo "---creating /tmp/cwlogs/logstreams.conf"
 mkdir -p /tmp/cwlogs
 
-cat > /tmp/cwlogs/logstreams.conf <<-EOC2
+cat > /tmp/cwlogs/logstreams.conf <<-EOC3
 [general]
 state_file = /var/awslogs/agent-state
 
@@ -142,8 +142,17 @@ log_stream_name = {instance_id}
 file = /home/oracle/scripts/logs/cdc_check.log
 log_group_name = $APPNAME-CDCstatus
 log_stream_name = {instance_id}
-EOC2
+EOC3
+
+
+# Create directories if they don't exist
+mkdir -p /home/oracle/scripts/logs
 
+# Create the log files if they don't exist
+touch /home/oracle/scripts/logs/freespace_alert.log
+touch /home/oracle/scripts/logs/pmon_status_alert.log
+touch /home/oracle/scripts/logs/cdc_check.log
+touch /home/oracle/scripts/logs/aws_ebs_backup.log
 sudo chmod 755 /home/oracle/scripts/logs
 sudo chmod 755 /etc/awslogs
 sudo chmod 755 /tmp/cwlogs
@@ -167,7 +176,7 @@ echo "---setup_file_systems"
 sudo yum install e2fsprogs
 
 echo "Updating /etc/fstab file and mount"
-cat <<EOT > /etc/fstab
+cat <<EOC4 > /etc/fstab
 /dev/VolGroup00/LogVol00	/	ext3	defaults	1 1
 LABEL=/boot	/boot	ext3	defaults	1 2
 tmpfs	/dev/shm	tmpfs	defaults	0 0
@@ -181,7 +190,7 @@ proc	/proc	proc	defaults	0 0
 /dev/xvdi /oracle/software ext4 defaults 0 0
 /dev/xvdj /oracle/temp_undo ext4 defaults 0 0
 $EFS.efs.eu-west-2.amazonaws.com:/ /backups nfs4 rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2
-EOT
+EOC4
 
 # Create file systems 
 sudo mkdir -p /oracle/dbf
@@ -278,55 +287,86 @@ chmod -R 777 /home/oracle
 # Set permissions for staging directory
 chmod -R 777 /stage/owb/
 
-# Replace the secret in the rootrotate.sh script
-sed -i "s|--secret-id .* --query|--secret-id ${aws_secretsmanager_secret.edw_db_ec2_root_secret.id} --query|g" /root/scripts/rootrotate.sh
-
 #### setup_backups:
 
 # setup efs backup mount point
 sudo mkdir -p /home/oracle/backup_logs/
 sudo mkdir -p /backups/$APPNAME_RMAN
 chmod 777 /backups/EDW_RMAN
-sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/oracle/backup_scripts/rman_s3_arch_backup_v2_1.sh
+sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/oracle/backup_scripts/rman_arch_backup_v2_1.sh
 sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/oracle/backup_scripts/rman_full_backup.sh
 chown -R oracle:dba /home/oracle/backup*
 chmod -R 740 /home/oracle/backup*
 
 echo "Setting up AWS EBS backup"
 INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id)
-cat <<EOC25 > /home/oracle/scripts/aws_ebs_backup.sh
+
+cat <<EOC5 > /home/oracle/scripts/aws_ebs_backup.sh
 #!/bin/bash
+
+# Check if the environment parameter is provided
+if [ $# -ne 1 ]; then
+  echo "Usage: \$0 <ENV>"
+  exit 1
+fi
+
+# Get the environment parameter
+ENV=\$1
+
+LOG_FILE="/home/oracle/scripts/logs/aws_ebs_backup.log"
+
+# Recreate log file and log start time
+echo "Backup started at \$(date) for environment: \$ENV" > \$LOG_FILE
+
+INSTANCE_ID=\$(curl http://169.254.169.254/latest/meta-data/instance-id)
+
+# Create snapshot
 /usr/local/bin/aws ec2 create-snapshots \
---instance-specification InstanceId=$INSTANCE_ID \
---description "AWS crash-consistent snapshots of EDW database volumes, automatically created snapshot from oracle_cron inside EC2" \
+--instance-specification InstanceId=\$INSTANCE_ID \
+--description "AWS crash-consistent snapshots of EDW database volumes, automatically created snapshot from oracle_cron inside EC2 for environment: \$ENV" \
 --copy-tags-from-source volume
-EOC25
+
+# Check if the backup command was successful
+if [ \$? -eq 0 ]; then
+  echo "Backup completed successfully at \$(date) for environment: \$ENV" >> \$LOG_FILE
+else
+  echo "Backup failed at \$(date) for environment: \$ENV" >> \$LOG_FILE
+  mailx -s "Backup for EC2 instance \$INSTANCE_ID on \$ENV failed at \$(date)" SLACK_ALERT_URL -- < \$LOG_FILE
+fi
+
+EOC5
 
 echo "Adding cron job scripts"
 /usr/local/bin/aws s3 cp s3://${aws_s3_bucket.scripts.id}/ /home/oracle/scripts --recursive
 chown -R oracle:dba /home/oracle/scripts/
 chmod -R 755 /home/oracle/scripts/*.sh
 
+sudo mkdir -p /root/scripts/
+sudo mv /home/oracle/scripts/rootrotate.sh /root/scripts/
+
+# Replace the secret in the rootrotate.sh script
+sed -i "s|--secret-id .* --query|--secret-id ${aws_secretsmanager_secret.edw_db_ec2_root_secret.id} --query|g" /root/scripts/rootrotate.sh
+
 echo "Update Slack alert URL for Oracle scripts"
 export SLACK_ALERT_URL=`/usr/local/bin/aws --region eu-west-2 ssm get-parameter --name SLACK_ALERT_URL --with-decryption --query Parameter.Value --output text`
 sed -i "s/SLACK_ALERT_URL/$SLACK_ALERT_URL/g" /home/oracle/scripts/*.sh
 
 # Create /etc/cron.d/backup_cron with the cron jobs
-cat <<EOC3 > /etc/cron.d/backup_cron
-0 */3 * * * /home/oracle/backup_scripts/rman_s3_arch_backup_v2_1.sh $APPNAME
+cat <<EOC6 > /etc/cron.d/backup_cron
+0 */3 * * * /home/oracle/backup_scripts/rman_arch_backup_v2_1.sh $APPNAME
 0 06 * * 01 /home/oracle/backup_scripts/rman_full_backup.sh $APPNAME
 00 07,10,13,16 * * * /home/oracle/scripts/freespace_alert.sh ${upper(local.application_data.accounts[local.environment].edw_environment)}
 00,15,30,45 * * * * /home/oracle/scripts/pmon_check.sh
 # 0 7 * * 1 /home/oracle/scripts/maat_05365_ware_db_changes.sh ${upper(local.application_data.accounts[local.environment].edw_environment)}
-00 02 * * * /home/oracle/scripts/aws_ebs_backup.sh > /tmp/aws_ebs_backup.log
+00 02 * * * /home/oracle/scripts/aws_ebs_backup.sh ${upper(local.application_data.accounts[local.environment].edw_environment)} > /tmp/aws_ebs_backup.log
 10,40 08-17 * * * /home/oracle/scripts/disk_space_alert.sh ${upper(local.application_data.accounts[local.environment].edw_environment)} 97  >/tmp/disk_space.trc 2>&1
-EOC3
+EOC6
 
 chown root:root /etc/cron.d/backup_cron
 chmod 644 /etc/cron.d/backup_cron
 
 # Add backup_cron to crontab for oracle user
-yes | cp -f /etc/cron.d/backup_cron /home/oracle/crecrontab.txt
+cp -f /etc/cron.d/backup_cron /home/oracle/crecrontab.txt
 chown oracle:dba /home/oracle/crecrontab.txt
 chmod 744 /home/oracle/crecrontab.txt
 su oracle -c "crontab /home/oracle/crecrontab.txt"
@@ -335,9 +375,9 @@ chown root:root /var/cw-custom.sh
 chmod 700 /var/cw-custom.sh
 
 # Create /etc/cron.d/custom_cloudwatch_metrics with the cron job
-cat <<EOC4 > /etc/cron.d/custom_cloudwatch_metrics
+cat <<EOC7 > /etc/cron.d/custom_cloudwatch_metrics
 */1 * * * * root /var/cw-custom.sh
-EOC4
+EOC7
 
 chown root:root /etc/cron.d/custom_cloudwatch_metrics
 chmod 600 /etc/cron.d/custom_cloudwatch_metrics
@@ -347,9 +387,9 @@ chown oracle:dba /home/oracle/scripts/alert_rota.sh
 chmod 755 /home/oracle/scripts/alert_rota.sh
 
 # Create /etc/cron.d/oracle_rotation with the cron jobs
-cat <<EOC5 > /etc/cron.d/oracle_rotation
+cat <<EOC8 > /etc/cron.d/oracle_rotation
 00 07 * * * /home/oracle/scripts/alert_rota.sh $APPNAME
-EOC5
+EOC8
 
 chown root:root /etc/cron.d/oracle_rotation
 chmod 644 /etc/cron.d/oracle_rotation
@@ -360,6 +400,18 @@ chown oracle:dba /home/oracle/crecrontab.txt
 chmod 777 /home/oracle/crecrontab.txt
 su oracle -c "crontab /home/oracle/crecrontab.txt"
 
+chown root:root /root/scripts/rootrotate.sh
+chmod 700 /root/scripts/rootrotate.sh
+
+# Create /etc/cron.d/rootrotate with the cron job
+cat <<EOC9 > /etc/cron.d/rootrotate
+0 6 28 * * root /root/scripts/rootrotate.sh
+EOC9
+
+chown root:root /etc/cron.d/rootrotate
+chmod 644 /etc/cron.d/rootrotate
+
+
 #Update send mail URL 
 echo "Update Sendmail configurations"
 sed -i 's/${local.application_data.accounts[local.environment].old_mail_server_url}/${local.application_data.accounts[local.environment].laa_mail_relay_url}/g' /etc/mail/sendmail.cf
@@ -368,11 +420,11 @@ sed -i 's/${local.application_data.accounts[local.environment].old_mail_server_u
 sed -i 's/${local.application_data.accounts[local.environment].old_domain_name}/${data.aws_route53_zone.external.name}/g' /etc/mail/sendmail.mc
 /etc/init.d/sendmail restart
 
-sudo su - oracle -c "sqlplus / as sysdba << EOC6
+sudo su - oracle -c "sqlplus / as sysdba << EOC10
 shutdown abort;
 startup;
 exit;
-EOC6"
+EOC10"
  
 EOF
 }
@@ -721,15 +773,6 @@ resource "aws_vpc_security_group_ingress_rule" "db_bastion_ssh" {
   to_port                      = 22
 }
 
-resource "aws_vpc_security_group_ingress_rule" "db_lambda" {
-  security_group_id            = aws_security_group.edw_db_security_group.id
-  description                  = "Allow Lambda SSH access for backup snapshots"
-  referenced_security_group_id = aws_security_group.backup_lambda.id
-  from_port                    = 22
-  ip_protocol                  = "tcp"
-  to_port                      = 22
-}
-
 resource "aws_vpc_security_group_ingress_rule" "RDS_Appstream" {
   security_group_id = aws_security_group.edw_db_security_group.id
   description       = "RDS Appstream access"

terraform/environments/edw/s3.tf

@@ -56,6 +56,13 @@ resource "aws_s3_object" "free_space_script" {
   source_hash = filemd5("./scripts/freespace_alert.sh")
 }
 
+resource "aws_s3_object" "free_space_sql_script" {
+  bucket      = aws_s3_bucket.scripts.id
+  key         = "freespace_alert.sql"
+  source      = "./scripts/freespace_alert.sql"
+  source_hash = filemd5("./scripts/freespace_alert.sql")
+}
+
 resource "aws_s3_object" "maat_sh_script" {
   bucket      = aws_s3_bucket.scripts.id
   key         = "maat_05365_ware_db_changes.sh"
@@ -69,3 +76,25 @@ resource "aws_s3_object" "maat_sql_script" {
   source      = "./scripts/maat_05365_ware_db_changes.sql"
   source_hash = filemd5("./scripts/maat_05365_ware_db_changes.sql")
 }
+
+resource "aws_s3_object" "pmon_check_script" {
+  bucket      = aws_s3_bucket.scripts.id
+  key         = "pmon_check.sh"
+  source      = "./scripts/pmon_check.sh"
+  source_hash = filemd5("./scripts/pmon_check.sh")
+}
+
+resource "aws_s3_object" "rootrotate_script" {
+  bucket      = aws_s3_bucket.scripts.id
+  key         = "rootrotate.sh"
+  source      = "./scripts/rootrotate.sh"
+  source_hash = filemd5("./scripts/rootrotate.sh")
+}
+
+resource "aws_s3_object" "alert_rota_script" {
+  bucket      = aws_s3_bucket.scripts.id
+  key         = "alert_rota.sh"
+  source      = "./scripts/alert_rota.sh"
+  source_hash = filemd5("./scripts/alert_rota.sh")
+}
+

terraform/environments/edw/scripts/alert_rota.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+chown -R oracle:dba /home/oracle/scripts
+ORACLE_SID=EDW;export ORACLE_SID
+ORACLE_HOME=/oracle/software/product/10.2.0
+ORACLE_BASE=/oracle/software/product/; export ORACLE_BASE
+LD_LIBRARY_PATH=$ORACLE_HOME/lib:/usr/lib; export LD_LIBRARY_PATH
+PATH=$ORACLE_HOME/bin:$PATH;export PATH
+TO_DATE="20`date +%y%m%d`"; export TO_DATE
+
+echo =======
+echo Extract Alert log location
+echo =======
+export VAL_DUMP=$(${ORACLE_HOME}/bin/sqlplus -S /nolog <<EOF
+conn /as sysdba
+set pages 0 feedback off;
+prompt
+SELECT value from v\$parameter where NAME='background_dump_dest';
+exit;
+EOF
+)
+export LOCATION=`echo ${VAL_DUMP} | perl -lpe'$_ = reverse' |awk '{print $1}'|perl -lpe'$_ = reverse'`
+export ALERTDB=${LOCATION}/alert_$ORACLE_SID.log
+export ELOG=$( echo ${ALERTDB} | sed s/cdump/trace/)
+
+echo =======
+echo Compress current
+echo =======
+
+if [ -e "$ELOG" ] ; then
+mv ${ELOG} ${ELOG}_${TO_DATE};
+gzip ${ELOG}_${TO_DATE};
+> ${ELOG}
+else
+echo not found
+fi
+
+exit

terraform/environments/edw/scripts/freespace_alert.sql

@@ -0,0 +1,12 @@
+prompt Find Tablespace used space > 95 percent used  with 'autoextend on'.
+set pages 90
+set lines 132
+select TABLESPACE_NAME,round(TABLESPACE_SIZE*8192/1024/1024/1024,2) "TS_SIZE(GB)",
+round(USED_SPACE*8192/1024/1024/1024,2) "USED_SPACE(GB)",
+round((TABLESPACE_SIZE*8192/1024/1024/1024)-(USED_SPACE*8192/1024/1024/1024),2) "FREE_SPACE(GB)",
+round(USED_PERCENT,2) "USED%",'ALERT' as status
+from dba_tablespace_usage_metrics
+where round(USED_PERCENT,2) > 94.75
+and TABLESPACE_NAME not like 'UNDO%'
+order by 5 desc
+/