From 8c0ade0b0120434ef8556800f54c09114da43b79 Mon Sep 17 00:00:00 2001 From: Fani Foteva Date: Fri, 27 Dec 2024 23:32:10 +0000 Subject: [PATCH 1/7] TM-606 migrate data Signed-off-by: Fani Foteva --- .../environments/edw/application_variables.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/environments/edw/application_variables.json b/terraform/environments/edw/application_variables.json index 6af3cf6c412..b11a171ecb8 100644 --- a/terraform/environments/edw/application_variables.json +++ b/terraform/environments/edw/application_variables.json @@ -4,7 +4,7 @@ "edw_AppName": "EDW", "edw_environment": "dev", "edw_region": "eu-west-2", - "edw_ec2_ami_id": "ami-012aa1576af4948d1", + "edw_ec2_ami_id": "ami-0935a2693d3e9f859", "edw_ec2_instance_type": "m4.xlarge", "edw_ssh_key_name": "development-general", "edw_OrahomeVolumeSize": "150", @@ -60,11 +60,11 @@ "database_ec2_name": "edw-db-instance", "edw_ec2_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCTLaLYlRp9nXnprXkvSd9pxHxJq4KrZENz0PQ4fy8xe93x54TRfqzMEJdG4tl9Yb5Uv0HEmvi9kMSKZ8uorM67v9oFI5/xtnFoxO9M/cDLXbUmIQ5QFP6+BlDAPWPTV8hh9jAhY1VhkFJTGF0vMBCxdqfIwKH4l6wTciP/lrPYqp4uKddicg0T9mfk6y1EvaZhVTgpMdtj374xC+t57JN6TEuYxwcxsdUA4tB9y4B51LE9WEG7w1up8SeTduCAyGNpUu0lKVnLgpkcrJ3hD/pavUzdAmFESYmMjJkvtKHIuEaAVlxaWQDhHFAJ0BdMlJV4jAARaqgtqgLpsjp3NPnM4FOu3wXRY6+gw3cwDyg7TFlwksnayCHG3ZrSs3fF2cVGF/VQ9L0QZyayPNxnh1g+B7Nnuly3eXLOwD0KQ8XtNDvAX0kV7e2QGiDO0//n+LQHqDMFMTvVWo5g2Ct2HjP97Wf+QgYZ2wbAxb/JSnlHMxbrjeAo0DejscoYzp9f3AM= fani.foteva@MJ004598", "edw_pagerduty_integration_key_name": "laa_edw_nonprod_alarms", - "oradata_snapshot_id": "snap-095a18300a7758e86", - "orahome_snapshot_id": "snap-0d993d189ad092cac", - "oraarch_snapshot_id": "snap-0cf815da06fcc1115", - "software_snapshot_id": "snap-05c64a90c08debf2a", - "oraredo_snapshot_id": "snap-09bdeb4217c0bdab0", + "oradata_snapshot_id": "snap-0009509aa2f260b9e", + "orahome_snapshot_id": "snap-0a3414c321a9ee084", + "oraarch_snapshot_id": "snap-014baa633e2684d43", + "software_snapshot_id": "snap-0c5d946e0a884325d", + "oraredo_snapshot_id": "snap-09fcb4cc0c27bbf31", "old_mail_server_url": "mail.aws.dev.legalservices.gov.uk", "old_domain_name": "dev.legalservices.gov.uk", "laa_mail_relay_url": "laa-mail.laa-development.modernisation-platform.service.justice.gov.uk" From 96bda81469fe8bf824545195499cde282c363019 Mon Sep 17 00:00:00 2001 From: Fani Foteva Date: Fri, 27 Dec 2024 23:34:27 +0000 Subject: [PATCH 2/7] remove lambda rule Signed-off-by: Fani Foteva --- terraform/environments/edw/ec2.tf | 9 --------- 1 file changed, 9 deletions(-) diff --git a/terraform/environments/edw/ec2.tf b/terraform/environments/edw/ec2.tf index e0d8c2fd348..477dae7f277 100644 --- a/terraform/environments/edw/ec2.tf +++ b/terraform/environments/edw/ec2.tf @@ -721,15 +721,6 @@ resource "aws_vpc_security_group_ingress_rule" "db_bastion_ssh" { to_port = 22 } -resource "aws_vpc_security_group_ingress_rule" "db_lambda" { - security_group_id = aws_security_group.edw_db_security_group.id - description = "Allow Lambda SSH access for backup snapshots" - referenced_security_group_id = aws_security_group.backup_lambda.id - from_port = 22 - ip_protocol = "tcp" - to_port = 22 -} - resource "aws_vpc_security_group_ingress_rule" "RDS_Appstream" { security_group_id = aws_security_group.edw_db_security_group.id description = "RDS Appstream access" From 34d6d752f3d86b67a816c91c3feafa9a2917c444 Mon Sep 17 00:00:00 2001 From: Fani Foteva Date: Sat, 28 Dec 2024 01:43:32 +0000 Subject: [PATCH 3/7] fixes after tst data and moved rootrotate to s3 Signed-off-by: Fani Foteva --- terraform/environments/edw/ec2.tf | 32 ++++++++++++++++--- terraform/environments/edw/s3.tf | 14 ++++++++ .../environments/edw/scripts/pmon_check.sh | 11 +++++++ .../environments/edw/scripts/rootrotate.sh | 2 ++ 4 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 terraform/environments/edw/scripts/pmon_check.sh create mode 100644 terraform/environments/edw/scripts/rootrotate.sh diff --git a/terraform/environments/edw/ec2.tf b/terraform/environments/edw/ec2.tf index 477dae7f277..2512deac815 100644 --- a/terraform/environments/edw/ec2.tf +++ b/terraform/environments/edw/ec2.tf @@ -144,6 +144,14 @@ log_group_name = $APPNAME-CDCstatus log_stream_name = {instance_id} EOC2 + +# Create directories if they don't exist +mkdir -p /home/oracle/scripts/logs + +# Create the log files if they don't exist +touch /home/oracle/scripts/logs/freespace_alert.log +touch /home/oracle/scripts/logs/pmon_status_alert.log +touch /home/oracle/scripts/logs/cdc_check.log sudo chmod 755 /home/oracle/scripts/logs sudo chmod 755 /etc/awslogs sudo chmod 755 /tmp/cwlogs @@ -278,16 +286,13 @@ chmod -R 777 /home/oracle # Set permissions for staging directory chmod -R 777 /stage/owb/ -# Replace the secret in the rootrotate.sh script -sed -i "s|--secret-id .* --query|--secret-id ${aws_secretsmanager_secret.edw_db_ec2_root_secret.id} --query|g" /root/scripts/rootrotate.sh - #### setup_backups: # setup efs backup mount point sudo mkdir -p /home/oracle/backup_logs/ sudo mkdir -p /backups/$APPNAME_RMAN chmod 777 /backups/EDW_RMAN -sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/oracle/backup_scripts/rman_s3_arch_backup_v2_1.sh +sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/oracle/rman_arch_backup_v2_1.sh sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/oracle/backup_scripts/rman_full_backup.sh chown -R oracle:dba /home/oracle/backup* chmod -R 740 /home/oracle/backup* @@ -307,13 +312,18 @@ echo "Adding cron job scripts" chown -R oracle:dba /home/oracle/scripts/ chmod -R 755 /home/oracle/scripts/*.sh +sudo mv /home/oracle/scripts/rootrotate.sh /root/scripts/ + +# Replace the secret in the rootrotate.sh script +sed -i "s|--secret-id .* --query|--secret-id ${aws_secretsmanager_secret.edw_db_ec2_root_secret.id} --query|g" /root/scripts/rootrotate.sh + echo "Update Slack alert URL for Oracle scripts" export SLACK_ALERT_URL=`/usr/local/bin/aws --region eu-west-2 ssm get-parameter --name SLACK_ALERT_URL --with-decryption --query Parameter.Value --output text` sed -i "s/SLACK_ALERT_URL/$SLACK_ALERT_URL/g" /home/oracle/scripts/*.sh # Create /etc/cron.d/backup_cron with the cron jobs cat < /etc/cron.d/backup_cron -0 */3 * * * /home/oracle/backup_scripts/rman_s3_arch_backup_v2_1.sh $APPNAME +0 */3 * * * /home/oracle/backup_scripts/rman_arch_backup_v2_1.sh $APPNAME 0 06 * * 01 /home/oracle/backup_scripts/rman_full_backup.sh $APPNAME 00 07,10,13,16 * * * /home/oracle/scripts/freespace_alert.sh ${upper(local.application_data.accounts[local.environment].edw_environment)} 00,15,30,45 * * * * /home/oracle/scripts/pmon_check.sh @@ -360,6 +370,18 @@ chown oracle:dba /home/oracle/crecrontab.txt chmod 777 /home/oracle/crecrontab.txt su oracle -c "crontab /home/oracle/crecrontab.txt" +chown root:root /root/scripts/rootrotate.sh +chmod 700 /root/scripts/rootrotate.sh + +# Create /etc/cron.d/rootrotate with the cron job +cat < /etc/cron.d/rootrotate +0 6 28 * * /root/scripts/rootrotate.sh +EOC30 + +chown root:root /etc/cron.d/rootrotate +chmod 644 /etc/cron.d/rootrotate + + #Update send mail URL echo "Update Sendmail configurations" sed -i 's/${local.application_data.accounts[local.environment].old_mail_server_url}/${local.application_data.accounts[local.environment].laa_mail_relay_url}/g' /etc/mail/sendmail.cf diff --git a/terraform/environments/edw/s3.tf b/terraform/environments/edw/s3.tf index b08c9c65eda..ec33c244481 100644 --- a/terraform/environments/edw/s3.tf +++ b/terraform/environments/edw/s3.tf @@ -69,3 +69,17 @@ resource "aws_s3_object" "maat_sql_script" { source = "./scripts/maat_05365_ware_db_changes.sql" source_hash = filemd5("./scripts/maat_05365_ware_db_changes.sql") } + +resource "aws_s3_object" "pmon_check_script" { + bucket = aws_s3_bucket.scripts.id + key = "pmon_check.sh" + source = "./scripts/pmon_check.sh" + source_hash = filemd5("./scripts/pmon_check.sh") +} + +resource "aws_s3_object" "rootrotate_script" { + bucket = aws_s3_bucket.scripts.id + key = "rootrotate.sh" + source = "./scripts/rootrotate.sh" + source_hash = filemd5("./scripts/rootrotate.sh") +} diff --git a/terraform/environments/edw/scripts/pmon_check.sh b/terraform/environments/edw/scripts/pmon_check.sh new file mode 100644 index 00000000000..ca40128e7ea --- /dev/null +++ b/terraform/environments/edw/scripts/pmon_check.sh @@ -0,0 +1,11 @@ +#!/bin/ksh + +# fixed variables +LOCATE=$HOME/scripts +export curdate=$(date) +logz=$LOCATE/logs/pmon_status_alert.log + +cd $LOCATE + +echo "PMON status as of " $curdate >$logz +ps -ef | grep -v check | grep -v grep | grep -c pmon >/dev/null && echo "PMON process is running" >>$logz || echo "PMON process is DOWN" >>$logz \ No newline at end of file diff --git a/terraform/environments/edw/scripts/rootrotate.sh b/terraform/environments/edw/scripts/rootrotate.sh new file mode 100644 index 00000000000..36778224213 --- /dev/null +++ b/terraform/environments/edw/scripts/rootrotate.sh @@ -0,0 +1,2 @@ +export SECRET2=`/usr/local/bin/aws --region eu-west-2 secretsmanager get-secret-value --secret-id EDW/app/db-EC2-root-password --query SecretString --output text` +echo "$SECRET2" | passwd root --stdin \ No newline at end of file From 0a8480ec5528a01057e3c3f2358b281092c5b677 Mon Sep 17 00:00:00 2001 From: Fani Foteva Date: Sun, 29 Dec 2024 23:42:59 +0000 Subject: [PATCH 4/7] add fix after data migrate Signed-off-by: Fani Foteva --- terraform/environments/edw/ec2.tf | 5 ++- terraform/environments/edw/s3.tf | 15 ++++++++ .../environments/edw/scripts/alert_rota.sh | 38 +++++++++++++++++++ .../edw/scripts/freespace_alert.sql | 12 ++++++ .../edw/scripts/maat_05365_ware_db_changes.sh | 10 +++-- 5 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 terraform/environments/edw/scripts/alert_rota.sh create mode 100644 terraform/environments/edw/scripts/freespace_alert.sql diff --git a/terraform/environments/edw/ec2.tf b/terraform/environments/edw/ec2.tf index 2512deac815..4fcfdb6eb67 100644 --- a/terraform/environments/edw/ec2.tf +++ b/terraform/environments/edw/ec2.tf @@ -292,7 +292,7 @@ chmod -R 777 /stage/owb/ sudo mkdir -p /home/oracle/backup_logs/ sudo mkdir -p /backups/$APPNAME_RMAN chmod 777 /backups/EDW_RMAN -sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/oracle/rman_arch_backup_v2_1.sh +sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/oracle/backup_scripts/rman_arch_backup_v2_1.sh sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/oracle/backup_scripts/rman_full_backup.sh chown -R oracle:dba /home/oracle/backup* chmod -R 740 /home/oracle/backup* @@ -312,6 +312,7 @@ echo "Adding cron job scripts" chown -R oracle:dba /home/oracle/scripts/ chmod -R 755 /home/oracle/scripts/*.sh +sudo mkdir -p /root/scripts/ sudo mv /home/oracle/scripts/rootrotate.sh /root/scripts/ # Replace the secret in the rootrotate.sh script @@ -375,7 +376,7 @@ chmod 700 /root/scripts/rootrotate.sh # Create /etc/cron.d/rootrotate with the cron job cat < /etc/cron.d/rootrotate -0 6 28 * * /root/scripts/rootrotate.sh +0 6 28 * * root /root/scripts/rootrotate.sh EOC30 chown root:root /etc/cron.d/rootrotate diff --git a/terraform/environments/edw/s3.tf b/terraform/environments/edw/s3.tf index ec33c244481..fb2558a0d17 100644 --- a/terraform/environments/edw/s3.tf +++ b/terraform/environments/edw/s3.tf @@ -56,6 +56,13 @@ resource "aws_s3_object" "free_space_script" { source_hash = filemd5("./scripts/freespace_alert.sh") } +resource "aws_s3_object" "free_space_sql_script" { + bucket = aws_s3_bucket.scripts.id + key = "freespace_alert.sql" + source = "./scripts/freespace_alert.sql" + source_hash = filemd5("./scripts/freespace_alert.sql") +} + resource "aws_s3_object" "maat_sh_script" { bucket = aws_s3_bucket.scripts.id key = "maat_05365_ware_db_changes.sh" @@ -83,3 +90,11 @@ resource "aws_s3_object" "rootrotate_script" { source = "./scripts/rootrotate.sh" source_hash = filemd5("./scripts/rootrotate.sh") } + +resource "aws_s3_object" "alert_rota_script" { + bucket = aws_s3_bucket.scripts.id + key = "alert_rota.sh" + source = "./scripts/alert_rota.sh" + source_hash = filemd5("./scripts/alert_rota.sh") +} + diff --git a/terraform/environments/edw/scripts/alert_rota.sh b/terraform/environments/edw/scripts/alert_rota.sh new file mode 100644 index 00000000000..3bd124d928c --- /dev/null +++ b/terraform/environments/edw/scripts/alert_rota.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +chown -R oracle:dba /home/oracle/scripts +ORACLE_SID=EDW;export ORACLE_SID +ORACLE_HOME=/oracle/software/product/10.2.0 +ORACLE_BASE=/oracle/software/product/; export ORACLE_BASE +LD_LIBRARY_PATH=$ORACLE_HOME/lib:/usr/lib; export LD_LIBRARY_PATH +PATH=$ORACLE_HOME/bin:$PATH;export PATH +TO_DATE="20`date +%y%m%d`"; export TO_DATE + +echo ======= +echo Extract Alert log location +echo ======= +export VAL_DUMP=$(${ORACLE_HOME}/bin/sqlplus -S /nolog < ${ELOG} +else +echo not found +fi + +exit \ No newline at end of file diff --git a/terraform/environments/edw/scripts/freespace_alert.sql b/terraform/environments/edw/scripts/freespace_alert.sql new file mode 100644 index 00000000000..c2a08f371e3 --- /dev/null +++ b/terraform/environments/edw/scripts/freespace_alert.sql @@ -0,0 +1,12 @@ +prompt Find Tablespace used space > 95 percent used with 'autoextend on'. +set pages 90 +set lines 132 +select TABLESPACE_NAME,round(TABLESPACE_SIZE*8192/1024/1024/1024,2) "TS_SIZE(GB)", +round(USED_SPACE*8192/1024/1024/1024,2) "USED_SPACE(GB)", +round((TABLESPACE_SIZE*8192/1024/1024/1024)-(USED_SPACE*8192/1024/1024/1024),2) "FREE_SPACE(GB)", +round(USED_PERCENT,2) "USED%",'ALERT' as status +from dba_tablespace_usage_metrics +where round(USED_PERCENT,2) > 94.75 +and TABLESPACE_NAME not like 'UNDO%' +order by 5 desc +/ \ No newline at end of file diff --git a/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh b/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh index 47aa873da97..70e2518bfa9 100644 --- a/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh +++ b/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh @@ -7,17 +7,19 @@ else chown -R oracle:dba /home/oracle/scripts LOCATE=/home/oracle/scripts -ORACLE_SID=$1; export ORACLE_SID +ORACLE_SID=EDW;export ORACLE_SID ORACLE_HOME=/oracle/software/product/10.2.0; export ORACLE_HOME -PATH=$ORACLE_HOME/bin:$PATH; export PATH +PATH=$PATH:$ORACLE_HOME/bin; export PATH + +echo 'Script run on $(date)' >> maat_05365_ware_db_changes.log # Append run time to log cd $LOCATE -sqlplus -s /nolog <rundatafix.log +sqlplus -s /nolog <maat_05365_ware_db_changes.log conn warehouse/whouse_prod @maat_05365_ware_db_changes.sql exit eosql -mailx -s "MI $1 (EDW005) datafix 3079 \`date\`" SLACK_ALERT_URL < rundatafix.log +mailx -s "MI $1 (EDW005) datafix 3079 \`date\`" SLACK_ALERT_URL < maat_05365_ware_db_changes.log fi \ No newline at end of file From 8c679caf394631945af6809f95abd6143e654e6e Mon Sep 17 00:00:00 2001 From: Fani Foteva Date: Mon, 30 Dec 2024 01:03:56 +0000 Subject: [PATCH 5/7] Last update Signed-off-by: Fani Foteva --- terraform/environments/edw/ec2.tf | 2 +- .../environments/edw/scripts/maat_05365_ware_db_changes.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/edw/ec2.tf b/terraform/environments/edw/ec2.tf index 4fcfdb6eb67..a85bd1d035d 100644 --- a/terraform/environments/edw/ec2.tf +++ b/terraform/environments/edw/ec2.tf @@ -337,7 +337,7 @@ chown root:root /etc/cron.d/backup_cron chmod 644 /etc/cron.d/backup_cron # Add backup_cron to crontab for oracle user -yes | cp -f /etc/cron.d/backup_cron /home/oracle/crecrontab.txt +cp -f /etc/cron.d/backup_cron /home/oracle/crecrontab.txt chown oracle:dba /home/oracle/crecrontab.txt chmod 744 /home/oracle/crecrontab.txt su oracle -c "crontab /home/oracle/crecrontab.txt" diff --git a/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh b/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh index 70e2518bfa9..7babe2c4f69 100644 --- a/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh +++ b/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh @@ -16,7 +16,7 @@ echo 'Script run on $(date)' >> maat_05365_ware_db_changes.log # Append run tim cd $LOCATE sqlplus -s /nolog <maat_05365_ware_db_changes.log -conn warehouse/whouse_prod +conn warehouse/password @maat_05365_ware_db_changes.sql exit eosql From 38eba1e64b480746523b7fc60d4c9215185a3f91 Mon Sep 17 00:00:00 2001 From: Fani Foteva Date: Mon, 30 Dec 2024 13:48:31 +0000 Subject: [PATCH 6/7] Last commit Signed-off-by: Fani Foteva --- terraform/environments/edw/ec2.tf | 79 +++++++++++++------ .../edw/scripts/maat_05365_ware_db_changes.sh | 30 ++++--- 2 files changed, 73 insertions(+), 36 deletions(-) diff --git a/terraform/environments/edw/ec2.tf b/terraform/environments/edw/ec2.tf index a85bd1d035d..82820e93859 100644 --- a/terraform/environments/edw/ec2.tf +++ b/terraform/environments/edw/ec2.tf @@ -38,7 +38,7 @@ fi # Check if the chrony.conf file exists and is properly configured if ! grep -q "server 169.254.169.123" /etc/chrony.conf; then - sudo bash -c 'cat << EOC9 > /etc/chrony.conf + sudo bash -c 'cat << EOC1 > /etc/chrony.conf server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4 # Record the rate at which the system clock gains/losses time. driftfile /var/lib/chrony/drift @@ -50,7 +50,7 @@ makestep 1.0 3 logdir /var/log/chrony # Select which information is logged log measurements statistics tracking -EOC9' +EOC1' fi # Start chronyd service @@ -94,17 +94,17 @@ sudo ln -s /bin/mail /bin/mailx # Set up log files echo "---creating /etc/awslogs/awscli.conf" mkdir -p /etc/awslogs -cat > /etc/awslogs/awscli.conf <<-EOC1 +cat > /etc/awslogs/awscli.conf <<-EOC2 [plugins] cwlogs = cwlogs [default] region = $REGION -EOC1 +EOC2 echo "---creating /tmp/cwlogs/logstreams.conf" mkdir -p /tmp/cwlogs -cat > /tmp/cwlogs/logstreams.conf <<-EOC2 +cat > /tmp/cwlogs/logstreams.conf <<-EOC3 [general] state_file = /var/awslogs/agent-state @@ -142,7 +142,7 @@ log_stream_name = {instance_id} file = /home/oracle/scripts/logs/cdc_check.log log_group_name = $APPNAME-CDCstatus log_stream_name = {instance_id} -EOC2 +EOC3 # Create directories if they don't exist @@ -175,7 +175,7 @@ echo "---setup_file_systems" sudo yum install e2fsprogs echo "Updating /etc/fstab file and mount" -cat < /etc/fstab +cat < /etc/fstab /dev/VolGroup00/LogVol00 / ext3 defaults 1 1 LABEL=/boot /boot ext3 defaults 1 2 tmpfs /dev/shm tmpfs defaults 0 0 @@ -189,7 +189,7 @@ proc /proc proc defaults 0 0 /dev/xvdi /oracle/software ext4 defaults 0 0 /dev/xvdj /oracle/temp_undo ext4 defaults 0 0 $EFS.efs.eu-west-2.amazonaws.com:/ /backups nfs4 rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 -EOT +EOC4 # Create file systems sudo mkdir -p /oracle/dbf @@ -297,15 +297,48 @@ sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/o chown -R oracle:dba /home/oracle/backup* chmod -R 740 /home/oracle/backup* +mkdir -p /home/oracle/scripts/logs/aws-ebs-backup.log +chmod 644 /home/oracle/scripts/logs/aws-ebs-backup.log + echo "Setting up AWS EBS backup" INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) -cat < /home/oracle/scripts/aws_ebs_backup.sh + +cat < /home/oracle/scripts/aws_ebs_backup.sh #!/bin/bash + +# Check if the environment parameter is provided +if [ $# -ne 1 ]; then + echo "Usage: \$0 " + exit 1 +fi + +# Get the environment parameter +ENV=\$1 + +LOG_FILE="/home/oracle/scripts/logs/aws-ebs-backup.log" + +# Recreate log file and log start time +echo "Backup started at \$(date) for environment: \$ENV" > \$LOG_FILE + +INSTANCE_ID=\$(curl http://169.254.169.254/latest/meta-data/instance-id) + +# Create snapshot /usr/local/bin/aws ec2 create-snapshots \ ---instance-specification InstanceId=$INSTANCE_ID \ ---description "AWS crash-consistent snapshots of EDW database volumes, automatically created snapshot from oracle_cron inside EC2" \ +--instance-specification InstanceId=\$INSTANCE_ID \ +--description "AWS crash-consistent snapshots of EDW database volumes, automatically created snapshot from oracle_cron inside EC2 for environment: \$ENV" \ --copy-tags-from-source volume -EOC25 + +# Check if the backup command was successful +if [ \$? -eq 0 ]; then + echo "Backup completed successfully at \$(date) for environment: \$ENV" >> \$LOG_FILE +else + echo "Backup failed at \$(date) for environment: \$ENV" >> \$LOG_FILE + mailx -s "Backup for EC2 instance \$INSTANCE_ID on \$ENV failed at \$(date)" SLACK_ALERT_URL -- < \$LOG_FILE +fi + +# Log end time +echo "Backup ended at \$(date) for environment: \$ENV" >> \$LOG_FILE +EOC5 echo "Adding cron job scripts" /usr/local/bin/aws s3 cp s3://${aws_s3_bucket.scripts.id}/ /home/oracle/scripts --recursive @@ -323,15 +356,15 @@ export SLACK_ALERT_URL=`/usr/local/bin/aws --region eu-west-2 ssm get-parameter sed -i "s/SLACK_ALERT_URL/$SLACK_ALERT_URL/g" /home/oracle/scripts/*.sh # Create /etc/cron.d/backup_cron with the cron jobs -cat < /etc/cron.d/backup_cron +cat < /etc/cron.d/backup_cron 0 */3 * * * /home/oracle/backup_scripts/rman_arch_backup_v2_1.sh $APPNAME 0 06 * * 01 /home/oracle/backup_scripts/rman_full_backup.sh $APPNAME 00 07,10,13,16 * * * /home/oracle/scripts/freespace_alert.sh ${upper(local.application_data.accounts[local.environment].edw_environment)} 00,15,30,45 * * * * /home/oracle/scripts/pmon_check.sh # 0 7 * * 1 /home/oracle/scripts/maat_05365_ware_db_changes.sh ${upper(local.application_data.accounts[local.environment].edw_environment)} -00 02 * * * /home/oracle/scripts/aws_ebs_backup.sh > /tmp/aws_ebs_backup.log +00 02 * * * /home/oracle/scripts/aws_ebs_backup.sh ${upper(local.application_data.accounts[local.environment].edw_environment)} > /tmp/aws_ebs_backup.log 10,40 08-17 * * * /home/oracle/scripts/disk_space_alert.sh ${upper(local.application_data.accounts[local.environment].edw_environment)} 97 >/tmp/disk_space.trc 2>&1 -EOC3 +EOC6 chown root:root /etc/cron.d/backup_cron chmod 644 /etc/cron.d/backup_cron @@ -346,9 +379,9 @@ chown root:root /var/cw-custom.sh chmod 700 /var/cw-custom.sh # Create /etc/cron.d/custom_cloudwatch_metrics with the cron job -cat < /etc/cron.d/custom_cloudwatch_metrics +cat < /etc/cron.d/custom_cloudwatch_metrics */1 * * * * root /var/cw-custom.sh -EOC4 +EOC7 chown root:root /etc/cron.d/custom_cloudwatch_metrics chmod 600 /etc/cron.d/custom_cloudwatch_metrics @@ -358,9 +391,9 @@ chown oracle:dba /home/oracle/scripts/alert_rota.sh chmod 755 /home/oracle/scripts/alert_rota.sh # Create /etc/cron.d/oracle_rotation with the cron jobs -cat < /etc/cron.d/oracle_rotation +cat < /etc/cron.d/oracle_rotation 00 07 * * * /home/oracle/scripts/alert_rota.sh $APPNAME -EOC5 +EOC8 chown root:root /etc/cron.d/oracle_rotation chmod 644 /etc/cron.d/oracle_rotation @@ -375,9 +408,9 @@ chown root:root /root/scripts/rootrotate.sh chmod 700 /root/scripts/rootrotate.sh # Create /etc/cron.d/rootrotate with the cron job -cat < /etc/cron.d/rootrotate +cat < /etc/cron.d/rootrotate 0 6 28 * * root /root/scripts/rootrotate.sh -EOC30 +EOC9 chown root:root /etc/cron.d/rootrotate chmod 644 /etc/cron.d/rootrotate @@ -391,11 +424,11 @@ sed -i 's/${local.application_data.accounts[local.environment].old_mail_server_u sed -i 's/${local.application_data.accounts[local.environment].old_domain_name}/${data.aws_route53_zone.external.name}/g' /etc/mail/sendmail.mc /etc/init.d/sendmail restart -sudo su - oracle -c "sqlplus / as sysdba << EOC6 +sudo su - oracle -c "sqlplus / as sysdba << EOC10 shutdown abort; startup; exit; -EOC6" +EOC10" EOF } diff --git a/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh b/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh index 7babe2c4f69..f414194dd55 100644 --- a/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh +++ b/terraform/environments/edw/scripts/maat_05365_ware_db_changes.sh @@ -1,25 +1,29 @@ #!/bin/ksh if [ $# -ne 1 ]; then - echo "1st parameter is ENV" + echo "1st parameter is ENV" else -# fixed variables -chown -R oracle:dba /home/oracle/scripts + # fixed variables + chown -R oracle:dba /home/oracle/scripts -LOCATE=/home/oracle/scripts -ORACLE_SID=EDW;export ORACLE_SID -ORACLE_HOME=/oracle/software/product/10.2.0; export ORACLE_HOME -PATH=$PATH:$ORACLE_HOME/bin; export PATH + LOCATE=/home/oracle/scripts + ORACLE_SID=EDW; export ORACLE_SID + ORACLE_HOME=/oracle/software/product/10.2.0; export ORACLE_HOME + PATH=$PATH:$ORACLE_HOME/bin; export PATH -echo 'Script run on $(date)' >> maat_05365_ware_db_changes.log # Append run time to log + # Get the current warehouse password from passwds.sql + warehouse_password=$(grep 'define EDW_WAREHOUSE=' /var/opt/oracle/passwds.sql | cut -d'=' -f2) -cd $LOCATE + echo "Script run on $(date)" >> maat_05365_ware_db_changes.log # Append run time to log -sqlplus -s /nolog <maat_05365_ware_db_changes.log -conn warehouse/password + cd $LOCATE + + # Use the extracted warehouse password in the SQL*Plus connection + sqlplus -s /nolog < maat_05365_ware_db_changes.log +conn warehouse/${warehouse_password} @maat_05365_ware_db_changes.sql exit eosql -mailx -s "MI $1 (EDW005) datafix 3079 \`date\`" SLACK_ALERT_URL < maat_05365_ware_db_changes.log -fi \ No newline at end of file + mailx -s "MI $1 (EDW005) datafix 3079 \`date\`" SLACK_ALERT_URL < maat_05365_ware_db_changes.log +fi From f079e0022a7d9ce69b7ab3f93b4933ecce7c2038 Mon Sep 17 00:00:00 2001 From: Fani Foteva Date: Mon, 30 Dec 2024 14:17:23 +0000 Subject: [PATCH 7/7] small fix Signed-off-by: Fani Foteva --- terraform/environments/edw/ec2.tf | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/terraform/environments/edw/ec2.tf b/terraform/environments/edw/ec2.tf index 82820e93859..fa40f53f1ed 100644 --- a/terraform/environments/edw/ec2.tf +++ b/terraform/environments/edw/ec2.tf @@ -152,6 +152,7 @@ mkdir -p /home/oracle/scripts/logs touch /home/oracle/scripts/logs/freespace_alert.log touch /home/oracle/scripts/logs/pmon_status_alert.log touch /home/oracle/scripts/logs/cdc_check.log +touch /home/oracle/scripts/logs/aws_ebs_backup.log sudo chmod 755 /home/oracle/scripts/logs sudo chmod 755 /etc/awslogs sudo chmod 755 /tmp/cwlogs @@ -297,9 +298,6 @@ sed -i "s/\/backups\/production\/MIDB_RMAN\//\/backups\/$APPNAME_RMAN/g" /home/o chown -R oracle:dba /home/oracle/backup* chmod -R 740 /home/oracle/backup* -mkdir -p /home/oracle/scripts/logs/aws-ebs-backup.log -chmod 644 /home/oracle/scripts/logs/aws-ebs-backup.log - echo "Setting up AWS EBS backup" INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) @@ -315,7 +313,7 @@ fi # Get the environment parameter ENV=\$1 -LOG_FILE="/home/oracle/scripts/logs/aws-ebs-backup.log" +LOG_FILE="/home/oracle/scripts/logs/aws_ebs_backup.log" # Recreate log file and log start time echo "Backup started at \$(date) for environment: \$ENV" > \$LOG_FILE @@ -336,8 +334,6 @@ else mailx -s "Backup for EC2 instance \$INSTANCE_ID on \$ENV failed at \$(date)" SLACK_ALERT_URL -- < \$LOG_FILE fi -# Log end time -echo "Backup ended at \$(date) for environment: \$ENV" >> \$LOG_FILE EOC5 echo "Adding cron job scripts"