Merge pull request #1372 from NASA-AMMOS/refactor/merge-the-dbs

Merge the Databases
NASA-AMMOS · Apr 15, 2024 · 19ceee2 · 19ceee2
2 parents 7c144ef + 3a285c9
commit 19ceee2
Show file tree

Hide file tree

Showing 412 changed files with 10,826 additions and 6,734 deletions.
diff --git a/.env.template b/.env.template
@@ -1,6 +1,20 @@
 AERIE_USERNAME=
 AERIE_PASSWORD=
+
+GATEWAY_USERNAME=
+GATEWAY_PASSWORD=
+
+MERLIN_USERNAME=
+MERLIN_PASSWORD=
+
+SCHEDULER_USERNAME=
+SCHEDULER_PASSWORD=
+
+SEQUENCING_USERNAME=
+SEQUENCING_PASSWORD=
+
+POSTGRES_PASSWORD=
+POSTGRES_USER=
+
 HASURA_GRAPHQL_ADMIN_SECRET=
 HASURA_GRAPHQL_JWT_SECRET=
-POSTGRES_USER=
-POSTGRES_PASSWORD=
diff --git a/.github/scripts/compareDatabasesDown.sh b/.github/scripts/compareDatabasesDown.sh
@@ -3,96 +3,27 @@
 mkdir results
 mkdir comparison
 
-PGCMPINPUT1=./pgdumpv1_0_1/AerieMerlinV1_0_1 PGCMPINPUT2=./pgdumpmigrateddown/AerieMerlinMigratedDown PGCLABEL1=AerieMerlinV1_0_1 PGCLABEL2=AerieMerlinMigratedDown PGCFULLOUTPUT=./comparison/fulloutputMerlin.txt PGCUNEXPLAINED=./comparison/unexplainedMerlin.txt PGCBADEXPLAIN=./comparison/badexplanationsMerlin.txt PGDB=postgres PGBINDIR=/usr/bin PGCEXPLANATIONS=./explanations_merlin ./pgcmp
+PGCMPINPUT1=./pgdumpV2_8_0/AerieV2_8_0 \
+PGCMPINPUT2=./pgdumpmigrateddown/AerieMigratedDown \
+PGCLABEL1=AerieV2_8_0 \
+PGCLABEL2=AerieMigratedDown \
+PGCFULLOUTPUT=./comparison/fulloutput.txt \
+PGCUNEXPLAINED=./comparison/unexplained.txt \
+PGCBADEXPLAIN=./comparison/badexplanations.txt \
+PGDB=postgres \
+PGBINDIR=/usr/bin \
+PGCOMITSCHEMAS="('hdb_catalog'),('pg_catalog'),('information_schema')" \
+./pgcmp
 return_code=$?
-if [ $return_code -ne 0 ]; then
-    echo "AerieMerlin comparison failed - return code=$return_code"
-    touch results/fulloutput
-    touch results/unexplained
-    touch results/badexplanations
-    touch results/perform-comparison.log
-    printf "==================\nAerieMerlin Results\n==================\n" >> results/fulloutput
-    printf "==================\nAerieMerlin Results\n==================\n" >> results/unexplained
-    printf "==================\nAerieMerlin Results\n==================\n" >> results/badexplanations
-    printf "==================\nAerieMerlin Results\n==================\n" >> results/perform-comparison.log
-    cat ./comparison/fulloutputMerlin.txt >> results/fulloutput
-    cat ./comparison/unexplainedMerlin.txt >> results/unexplained
-    cat ./comparison/badexplanationsMerlin.txt >> results/badexplanations
-    cat /tmp/perform-comparison.log >> results/perform-comparison.log
-else
-    echo "AerieMerlin comparison succeeded"
-fi
-
-PGCMPINPUT1=./pgdumpv1_0_1/AerieSchedulerV1_0_1 PGCMPINPUT2=./pgdumpmigrateddown/AerieSchedulerMigratedDown PGCLABEL1=AerieSchedulerV1_0_1 PGCLABEL2=AerieSchedulerMigratedDown PGCFULLOUTPUT=./comparison/fulloutputScheduler.txt PGCUNEXPLAINED=./comparison/unexplainedScheduler.txt PGCBADEXPLAIN=./comparison/badexplanationsScheduler.txt PGDB=postgres PGBINDIR=/usr/bin PGCEXPLANATIONS=./explanations ./pgcmp
-retcode=$?
-if [ $retcode -gt $return_code ]; then
-return_code=$retcode
-fi
-
-if [ $retcode -ne 0 ]; then
-    echo "AerieScheduler comparison failed - return code=$retcode"
-    touch results/fulloutput
-    touch results/unexplained
-    touch results/badexplanations
-    touch results/perform-comparison.log
-    printf "\n==================\nAerieScheduler Results\n==================\n" >> results/fulloutput
-    printf "\n==================\nAerieScheduler Results\n==================\n" >> results/unexplained
-    printf "\n==================\nAerieScheduler Results\n==================\n" >> results/badexplanations
-    printf "\n==================\nAerieScheduler Results\n==================\n" >> results/perform-comparison.log
-    cat ./comparison/fulloutputScheduler.txt >> results/fulloutput
-    cat ./comparison/unexplainedScheduler.txt >> results/unexplained
-    cat ./comparison/badexplanationsScheduler.txt >> results/badexplanations
-    cat /tmp/perform-comparison.log >> results/perform-comparison.log
-else
-    echo "AerieScheduler comparison succeeded"
-fi
 
-PGCMPINPUT1=./pgdumpv1_0_1/AerieSequencingV1_0_1 PGCMPINPUT2=./pgdumpmigrateddown/AerieSequencingMigratedDown PGCLABEL1=AerieSequencingV1_0_1 PGCLABEL2=AerieSequencingMigratedDown PGCFULLOUTPUT=./comparison/fulloutputSequencing.txt PGCUNEXPLAINED=./comparison/unexplainedSequencing.txt PGCBADEXPLAIN=./comparison/badexplanationsSequencing.txt PGDB=postgres PGBINDIR=/usr/bin PGCEXPLANATIONS=./explanations ./pgcmp
-retcode=$?
-if [ $retcode -gt $return_code ]; then
-return_code=$retcode
-fi
-
-if [ $retcode -ne 0 ]; then
-    echo "AerieSequencing comparison failed - return code=$retcode"
-    touch results/fulloutput
-    touch results/unexplained
-    touch results/badexplanations
-    touch results/perform-comparison.log
-    printf "\n==================\nAerieSequencing Results\n==================\n" >> results/fulloutput
-    printf "\n==================\nAerieSequencing Results\n==================\n" >> results/unexplained
-    printf "\n==================\nAerieSequencing Results\n==================\n" >> results/badexplanations
-    printf "\n==================\nAerieSequencing Results\n==================\n" >> results/perform-comparison.log
-    cat ./comparison/fulloutputSequencing.txt >> results/fulloutput
-    cat ./comparison/unexplainedSequencing.txt >> results/unexplained
-    cat ./comparison/badexplanationsSequencing.txt >> results/badexplanations
-    cat /tmp/perform-comparison.log >> results/perform-comparison.log
-else
-    echo "AerieSequencing comparison succeeded"
-fi
-
-PGCMPINPUT1=./pgdumpv1_0_1/AerieUIV1_0_1 PGCMPINPUT2=./pgdumpmigrateddown/AerieUIMigratedDown PGCLABEL1=AerieUIV1_0_1 PGCLABEL2=AerieUIMigratedDown PGCFULLOUTPUT=./comparison/fulloutputUI.txt PGCUNEXPLAINED=./comparison/unexplainedUI.txt PGCBADEXPLAIN=./comparison/badexplanationsUI.txt PGDB=postgres PGBINDIR=/usr/bin PGCEXPLANATIONS=./explanations ./pgcmp
-retcode=$?
-if [ $retcode -gt $return_code ]; then
-return_code=$retcode
-fi
-
-if [ $retcode -ne 0 ]; then
-    echo "AerieUI comparison failed - return code=$retcode"
-    touch results/fulloutput
-    touch results/unexplained
-    touch results/badexplanations
-    touch results/perform-comparison.log
-    printf "\n==================\nAerieUI Results\n==================\n" >> results/fulloutput
-    printf "\n==================\nAerieUI Results\n==================\n" >> results/unexplained
-    printf "\n==================\nAerieUI Results\n==================\n" >> results/badexplanations
-    printf "\n==================\nAerieUI Results\n==================\n" >> results/perform-comparison.log
-    cat ./comparison/fulloutputUI.txt >> results/fulloutput
-    cat ./comparison/unexplainedUI.txt >> results/unexplained
-    cat ./comparison/badexplanationsUI.txt >> results/badexplanations
-    cat /tmp/perform-comparison.log >> results/perform-comparison.log
+if [ $return_code -ne 0 ]; then
+  echo "Database schema comparison failed - return code=$return_code"
+  mv ./comparison/fulloutput.txt results/fulloutput
+  mv ./comparison/unexplained.txt results/unexplained
+  mv ./comparison/badexplanations.txt results/badexplanations
+  mv /tmp/perform-comparison.log results/perform-comparison.log
 else
-    echo "AerieUI comparison succeeded"
+  echo "Database schema comparison succeeded"
 fi
 
 exit $return_code
diff --git a/.github/scripts/compareDatabasesUp.sh b/.github/scripts/compareDatabasesUp.sh
@@ -3,96 +3,27 @@
 mkdir results
 mkdir comparison
 
-PGCMPINPUT1=./pgdumpmigrated/AerieMerlinMigrated PGCMPINPUT2=./pgdumpraw/AerieMerlinRaw PGCLABEL1=AerieMerlinMigrated PGCLABEL2=AerieMerlinRaw PGCFULLOUTPUT=./comparison/fulloutputMerlin.txt PGCUNEXPLAINED=./comparison/unexplainedMerlin.txt PGCBADEXPLAIN=./comparison/badexplanationsMerlin.txt PGDB=postgres PGBINDIR=/usr/bin ./pgcmp
+PGCMPINPUT1=./pgdumpmigrated/AerieMigrated \
+PGCMPINPUT2=./pgdumpraw/AerieRaw \
+PGCLABEL1=AerieMigrated \
+PGCLABEL2=AerieRaw \
+PGCFULLOUTPUT=./comparison/fulloutput.txt \
+PGCUNEXPLAINED=./comparison/unexplained.txt \
+PGCBADEXPLAIN=./comparison/badexplanations.txt \
+PGDB=postgres \
+PGBINDIR=/usr/bin \
+PGCOMITSCHEMAS="('hdb_catalog'),('pg_catalog'),('information_schema')" \
+./pgcmp
 return_code=$?
-if [ $return_code -ne 0 ]; then
-    echo "AerieMerlin comparison failed - return code=$return_code"
-    touch results/fulloutput
-    touch results/unexplained
-    touch results/badexplanations
-    touch results/perform-comparison.log
-    printf "==================\nAerieMerlin Results\n==================\n" >> results/fulloutput
-    printf "==================\nAerieMerlin Results\n==================\n" >> results/unexplained
-    printf "==================\nAerieMerlin Results\n==================\n" >> results/badexplanations
-    printf "==================\nAerieMerlin Results\n==================\n" >> results/perform-comparison.log
-    cat ./comparison/fulloutputMerlin.txt >> results/fulloutput
-    cat ./comparison/unexplainedMerlin.txt >> results/unexplained
-    cat ./comparison/badexplanationsMerlin.txt >> results/badexplanations
-    cat /tmp/perform-comparison.log >> results/perform-comparison.log
-else
-    echo "AerieMerlin comparison succeeded"
-fi
-
-PGCMPINPUT1=./pgdumpmigrated/AerieSchedulerMigrated PGCMPINPUT2=./pgdumpraw/AerieSchedulerRaw PGCLABEL1=AerieSchedulerMigrated PGCLABEL2=AerieSchedulerRaw PGCFULLOUTPUT=./comparison/fulloutputScheduler.txt PGCUNEXPLAINED=./comparison/unexplainedScheduler.txt PGCBADEXPLAIN=./comparison/badexplanationsScheduler.txt PGDB=postgres PGBINDIR=/usr/bin ./pgcmp
-retcode=$?
-if [ $retcode -gt $return_code ]; then
-return_code=$retcode
-fi
-
-if [ $retcode -ne 0 ]; then
-    echo "AerieScheduler comparison failed - return code=$retcode"
-    touch results/fulloutput
-    touch results/unexplained
-    touch results/badexplanations
-    touch results/perform-comparison.log
-    printf "\n==================\nAerieScheduler Results\n==================\n" >> results/fulloutput
-    printf "\n==================\nAerieScheduler Results\n==================\n" >> results/unexplained
-    printf "\n==================\nAerieScheduler Results\n==================\n" >> results/badexplanations
-    printf "\n==================\nAerieScheduler Results\n==================\n" >> results/perform-comparison.log
-    cat ./comparison/fulloutputScheduler.txt >> results/fulloutput
-    cat ./comparison/unexplainedScheduler.txt >> results/unexplained
-    cat ./comparison/badexplanationsScheduler.txt >> results/badexplanations
-    cat /tmp/perform-comparison.log >> results/perform-comparison.log
-else
-    echo "AerieScheduler comparison succeeded"
-fi
 
-PGCMPINPUT1=./pgdumpmigrated/AerieSequencingMigrated PGCMPINPUT2=./pgdumpraw/AerieSequencingRaw PGCLABEL1=AerieSequencingMigrated PGCLABEL2=AerieSequencingRaw PGCFULLOUTPUT=./comparison/fulloutputSequencing.txt PGCUNEXPLAINED=./comparison/unexplainedSequencing.txt PGCBADEXPLAIN=./comparison/badexplanationsSequencing.txt PGDB=postgres PGBINDIR=/usr/bin ./pgcmp
-retcode=$?
-if [ $retcode -gt $return_code ]; then
-return_code=$retcode
-fi
-
-if [ $retcode -ne 0 ]; then
-    echo "AerieSequencing comparison failed - return code=$retcode"
-    touch results/fulloutput
-    touch results/unexplained
-    touch results/badexplanations
-    touch results/perform-comparison.log
-    printf "\n==================\nAerieSequencing Results\n==================\n" >> results/fulloutput
-    printf "\n==================\nAerieSequencing Results\n==================\n" >> results/unexplained
-    printf "\n==================\nAerieSequencing Results\n==================\n" >> results/badexplanations
-    printf "\n==================\nAerieSequencing Results\n==================\n" >> results/perform-comparison.log
-    cat ./comparison/fulloutputSequencing.txt >> results/fulloutput
-    cat ./comparison/unexplainedSequencing.txt >> results/unexplained
-    cat ./comparison/badexplanationsSequencing.txt >> results/badexplanations
-    cat /tmp/perform-comparison.log >> results/perform-comparison.log
-else
-    echo "AerieSequencing comparison succeeded"
-fi
-
-PGCMPINPUT1=./pgdumpmigrated/AerieUIMigrated PGCMPINPUT2=./pgdumpraw/AerieUIRaw PGCLABEL1=AerieUIMigrated PGCLABEL2=AerieUIRaw PGCFULLOUTPUT=./comparison/fulloutputUI.txt PGCUNEXPLAINED=./comparison/unexplainedUI.txt PGCBADEXPLAIN=./comparison/badexplanationsUI.txt PGDB=postgres PGBINDIR=/usr/bin ./pgcmp
-retcode=$?
-if [ $retcode -gt $return_code ]; then
-return_code=$retcode
-fi
-
-if [ $retcode -ne 0 ]; then
-    echo "AerieUI comparison failed - return code=$retcode"
-    touch results/fulloutput
-    touch results/unexplained
-    touch results/badexplanations
-    touch results/perform-comparison.log
-    printf "\n==================\nAerieUI Results\n==================\n" >> results/fulloutput
-    printf "\n==================\nAerieUI Results\n==================\n" >> results/unexplained
-    printf "\n==================\nAerieUI Results\n==================\n" >> results/badexplanations
-    printf "\n==================\nAerieUI Results\n==================\n" >> results/perform-comparison.log
-    cat ./comparison/fulloutputUI.txt >> results/fulloutput
-    cat ./comparison/unexplainedUI.txt >> results/unexplained
-    cat ./comparison/badexplanationsUI.txt >> results/badexplanations
-    cat /tmp/perform-comparison.log >> results/perform-comparison.log
+if [ $return_code -ne 0 ]; then
+  echo "Database schema comparison failed - return code=$return_code"
+  mv ./comparison/fulloutput.txt results/fulloutput
+  mv ./comparison/unexplained.txt results/unexplained
+  mv ./comparison/badexplanations.txt results/badexplanations
+  mv /tmp/perform-comparison.log results/perform-comparison.log
 else
-    echo "AerieUI comparison succeeded"
+  echo "Database schema comparison succeeded"
 fi
 
 exit $return_code
diff --git a/.github/scripts/explanations b/.github/scripts/explanations
diff --git a/.github/scripts/explanations_merlin_down b/.github/scripts/explanations_merlin_down
diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml
@@ -10,6 +10,14 @@ env:
   HASURA_GRAPHQL_JWT_SECRET: '{ "type": "HS256", "key": "oursupersecretsupersecurekey1234567890" }'
   POSTGRES_USER: "postgres"
   POSTGRES_PASSWORD: "postgres"
+  GATEWAY_USERNAME: "${{secrets.GATEWAY_USERNAME}}"
+  GATEWAY_PASSWORD: "${{secrets.GATEWAY_PASSWORD}}"
+  MERLIN_USERNAME: "${{secrets.MERLIN_USERNAME}}"
+  MERLIN_PASSWORD: "${{secrets.MERLIN_PASSWORD}}"
+  SCHEDULER_USERNAME: "${{secrets.SCHEDULER_USERNAME}}"
+  SCHEDULER_PASSWORD: "${{secrets.SCHEDULER_PASSWORD}}"
+  SEQUENCING_USERNAME: "${{secrets.SEQUENCING_USERNAME}}"
+  SEQUENCING_PASSWORD: "${{secrets.SEQUENCING_PASSWORD}}"
 
 jobs:
   load-test: