From eb55e129c8f5c6329559af39316fb16df2770bc5 Mon Sep 17 00:00:00 2001 From: Splines Date: Sat, 28 Dec 2024 00:44:57 +0100 Subject: [PATCH 1/8] Init first attempt to download latest database dump --- .config/commands/docker.justfile | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.config/commands/docker.justfile b/.config/commands/docker.justfile index bec8465f9..84b41bfc4 100644 --- a/.config/commands/docker.justfile +++ b/.config/commands/docker.justfile @@ -30,6 +30,40 @@ up-reseed *args: export UPLOADS_PRESEED_URL="https://github.com/MaMpf-HD/mampf-init-data/raw/main/data/uploads.zip" docker compose rm --stop --force mampf && docker compose up {{args}} +# Downloads the latest database dump from the production server and reseeds the local database with it +[confirm("This will reset all your data in the database locally. Continue? (y/n)")] +up-reseed-prod *args: + #!/usr/bin/env bash + # Get to the folder with the database dump + echo "Please enter the SSH command to log in to the MaMpf server and cd into the folder with the database dump (e.g. ssh ... && cd ...):" + read get_to_dump_folder_command + echo "This is the command you entered:" + echo $get_to_dump_folder_command + echo -n "Are you sure you want to execute that command (y/n)" + read confirmation + if [ "$confirmation" != "y" ]; then + echo "Operation cancelled." + exit 1 + fi + $get_to_dump_folder_command + + # Find the latest file in the folder + latest_file=$(ls -t | head -n 1) + echo "Latest file found: $latest_file" + + # Download the file to the local machine + echo "We will now execute the following command to download the file:" + download_command = "scp ./path/to/dump/folder/$latest_file ./local/path/" + echo $download_command + echo -n "Are you sure you want to continue (y/n)" + read confirmation + if [ "$confirmation" != "y" ]; then + echo "Operation cancelled." + exit 1 + fi + $download_command + + # Removes the development docker containers @down: #!/usr/bin/env bash From e1ad9a9ae07cba291c114e3e4adf01ec7120e1bb Mon Sep 17 00:00:00 2001 From: Splines Date: Sat, 28 Dec 2024 01:20:04 +0100 Subject: [PATCH 2/8] Overhaul just command to download the latest database dump --- .config/commands/docker.justfile | 45 +++++++++++++++++--------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/.config/commands/docker.justfile b/.config/commands/docker.justfile index 84b41bfc4..5ae0b3397 100644 --- a/.config/commands/docker.justfile +++ b/.config/commands/docker.justfile @@ -31,38 +31,41 @@ up-reseed *args: docker compose rm --stop --force mampf && docker compose up {{args}} # Downloads the latest database dump from the production server and reseeds the local database with it -[confirm("This will reset all your data in the database locally. Continue? (y/n)")] up-reseed-prod *args: #!/usr/bin/env bash - # Get to the folder with the database dump - echo "Please enter the SSH command to log in to the MaMpf server and cd into the folder with the database dump (e.g. ssh ... && cd ...):" - read get_to_dump_folder_command - echo "This is the command you entered:" - echo $get_to_dump_folder_command - echo -n "Are you sure you want to execute that command (y/n)" - read confirmation - if [ "$confirmation" != "y" ]; then - echo "Operation cancelled." + set -e + + # User input: proxy jump + echo "To connect to the remote server you might need a proxy jump. Enter the host name (or leave empty):" + read proxy_jump_destination + proxy_jump_cmd="-J $proxy_jump_destination" + + # User input for remote server & dump folder + echo "Enter the remote user and host in the format user@host " + read remote_user_host + echo "Enter the path to the folder that contains the database dumps on the remote server, e.g. /a/b/db" + read remote_dump_folder + + # Latest file + latest_file=$(ssh $proxy_jump_cmd "$remote_user_host" "ls -t $remote_dump_folder | head -n 1") + if [ -z "$latest_file" ]; then + echo "No files found in the remote folder." exit 1 fi - $get_to_dump_folder_command - - # Find the latest file in the folder - latest_file=$(ls -t | head -n 1) + echo "" echo "Latest file found: $latest_file" - # Download the file to the local machine - echo "We will now execute the following command to download the file:" - download_command = "scp ./path/to/dump/folder/$latest_file ./local/path/" - echo $download_command - echo -n "Are you sure you want to continue (y/n)" + # Download file + echo "We will now download this file to the local machine into the folder tmp/db/." + echo -n "Are you sure you want to continue (y/n) " read confirmation if [ "$confirmation" != "y" ]; then echo "Operation cancelled." exit 1 fi - $download_command - + local_dir={{justfile_directory()}}/tmp/db/ + mkdir -p "$local_dir" + scp -C $proxy_jump_cmd "$remote_user_host:$remote_dump_folder/$latest_file" "$local_dir" # Removes the development docker containers @down: From da08beeb72379ab88b0ac558da985836adcec324 Mon Sep 17 00:00:00 2001 From: Splines Date: Sat, 28 Dec 2024 01:23:00 +0100 Subject: [PATCH 3/8] Repurpose the download command to just download the dump --- .config/commands/docker.justfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/commands/docker.justfile b/.config/commands/docker.justfile index 5ae0b3397..ccb24638c 100644 --- a/.config/commands/docker.justfile +++ b/.config/commands/docker.justfile @@ -30,8 +30,8 @@ up-reseed *args: export UPLOADS_PRESEED_URL="https://github.com/MaMpf-HD/mampf-init-data/raw/main/data/uploads.zip" docker compose rm --stop --force mampf && docker compose up {{args}} -# Downloads the latest database dump from the production server and reseeds the local database with it -up-reseed-prod *args: +# Downloads the latest database dump from the production server +download-db-dump: #!/usr/bin/env bash set -e From 60d5a7f5bef0440d7bbf865b70bc6e4bf49dd1bb Mon Sep 17 00:00:00 2001 From: Splines Date: Sat, 28 Dec 2024 01:31:39 +0100 Subject: [PATCH 4/8] Use existing /db/backups/ folder for storage --- .config/commands/docker.justfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/commands/docker.justfile b/.config/commands/docker.justfile index ccb24638c..6dda8f9c6 100644 --- a/.config/commands/docker.justfile +++ b/.config/commands/docker.justfile @@ -56,14 +56,14 @@ download-db-dump: echo "Latest file found: $latest_file" # Download file - echo "We will now download this file to the local machine into the folder tmp/db/." + echo "We will now download this file to the local machine into the folder db/backups/prod/." echo -n "Are you sure you want to continue (y/n) " read confirmation if [ "$confirmation" != "y" ]; then echo "Operation cancelled." exit 1 fi - local_dir={{justfile_directory()}}/tmp/db/ + local_dir={{justfile_directory()}}/db/backups/prod/ mkdir -p "$local_dir" scp -C $proxy_jump_cmd "$remote_user_host:$remote_dump_folder/$latest_file" "$local_dir" From 22bab998160c8d726ae476c2fc59b3d3ea9688d4 Mon Sep 17 00:00:00 2001 From: Splines Date: Sat, 28 Dec 2024 01:33:04 +0100 Subject: [PATCH 5/8] Init dummy reseed from file command --- .config/commands/docker.justfile | 9 +++++++++ .vscode/settings.json | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.config/commands/docker.justfile b/.config/commands/docker.justfile index 6dda8f9c6..9c166977d 100644 --- a/.config/commands/docker.justfile +++ b/.config/commands/docker.justfile @@ -30,6 +30,15 @@ up-reseed *args: export UPLOADS_PRESEED_URL="https://github.com/MaMpf-HD/mampf-init-data/raw/main/data/uploads.zip" docker compose rm --stop --force mampf && docker compose up {{args}} +# Starts the dev docker containers and preseeds the database from a local file +[confirm("This will reset all your data in the database locally. Continue? (y/n)")] +up-reseed-from-file preseed_file *args: + #!/usr/bin/env bash + cd {{justfile_directory()}}/docker/development/ + export DB_SQL_PRESEED_URL="$preseed_file" + export UPLOADS_PRESEED_URL="" + docker compose rm --stop --force mampf && docker compose up {{args}} + # Downloads the latest database dump from the production server download-db-dump: #!/usr/bin/env bash diff --git a/.vscode/settings.json b/.vscode/settings.json index 40c9d2aeb..d21a20098 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -118,12 +118,13 @@ "factorybot", "helpdesk", "katex", + "preseeds", "preselection", "selectize", "Timecop", "turbolinks", - "Unsets", "uncached", + "Unsets", "whitespaces" ], "cSpell.enableFiletypes": [ From 016b58f0ee5e9bb238ced75fcb524d6543f005ac Mon Sep 17 00:00:00 2001 From: Splines Date: Sat, 28 Dec 2024 02:17:28 +0100 Subject: [PATCH 6/8] Check if preseed is url or file --- .config/commands/docker.justfile | 2 +- .vscode/settings.json | 2 ++ initialize.sh | 17 +++++++++++------ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.config/commands/docker.justfile b/.config/commands/docker.justfile index 9c166977d..72e8e9f10 100644 --- a/.config/commands/docker.justfile +++ b/.config/commands/docker.justfile @@ -35,7 +35,7 @@ up-reseed *args: up-reseed-from-file preseed_file *args: #!/usr/bin/env bash cd {{justfile_directory()}}/docker/development/ - export DB_SQL_PRESEED_URL="$preseed_file" + export DB_SQL_PRESEED_URL="{{preseed_file}}" export UPLOADS_PRESEED_URL="" docker compose rm --stop --force mampf && docker compose up {{args}} diff --git a/.vscode/settings.json b/.vscode/settings.json index d21a20098..9c52f05cf 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -118,6 +118,8 @@ "factorybot", "helpdesk", "katex", + "mampf", + "preseed", "preseeds", "preselection", "selectize", diff --git a/initialize.sh b/initialize.sh index 567901a29..547c4fb20 100755 --- a/initialize.sh +++ b/initialize.sh @@ -5,12 +5,17 @@ check_for_preseeds() { # Database preseed if [[ "${DB_SQL_PRESEED_URL}" ]]; then - echo "💾 Found DB preseed at URL: $DB_SQL_PRESEED_URL" - mkdir -pv db/backups/docker_development - wget --content-disposition --directory-prefix=db/backups/docker_development/ --timestamping $DB_SQL_PRESEED_URL - for file in db/backups/docker_development/*.sql; do - [[ $file -nt $latest ]] && latest=$file - done + if [[ -f "${DB_SQL_PRESEED_URL}" ]]; then + echo "💾 Found DB preseed file: $DB_SQL_PRESEED_URL" + latest=$DB_SQL_PRESEED_URL + else + echo "💾 Found DB preseed at URL: $DB_SQL_PRESEED_URL" + mkdir -pv db/backups/docker_development + wget --content-disposition --directory-prefix=db/backups/docker_development/ --timestamping $DB_SQL_PRESEED_URL + for file in db/backups/docker_development/*.sql; do + [[ $file -nt $latest ]] && latest=$file + done + fi bundle exec rails db:restore pattern=$(echo $latest | rev | cut -d "/" -f1 | rev | cut -d "_" -f1) bundle exec rails db:migrate From a1ef60720ce18e413e0de3926a65cb7859bb32e4 Mon Sep 17 00:00:00 2001 From: Splines Date: Sat, 28 Dec 2024 02:20:15 +0100 Subject: [PATCH 7/8] Move download db command to new `prod.justfile` --- .config/commands/docker.justfile | 37 ----------------------------- .config/commands/prod.justfile | 40 ++++++++++++++++++++++++++++++++ .justfile | 3 +++ 3 files changed, 43 insertions(+), 37 deletions(-) create mode 100644 .config/commands/prod.justfile diff --git a/.config/commands/docker.justfile b/.config/commands/docker.justfile index 72e8e9f10..62e3e55c8 100644 --- a/.config/commands/docker.justfile +++ b/.config/commands/docker.justfile @@ -39,43 +39,6 @@ up-reseed-from-file preseed_file *args: export UPLOADS_PRESEED_URL="" docker compose rm --stop --force mampf && docker compose up {{args}} -# Downloads the latest database dump from the production server -download-db-dump: - #!/usr/bin/env bash - set -e - - # User input: proxy jump - echo "To connect to the remote server you might need a proxy jump. Enter the host name (or leave empty):" - read proxy_jump_destination - proxy_jump_cmd="-J $proxy_jump_destination" - - # User input for remote server & dump folder - echo "Enter the remote user and host in the format user@host " - read remote_user_host - echo "Enter the path to the folder that contains the database dumps on the remote server, e.g. /a/b/db" - read remote_dump_folder - - # Latest file - latest_file=$(ssh $proxy_jump_cmd "$remote_user_host" "ls -t $remote_dump_folder | head -n 1") - if [ -z "$latest_file" ]; then - echo "No files found in the remote folder." - exit 1 - fi - echo "" - echo "Latest file found: $latest_file" - - # Download file - echo "We will now download this file to the local machine into the folder db/backups/prod/." - echo -n "Are you sure you want to continue (y/n) " - read confirmation - if [ "$confirmation" != "y" ]; then - echo "Operation cancelled." - exit 1 - fi - local_dir={{justfile_directory()}}/db/backups/prod/ - mkdir -p "$local_dir" - scp -C $proxy_jump_cmd "$remote_user_host:$remote_dump_folder/$latest_file" "$local_dir" - # Removes the development docker containers @down: #!/usr/bin/env bash diff --git a/.config/commands/prod.justfile b/.config/commands/prod.justfile new file mode 100644 index 000000000..a15462ae7 --- /dev/null +++ b/.config/commands/prod.justfile @@ -0,0 +1,40 @@ +[private] +help: + @just --list --justfile {{source_file()}} + +# Downloads the latest database dump from the production server +download-db-dump: + #!/usr/bin/env bash + set -e + + # User input: proxy jump + echo "To connect to the remote server you might need a proxy jump. Enter the host name (or leave empty):" + read proxy_jump_destination + proxy_jump_cmd="-J $proxy_jump_destination" + + # User input for remote server & dump folder + echo "Enter the remote user and host in the format user@host " + read remote_user_host + echo "Enter the path to the folder that contains the database dumps on the remote server, e.g. /a/b/db" + read remote_dump_folder + + # Latest file + latest_file=$(ssh $proxy_jump_cmd "$remote_user_host" "ls -t $remote_dump_folder | head -n 1") + if [ -z "$latest_file" ]; then + echo "No files found in the remote folder." + exit 1 + fi + echo "" + echo "Latest file found: $latest_file" + + # Download file + echo "We will now download this file to the local machine into the folder db/backups/prod/." + echo -n "Are you sure you want to continue (y/n) " + read confirmation + if [ "$confirmation" != "y" ]; then + echo "Operation cancelled." + exit 1 + fi + local_dir={{justfile_directory()}}/db/backups/prod/ + mkdir -p "$local_dir" + scp -C $proxy_jump_cmd "$remote_user_host:$remote_dump_folder/$latest_file" "$local_dir" diff --git a/.justfile b/.justfile index fe8e56d53..478999961 100644 --- a/.justfile +++ b/.justfile @@ -18,6 +18,9 @@ mod deps ".config/commands/deps.justfile" # Some utils, e.g. ERD-generation etc. mod utils ".config/commands/utils.justfile" +# Commands to interact with the production server +mod prod ".config/commands/prod.justfile" + # Opens the MaMpf wiki in the default browser wiki: #!/usr/bin/env bash From c5238445e622da82611128ab7a9ff04cad22ae3b Mon Sep 17 00:00:00 2001 From: Splines Date: Sat, 28 Dec 2024 03:01:38 +0100 Subject: [PATCH 8/8] Fix punctuation --- .config/commands/prod.justfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/commands/prod.justfile b/.config/commands/prod.justfile index a15462ae7..fec82f587 100644 --- a/.config/commands/prod.justfile +++ b/.config/commands/prod.justfile @@ -28,8 +28,8 @@ download-db-dump: echo "Latest file found: $latest_file" # Download file - echo "We will now download this file to the local machine into the folder db/backups/prod/." - echo -n "Are you sure you want to continue (y/n) " + echo "We will now download this file to the local machine into the folder db/backups/prod/" + echo -n "Are you sure you want to continue? (y/n) " read confirmation if [ "$confirmation" != "y" ]; then echo "Operation cancelled."