diff --git a/.config/commands/docker.justfile b/.config/commands/docker.justfile index bec8465f9..73b292aab 100644 --- a/.config/commands/docker.justfile +++ b/.config/commands/docker.justfile @@ -2,17 +2,21 @@ help: @just --list --justfile {{source_file()}} -# Starts the dev docker containers -@up *args: +# Starts the dev containers (assumes a valid database) +up *args: #!/usr/bin/env bash + just docker ensure-db-container-running-and-postgres-ready + just docker create-empty-mampf-db-if-not-exists + cd {{justfile_directory()}}/docker/development/ docker compose up {{args}} # Starts the dev docker containers (detached) & shows MaMpf logs up-logs *args: #!/usr/bin/env bash + just docker up -d {{args}} + cd {{justfile_directory()}}/docker/development/ - docker compose up -d {{args}} docker compose logs -f mampf # Shows the log of the specified container @@ -25,10 +29,93 @@ up-logs *args: [confirm("This will reset all your data in the database locally. Continue? (y/n)")] up-reseed *args: #!/usr/bin/env bash + # (pgadmin issue: https://github.com/pgadmin-org/pgadmin4/issues/8071) + + set -e + just --yes docker db-tear-down + cd {{justfile_directory()}}/docker/development/ export DB_SQL_PRESEED_URL="https://github.com/MaMpf-HD/mampf-init-data/raw/main/data/20220923120841_mampf.sql" export UPLOADS_PRESEED_URL="https://github.com/MaMpf-HD/mampf-init-data/raw/main/data/uploads.zip" - docker compose rm --stop --force mampf && docker compose up {{args}} + docker compose rm --stop --force mampf && just docker up {{args}} + +# Starts the dev docker containers and preseeds the database from an .sql file +[confirm("This will reset all your data in the database locally. Continue? (y/n)")] +up-reseed-from-file preseed_file *args: + #!/usr/bin/env bash + if [[ {{preseed_file}} != *.sql ]]; then + echo "The file must be an .sql file." + exit 1 + fi + + cd {{justfile_directory()}}/docker/development/ + export DB_SQL_PRESEED_URL="{{preseed_file}}" + export UPLOADS_PRESEED_URL="" + docker compose rm --stop --force mampf && just docker up {{args}} + +# Restores a postgres backup file that was made using pg_dump or pg_dumpall +[no-cd] +up-reseed-from-dump preseed_file: + #!/usr/bin/env bash + set -e + just docker db-tear-down + + # If file is gzipped, unzip it + if [[ {{preseed_file}} == *.gz ]]; then + unzipped=$(echo {{preseed_file}} | sed 's/\.gz$//') + if [[ -f $unzipped ]]; then + echo "Using already existing unzipped file $unzipped" + else + echo "Unzipping {{preseed_file}}" + gunzip {{preseed_file}} + fi + else + unzipped={{preseed_file}} + fi + + if [[ $unzipped != *.pg_dump ]]; then + echo "The file must be a .pg_dump file." + exit 1 + fi + + file=$(realpath ${unzipped}) + echo "Will restore database from $file" + echo -n "Is this correct? (y/n) " + read confirmation + if [ "$confirmation" != "y" ]; then + exit 1 + fi + + just docker ensure-db-container-running-and-postgres-ready + + echo "Copy file over to docker container" + docker compose cp ${file} db:/tmp/backup.pg_dump + + # This is necessary because somehow the last line is not properly read, + # probably due to some missing newline character + # As the last line is really not that important, we just get rid of it + echo "Removing last line from dump" + docker compose exec -T db bash -c "head -n -1 /tmp/backup.pg_dump > /tmp/backup.pg_dump.tmp && mv /tmp/backup.pg_dump.tmp /tmp/backup.pg_dump" + + echo "Restoring database from dump" + # ON_ERROR_STOP=1 because of "\N" errors, see https://stackoverflow.com/questions/20427689/psql-invalid-command-n-while-restore-sql#comment38644877_20428547 + docker compose exec -T db bash -c "psql -v ON_ERROR_STOP=1 -h localhost -p 5432 -U localroot -f /tmp/backup.pg_dump" + + echo "Restarting containers" + just docker stop + just docker up -d + +# Removes all database data in the db docker container +[confirm("This will completely destroy your local database, including all tables and users. Continue? (y/n)")] +db-tear-down: + #!/usr/bin/env bash + just docker ensure-db-container-running + + cd {{justfile_directory()}}/docker/development/ + docker compose exec -T db bash -c "rm -rf /var/lib/postgresql/data/*" + docker-compose up -d --force-recreate --build db + + just docker wait-for-postgres # Removes the development docker containers @down: @@ -55,10 +142,10 @@ up-reseed *args: docker compose exec -it {{name}} {{shell}} # Puts you into the rails console of the dev docker mampf container -@rails-c: +@rails-c *args: #!/usr/bin/env bash cd {{justfile_directory()}}/docker/development/ - docker compose exec mampf bundle exec rails c + docker compose exec mampf bundle exec rails c {{args}} # Rebuilds the most essential containers in the dev or test environment rebuild env="dev": @@ -78,3 +165,61 @@ rebuild env="dev": if [ "$environment" = "development" ]; then docker compose build webpacker fi + +# Creates an empty mampf db (assumes that the db container is running) +[private] +create-empty-mampf-db-if-not-exists: + #!/usr/bin/env bash + set -e + cd {{justfile_directory()}}/docker/development/ + + # Early return if mampf db already exists + if [ "$(docker compose exec -T db bash -c "psql -h localhost -U localroot -XtA -c \"SELECT 1 FROM pg_database WHERE datname='mampf'\"")" == "1" ]; then + exit 0 + fi + + # Create an empty mampf database. This is necessary since the default user + # is called localroot and not mampf. (It it were called mampf, postgresql + # would created the mampf db automatically.) + # see also https://stackoverflow.com/a/68091072/ + docker compose exec -T db bash -c "psql -v ON_ERROR_STOP=1 -h localhost -p 5432 -U localroot -c 'CREATE DATABASE mampf;'" + + # Make sure mampf db exists now + if [ "$(docker compose exec -T db bash -c "psql -h localhost -U localroot -XtA -c \"SELECT 1 FROM pg_database WHERE datname='mampf'\"")" != "1" ]; then + echo "Could not create the empty mampf database. Exiting." + exit 1 + fi + +# Waits for postgres to be available (assumes that the db container is running) +[private] +wait-for-postgres: + #!/usr/bin/env bash + set -e + cd {{justfile_directory()}}/docker/development/ + + # https://stackoverflow.com/a/77582897/ + # Wait for the db container to be up + until docker compose exec -T db bash -c "pg_isready -h localhost -U localroot"; do + >&2 echo "Postgres is unavailable (waiting...)" + sleep 1 + done + >&2 echo "Postgres is up, will continue" + +# Ensures that the db container is running (does not wait for postgres; starts the db container if not running) +[private] +ensure-db-container-running: + #!/usr/bin/env bash + cd {{justfile_directory()}}/docker/development/ + + # If the db container is not running, start it first + if [ -z "$(docker compose ps --services --filter 'status=running' | grep db)" ]; then + docker compose up -d db + fi + + +# Ensures that the db container is running and postgres is ready (if not, starts the db container and waits for postgres) +[private] +ensure-db-container-running-and-postgres-ready: + #!/usr/bin/env bash + just docker ensure-db-container-running + just docker wait-for-postgres diff --git a/.config/commands/prod.justfile b/.config/commands/prod.justfile new file mode 100644 index 000000000..fec82f587 --- /dev/null +++ b/.config/commands/prod.justfile @@ -0,0 +1,40 @@ +[private] +help: + @just --list --justfile {{source_file()}} + +# Downloads the latest database dump from the production server +download-db-dump: + #!/usr/bin/env bash + set -e + + # User input: proxy jump + echo "To connect to the remote server you might need a proxy jump. Enter the host name (or leave empty):" + read proxy_jump_destination + proxy_jump_cmd="-J $proxy_jump_destination" + + # User input for remote server & dump folder + echo "Enter the remote user and host in the format user@host " + read remote_user_host + echo "Enter the path to the folder that contains the database dumps on the remote server, e.g. /a/b/db" + read remote_dump_folder + + # Latest file + latest_file=$(ssh $proxy_jump_cmd "$remote_user_host" "ls -t $remote_dump_folder | head -n 1") + if [ -z "$latest_file" ]; then + echo "No files found in the remote folder." + exit 1 + fi + echo "" + echo "Latest file found: $latest_file" + + # Download file + echo "We will now download this file to the local machine into the folder db/backups/prod/" + echo -n "Are you sure you want to continue? (y/n) " + read confirmation + if [ "$confirmation" != "y" ]; then + echo "Operation cancelled." + exit 1 + fi + local_dir={{justfile_directory()}}/db/backups/prod/ + mkdir -p "$local_dir" + scp -C $proxy_jump_cmd "$remote_user_host:$remote_dump_folder/$latest_file" "$local_dir" diff --git a/.justfile b/.justfile index fe8e56d53..478999961 100644 --- a/.justfile +++ b/.justfile @@ -18,6 +18,9 @@ mod deps ".config/commands/deps.justfile" # Some utils, e.g. ERD-generation etc. mod utils ".config/commands/utils.justfile" +# Commands to interact with the production server +mod prod ".config/commands/prod.justfile" + # Opens the MaMpf wiki in the default browser wiki: #!/usr/bin/env bash diff --git a/.vscode/settings.json b/.vscode/settings.json index e6c8d5859..c49e2122d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -120,8 +120,16 @@ "factorybot", "helpdesk", "javascripts", + "justfile", "katex", + "localroot", + "mampf", + "pgadmin", + "preseed", + "preseeds", "preselection", + "psql", + "realpath", "selectize", "Timecop", "turbolinks", diff --git a/config/db-pgadmin.json b/config/db-pgadmin.json index 64a214adc..a3b520671 100644 --- a/config/db-pgadmin.json +++ b/config/db-pgadmin.json @@ -6,7 +6,7 @@ "Host": "db", "Port": 5432, "MaintenanceDB": "mampf", - "Username": "mampf" + "Username": "localroot" } } } \ No newline at end of file diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 103ffb436..680653b00 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -23,7 +23,7 @@ services: db: image: postgres:13 environment: - - POSTGRES_USER=mampf + - POSTGRES_USER=localroot - POSTGRES_HOST_AUTH_METHOD=trust volumes: - type: volume @@ -43,6 +43,8 @@ services: # the web interface is required). See "pgadmin4/servers.json" here: # -> https://www.pgadmin.org/docs/pgadmin4/latest/container_deployment.html#mapped-files-and-directories # -> https://forums.docker.com/t/automatically-connect-pgadmin-to-a-postgresql-volume-on-which-there-is-a-database-and-automatically-load-a-schema-present-on-a-sql-file-with-docker-compose/124647/2 + # + # This only loads once, see: https://github.com/pgadmin-org/pgadmin4/issues/8071 - type: bind source: ../../config/db-pgadmin.json target: /pgadmin4/servers.json @@ -75,8 +77,8 @@ services: # config/db-pgadmin-config.json DEVELOPMENT_DATABASE_ADAPTER: postgresql DEVELOPMENT_DATABASE_DATABASE: mampf - DEVELOPMENT_DATABASE_INTERACTIONS: interactions - DEVELOPMENT_DATABASE_USERNAME: mampf + DEVELOPMENT_DATABASE_INTERACTIONS: mampf_interactions + DEVELOPMENT_DATABASE_USERNAME: localroot DEVELOPMENT_DATABASE_HOST: db DEVELOPMENT_DATABASE_PORT: 5432 TEST_DATABASE_ADAPTER: postgresql diff --git a/initialize.sh b/initialize.sh index 567901a29..547c4fb20 100755 --- a/initialize.sh +++ b/initialize.sh @@ -5,12 +5,17 @@ check_for_preseeds() { # Database preseed if [[ "${DB_SQL_PRESEED_URL}" ]]; then - echo "💾 Found DB preseed at URL: $DB_SQL_PRESEED_URL" - mkdir -pv db/backups/docker_development - wget --content-disposition --directory-prefix=db/backups/docker_development/ --timestamping $DB_SQL_PRESEED_URL - for file in db/backups/docker_development/*.sql; do - [[ $file -nt $latest ]] && latest=$file - done + if [[ -f "${DB_SQL_PRESEED_URL}" ]]; then + echo "💾 Found DB preseed file: $DB_SQL_PRESEED_URL" + latest=$DB_SQL_PRESEED_URL + else + echo "💾 Found DB preseed at URL: $DB_SQL_PRESEED_URL" + mkdir -pv db/backups/docker_development + wget --content-disposition --directory-prefix=db/backups/docker_development/ --timestamping $DB_SQL_PRESEED_URL + for file in db/backups/docker_development/*.sql; do + [[ $file -nt $latest ]] && latest=$file + done + fi bundle exec rails db:restore pattern=$(echo $latest | rev | cut -d "/" -f1 | rev | cut -d "_" -f1) bundle exec rails db:migrate