From 5c5f8db05d0359a448d123dcce039ecaf1a76cdd Mon Sep 17 00:00:00 2001 From: Dark Decoy Date: Mon, 6 Mar 2023 20:42:36 +0000 Subject: [PATCH 01/10] Rewriting script to take advantage of functions and for loops --- backup.sh | 349 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 221 insertions(+), 128 deletions(-) diff --git a/backup.sh b/backup.sh index ac9fd21..8aeb7e6 100755 --- a/backup.sh +++ b/backup.sh @@ -1,66 +1,5 @@ #!/usr/bin/env bash -set -Eeo pipefail - -HOOKS_DIR="/hooks" -if [ -d "${HOOKS_DIR}" ]; then - on_error(){ - run-parts -a "error" "${HOOKS_DIR}" - } - trap 'on_error' ERR -fi - -if [ "${POSTGRES_DB}" = "**None**" -a "${POSTGRES_DB_FILE}" = "**None**" ]; then - echo "You need to set the POSTGRES_DB or POSTGRES_DB_FILE environment variable." - exit 1 -fi - -if [ "${POSTGRES_HOST}" = "**None**" ]; then - if [ -n "${POSTGRES_PORT_5432_TCP_ADDR}" ]; then - POSTGRES_HOST=${POSTGRES_PORT_5432_TCP_ADDR} - POSTGRES_PORT=${POSTGRES_PORT_5432_TCP_PORT} - else - echo "You need to set the POSTGRES_HOST environment variable." - exit 1 - fi -fi - -if [ "${POSTGRES_USER}" = "**None**" -a "${POSTGRES_USER_FILE}" = "**None**" ]; then - echo "You need to set the POSTGRES_USER or POSTGRES_USER_FILE environment variable." - exit 1 -fi - -if [ "${POSTGRES_PASSWORD}" = "**None**" -a "${POSTGRES_PASSWORD_FILE}" = "**None**" -a "${POSTGRES_PASSFILE_STORE}" = "**None**" ]; then - echo "You need to set the POSTGRES_PASSWORD or POSTGRES_PASSWORD_FILE or POSTGRES_PASSFILE_STORE environment variable or link to a container named POSTGRES." - exit 1 -fi - -#Process vars -if [ "${POSTGRES_DB_FILE}" = "**None**" ]; then - POSTGRES_DBS=$(echo "${POSTGRES_DB}" | tr , " ") -elif [ -r "${POSTGRES_DB_FILE}" ]; then - POSTGRES_DBS=$(cat "${POSTGRES_DB_FILE}") -else - echo "Missing POSTGRES_DB_FILE file." - exit 1 -fi -if [ "${POSTGRES_USER_FILE}" = "**None**" ]; then - export PGUSER="${POSTGRES_USER}" -elif [ -r "${POSTGRES_USER_FILE}" ]; then - export PGUSER=$(cat "${POSTGRES_USER_FILE}") -else - echo "Missing POSTGRES_USER_FILE file." - exit 1 -fi -if [ "${POSTGRES_PASSWORD_FILE}" = "**None**" -a "${POSTGRES_PASSFILE_STORE}" = "**None**" ]; then - export PGPASSWORD="${POSTGRES_PASSWORD}" -elif [ -r "${POSTGRES_PASSWORD_FILE}" ]; then - export PGPASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}") -elif [ -r "${POSTGRES_PASSFILE_STORE}" ]; then - export PGPASSFILE="${POSTGRES_PASSFILE_STORE}" -else - echo "Missing POSTGRES_PASSWORD_FILE or POSTGRES_PASSFILE_STORE file." - exit 1 -fi + export PGHOST="${POSTGRES_HOST}" export PGPORT="${POSTGRES_PORT}" KEEP_MINS=${BACKUP_KEEP_MINS} @@ -68,78 +7,232 @@ KEEP_DAYS=${BACKUP_KEEP_DAYS} KEEP_WEEKS=`expr $(((${BACKUP_KEEP_WEEKS} * 7) + 1))` KEEP_MONTHS=`expr $(((${BACKUP_KEEP_MONTHS} * 31) + 1))` -# Pre-backup hook -if [ -d "${HOOKS_DIR}" ]; then - run-parts -a "pre-backup" --exit-on-error "${HOOKS_DIR}" -fi - -#Initialize dirs -mkdir -p "${BACKUP_DIR}/last/" "${BACKUP_DIR}/daily/" "${BACKUP_DIR}/weekly/" "${BACKUP_DIR}/monthly/" - -#Loop all databases -for DB in ${POSTGRES_DBS}; do - #Initialize filename vers - LAST_FILENAME="${DB}-`date +%Y%m%d-%H%M%S`${BACKUP_SUFFIX}" - DAILY_FILENAME="${DB}-`date +%Y%m%d`${BACKUP_SUFFIX}" - WEEKLY_FILENAME="${DB}-`date +%G%V`${BACKUP_SUFFIX}" - MONTHY_FILENAME="${DB}-`date +%Y%m`${BACKUP_SUFFIX}" - FILE="${BACKUP_DIR}/last/${LAST_FILENAME}" - DFILE="${BACKUP_DIR}/daily/${DAILY_FILENAME}" - WFILE="${BACKUP_DIR}/weekly/${WEEKLY_FILENAME}" - MFILE="${BACKUP_DIR}/monthly/${MONTHY_FILENAME}" - #Create dump +setup () { + set -Eeo pipefail + + HOOKS_DIR="/hooks" + if [ -d "${HOOKS_DIR}" ]; then + on_error(){ + run-parts -a "error" "${HOOKS_DIR}" + } + trap 'on_error' ERR + fi + + if [ "${POSTGRES_DB}" = "**None**" -a "${POSTGRES_DB_FILE}" = "**None**" ]; then + echo "You need to set the POSTGRES_DB or POSTGRES_DB_FILE environment variable." + exit 1 + fi + + if [ "${POSTGRES_HOST}" = "**None**" ]; then + if [ -n "${POSTGRES_PORT_5432_TCP_ADDR}" ]; then + POSTGRES_HOST=${POSTGRES_PORT_5432_TCP_ADDR} + POSTGRES_PORT=${POSTGRES_PORT_5432_TCP_PORT} + else + echo "You need to set the POSTGRES_HOST environment variable." + exit 1 + fi + fi + + if [ "${POSTGRES_USER}" = "**None**" -a "${POSTGRES_USER_FILE}" = "**None**" ]; then + echo "You need to set the POSTGRES_USER or POSTGRES_USER_FILE environment variable." + exit 1 + fi + + if [ "${POSTGRES_PASSWORD}" = "**None**" -a "${POSTGRES_PASSWORD_FILE}" = "**None**" -a "${POSTGRES_PASSFILE_STORE}" = "**None**" ]; then + echo "You need to set the POSTGRES_PASSWORD or POSTGRES_PASSWORD_FILE or POSTGRES_PASSFILE_STORE environment variable or link to a container named POSTGRES." + exit 1 + fi + + #Process vars + if [ "${POSTGRES_DB_FILE}" = "**None**" ]; then + POSTGRES_DBS=$(echo "${POSTGRES_DB}" | tr , " ") + elif [ -r "${POSTGRES_DB_FILE}" ]; then + POSTGRES_DBS=$(cat "${POSTGRES_DB_FILE}") + else + echo "Missing POSTGRES_DB_FILE file." + exit 1 + fi + if [ "${POSTGRES_USER_FILE}" = "**None**" ]; then + export PGUSER="${POSTGRES_USER}" + elif [ -r "${POSTGRES_USER_FILE}" ]; then + export PGUSER=$(cat "${POSTGRES_USER_FILE}") + else + echo "Missing POSTGRES_USER_FILE file." + exit 1 + fi + if [ "${POSTGRES_PASSWORD_FILE}" = "**None**" -a "${POSTGRES_PASSFILE_STORE}" = "**None**" ]; then + export PGPASSWORD="${POSTGRES_PASSWORD}" + elif [ -r "${POSTGRES_PASSWORD_FILE}" ]; then + export PGPASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}") + elif [ -r "${POSTGRES_PASSFILE_STORE}" ]; then + export PGPASSFILE="${POSTGRES_PASSFILE_STORE}" + else + echo "Missing POSTGRES_PASSWORD_FILE or POSTGRES_PASSFILE_STORE file." + exit 1 + fi + + + # Pre-backup hook + if [ -d "${HOOKS_DIR}" ]; then + run-parts -a "pre-backup" --exit-on-error "${HOOKS_DIR}" + fi + + #Initialize dirs + FREQUENCY=( last daily weekly monthly ) + + for f in ${FREQUENCY[@]} + do + + mkdir -p "${BACKUP_DIR}/${f}/" + + done + +} + +#Create Backups +create_backups () { + + for DB in ${POSTGRES_DBS} + do + + LAST_FILENAME="${DB}-`date +%Y%m%d-%H%M%S`${BACKUP_SUFFIX}" + FILE="${BACKUP_DIR}/last/${LAST_FILENAME}" + + create_dump + + echo "Point last backup file to this last backup..." + ln -svf "${LAST_FILENAME}" "${BACKUP_DIR}/last/${DB}-latest${BACKUP_SUFFIX}" + + create_hardlinks "${FILE}" "daily" + + if [ "${BACKUP_MONTH_DAY}" = "${MONTH_DAY}" ] + then + + create_hardlinks "${FILE}" "monthly" + + elif [ "${BACKUP_WEEK_DAY}" = "${WEEK_DAY}" ] + then + + create_hardlinks "${FILE}" "weekly" + + fi + + einfo "SQL backup created successfully" + + done + + # Post-backup hook + if [ -d "${HOOKS_DIR}" ]; then + run-parts -a "post-backup" --reverse --exit-on-error "${HOOKS_DIR}" + fi + +} + +#Create dump +create_dump () { + if [ "${POSTGRES_CLUSTER}" = "TRUE" ]; then - echo "Creating cluster dump of ${DB} database from ${POSTGRES_HOST}..." + einfo "Creating cluster dump of ${DB} database from ${POSTGRES_HOST}..." pg_dumpall -l "${DB}" ${POSTGRES_EXTRA_OPTS} | gzip > "${FILE}" else - echo "Creating dump of ${DB} database from ${POSTGRES_HOST}..." + einfo "Creating dump of ${DB} database from ${POSTGRES_HOST}..." pg_dump -d "${DB}" -f "${FILE}" ${POSTGRES_EXTRA_OPTS} fi + +} + +create_hardlinks () { + + SRC=$1 + INCREMENT=$2 + + if [ "${INCREMENT}" = "daily" ] + then + + FILENAME="${DB}-`date +%Y%m%d`${BACKUP_SUFFIX}" + + elif [ "${INCREMENT}" = "weekly" ] + then + + FILENAME="${DB}-`date +%G%V`${BACKUP_SUFFIX}" + + elif [ "${INCREMENT}" = "monthly"] + then + + FILENAME="${DB}-`date +%Y%m`${BACKUP_SUFFIX}" + + fi + + DEST="${BACKUP_DIR}/${INCREMENT}/${FILENAME}" + #Copy (hardlink) for each entry - if [ -d "${FILE}" ]; then - DFILENEW="${DFILE}-new" - WFILENEW="${WFILE}-new" - MFILENEW="${MFILE}-new" - rm -rf "${DFILENEW}" "${WFILENEW}" "${MFILENEW}" - mkdir "${DFILENEW}" "${WFILENEW}" "${MFILENEW}" - ln -f "${FILE}/"* "${DFILENEW}/" - ln -f "${FILE}/"* "${WFILENEW}/" - ln -f "${FILE}/"* "${MFILENEW}/" - rm -rf "${DFILE}" "${WFILE}" "${MFILE}" - echo "Replacing daily backup ${DFILE} folder this last backup..." - mv "${DFILENEW}" "${DFILE}" - echo "Replacing weekly backup ${WFILE} folder this last backup..." - mv "${WFILENEW}" "${WFILE}" - echo "Replacing monthly backup ${MFILE} folder this last backup..." - mv "${MFILENEW}" "${MFILE}" + if [ -d "${SRC}" ] + then + DESTNEW="${DEST}-new" + rm -rf "${DESTNEW}" + mkdir "${DESTNEW}" + ln -f "${SRC}/"* "${DESTNEW}/" + rm -rf "${DEST}" + echo "Replacing ${INCREMENT} backup ${DEST} file this last backup..." + mv "${DESTNEW}" "${DEST}" else - echo "Replacing daily backup ${DFILE} file this last backup..." - ln -vf "${FILE}" "${DFILE}" - echo "Replacing weekly backup ${WFILE} file this last backup..." - ln -vf "${FILE}" "${WFILE}" - echo "Replacing monthly backup ${MFILE} file this last backup..." - ln -vf "${FILE}" "${MFILE}" + echo "Replacing ${INCREMENT} backup ${DEST} file this last backup..." + ln -vf "${SRC}" "${DEST}" fi # Update latest symlinks - echo "Point last backup file to this last backup..." - ln -svf "${LAST_FILENAME}" "${BACKUP_DIR}/last/${DB}-latest${BACKUP_SUFFIX}" - echo "Point latest daily backup to this last backup..." - ln -svf "${DAILY_FILENAME}" "${BACKUP_DIR}/daily/${DB}-latest${BACKUP_SUFFIX}" - echo "Point latest weekly backup to this last backup..." - ln -svf "${WEEKLY_FILENAME}" "${BACKUP_DIR}/weekly/${DB}-latest${BACKUP_SUFFIX}" - echo "Point latest monthly backup to this last backup..." - ln -svf "${MONTHY_FILENAME}" "${BACKUP_DIR}/monthly/${DB}-latest${BACKUP_SUFFIX}" - #Clean old files - echo "Cleaning older files for ${DB} database from ${POSTGRES_HOST}..." - find "${BACKUP_DIR}/last" -maxdepth 1 -mmin "+${KEEP_MINS}" -name "${DB}-*${BACKUP_SUFFIX}" -exec rm -rvf '{}' ';' - find "${BACKUP_DIR}/daily" -maxdepth 1 -mtime "+${KEEP_DAYS}" -name "${DB}-*${BACKUP_SUFFIX}" -exec rm -rvf '{}' ';' - find "${BACKUP_DIR}/weekly" -maxdepth 1 -mtime "+${KEEP_WEEKS}" -name "${DB}-*${BACKUP_SUFFIX}" -exec rm -rvf '{}' ';' - find "${BACKUP_DIR}/monthly" -maxdepth 1 -mtime "+${KEEP_MONTHS}" -name "${DB}-*${BACKUP_SUFFIX}" -exec rm -rvf '{}' ';' -done - -echo "SQL backup created successfully" - -# Post-backup hook -if [ -d "${HOOKS_DIR}" ]; then - run-parts -a "post-backup" --reverse --exit-on-error "${HOOKS_DIR}" -fi + echo "Replacing lastest ${INCREMENT} backup to this last backup..." + ln -svf "${DEST}" "${BACKUP_DIR}/${INCREMENT}/${DB}-latest" + +} + +#Clean up old backups +cleanup_backups () { + + for folder in "${FREQUENCY[@]}" + do + if [ $folder == 'last' ] + then + KEEP=$KEEP_MINS + elif [ $folder == 'daily' ] + then + KEEP=$KEEP_DAYS + elif [ $folder == "weekly" ] + then + KEEP=$KEEP_WEEKS + elif [ $folder == 'monthly' ] + then + KEEP=$KEEP_MONTHS + fi + + for DB in ${POSTGRES_DBS} + do + + #Clean old files + local all=( `find "${BACKUP_DIR}/${folder}" -maxdepth 1 -name "${DB}-*"` ) + local files=() + + if [ $KEEP -gt 0 ] + then + + echo "Cleaning older files in ${folder} for ${DB} database from ${POSTGRES_HOST}..." + local files=( `find "${BACKUP_DIR}/${folder}" -maxdepth 1 -mtime "+$((${KEEP}-1))" -name "${DB}-*"` ) + local files=( `printf "%s\n" "${files[@]}" | sort -t/ -k3` ) + + fi + + for file in "${files[@]}" + do + + echo "Deleting $file" + rm $file + + done + + done + + done +} + +setup +create_backups +cleanup_backups \ No newline at end of file From 62dbae26e7d0ce7d8d16ecac4e60eb86e3bedfc1 Mon Sep 17 00:00:00 2001 From: Dark Decoy Date: Mon, 6 Mar 2023 20:47:55 +0000 Subject: [PATCH 02/10] Fixed command that doesn't exist --- backup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backup.sh b/backup.sh index 8aeb7e6..e86afd8 100755 --- a/backup.sh +++ b/backup.sh @@ -117,7 +117,7 @@ create_backups () { fi - einfo "SQL backup created successfully" + echo "SQL backup created successfully" done @@ -132,10 +132,10 @@ create_backups () { create_dump () { if [ "${POSTGRES_CLUSTER}" = "TRUE" ]; then - einfo "Creating cluster dump of ${DB} database from ${POSTGRES_HOST}..." + echo "Creating cluster dump of ${DB} database from ${POSTGRES_HOST}..." pg_dumpall -l "${DB}" ${POSTGRES_EXTRA_OPTS} | gzip > "${FILE}" else - einfo "Creating dump of ${DB} database from ${POSTGRES_HOST}..." + echo "Creating dump of ${DB} database from ${POSTGRES_HOST}..." pg_dump -d "${DB}" -f "${FILE}" ${POSTGRES_EXTRA_OPTS} fi From abccdd9f7376ecbe24a43f6b45c5b839e84a46b1 Mon Sep 17 00:00:00 2001 From: Dark Decoy Date: Mon, 6 Mar 2023 22:30:58 +0000 Subject: [PATCH 03/10] Removed changes to when daily, monthly and weekly backups are taken --- backup.sh | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/backup.sh b/backup.sh index e86afd8..50cbf8c 100755 --- a/backup.sh +++ b/backup.sh @@ -104,18 +104,8 @@ create_backups () { ln -svf "${LAST_FILENAME}" "${BACKUP_DIR}/last/${DB}-latest${BACKUP_SUFFIX}" create_hardlinks "${FILE}" "daily" - - if [ "${BACKUP_MONTH_DAY}" = "${MONTH_DAY}" ] - then - - create_hardlinks "${FILE}" "monthly" - - elif [ "${BACKUP_WEEK_DAY}" = "${WEEK_DAY}" ] - then - - create_hardlinks "${FILE}" "weekly" - - fi + create_hardlinks "${FILE}" "monthly" + create_hardlinks "${FILE}" "weekly" echo "SQL backup created successfully" From 78e838905bc937a6898592910aff431cdf3e931f Mon Sep 17 00:00:00 2001 From: Dark Decoy Date: Mon, 6 Mar 2023 23:34:50 +0000 Subject: [PATCH 04/10] Fixed build errors --- docker-bake.hcl | 62 ++++++++++++++--------------------------- generate-docker-bake.sh | 2 +- 2 files changed, 22 insertions(+), 42 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 5b67efb..7188206 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,5 +1,5 @@ group "default" { - targets = ["debian-latest", "alpine-latest", "debian-14", "debian-13", "debian-12", "debian-11", "debian-10", "alpine-14", "alpine-13", "alpine-12", "alpine-11", "alpine-10"] + targets = ["debian-latest", "alpine-latest", "debian-14", "debian-13", "debian-12", "alpine-14", "alpine-13", "alpine-12"] } variable "REGISTRY_PREFIX" { @@ -46,6 +46,26 @@ target "alpine-latest" { ] } +target "debian-15" { + inherits = ["debian"] + platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x", "linux/ppc64le"] + args = {"BASETAG" = "15"} + tags = [ + "${REGISTRY_PREFIX}${IMAGE_NAME}:15", + notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:15-debian-${BUILD_REVISION}" : "" + ] +} + +target "alpine-15" { + inherits = ["alpine"] + platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x", "linux/ppc64le"] + args = {"BASETAG" = "15-alpine"} + tags = [ + "${REGISTRY_PREFIX}${IMAGE_NAME}:15-alpine", + notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:15-alpine-${BUILD_REVISION}" : "" + ] +} + target "debian-14" { inherits = ["debian"] platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x", "linux/ppc64le"] @@ -105,43 +125,3 @@ target "alpine-12" { notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:12-alpine-${BUILD_REVISION}" : "" ] } - -target "debian-11" { - inherits = ["debian"] - platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7"] - args = {"BASETAG" = "11"} - tags = [ - "${REGISTRY_PREFIX}${IMAGE_NAME}:11", - notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:11-debian-${BUILD_REVISION}" : "" - ] -} - -target "alpine-11" { - inherits = ["alpine"] - platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x", "linux/ppc64le"] - args = {"BASETAG" = "11-alpine"} - tags = [ - "${REGISTRY_PREFIX}${IMAGE_NAME}:11-alpine", - notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:11-alpine-${BUILD_REVISION}" : "" - ] -} - -target "debian-10" { - inherits = ["debian"] - platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7"] - args = {"BASETAG" = "10"} - tags = [ - "${REGISTRY_PREFIX}${IMAGE_NAME}:10", - notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:10-debian-${BUILD_REVISION}" : "" - ] -} - -target "alpine-10" { - inherits = ["alpine"] - platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x", "linux/ppc64le"] - args = {"BASETAG" = "10-alpine"} - tags = [ - "${REGISTRY_PREFIX}${IMAGE_NAME}:10-alpine", - notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:10-alpine-${BUILD_REVISION}" : "" - ] -} diff --git a/generate-docker-bake.sh b/generate-docker-bake.sh index e8193f2..70fcae8 100755 --- a/generate-docker-bake.sh +++ b/generate-docker-bake.sh @@ -4,7 +4,7 @@ set -e GOCRONVER="v0.0.10" MAIN_TAG="15" -TAGS_EXTRA="14 13 12" +TAGS_EXTRA="15 14 13 12" PLATFORMS="linux/amd64 linux/arm64 linux/arm/v7 linux/s390x linux/ppc64le" TAGS_EXTRA_2="11 10" PLATFORMS_DEBIAN_2="linux/amd64 linux/arm64 linux/arm/v7" From 9babdb805d40fd4703530a980e638aaf3647911b Mon Sep 17 00:00:00 2001 From: Dark Decoy Date: Tue, 7 Mar 2023 00:12:17 +0000 Subject: [PATCH 05/10] Fixed issue when it is non empty folder --- backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.sh b/backup.sh index 50cbf8c..0535f50 100755 --- a/backup.sh +++ b/backup.sh @@ -214,7 +214,7 @@ cleanup_backups () { do echo "Deleting $file" - rm $file + rm -r $file done From 12296e2b01ab102e2ed7aab8c815a32f61d99783 Mon Sep 17 00:00:00 2001 From: Dark Decoy Date: Tue, 7 Mar 2023 00:13:01 +0000 Subject: [PATCH 06/10] Removed 15 as it is already in the main tag --- generate-docker-bake.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate-docker-bake.sh b/generate-docker-bake.sh index 70fcae8..e8193f2 100755 --- a/generate-docker-bake.sh +++ b/generate-docker-bake.sh @@ -4,7 +4,7 @@ set -e GOCRONVER="v0.0.10" MAIN_TAG="15" -TAGS_EXTRA="15 14 13 12" +TAGS_EXTRA="14 13 12" PLATFORMS="linux/amd64 linux/arm64 linux/arm/v7 linux/s390x linux/ppc64le" TAGS_EXTRA_2="11 10" PLATFORMS_DEBIAN_2="linux/amd64 linux/arm64 linux/arm/v7" From 384b796800454e5dee5c8116ef3d09764f5f4ad5 Mon Sep 17 00:00:00 2001 From: Dark Decoy Date: Tue, 7 Mar 2023 00:14:16 +0000 Subject: [PATCH 07/10] Generated the hcl file --- docker-bake.hcl | 62 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 7188206..5b67efb 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,5 +1,5 @@ group "default" { - targets = ["debian-latest", "alpine-latest", "debian-14", "debian-13", "debian-12", "alpine-14", "alpine-13", "alpine-12"] + targets = ["debian-latest", "alpine-latest", "debian-14", "debian-13", "debian-12", "debian-11", "debian-10", "alpine-14", "alpine-13", "alpine-12", "alpine-11", "alpine-10"] } variable "REGISTRY_PREFIX" { @@ -46,26 +46,6 @@ target "alpine-latest" { ] } -target "debian-15" { - inherits = ["debian"] - platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x", "linux/ppc64le"] - args = {"BASETAG" = "15"} - tags = [ - "${REGISTRY_PREFIX}${IMAGE_NAME}:15", - notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:15-debian-${BUILD_REVISION}" : "" - ] -} - -target "alpine-15" { - inherits = ["alpine"] - platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x", "linux/ppc64le"] - args = {"BASETAG" = "15-alpine"} - tags = [ - "${REGISTRY_PREFIX}${IMAGE_NAME}:15-alpine", - notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:15-alpine-${BUILD_REVISION}" : "" - ] -} - target "debian-14" { inherits = ["debian"] platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x", "linux/ppc64le"] @@ -125,3 +105,43 @@ target "alpine-12" { notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:12-alpine-${BUILD_REVISION}" : "" ] } + +target "debian-11" { + inherits = ["debian"] + platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7"] + args = {"BASETAG" = "11"} + tags = [ + "${REGISTRY_PREFIX}${IMAGE_NAME}:11", + notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:11-debian-${BUILD_REVISION}" : "" + ] +} + +target "alpine-11" { + inherits = ["alpine"] + platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x", "linux/ppc64le"] + args = {"BASETAG" = "11-alpine"} + tags = [ + "${REGISTRY_PREFIX}${IMAGE_NAME}:11-alpine", + notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:11-alpine-${BUILD_REVISION}" : "" + ] +} + +target "debian-10" { + inherits = ["debian"] + platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7"] + args = {"BASETAG" = "10"} + tags = [ + "${REGISTRY_PREFIX}${IMAGE_NAME}:10", + notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:10-debian-${BUILD_REVISION}" : "" + ] +} + +target "alpine-10" { + inherits = ["alpine"] + platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7", "linux/s390x", "linux/ppc64le"] + args = {"BASETAG" = "10-alpine"} + tags = [ + "${REGISTRY_PREFIX}${IMAGE_NAME}:10-alpine", + notequal("", BUILD_REVISION) ? "${REGISTRY_PREFIX}${IMAGE_NAME}:10-alpine-${BUILD_REVISION}" : "" + ] +} From 106d701cfac047193fda9dafb032c241530dbb31 Mon Sep 17 00:00:00 2001 From: Dark Decoy Date: Tue, 7 Mar 2023 15:28:08 +0000 Subject: [PATCH 08/10] Removing functions to provide a execution straight from top to bottom --- backup.sh | 328 +++++++++++++++++++++++++----------------------------- 1 file changed, 151 insertions(+), 177 deletions(-) diff --git a/backup.sh b/backup.sh index 0535f50..df4aeb0 100755 --- a/backup.sh +++ b/backup.sh @@ -7,222 +7,196 @@ KEEP_DAYS=${BACKUP_KEEP_DAYS} KEEP_WEEKS=`expr $(((${BACKUP_KEEP_WEEKS} * 7) + 1))` KEEP_MONTHS=`expr $(((${BACKUP_KEEP_MONTHS} * 31) + 1))` -setup () { - set -Eeo pipefail - - HOOKS_DIR="/hooks" - if [ -d "${HOOKS_DIR}" ]; then - on_error(){ - run-parts -a "error" "${HOOKS_DIR}" - } - trap 'on_error' ERR - fi - - if [ "${POSTGRES_DB}" = "**None**" -a "${POSTGRES_DB_FILE}" = "**None**" ]; then - echo "You need to set the POSTGRES_DB or POSTGRES_DB_FILE environment variable." +set -Eeo pipefail + +HOOKS_DIR="/hooks" +if [ -d "${HOOKS_DIR}" ]; then + on_error(){ + run-parts -a "error" "${HOOKS_DIR}" + } + trap 'on_error' ERR +fi + +if [ "${POSTGRES_DB}" = "**None**" -a "${POSTGRES_DB_FILE}" = "**None**" ]; then + echo "You need to set the POSTGRES_DB or POSTGRES_DB_FILE environment variable." + exit 1 +fi + +if [ "${POSTGRES_HOST}" = "**None**" ]; then + if [ -n "${POSTGRES_PORT_5432_TCP_ADDR}" ]; then + POSTGRES_HOST=${POSTGRES_PORT_5432_TCP_ADDR} + POSTGRES_PORT=${POSTGRES_PORT_5432_TCP_PORT} + else + echo "You need to set the POSTGRES_HOST environment variable." exit 1 fi +fi + +if [ "${POSTGRES_USER}" = "**None**" -a "${POSTGRES_USER_FILE}" = "**None**" ]; then + echo "You need to set the POSTGRES_USER or POSTGRES_USER_FILE environment variable." + exit 1 +fi + +if [ "${POSTGRES_PASSWORD}" = "**None**" -a "${POSTGRES_PASSWORD_FILE}" = "**None**" -a "${POSTGRES_PASSFILE_STORE}" = "**None**" ]; then + echo "You need to set the POSTGRES_PASSWORD or POSTGRES_PASSWORD_FILE or POSTGRES_PASSFILE_STORE environment variable or link to a container named POSTGRES." + exit 1 +fi + +#Process vars +if [ "${POSTGRES_DB_FILE}" = "**None**" ]; then + POSTGRES_DBS=$(echo "${POSTGRES_DB}" | tr , " ") +elif [ -r "${POSTGRES_DB_FILE}" ]; then + POSTGRES_DBS=$(cat "${POSTGRES_DB_FILE}") +else + echo "Missing POSTGRES_DB_FILE file." + exit 1 +fi +if [ "${POSTGRES_USER_FILE}" = "**None**" ]; then + export PGUSER="${POSTGRES_USER}" +elif [ -r "${POSTGRES_USER_FILE}" ]; then + export PGUSER=$(cat "${POSTGRES_USER_FILE}") +else + echo "Missing POSTGRES_USER_FILE file." + exit 1 +fi +if [ "${POSTGRES_PASSWORD_FILE}" = "**None**" -a "${POSTGRES_PASSFILE_STORE}" = "**None**" ]; then + export PGPASSWORD="${POSTGRES_PASSWORD}" +elif [ -r "${POSTGRES_PASSWORD_FILE}" ]; then + export PGPASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}") +elif [ -r "${POSTGRES_PASSFILE_STORE}" ]; then + export PGPASSFILE="${POSTGRES_PASSFILE_STORE}" +else + echo "Missing POSTGRES_PASSWORD_FILE or POSTGRES_PASSFILE_STORE file." + exit 1 +fi + +# Pre-backup hook +if [ -d "${HOOKS_DIR}" ]; then + run-parts -a "pre-backup" --exit-on-error "${HOOKS_DIR}" +fi + +#Initialize dirs +FREQUENCY=( last daily weekly monthly ) + +for f in ${FREQUENCY[@]} +do + + mkdir -p "${BACKUP_DIR}/${f}/" + +done - if [ "${POSTGRES_HOST}" = "**None**" ]; then - if [ -n "${POSTGRES_PORT_5432_TCP_ADDR}" ]; then - POSTGRES_HOST=${POSTGRES_PORT_5432_TCP_ADDR} - POSTGRES_PORT=${POSTGRES_PORT_5432_TCP_PORT} - else - echo "You need to set the POSTGRES_HOST environment variable." - exit 1 - fi - fi - - if [ "${POSTGRES_USER}" = "**None**" -a "${POSTGRES_USER_FILE}" = "**None**" ]; then - echo "You need to set the POSTGRES_USER or POSTGRES_USER_FILE environment variable." - exit 1 - fi +#Create Backups +links=( "daily" "monthly" "weekly" ) +for DB in ${POSTGRES_DBS} +do - if [ "${POSTGRES_PASSWORD}" = "**None**" -a "${POSTGRES_PASSWORD_FILE}" = "**None**" -a "${POSTGRES_PASSFILE_STORE}" = "**None**" ]; then - echo "You need to set the POSTGRES_PASSWORD or POSTGRES_PASSWORD_FILE or POSTGRES_PASSFILE_STORE environment variable or link to a container named POSTGRES." - exit 1 - fi + LAST_FILENAME="${DB}-`date +%Y%m%d-%H%M%S`${BACKUP_SUFFIX}" + FILE="${BACKUP_DIR}/last/${LAST_FILENAME}" - #Process vars - if [ "${POSTGRES_DB_FILE}" = "**None**" ]; then - POSTGRES_DBS=$(echo "${POSTGRES_DB}" | tr , " ") - elif [ -r "${POSTGRES_DB_FILE}" ]; then - POSTGRES_DBS=$(cat "${POSTGRES_DB_FILE}") - else - echo "Missing POSTGRES_DB_FILE file." - exit 1 - fi - if [ "${POSTGRES_USER_FILE}" = "**None**" ]; then - export PGUSER="${POSTGRES_USER}" - elif [ -r "${POSTGRES_USER_FILE}" ]; then - export PGUSER=$(cat "${POSTGRES_USER_FILE}") - else - echo "Missing POSTGRES_USER_FILE file." - exit 1 - fi - if [ "${POSTGRES_PASSWORD_FILE}" = "**None**" -a "${POSTGRES_PASSFILE_STORE}" = "**None**" ]; then - export PGPASSWORD="${POSTGRES_PASSWORD}" - elif [ -r "${POSTGRES_PASSWORD_FILE}" ]; then - export PGPASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}") - elif [ -r "${POSTGRES_PASSFILE_STORE}" ]; then - export PGPASSFILE="${POSTGRES_PASSFILE_STORE}" + #Create dump + if [ "${POSTGRES_CLUSTER}" = "TRUE" ]; then + echo "Creating cluster dump of ${DB} database from ${POSTGRES_HOST}..." + pg_dumpall -l "${DB}" ${POSTGRES_EXTRA_OPTS} | gzip > "${FILE}" else - echo "Missing POSTGRES_PASSWORD_FILE or POSTGRES_PASSFILE_STORE file." - exit 1 - fi - - - # Pre-backup hook - if [ -d "${HOOKS_DIR}" ]; then - run-parts -a "pre-backup" --exit-on-error "${HOOKS_DIR}" + echo "Creating dump of ${DB} database from ${POSTGRES_HOST}..." + pg_dump -d "${DB}" -f "${FILE}" ${POSTGRES_EXTRA_OPTS} fi - #Initialize dirs - FREQUENCY=( last daily weekly monthly ) + echo "Point last backup file to this last backup..." + ln -svf "${LAST_FILENAME}" "${BACKUP_DIR}/last/${DB}-latest${BACKUP_SUFFIX}" - for f in ${FREQUENCY[@]} + for link in ${links} do - mkdir -p "${BACKUP_DIR}/${f}/" - - done + if [ "${link}" == "daily" ] + then -} + FILENAME="${DB}-`date +%Y%m%d`${BACKUP_SUFFIX}" -#Create Backups -create_backups () { + elif [ "${link}" == "weekly" ] + then - for DB in ${POSTGRES_DBS} - do + FILENAME="${DB}-`date +%G%V`${BACKUP_SUFFIX}" - LAST_FILENAME="${DB}-`date +%Y%m%d-%H%M%S`${BACKUP_SUFFIX}" - FILE="${BACKUP_DIR}/last/${LAST_FILENAME}" + elif [ "${link}" == "monthly"] + then - create_dump + FILENAME="${DB}-`date +%Y%m`${BACKUP_SUFFIX}" - echo "Point last backup file to this last backup..." - ln -svf "${LAST_FILENAME}" "${BACKUP_DIR}/last/${DB}-latest${BACKUP_SUFFIX}" + fi - create_hardlinks "${FILE}" "daily" - create_hardlinks "${FILE}" "monthly" - create_hardlinks "${FILE}" "weekly" + DEST="${BACKUP_DIR}/${link}/${FILENAME}" - echo "SQL backup created successfully" + #Copy (hardlink) for each entry + if [ -d "${FILE}" ] + then + DESTNEW="${DEST}-new" + rm -rf "${DESTNEW}" + mkdir "${DESTNEW}" + ln -f "${FILE}/"* "${DESTNEW}/" + rm -rf "${DEST}" + echo "Replacing ${link} backup ${DEST} file this last backup..." + mv "${DESTNEW}" "${DEST}" + else + echo "Replacing ${link} backup ${DEST} file this last backup..." + ln -vf "${FILE}" "${DEST}" + fi + # Update latest symlinks + echo "Replacing lastest ${link} backup to this last backup..." + ln -svf "${DEST}" "${BACKUP_DIR}/${link}/${DB}-latest" done - # Post-backup hook - if [ -d "${HOOKS_DIR}" ]; then - run-parts -a "post-backup" --reverse --exit-on-error "${HOOKS_DIR}" - fi - -} - -#Create dump -create_dump () { + echo "SQL backup created successfully" - if [ "${POSTGRES_CLUSTER}" = "TRUE" ]; then - echo "Creating cluster dump of ${DB} database from ${POSTGRES_HOST}..." - pg_dumpall -l "${DB}" ${POSTGRES_EXTRA_OPTS} | gzip > "${FILE}" - else - echo "Creating dump of ${DB} database from ${POSTGRES_HOST}..." - pg_dump -d "${DB}" -f "${FILE}" ${POSTGRES_EXTRA_OPTS} - fi +done -} +# Post-backup hook +if [ -d "${HOOKS_DIR}" ]; then + run-parts -a "post-backup" --reverse --exit-on-error "${HOOKS_DIR}" +fi -create_hardlinks () { - - SRC=$1 - INCREMENT=$2 - - if [ "${INCREMENT}" = "daily" ] +#Clean up old backups +for folder in "${FREQUENCY[@]}" +do + if [ $folder == 'last' ] then - - FILENAME="${DB}-`date +%Y%m%d`${BACKUP_SUFFIX}" - - elif [ "${INCREMENT}" = "weekly" ] + KEEP=$KEEP_MINS + elif [ $folder == 'daily' ] then - - FILENAME="${DB}-`date +%G%V`${BACKUP_SUFFIX}" - - elif [ "${INCREMENT}" = "monthly"] + KEEP=$KEEP_DAYS + elif [ $folder == "weekly" ] then - - FILENAME="${DB}-`date +%Y%m`${BACKUP_SUFFIX}" - - fi - - DEST="${BACKUP_DIR}/${INCREMENT}/${FILENAME}" - - #Copy (hardlink) for each entry - if [ -d "${SRC}" ] + KEEP=$KEEP_WEEKS + elif [ $folder == 'monthly' ] then - DESTNEW="${DEST}-new" - rm -rf "${DESTNEW}" - mkdir "${DESTNEW}" - ln -f "${SRC}/"* "${DESTNEW}/" - rm -rf "${DEST}" - echo "Replacing ${INCREMENT} backup ${DEST} file this last backup..." - mv "${DESTNEW}" "${DEST}" - else - echo "Replacing ${INCREMENT} backup ${DEST} file this last backup..." - ln -vf "${SRC}" "${DEST}" + KEEP=$KEEP_MONTHS fi - # Update latest symlinks - echo "Replacing lastest ${INCREMENT} backup to this last backup..." - ln -svf "${DEST}" "${BACKUP_DIR}/${INCREMENT}/${DB}-latest" - -} -#Clean up old backups -cleanup_backups () { - - for folder in "${FREQUENCY[@]}" + for DB in ${POSTGRES_DBS} do - if [ $folder == 'last' ] - then - KEEP=$KEEP_MINS - elif [ $folder == 'daily' ] - then - KEEP=$KEEP_DAYS - elif [ $folder == "weekly" ] - then - KEEP=$KEEP_WEEKS - elif [ $folder == 'monthly' ] - then - KEEP=$KEEP_MONTHS - fi - - for DB in ${POSTGRES_DBS} - do - #Clean old files - local all=( `find "${BACKUP_DIR}/${folder}" -maxdepth 1 -name "${DB}-*"` ) - local files=() + #Clean old files + local all=( `find "${BACKUP_DIR}/${folder}" -maxdepth 1 -name "${DB}-*"` ) + local files=() - if [ $KEEP -gt 0 ] - then - - echo "Cleaning older files in ${folder} for ${DB} database from ${POSTGRES_HOST}..." - local files=( `find "${BACKUP_DIR}/${folder}" -maxdepth 1 -mtime "+$((${KEEP}-1))" -name "${DB}-*"` ) - local files=( `printf "%s\n" "${files[@]}" | sort -t/ -k3` ) - - fi + if [ $KEEP -gt 0 ] + then + + echo "Cleaning older files in ${folder} for ${DB} database from ${POSTGRES_HOST}..." + local files=( `find "${BACKUP_DIR}/${folder}" -maxdepth 1 -mtime "+$((${KEEP}-1))" -name "${DB}-*"` ) + local files=( `printf "%s\n" "${files[@]}" | sort -t/ -k3` ) - for file in "${files[@]}" - do + fi - echo "Deleting $file" - rm -r $file + for file in "${files[@]}" + do - done + echo "Deleting $file" + rm -r $file done done -} -setup -create_backups -cleanup_backups \ No newline at end of file +done From 400473cf96a8b495143e5980dcda675c4d6f1b91 Mon Sep 17 00:00:00 2001 From: Dark Decoy Date: Tue, 7 Mar 2023 15:30:01 +0000 Subject: [PATCH 09/10] Removed local variable designation --- backup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backup.sh b/backup.sh index df4aeb0..de305f3 100755 --- a/backup.sh +++ b/backup.sh @@ -177,15 +177,15 @@ do do #Clean old files - local all=( `find "${BACKUP_DIR}/${folder}" -maxdepth 1 -name "${DB}-*"` ) - local files=() + all=( `find "${BACKUP_DIR}/${folder}" -maxdepth 1 -name "${DB}-*"` ) + files=() if [ $KEEP -gt 0 ] then echo "Cleaning older files in ${folder} for ${DB} database from ${POSTGRES_HOST}..." - local files=( `find "${BACKUP_DIR}/${folder}" -maxdepth 1 -mtime "+$((${KEEP}-1))" -name "${DB}-*"` ) - local files=( `printf "%s\n" "${files[@]}" | sort -t/ -k3` ) + files=( `find "${BACKUP_DIR}/${folder}" -maxdepth 1 -mtime "+$((${KEEP}-1))" -name "${DB}-*"` ) + files=( `printf "%s\n" "${files[@]}" | sort -t/ -k3` ) fi From d195836f6ad611dc4c0d5ed9cbc7acdc4b43b5bf Mon Sep 17 00:00:00 2001 From: Dark Decoy Date: Tue, 7 Mar 2023 22:46:44 +0000 Subject: [PATCH 10/10] Fixed missing space typo --- backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.sh b/backup.sh index de305f3..84e7e3b 100755 --- a/backup.sh +++ b/backup.sh @@ -118,7 +118,7 @@ do FILENAME="${DB}-`date +%G%V`${BACKUP_SUFFIX}" - elif [ "${link}" == "monthly"] + elif [ "${link}" == "monthly" ] then FILENAME="${DB}-`date +%Y%m`${BACKUP_SUFFIX}"