Skip to content

Commit

Permalink
Merge branch 'hotfix/1.5.2'
Browse files Browse the repository at this point in the history
  • Loading branch information
lucas-barbosa-oliveira committed Apr 21, 2020
2 parents 7d7ab6d + b70e5f1 commit 9b180c7
Show file tree
Hide file tree
Showing 7 changed files with 55 additions and 35 deletions.
2 changes: 1 addition & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -107,5 +107,5 @@ LOCAL_TARGET=/path/to/backup
RESTORE_TARGET=LOCAL

# BACKUP_DATA_RETENTION: Time the data backup will remain stored.
# Default value (15 days): 15d
# Default value (15 days): 15D
BACKUP_DATA_RETENTION=15D
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@ Repository with configuration files required for OCARIoT platform **deployment i
All software installation is performed using the following command:

```sh
curl -o- https://raw.githubusercontent.com/ocariot/docker-swarm/1.5.1/install.sh | sudo bash
curl -o- https://raw.githubusercontent.com/ocariot/docker-swarm/1.5.2/install.sh | sudo bash
```

```sh
wget -qO- https://raw.githubusercontent.com/ocariot/docker-swarm/1.5.1/install.sh | sudo bash
wget -qO- https://raw.githubusercontent.com/ocariot/docker-swarm/1.5.2/install.sh | sudo bash
```


Expand Down Expand Up @@ -127,7 +127,7 @@ Variables responsible for defining backup settings. The variables with prefix `C
In order for backup and restore operations to be successful, credentials must be granted permissions to manipulate the cloud storage location:

- [Google Drive](https://console.developers.google.com/apis/credentials)
When performing the first backup, a link will be provided that redirects the browser to a user's authentication screen at Google, thus granting permission to manipulate Google Drive. In future `backup` or `restore` operations, authentication is not required unless the `google_credentials` volume is removed.
When performing the first backup, a link will be provided that redirects the browser to a user's authentication screen at Google, thus granting permission to manipulate Google Drive. In future `backup` or `restore` operations, authentication is not required unless the `ocariot-credentials-data` volume is removed.

- [AWS S3](https://docs.aws.amazon.com/pt_br/sdk-for-java/v1/developer-guide/signup-create-iam-user.html)
To use the `backup` or` restore` operations, it is necessary to associate the following policy with the created user:
Expand Down Expand Up @@ -278,7 +278,7 @@ Variables responsible for defining backup settings. The variables with prefix `C
In order for backup and restore operations to be successful, credentials must be granted permissions to manipulate the cloud storage location:

- [Google Drive](https://console.developers.google.com/apis/credentials)
When performing the first backup, a link will be provided that redirects the browser to a user's authentication screen at Google, thus granting permission to manipulate Google Drive. In future `backup` or `restore` operations, authentication is not required unless the `google_credentials` volume is removed.
When performing the first backup, a link will be provided that redirects the browser to a user's authentication screen at Google, thus granting permission to manipulate Google Drive. In future `backup` or `restore` operations, authentication is not required unless the `ocariot-monitor-credentials-data` volume is removed.

- [AWS S3](https://docs.aws.amazon.com/pt_br/sdk-for-java/v1/developer-guide/signup-create-iam-user.html)
To use the `backup` or` restore` operations, it is necessary to associate the following policy with the created user:
Expand Down
2 changes: 1 addition & 1 deletion install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ INSTALL_PATH="/opt/ocariot-swarm"

version()
{
echo "1.5.1"
echo "1.5.2"
}

isInstalled()
Expand Down
20 changes: 12 additions & 8 deletions scripts/general_functions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ MONITOR_STACK_NAME="ocariot_monitor"
ENV_OCARIOT=".env"
ENV_MONITOR=".env.monitor"
NETWORK_NAME="ocariot"
BACKUP_CONTAINER_NAME="volumerize"
OCARIOT_BACKUP_CONTAINER="${OCARIOT_STACK_NAME}_volumerize"
MONITOR_BACKUP_CONTAINER="${MONITOR_STACK_NAME}_volumerize"
CLOUD_TARGET=$(echo ${CLOUD_TARGET} | sed 's/\/$//g')
OCARIOT_CREDS_DRIVER="ocariot-credentials-data"
MONITOR_CREDS_DRIVER="ocariot-monitor-credentials-data"
Expand All @@ -26,7 +27,10 @@ start_watchdog()

stop_process()
{
pgrep -f "$1" | xargs kill -9
PID=$(pgrep -f "$1")
if [ "${PID}" ]; then
kill -SIGINT ${PID}
fi
}

create_network()
Expand Down Expand Up @@ -202,15 +206,15 @@ validate_file_path() {
}

cloud_bkps() {
docker run -t $1 --rm --name ${BACKUP_CONTAINER_NAME} \
-v $2:/credentials \
docker run -t $1 --rm --name $2 \
-v $3:/credentials \
-e "VOLUMERIZE_SOURCE=/source" \
-e "VOLUMERIZE_TARGET=${CLOUD_TARGET}" \
-e "GOOGLE_DRIVE_ID=${CLOUD_ACCESS_KEY_ID}" \
-e "GOOGLE_DRIVE_SECRET=${CLOUD_SECRET_ACCESS_KEY}" \
-e "AWS_ACCESS_KEY_ID=${CLOUD_ACCESS_KEY_ID}" \
-e "AWS_SECRET_ACCESS_KEY=${CLOUD_SECRET_ACCESS_KEY}" \
blacklabelops/volumerize "${@:3}"
blacklabelops/volumerize "${@:4}"
}

validate_bkp_target() {
Expand Down Expand Up @@ -268,9 +272,9 @@ check_backup_target_config() {
exit
fi
CREDS_FILE_NAME="googledrive.cred"
CREDS_FILE="$(cloud_bkps "" $1 find /credentials -name ${CREDS_FILE_NAME})"
CREDS_FILE="$(cloud_bkps "" $1 $2 find /credentials -name ${CREDS_FILE_NAME})"
if [ -z "$(echo ${CREDS_FILE} | grep ${CREDS_FILE_NAME})" ]; then
cloud_bkps "-i" $1 list
cloud_bkps "-i" $1 $2 list
fi
fi

Expand Down Expand Up @@ -323,5 +327,5 @@ EOF

backup_container_operation()
{
docker container "$1" "${BACKUP_CONTAINER_NAME}" > /dev/null
docker container "$1" "$2" > /dev/null
}
26 changes: 16 additions & 10 deletions scripts/monitor/volumes.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ source ${INSTALL_PATH}/scripts/general_functions.sh

set_variables_environment "${ENV_MONITOR}"

backup_container_operation stop &> /dev/null

VALIDATING_OPTS=$(echo "$@" | sed 's/ /\n/g' |
grep -P "(\-\-services|\-\-time|\-\-expression|\-\-keys).*" -v | grep '\-\-')

Expand All @@ -29,12 +27,20 @@ if ([ "$1" != "backup" ] && [ "$1" != "restore" ]) ||
monitor_help
fi

if [ "$(pgrep -f "$0 backup" | wc -l)" -gt "2" ] \
|| [ "$(pgrep -f "$0 restore" | wc -l)" -gt "2" ]; then
echo "A backup or restore operation is being performed. Please wait for completion to initialize this operation."
exit
fi

backup_container_operation stop "${MONITOR_BACKUP_CONTAINER}" &> /dev/null

if ([ $1 = "backup" ] && [ ${CHECK_TIME_OPT} ]) ||
([ $1 = "restore" ] && [ ${CHECK_AUTO_BKP_OPT} ]); then
monitor_help
fi

check_backup_target_config "${MONITOR_CREDS_DRIVER}"
check_backup_target_config "${MONITOR_BACKUP_CONTAINER}" "${MONITOR_CREDS_DRIVER}"

if [ ${RESTORE_TIME} ]; then
RESTORE_TIME="--time ${RESTORE_TIME}"
Expand All @@ -48,7 +54,7 @@ if [ "$1" = "restore" ]; then
COMMAND="restore ${RESTORE_TIME}"
BACKUP_VOLUME_PROPERTY=":ro"
SOURCE_VOLUME_PROPERTY=""

check_restore_target_config
fi

Expand Down Expand Up @@ -116,7 +122,7 @@ if [ -z "${SERVICES}" ]; then
grep -oE "${EXPRESSION_GREP}" |
sed 's/\(ocariot-monitor-\|-data\)//g')
elif [ "$1" = "restore" ] && [ "${RESTORE_TARGET}" != "LOCAL" ]; then
SERVICES=$(cloud_bkps "" "${MONITOR_CREDS_DRIVER}" ${VOLUME_COMMAND} |
SERVICES=$(cloud_bkps "" "${MONITOR_BACKUP_CONTAINER}" "${MONITOR_CREDS_DRIVER}" ${VOLUME_COMMAND} |
grep -oE "${EXPRESSION_GREP}" |
sed 's/\(ocariot-monitor-\|-data\)//g')
else
Expand All @@ -141,7 +147,7 @@ for SERVICE in ${SERVICES}; do
grep -w ${SERVICE})
elif [ "$1" = "restore" ] && [ "${RESTORE_TARGET}" != "LOCAL" ]; then
if [ -z "${CLOUD_BACKUPS}" ];then
CLOUD_BACKUPS=$(cloud_bkps "" "${MONITOR_CREDS_DRIVER}" ${VOLUME_COMMAND})
CLOUD_BACKUPS=$(cloud_bkps "" "${MONITOR_BACKUP_CONTAINER}" "${MONITOR_CREDS_DRIVER}" ${VOLUME_COMMAND})
fi
MESSAGE="Volume BKP ${SERVICE} not found!"
VOLUME_NAME="$(echo ${CLOUD_BACKUPS} |
Expand Down Expand Up @@ -186,7 +192,7 @@ PROCESS_BKP="OK"
BKP_CONFIG_MODEL=$(mktemp --suffix=.json)

docker run -d --rm \
--name ${BACKUP_CONTAINER_NAME} \
--name ${MONITOR_BACKUP_CONTAINER} \
${VOLUMES} \
${VOLUMES_CACHE} \
-v ${LOCAL_TARGET}:/local-backup${BACKUP_VOLUME_PROPERTY} \
Expand All @@ -206,7 +212,7 @@ for VOLUME in ${VOLUMES_BKP}; do
restore_config "${BKP_CONFIG_MODEL}" "${VOLUME}"
fi

backup_container_operation restart
backup_container_operation restart "${MONITOR_BACKUP_CONTAINER}"

echo "======Backup of ${VOLUME} volume======"

Expand All @@ -218,7 +224,7 @@ for VOLUME in ${VOLUMES_BKP}; do
-e GOOGLE_DRIVE_SECRET=${CLOUD_SECRET_ACCESS_KEY} \
-e AWS_ACCESS_KEY_ID=${CLOUD_ACCESS_KEY_ID} \
-e AWS_SECRET_ACCESS_KEY=${CLOUD_SECRET_ACCESS_KEY} \
${BACKUP_CONTAINER_NAME} bash -c "${COMMAND} && remove-older-than ${BACKUP_DATA_RETENTION} --force"
${MONITOR_BACKUP_CONTAINER} bash -c "${COMMAND} && remove-older-than ${BACKUP_DATA_RETENTION} --force"

if [ $? != 0 ]; then
PROCESS_BKP=FALSE
Expand All @@ -228,7 +234,7 @@ for VOLUME in ${VOLUMES_BKP}; do
INCREMENT=$((INCREMENT + 1))
done

backup_container_operation stop
backup_container_operation stop "${MONITOR_BACKUP_CONTAINER}"

if [ "${PROCESS_BKP}" = "OK" ]; then
RUNNING_SERVICES=$(echo ${RUNNING_SERVICES} | sed 's/ //g')
Expand Down
8 changes: 6 additions & 2 deletions scripts/stack/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,14 @@ create_network
# Executing the services in mode swarm defined in docker-compose.yml file
docker stack deploy -c ${INSTALL_PATH}/docker-ocariot-stack.yml ${OCARIOT_STACK_NAME} --resolve-image changed

if [ $? != 0 ]; then
exit
fi

if [ -z "${STATUS_OCARIOT_STACK}" ]; then
validate_keys "${GENERATE_KEYS_FILE}" &
waiting_vault
# Monitoring Vault service
docker service logs ${OCARIOT_STACK_NAME}_vault -f 2> /dev/null &
stop_log
stop_log &
docker service logs ${OCARIOT_STACK_NAME}_vault -f 2> /dev/null
fi
24 changes: 15 additions & 9 deletions scripts/stack/volumes.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@ registre_bkp_vault() {

set_variables_environment "${ENV_OCARIOT}"

backup_container_operation stop &> /dev/null

BACKEND_VAULT="consul"

VALIDATING_OPTS=$(echo "$@" | sed 's/ /\n/g' |
Expand Down Expand Up @@ -42,12 +40,20 @@ if ([ "$1" != "backup" ] && [ "$1" != "restore" ]) ||
stack_help
fi

if [ "$(pgrep -f "$0 backup" | wc -l)" -gt "2" ] \
|| [ "$(pgrep -f "$0 restore" | wc -l)" -gt "2" ]; then
echo "A backup or restore operation is being performed. Please wait for completion to initialize this operation."
exit
fi

backup_container_operation stop "${OCARIOT_BACKUP_CONTAINER}" &> /dev/null

if ([ $1 = "backup" ] && [ ${CHECK_TIME_OPT} ]) ||
([ $1 = "restore" ] && [ ${CHECK_AUTO_BKP_OPT} ]); then
stack_help
fi

check_backup_target_config "${OCARIOT_CREDS_DRIVER}"
check_backup_target_config "${OCARIOT_BACKUP_CONTAINER}" "${OCARIOT_CREDS_DRIVER}"

if [ ${RESTORE_TIME} ]; then
RESTORE_TIME="--time ${RESTORE_TIME}"
Expand Down Expand Up @@ -132,7 +138,7 @@ if [ -z "${SERVICES}" ]; then
grep -oE "${EXPRESSION_GREP}" |
sed 's/\(psmdb-\|psmysql-\|ocariot-\|-data\|redis-\)//g')
elif [ "$1" = "restore" ] && [ "${RESTORE_TARGET}" != "LOCAL" ]; then
SERVICES=$(cloud_bkps "" "${OCARIOT_CREDS_DRIVER}" ${VOLUME_COMMAND} |
SERVICES=$(cloud_bkps "" "${OCARIOT_BACKUP_CONTAINER}" "${OCARIOT_CREDS_DRIVER}" ${VOLUME_COMMAND} |
grep -oE "${EXPRESSION_GREP}" |
sed 's/\(psmdb-\|psmysql-\|ocariot-\|-data\|redis-\)//g')
else
Expand All @@ -155,7 +161,7 @@ for SERVICE in ${SERVICES}; do
grep -w ${SERVICE})
elif [ "$1" = "restore" ] && [ "${RESTORE_TARGET}" != "LOCAL" ]; then
if [ -z "${CLOUD_BACKUPS}" ];then
CLOUD_BACKUPS=$(cloud_bkps "" "${OCARIOT_CREDS_DRIVER}" ${VOLUME_COMMAND})
CLOUD_BACKUPS=$(cloud_bkps "" "${OCARIOT_BACKUP_CONTAINER}" "${OCARIOT_CREDS_DRIVER}" ${VOLUME_COMMAND})
fi
MESSAGE="Volume BKP ${SERVICE} not found!"
VOLUME_NAME="$(echo ${CLOUD_BACKUPS} |
Expand Down Expand Up @@ -205,7 +211,7 @@ PROCESS_BKP="OK"
BKP_CONFIG_MODEL=$(mktemp --suffix=.json)

docker run -d --rm \
--name ${BACKUP_CONTAINER_NAME} \
--name ${OCARIOT_BACKUP_CONTAINER} \
${VOLUMES} \
${VOLUMES_CACHE} \
-v ${LOCAL_TARGET}:/local-backup${BACKUP_VOLUME_PROPERTY} \
Expand All @@ -225,7 +231,7 @@ for VOLUME in ${VOLUMES_BKP}; do
restore_config "${BKP_CONFIG_MODEL}" "${VOLUME}"
fi

backup_container_operation restart
backup_container_operation restart "${OCARIOT_BACKUP_CONTAINER}"

echo "======Backup of ${VOLUME} volume======"

Expand All @@ -237,7 +243,7 @@ for VOLUME in ${VOLUMES_BKP}; do
-e GOOGLE_DRIVE_SECRET=${CLOUD_SECRET_ACCESS_KEY} \
-e AWS_ACCESS_KEY_ID=${CLOUD_ACCESS_KEY_ID} \
-e AWS_SECRET_ACCESS_KEY=${CLOUD_SECRET_ACCESS_KEY} \
${BACKUP_CONTAINER_NAME} bash -c "${COMMAND} && remove-older-than ${BACKUP_DATA_RETENTION} --force"
${OCARIOT_BACKUP_CONTAINER} bash -c "${COMMAND} && remove-older-than ${BACKUP_DATA_RETENTION} --force"

if [ $? != 0 ]; then
PROCESS_BKP=FALSE
Expand All @@ -247,7 +253,7 @@ for VOLUME in ${VOLUMES_BKP}; do
INCREMENT=$((INCREMENT + 1))
done

backup_container_operation stop
backup_container_operation stop "${OCARIOT_BACKUP_CONTAINER}"

if [ "${PROCESS_BKP}" = "OK" ]; then
RUNNING_SERVICES=$(echo ${RUNNING_SERVICES} | sed 's/ //g')
Expand Down

0 comments on commit 9b180c7

Please sign in to comment.