Merge branch 'release/1.5.0'

ocariot · Apr 16, 2020 · 1dda757 · 1dda757
2 parents e9e891e + 0b90714
commit 1dda757
Show file tree

Hide file tree

Showing 10 changed files with 725 additions and 417 deletions.
diff --git a/.env.example b/.env.example
@@ -79,4 +79,33 @@ EXPRESSION_AUTO_SYNC="0 * * * *"
 
 # FIREBASE_KEYFILE_PATH:    File that must be obtained in the Firebase project settings.
 #                           example value: /home/myuser/firebase_keyfile.json
-FIREBASE_KEYFILE_PATH=/home/myuser/firebase_keyfile.json
+FIREBASE_KEYFILE_PATH=/home/myuser/firebase_keyfile.json
+
+###########################################################################################################
+###########################################  DATA BACKUP SETUP ############################################
+###########################################################################################################
+
+# LOCAL_TARGET:              Defined the path where the generated backup will be stored locally.
+#                            example value: /path/to/backup
+LOCAL_TARGET=/path/to/backup
+
+# CLOUD_TARGET:              Define a URL and the path where the backup will be stored in the Google Drive or AWS S3 cloud.
+#                            example value AWS S3: s3://s3.<bucket-region>.amazonaws.com/<bucket-name>[/<path>]
+#                            example value Google Drive: gdocs://youremail@gmail.com[/<path>]
+# CLOUD_TARGET=
+
+# CLOUD_ACCESS_KEY_ID:       Client Id for access Google Driver or AWS S3 service responsible to store backup data.
+#                            example value: AKIAYXGARMBIICAV23FE
+# CLOUD_ACCESS_KEY_ID=
+
+# CLOUD_SECRET_ACCESS_KEY:   Client Secret for access Google Driver or S3 service responsible to store backup data.
+#                            example value: J/YXk2xMaJQugb+vYm+c/TbTz+LpMnkxucdfv/Rh
+# CLOUD_SECRET_ACCESS_KEY=
+
+# RESTORE_TARGET:            Define the target used to restore the backup.
+#                            example value: LOCAL | GOOGLE_DRIVE | AWS
+RESTORE_TARGET=LOCAL
+
+# BACKUP_DATA_RETENTION:     Time the data backup will remain stored.
+#                            Default value (15 days): 15d
+BACKUP_DATA_RETENTION=15D
diff --git a/.env.monitor.example b/.env.monitor.example
@@ -37,6 +37,35 @@ GF_SMTP_PASSWORD=secret
 ##############################################  STORAGE DATA  #############################################
 ###########################################################################################################
 
-# RETENTION_DATA:      Time the data remained stored.
+# DATA_RETENTION:            Time the data remained stored in database.
 #                            default value (15 days): 15d
-RETENTION_DATA=15d
+DATA_RETENTION=15d
+
+###########################################################################################################
+###########################################  DATA BACKUP SETUP ############################################
+###########################################################################################################
+
+# LOCAL_TARGET:              Defined the path where the generated backup will be stored locally.
+#                            example value: /path/to/backup
+LOCAL_TARGET=/path/to/backup
+
+# CLOUD_TARGET:              Define a URL and the path where the backup will be stored in the Google Drive or AWS S3 cloud.
+#                            example value AWS S3: s3://s3.<bucket-region>.amazonaws.com/<bucket-name>[/<path>]
+#                            example value Google Drive: gdocs://youremail@gmail.com[/<path>]
+# CLOUD_TARGET=
+
+# CLOUD_ACCESS_KEY_ID:       Client Id for access Google Driver or AWS S3 service responsible to store backup data.
+#                            example value: AKIAYXGARMBIICAV23FE
+# CLOUD_ACCESS_KEY_ID=
+
+# CLOUD_SECRET_ACCESS_KEY:   Client Secret for access Google Driver or S3 service responsible to store backup data.
+#                            example value: J/YXk2xMaJQugb+vYm+c/TbTz+LpMnkxucdfv/Rh
+# CLOUD_SECRET_ACCESS_KEY=
+
+# RESTORE_TARGET:            Define the target used to restore the backup.
+#                            example value: LOCAL | GOOGLE_DRIVE | AWS
+RESTORE_TARGET=LOCAL
+
+# BACKUP_DATA_RETENTION:     Time the data backup will remain stored.
+#                            Default value (15 days): 15D
+BACKUP_DATA_RETENTION=15D
diff --git a/README.md b/README.md
@@ -44,11 +44,11 @@ Repository with configuration files required for OCARIoT platform **deployment i
 All software installation is performed using the following command:
 
 ```sh
-curl -o- https://raw.githubusercontent.com/ocariot/docker-swarm/1.4.3/install.sh | sudo bash
+curl -o- https://raw.githubusercontent.com/ocariot/docker-swarm/1.5.0/install.sh | sudo bash
 ```
 
 ```sh
-wget -qO- https://raw.githubusercontent.com/ocariot/docker-swarm/1.4.3/install.sh | sudo bash
+wget -qO- https://raw.githubusercontent.com/ocariot/docker-swarm/1.5.0/install.sh | sudo bash
 ```
 
 
@@ -120,6 +120,51 @@ Variables to define the administrator user's credentials the first time the plat
 | `ADMIN_USERNAME` | Username of the default admin user created automatically at the first time the OCARIoT platform is instantiated. | `admin` |
 | `ADMIN_PASSWORD` | Password of the default admin user created automatically at the first time the OCARIoT platform is instatiated. | `admin` |
 
+#### 2.1.6 Data Backup Setup
+
+Variables responsible for defining backup settings. The variables with prefix `CLOUD` are commented out by default, to activate them uncommented and set their respective value based on the values provided by the cloud service that you want to perform the backups and restores. The supported cloud storage services are Google Drive and AWS S3.
+
+In order for backup and restore operations to be successful, credentials must be granted permissions to manipulate the cloud storage location:
+
+- [Google Drive](https://console.developers.google.com/apis/credentials)
+When performing the first backup, a link will be provided that redirects the browser to a user's authentication screen at Google, thus granting permission to manipulate Google Drive. In future `backup` or `restore` operations, authentication is not required unless the `google_credentials` volume is removed.
+
+- [AWS S3](https://docs.aws.amazon.com/pt_br/sdk-for-java/v1/developer-guide/signup-create-iam-user.html)
+To use the `backup` or` restore` operations, it is necessary to associate the following policy with the created user:
+
+```json=
+{
+    "Version":"2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": "s3:ListAllMyBuckets",
+            "Resource": "arn:aws:s3:::*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": "s3:*",
+            "Resource": [
+                "arn:aws:s3:::BUCKET_NAME",
+                "arn:aws:s3:::BUCKET_NAME/*"
+            ]
+        }
+    ]
+}
+
+```
+> :warning: Note: Replace BUCKET_NAME with your bucket name!
+
+| Variable | Description | Example |
+| -------- | ----------- | ------- |
+| `LOCAL_TARGET` | Defined the path where the generated backup will be stored locally. | `/path/to/backup` |
+| `CLOUD_TARGET` | Define a URL and the path where the backup will be stored in the Google Drive or AWS S3 cloud. | `s3://s3.<bucket-region>.amazonaws.com/<bucket-name>[/<path>]` |
+| `CLOUD_ACCESS_KEY_ID` | Client Id for access Google Driver or AWS S3 service responsible to store backup data. | `AKIAYXGARMBIICAV23FE` |
+| `CLOUD_SECRET_ACCESS_KEY` | Client Secret for access Google Driver or S3 service responsible to store backup data. | `J/YXk2xMaJQugb+vYm+c/TbTz+LpMnkxucdfv/Rh` |
+| `RESTORE_TARGET` | Define the target used to restore the backup. example value: `LOCAL, GOOGLE_DRIVE, AWS`. | `AWS` |
+| `BACKUP_DATA_RETENTION` | Time the data backup will remain stored. Default value (15 days): `15D`. | `15D` |
+
+
 ### 2.2 Building and Deploying the containers
 
 #### 2.2.1 Start containers
@@ -162,7 +207,6 @@ $ sudo ocariot stack backup
 
 - `--services <values>` - Defines a set of services from which you want to generate the backup. The delimiter for specifying one more service is space. For example: `sudo ocariot stack backup --services account iot-tracking`;
 - `--expression <values>` - Parameter used to define a crontab expression that will schedule the generation of a backup. The value of this option must be passed in double quotes. Example: `sudo ocariot stack backup --expression "0 3 * * *"`;
-- `--path <values>` - Parameter used to specify the path where the backup will be saved. If this option is omitted, the backup files will be placed at the place of execution of the command currently described.
 
 #### 2.2.4 Restore
 In order to restore all backups of the volumes present in the current path, the following interface is reserved:
@@ -176,7 +220,6 @@ $ sudo ocariot stack restore
 *Optional parameters:*
 
 - `--keys` - Specifies the location of the file containing the encryption keys and root token used by the vault. This file was generated at the first start of the OCARIoT stack using the command [`sudo ocariot stack start`](#3-Building-and-Deploying-the-containers). To restore only the cryptographic keys, the backup path must not have any backup files;
-- `--path` - Parameter used to specify the path where the backup files will be searched for restoring from a previous backup performed. If this option is omitted, the backup files will be searched at the place of execution of the command currently described;
 - `--services <values>` - Defines a set of services that will have their volumes restored. The delimiter for specifying one more service is space. For example: `sudo ocariot stack restore --services account iot-tracking`;
 - `--time` - You can restore from a particular backup by adding a time parameter to the command restore. For example, using restore `--time 3D `at the end in the above command will restore a backup from 3 days ago. See the [Duplicity manual](http://duplicity.nongnu.org/vers7/duplicity.1.html#toc8) to view the accepted time formats.
 
@@ -226,7 +269,51 @@ Variables to define the administrator user's credentials the first time the Graf
 
 | Variable | Description | Example |
 | -------- | ----------- | ------- |
-| `RETENTION_DATA` | Time the data remained stored. | `15d` - corresponds to 15 days |
+| `DATA_RETENTION` | Time the data remained stored in database. | `15d` - corresponds to 15 days |
+
+#### 3.1.4 Data Backup Setup
+
+Variables responsible for defining backup settings. The variables with prefix `CLOUD` are commented out by default, to activate them uncommented and set their respective value based on the values provided by the cloud service that you want to perform the backups and restores. The supported cloud storage services are Google Drive and AWS S3.
+
+In order for backup and restore operations to be successful, credentials must be granted permissions to manipulate the cloud storage location:
+
+- [Google Drive](https://console.developers.google.com/apis/credentials)
+When performing the first backup, a link will be provided that redirects the browser to a user's authentication screen at Google, thus granting permission to manipulate Google Drive. In future `backup` or `restore` operations, authentication is not required unless the `google_credentials` volume is removed.
+
+- [AWS S3](https://docs.aws.amazon.com/pt_br/sdk-for-java/v1/developer-guide/signup-create-iam-user.html)
+To use the `backup` or` restore` operations, it is necessary to associate the following policy with the created user:
+
+```json=
+{
+    "Version":"2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": "s3:ListAllMyBuckets",
+            "Resource": "arn:aws:s3:::*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": "s3:*",
+            "Resource": [
+                "arn:aws:s3:::BUCKET_NAME",
+                "arn:aws:s3:::BUCKET_NAME/*"
+            ]
+        }
+    ]
+}
+
+```
+> :warning: Note: Replace BUCKET_NAME with your bucket name!
+
+| Variable | Description | Example |
+| -------- | ----------- | ------- |
+| `LOCAL_TARGET` | Defined the path where the generated backup will be stored locally. | `/path/to/backup` |
+| `CLOUD_TARGET` | Define a URL and the path where the backup will be stored in the Google Drive or AWS S3 cloud. | `s3://s3.<bucket-region>.amazonaws.com/<bucket-name>[/<path>]` |
+| `CLOUD_ACCESS_KEY_ID` | Client Id for access Google Driver or AWS S3 service responsible to store backup data. | `AKIAYXGARMBIICAV23FE` |
+| `CLOUD_SECRET_ACCESS_KEY` | Client Secret for access Google Driver or S3 service responsible to store backup data. | `J/YXk2xMaJQugb+vYm+c/TbTz+LpMnkxucdfv/Rh` |
+| `RESTORE_TARGET` | Define the target used to restore the backup. example value: `LOCAL, GOOGLE_DRIVE, AWS`. | `AWS` |
+| `BACKUP_DATA_RETENTION` | Time the data backup will remain stored. Default value (15 days): `15D`. | `15D` |
 
 ### 3.2 Building and Deploying the containers
 
@@ -266,7 +353,6 @@ $ sudo ocariot monitor backup
 
 - `--services <values>` - Defines a set of services from which you want to generate the backup. The delimiter for specifying one more service is space. For example: `sudo ocariot monitor backup --services grafana prometheus`;
 - `--expression <values>` - Parameter used to define a crontab expression that will schedule the generation of a backup. The value of this option must be passed in double quotes. Example: `sudo ocariot monitor backup --expression "0 3 * * *"`;
-- `--path <values>` - Parameter used to specify the path where the backup will be saved. If this option is omitted, the backup files will be placed at the place of execution of the command currently described.
 
 #### 3.2.4 Restore
 In order to restore all backups of the volumes present in the current path, the following interface is reserved:
@@ -279,7 +365,6 @@ $ sudo ocariot monitor restore
 
 *Optional parameters:*
 
-- `--path` - Parameter used to specify the path where the backup files will be searched for restoring from a previous backup performed. If this option is omitted, the backup files will be searched at the place of execution of the command currently described;
 - `--services <values>` - Defines a set of services that will have their volumes restored. The delimiter for specifying one more service is space. For example: `sudo ocariot monitor restore --services grafana prometheus`;
 - `--time` - You can restore from a particular backup by adding a time parameter to the command restore. For example, using restore `--time 3D `at the end in the above command will restore a backup from 3 days ago. See the [Duplicity manual](http://duplicity.nongnu.org/vers7/duplicity.1.html#toc8) to view the accepted time formats.
 

diff --git a/docker-monitor-stack.yml b/docker-monitor-stack.yml
@@ -67,7 +67,7 @@ services:
     command:
       - '--config.file=/etc/prometheus/config.yml'
       - '--storage.tsdb.path=/prometheus'
-      - '--storage.tsdb.retention.time=${RETENTION_DATA:-15d}'
+      - '--storage.tsdb.retention.time=${DATA_RETENTION:-15d}'
     volumes:
       - prometheus-data:/prometheus
     configs:

diff --git a/install.sh b/install.sh
@@ -4,7 +4,7 @@ INSTALL_PATH="/opt/ocariot-swarm"
 
 version()
 {
-  echo "1.4.3"
+  echo "1.5.0"
 }
 
 isInstalled()

diff --git a/ocariot b/ocariot
@@ -103,6 +103,7 @@ case $1 in
 		stack_options $(echo $@ | sed "s/$1 //g")
 		;;
   monitor)
+  	EXPRESSION_BKP="$(echo "$@" | grep -o -P '(?<=--expression ).*' | sed 's/--.*//g')"
     monitor_options $(echo $@ | sed "s/$1 //g")
 		;;
   version)