metabase and agraph

Makefile

@@ -144,6 +144,14 @@ stop-mongo:
 	@ echo -e "$(BUILD_PRINT)Stopping the Mongo services $(END_BUILD_PRINT)"
 	@ docker-compose -p ${ENVIRONMENT} --file ./infra/mongo/docker-compose.yml --env-file ${ENV_FILE} down
 
+start-metabase: build-externals
+	@ echo -e "$(BUILD_PRINT)Starting the Metabase services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/metabase/docker-compose.yml --env-file ${ENV_FILE} up -d
+
+stop-metabase:
+	@ echo -e "$(BUILD_PRINT)Stopping the Metabase services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/metabase/docker-compose.yml --env-file ${ENV_FILE} down
+
 init-rml-mapper:
 	@ echo -e "RMLMapper folder initialisation!"
 	@ wget -c https://github.com/RMLio/rmlmapper-java/releases/download/v5.0.0/rmlmapper-5.0.0-r362-all.jar -P .rmlmapper/
@@ -154,8 +162,8 @@ init-saxon:
 	@ wget -c https://kumisystems.dl.sourceforge.net/project/saxon/Saxon-HE/10/Java/SaxonHE10-6J.zip -P .saxon/
 	@ cd .saxon && unzip SaxonHE10-6J.zip && rm -rf SaxonHE10-6J.zip
 
-start-project-services: | start-airflow start-mongo init-rml-mapper
-stop-project-services: | stop-airflow stop-mongo
+start-project-services: | start-airflow start-mongo init-rml-mapper start-allegro-graph start-metabase
+stop-project-services: | stop-airflow stop-mongo stop-allegro-graph stop-metabase
 
 #-----------------------------------------------------------------------------
 # VAULT SERVICES
@@ -190,6 +198,8 @@ staging-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
 	@ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/staging >> .env
 	@ vault kv get -format="json" ted-staging/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
 	@ vault kv get -format="json" ted-staging/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+	@ vault kv get -format="json" ted-staging/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+	@ vault kv get -format="json" ted-staging/agraph | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
 
 dev-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
 	@ echo -e "$(BUILD_PRINT)Create .env file $(END_BUILD_PRINT)"
@@ -207,6 +217,8 @@ dev-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
 	@ echo AIRFLOW_INFRA_FOLDER=${AIRFLOW_INFRA_FOLDER} >> .env
 	@ vault kv get -format="json" ted-dev/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
 	@ vault kv get -format="json" ted-dev/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+	@ vault kv get -format="json" ted-dev/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+	@ vault kv get -format="json" ted-dev/agraph | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
 
 
 prod-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
@@ -225,6 +237,8 @@ prod-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
 	@ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/prod >> .env
 	@ vault kv get -format="json" ted-prod/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
 	@ vault kv get -format="json" ted-prod/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+	@ vault kv get -format="json" ted-prod/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+	@ vault kv get -format="json" ted-prod/agraph | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
 
 local-dotenv-file: rml-mapper-path-add-dotenv-file elk-add-dotenv-file logging-add-dotenv-file
 

infra/metabase/docker-compose.yml

@@ -0,0 +1,54 @@
+version: "3.8"
+
+services:
+
+  postgres:
+    container_name: metabase-postgres-${ENVIRONMENT}
+    restart: "always"
+    image: postgres
+    environment:
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - METABASE_PASSWORD=${METABASE_PASSWORD}
+    volumes:
+      - postgres_volume:/var/lib/postgresql/data/
+      - ./services/postgres/:/docker-entrypoint-initdb.d/
+    networks:
+      - metabase-net
+
+  metabase:
+    container_name: metabase-${ENVIRONMENT}
+    restart: "always"
+    image: metabase/metabase
+    environment:
+      - ENV_MB_DB_TYPE=postgres
+      - ENV_MB_DB_DBNAME=${ENV_MB_DB_DBNAME}
+      - ENV_MB_DB_PORT=${ENV_MB_DB_PORT}
+      - ENV_MB_DB_USER=${ENV_MB_DB_USER}
+      - ENV_MB_DB_PASS=${ENV_MB_DB_PASS}
+      - ENV_MB_DB_HOST=metabase-postgres-${ENVIRONMENT}
+      - ENV_MB_ENCRYPTION_SECRET_KEY=${ENV_MB_ENCRYPTION_SECRET_KEY}
+    networks:
+      - metabase-net
+      - proxy-net
+    labels:
+      #### Labels define the behavior and rules of the traefik proxy for this container ####
+      - "traefik.enable=true" # <== Enable traefik to proxy this container
+      - "traefik.http.routers.${ENVIRONMENT}-metabase.rule=Host(`metabase.${SUBDOMAIN}${DOMAIN}`)" # <== Your Domain Name goes here for the http rule
+      - "traefik.http.routers.${ENVIRONMENT}-metabase.entrypoints=web" # <== Defining the entrypoint for http, **ref: line 30
+      - "traefik.http.routers.${ENVIRONMENT}-metabase.middlewares=redirect@file" # <== This is a middleware to redirect to https
+      - "traefik.http.routers.${ENVIRONMENT}-metabase-secured.rule=Host(`metabase.${SUBDOMAIN}${DOMAIN}`)" # <== Your Domain Name for the https rule
+      - "traefik.http.routers.${ENVIRONMENT}-metabase-secured.entrypoints=web-secured" # <== Defining entrypoint for https, **ref: line 31
+      - "traefik.http.routers.${ENVIRONMENT}-metabase-secured.tls.certresolver=mytlschallenge" # <== Defining certsresolvers for https
+
+
+volumes:
+  postgres_volume:
+    name: postgres-volume-${ENVIRONMENT}
+
+networks:
+  metabase-net:
+    internal: false
+    name: metabase-net-${ENVIRONMENT}
+  proxy-net:
+    external:
+      name: proxy-net

infra/metabase/services/postgres/00_metabase_create_user.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e # exit if a command exits with a not-zero exit code
+
+POSTGRES="psql -U postgres"
+
+# create a shared role to read & write general datasets into postgres
+echo "Creating database role: metabase"
+$POSTGRES <<-EOSQL
+CREATE USER metabase WITH
+    LOGIN
+    NOSUPERUSER
+    NOCREATEDB
+    NOCREATEROLE
+    NOINHERIT
+    NOREPLICATION
+    PASSWORD '$METABASE_PASSWORD';
+EOSQL

infra/metabase/services/postgres/01_metabase_create_db.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e # exit immediately if a command exits with a non-zero status.
+
+POSTGRES="psql --username postgres"
+
+# create database for superset
+echo "Creating database: metabase"
+$POSTGRES <<EOSQL
+CREATE DATABASE metabase OWNER metabase;
+EOSQL