Refactor docker compose, .env file doc

.env.dev.example

@@ -0,0 +1,46 @@
+# Ports on the host Docker will listen on. Change them if they are clashing with other containers or processes that are running.
+HOST_API_PORT=8080
+HOST_APP_PORT=3000
+ES_PORT=9200
+YENTE_PORT=8000
+PG_PORT=5432
+
+# Set your license key here if you have one in order to access premium features.
+LICENSE_KEY=
+
+# Change the three settings below to create your initial organization and users.
+# The user must also have a matching account in the authentication store used (in Firebase, for example).
+# Subsequent users will be able to be created from the application.
+# The firebase emulator attached in the docker compose file is preconfigured with a `jbe@zorg.com` user.
+CREATE_ORG_NAME=Zorg
+CREATE_ORG_ADMIN_EMAIL=jbe@zorg.com
+
+# Configure the document blob storage backend. This example uses local files for a minimal working example.
+# See the .env.example file for a more complete documentation
+INGESTION_BUCKET_URL="file://./tempFiles/data-ingestion-bucket?create_dir=true"
+CASE_MANAGER_BUCKET_URL="file://./tempFiles/case-manager-bucket?create_dir=true"
+
+# Configure the connection details to your Metabase instance.
+#  - To retrieve the JWT signing key, go to your Metabase admin panel, in 'Settings', then 'Embedding', and click 'Manage' under 'Static embedding'
+METABASE_SITE_URL=
+METABASE_JWT_SIGNING_KEY=
+METABASE_GLOBAL_DASHBOARD_ID=
+
+# Set up connection details to Convoy to enable webhooks sending.
+# You can get your project ID and API key from your project settings page in Convoy's dashboard, in the "Secrets" section.
+# NB: CONVOY_API_URL should be {scheme}://{host}:{port}/api - forgetting the /api will result in unexpected errors.
+CONVOY_API_URL= 
+CONVOY_API_KEY=
+CONVOY_PROJECT_ID=
+
+# If you use the SaaS Open Sanctions API, only uncomment OPENSANCTIONS_API_KEY and provide its values.
+# If you self-host the API, uncomment all relevant settings to point to your deployment and configure authentication.
+#  - OPENSANCTIONS_AUTH_METHOD supports 'bearer' and 'basic'
+#    If basic, provide the username and password in the form of 'user:password'
+# OPENSANCTIONS_API_HOST=http://yente
+# OPENSANCTIONS_AUTH_METHOD=bearer
+# OPENSANCTIONS_API_KEY=
+
+# Configure session parameters on the frontend
+SESSION_SECRET=SESSION_SECRET
+SESSION_MAX_AGE=43200

.env.example

@@ -2,53 +2,61 @@
 # REQUIRED SETTINGS
 #
 
-ENV=development
-NODE_ENV=production
-
-# Ports respectively used by the backend API and the frontends.
+# If equal to "development", applies configurations on the backend and frontend containers that 
+# are not suitable for production, related to CORS, secure cookies and https.
+# Any other value works for a production environment, and will be used by sentry to mark the environment
+# an error occurred in.
+ENV=production
+
+# Ports respectively used by the backend API and the frontends containers
+# Maps to the "PORT" environment variable in the backend and frontend containers
+# respectively.
 API_PORT=8080
 APP_PORT=3000
 
-# Ports on the host Docker will listen on.
+# Ports on the host Docker will listen on. Specific to using docker-compose,
+# can be ignored for a deployment with managed containers or k8s.
 HOST_API_PORT=8080
 HOST_APP_PORT=3000
 
-# Set this to the domain and port used by your users to access Marble's frontend. The second one must include the scheme (e.g http://).
-MARBLE_APP_HOST="localhost:3000"
-MARBLE_APP_DOMAIN="http://localhost:3000"
+# Set this to the url used by your users to access Marble's frontend from the browser.
+# Must include the scheme (e.g http://), and the port if non-standard.
+MARBLE_APP_URL="http://localhost:3000"
 
 # Set those values to the URLs the Marble API can be reached at. Both must include the scheme (e.g http://)
-# "MARBLE_API_DOMAIN_SERVER" must be reachable from the frontend host.
-# "MARBLE_API_DOMAIN_CLIENT" must be reachable from the user's browser.
-MARBLE_API_DOMAIN_SERVER="http://api:8080"
-MARBLE_API_DOMAIN_CLIENT="http://localhost:8080"
-
-# Set your license key here if you have one in order to access premium features.
-LICENSE_KEY=
+# "MARBLE_API_URL_SERVER" must be reachable from the frontend host. With docker-compose, use the api container name as host name.
+# "MARBLE_API_URL_CLIENT" must be reachable from the user's browser.
+MARBLE_API_URL_SERVER="http://api:8080"
+MARBLE_API_URL_CLIENT="http://localhost:8080"
 
 # RSA private key, in PEM format, used for for signing authentication tokens. MUST be changed for production.
-# We recommend using AUTHENTICATION_JWT_SIGNING_KEY_FILE to point to a private key on disk.
+# We recommend using AUTHENTICATION_JWT_SIGNING_KEY_FILE to point to a private key on disk because multi-line
+# environment variables are not handled very gracefully.
+# 
+# If using AUTHENTICATION_JWT_SIGNING_KEY, line breaks should be replaced with '\n':
+# e.g. AUTHENTICATION_JWT_SIGNING_KEY="-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBA...
+# 
 # To generate a private key, use `openssl genrsa -out /path/to/private/key.pem 4096`.
-# If left empty, will be regenerated on every app restart.
+# If left empty, a key will be regenerated on every app restart, which may cause unexpected logouts.
 AUTHENTICATION_JWT_SIGNING_KEY=
+# AUTHENTICATION_JWT_SIGNING_KEY_FILE=/shared/jwt_key.pem
 
+# Configure your PostgreSQL database connection information, either by providing a DSN using this form:
+# PG_CONNECTION_STRING='postgres://postgres:marble@localhost:5432/marble?sslmode=prefer'
 # Or by setting each piece of information in those variables:
 PG_HOSTNAME=db
-PG_PORT=54321
+PG_PORT=5432
 PG_USER=postgres
-PG_PASSWORD=marble
-
-# Change the three settings below to create your initial organization and users.
-# The user must also have a matching account in the authentication store used (in Firebase, for example).
-# Subsequent users will be able to be created from the application.
-CREATE_ORG_NAME=Zorg
-CREATE_ORG_ADMIN_EMAIL=jbe@zorg.com
+PG_PASSWORD=changeme
+# For production, this SHOULD be set to 'require'
+PG_SSL_MODE=prefer
 
 # Configure the ID of your Firebase project for authentication and the path to the service account's JSON private key file.
 #  - The 'Project ID' can be found in the 'General' section of your Firebase project's settings page.
-#  - The private key must be generated in the 'Serv
-GOOGLE_CLOUD_PROJECT="test-project"
-# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/private/key.json
+#  - The private key must be generated in the 'Service accounts' tab by clicking 'Generate new private key'
+# If you are using GCS as a blob storage backend, the same service account key is used to sign URLs.
+GOOGLE_CLOUD_PROJECT=
+GOOGLE_APPLICATION_CREDENTIALS=/shared/service_account_key.json
 
 # Configure the document blob storage backend.
 # The example values MUST be changed for anything but a local test environment since the file provider does not support all features.
@@ -72,32 +80,67 @@ GOOGLE_CLOUD_PROJECT="test-project"
 INGESTION_BUCKET_URL="file://./tempFiles/data-ingestion-bucket?create_dir=true"
 CASE_MANAGER_BUCKET_URL="file://./tempFiles/case-manager-bucket?create_dir=true"
 
-# Configure the connection details to your Metabase instance.
-#  - To retrieve the JWT signing key, go to your Metabase admin panel, in 'Settings', then 'Embedding', and click 'Manage' under 'Static embedding'
-METABASE_SITE_URL="https://your_subdomain.metabaseapp.com"
-METABASE_JWT_SIGNING_KEY="dummy"
-METABASE_GLOBAL_DASHBOARD_ID=123
-
 # Configure your Firebase project to allow username and password authentication.
-FIREBASE_PROJECT_ID=
+# To retrieve this value, go into the settings of your Firebase project, and register 
+# a new app from the 'General' tab.
 FIREBASE_API_KEY=
 
-SESSION_SECRET=SESSION_SECRET
+# Generate a random string, for instance using `openssl rand -base64 128 | tr -d "\n"`
+SESSION_SECRET=changeme
 SESSION_MAX_AGE=43200
 
 #
 # OPTIONAL SETTINGS
 #
 
-# You can adjust the URLs used to access the Firebase emulator.
-#  - The "FIREBASE_AUTH_EMULATOR_HOST_CLIENT" URL needs to be accessible from your browser.
-#  - The "FIREBASE_AUTH_EMULATOR_HOST_SERVER" URL needs to be accessible from the API container.
-FIREBASE_AUTH_EMULATOR_HOST_CLIENT=localhost:9099
-FIREBASE_AUTH_EMULATOR_HOST_SERVER=firebase-auth:9099
-FIREBASE_AUTH_EMULATOR_PORT=9099
+# Set your license key here if you have one in order to access premium features.
+LICENSE_KEY=
+
+# Change the two settings below to create your initial organization and users.
+# The user must also have a matching account in the authentication store used (in Firebase, for example).
+# Subsequent users will be able to be created from the application, and the variables
+# can be left blank.
+CREATE_ORG_NAME=
+CREATE_ORG_ADMIN_EMAIL=
+
+# Configure the connection details to your Metabase instance.
+# - To retrieve the JWT signing key, go to your Metabase admin panel, in 'Settings', then 'Embedding', and click 'Manage' under 'Static embedding'
+# - The dashboard ID is the ID of the dashboard you want to embed in Marble. You will find it when you publish a dashboard as 
+#   a static embedding on Metabase.
+# - The site url can be on a metabase domain, or can use your self-hosted Metabase instance.
+METABASE_SITE_URL=
+METABASE_JWT_SIGNING_KEY=
+METABASE_GLOBAL_DASHBOARD_ID=
+
+# Set up connection details to Convoy to enable webhooks sending.
+# You can get your project ID and API key from your project settings page in Convoy's dashboard, in the "Secrets" section.
+# NB: CONVOY_API_URL should be {scheme}://{host}:{port}/api - forgetting the /api will result in unexpected errors.
+CONVOY_API_URL= 
+CONVOY_API_KEY=
+CONVOY_PROJECT_ID=
+
+# Configure the URL of the Elasticsearch instance used by yente for sanction screening.
+# Those are not specific to marble, and are only used to configure containers
+# when using docker-compose.
+YENTE_ELASTICSEARCH_URL=http://marble-es:9200
+YENTE_PORT=8000
+ES_PORT=9200
+
+# Configure access to the Open Sanctions API to use sanction checks
+#
+# If you use the SaaS Open Sanctions API, only uncomment OPENSANCTIONS_API_KEY and provide its values.
+# If you self-host the API, uncomment all relevant settings to point to your deployment and configure authentication.
+#  - OPENSANCTIONS_AUTH_METHOD supports 'bearer' and 'basic'
+#    If basic, provide the username and password in the form of 'user:password'
+# OPENSANCTIONS_API_HOST=http://self.hosted.local
+# OPENSANCTIONS_AUTH_METHOD=bearer
+# OPENSANCTIONS_API_KEY=
+
+# Configure various external integrations.
+SENTRY_DSN=
 
 # If you need to support federated authentication through Firebase, you will need to configure the following settings:
-# To retrieve those value, go into the settings of your Firebase project, and register a new app from the 'General' tab.
-# The two information below will be provided to you after that. 
+# To retrieve those values, go into the settings of your Firebase project, and register a new app from the 'General' tab.
+# The two pieces of information below will be provided to you after that. 
 # FIREBASE_APP_ID=
 # FIREBASE_AUTH_DOMAIN=

.gitignore

@@ -12,3 +12,4 @@
 .secrets
 key.json
 .env
+.env.dev

docker-compose-dev.yaml

@@ -0,0 +1,167 @@
+name: marble
+
+x-backend-image-version: &backend-image-version
+  image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:v0.38.2
+x-frontend-image-version: &frontend-image-version
+  image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend:v0.38.0
+
+x-shared-environment: &shared-env
+  ENV: development
+  MARBLE_APP_URL: http://localhost:${HOST_APP_PORT:-3000}
+x-backend-environment: &backend-env
+  LICENSE_KEY: ${LICENSE_KEY:-LICENSE_KEY}
+  PORT: 8080
+
+  PG_HOSTNAME: db
+  PG_PORT: ${PG_PORT:-5432}
+  PG_USER: postgres
+  PG_PASSWORD: changeme
+
+  INGESTION_BUCKET_URL: ${INGESTION_BUCKET_URL:-}
+  CASE_MANAGER_BUCKET_URL: ${CASE_MANAGER_BUCKET_URL:-}
+
+  FIREBASE_AUTH_EMULATOR_HOST: firebase-auth:9099
+
+  # default value of GOOGLE_CLOUD_PROJECT must be kept if working with the emulator (and the emulator is running in the docker image below)
+  GOOGLE_CLOUD_PROJECT: test-project
+
+  CREATE_ORG_NAME: ${CREATE_ORG_NAME:-}
+  CREATE_ORG_ADMIN_EMAIL: ${CREATE_ORG_ADMIN_EMAIL:-}
+
+  SEGMENT_WRITE_KEY: ${SEGMENT_WRITE_KEY_BACKEND:-UgkImFmHmBZAWh5fxIKBY3QtvlcBrhqQ}
+
+  METABASE_SITE_URL: ${METABASE_SITE_URL:-}
+  METABASE_JWT_SIGNING_KEY: ${METABASE_JWT_SIGNING_KEY:-}
+  METABASE_GLOBAL_DASHBOARD_ID: ${METABASE_GLOBAL_DASHBOARD_ID:-}
+
+  CONVOY_API_KEY: ${CONVOY_API_KEY:-}
+  CONVOY_API_URL: ${CONVOY_API_URL:-}
+  CONVOY_PROJECT_ID: ${CONVOY_PROJECT_ID:-}
+x-frontend-environment: &frontend-env
+  NODE_ENV: production
+  SESSION_SECRET: ${SESSION_SECRET:-}
+  SESSION_MAX_AGE: 43200
+
+  MARBLE_API_URL_CLIENT: http://localhost:${HOST_API_PORT:-8080}
+  MARBLE_API_URL_SERVER: http://api:${HOST_API_PORT:-8080}
+
+  FIREBASE_AUTH_EMULATOR_HOST: localhost:9099
+
+  SEGMENT_WRITE_KEY: ${SEGMENT_WRITE_KEY:-hC8qrY2OLhUpl1Xycw523tbuClxlQR6u}
+
+services:
+  db:
+    container_name: marble-postgres
+    image: postgres:15
+    restart: always
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: changeme
+      POSTGRES_DB: marble
+      PGDATA: /data/postgres
+      PGPORT: ${PG_PORT:-5432}
+    ports:
+      - ${PG_PORT:-5432}:${PG_PORT:-5432}
+    volumes:
+      - marble-db:/data/postgres
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 2s
+      timeout: 1s
+      retries: 5
+  api: &backend
+    <<: *backend-image-version
+    container_name: marble-api
+    platform: linux/amd64
+    restart: always
+    depends_on:
+      - db
+      - yente
+    entrypoint: ["./app", "--server", "--migrations"]
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8080/liveness || exit 1
+      interval: 10s
+      timeout: 60s
+      retries: 1
+      start_period: 30s
+    ports:
+      - ${HOST_API_PORT:-8080}:8080
+    volumes:
+      - marble-tempfiles:/tempFiles
+    environment:
+      <<: [*shared-env, *backend-env]
+  cron:
+    <<: *backend
+    container_name: marble-api-cron
+    depends_on:
+      - api
+    entrypoint: ["./app", "--worker"]
+    healthcheck:
+      disable: true
+    ports: []
+  app:
+    <<: *frontend-image-version
+    container_name: marble-app
+    platform: linux/amd64
+    restart: always
+    depends_on:
+      - api
+      - firebase_auth
+    healthcheck:
+      test: curl --fail http://localhost:8080/healthcheck || exit 1
+      interval: 10s
+      timeout: 60s
+      retries: 1
+      start_period: 30s
+    ports:
+      - ${HOST_APP_PORT:-3000}:8080
+    environment:
+      <<: [*shared-env, *frontend-env]
+  firebase_auth:
+    container_name: firebase-auth
+    image: europe-west1-docker.pkg.dev/marble-infra/marble/firebase-emulator:latest
+    restart: always
+    ports:
+      - 9099:9099
+      - 4000:4000
+  elasticsearch:
+    container_name: marble-es
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.14.3
+    ports:
+      - ${ES_PORT:-9200}:${ES_PORT:-9200}
+    environment:
+      - node.name=es
+      - cluster.name=marble-es
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - marble-es:/usr/share/elasticsearch/data
+  yente:
+    container_name: marble-yente
+    image: ghcr.io/opensanctions/yente:4.2.1
+    depends_on:
+      - elasticsearch
+    ports:
+      - ${YENTE_PORT:-8000}:${YENTE_PORT:-8000}
+    volumes:
+      # This configuration file will load only a reduced into the elastic search index.
+      # This is only a convenience for development purposes, and should be removed or adapted
+      # for production environments.
+      - ./contrib/yente-datasets.yml:/app/manifests/default.yml
+    environment:
+      YENTE_INDEX_TYPE: elasticsearch
+      YENTE_INDEX_URL: ${YENTE_ELASTICSEARCH_HOST:-http://marble-es:9200}
+      YENTE_UPDATE_TOKEN: ""
+
+volumes:
+  marble-db:
+    driver: local
+  marble-es:
+    driver: local
+  marble-tempfiles: