diff --git a/.env.dev.example b/.env.dev.example new file mode 100644 index 0000000..9a68055 --- /dev/null +++ b/.env.dev.example @@ -0,0 +1,46 @@ +# Ports on the host Docker will listen on. Change them if they are clashing with other containers or processes that are running. +HOST_API_PORT=8080 +HOST_APP_PORT=3000 +ES_PORT=9200 +YENTE_PORT=8000 +PG_PORT=5432 + +# Set your license key here if you have one in order to access premium features. +LICENSE_KEY= + +# Change the three settings below to create your initial organization and users. +# The user must also have a matching account in the authentication store used (in Firebase, for example). +# Subsequent users will be able to be created from the application. +# The firebase emulator attached in the docker compose file is preconfigured with a `jbe@zorg.com` user. +CREATE_ORG_NAME=Zorg +CREATE_ORG_ADMIN_EMAIL=jbe@zorg.com + +# Configure the document blob storage backend. This example uses local files for a minimal working example. +# See the .env.example file for a more complete documentation +INGESTION_BUCKET_URL="file://./tempFiles/data-ingestion-bucket?create_dir=true" +CASE_MANAGER_BUCKET_URL="file://./tempFiles/case-manager-bucket?create_dir=true" + +# Configure the connection details to your Metabase instance. +# - To retrieve the JWT signing key, go to your Metabase admin panel, in 'Settings', then 'Embedding', and click 'Manage' under 'Static embedding' +METABASE_SITE_URL= +METABASE_JWT_SIGNING_KEY= +METABASE_GLOBAL_DASHBOARD_ID= + +# Set up connection details to Convoy to enable webhooks sending. +# You can get your project ID and API key from your project settings page in Convoy's dashboard, in the "Secrets" section. +# NB: CONVOY_API_URL should be {scheme}://{host}:{port}/api - forgetting the /api will result in unexpected errors. +CONVOY_API_URL= +CONVOY_API_KEY= +CONVOY_PROJECT_ID= + +# If you use the SaaS Open Sanctions API, only uncomment OPENSANCTIONS_API_KEY and provide its values. +# If you self-host the API, uncomment all relevant settings to point to your deployment and configure authentication. +# - OPENSANCTIONS_AUTH_METHOD supports 'bearer' and 'basic' +# If basic, provide the username and password in the form of 'user:password' +# OPENSANCTIONS_API_HOST=http://yente +# OPENSANCTIONS_AUTH_METHOD=bearer +# OPENSANCTIONS_API_KEY= + +# Configure session parameters on the frontend +SESSION_SECRET=SESSION_SECRET +SESSION_MAX_AGE=43200 \ No newline at end of file diff --git a/.env.example b/.env.example index e9845c9..6437307 100644 --- a/.env.example +++ b/.env.example @@ -2,53 +2,61 @@ # REQUIRED SETTINGS # -ENV=development -NODE_ENV=production - -# Ports respectively used by the backend API and the frontends. +# If equal to "development", applies configurations on the backend and frontend containers that +# are not suitable for production, related to CORS, secure cookies and https. +# Any other value works for a production environment, and will be used by sentry to mark the environment +# an error occurred in. +ENV=production + +# Ports respectively used by the backend API and the frontends containers +# Maps to the "PORT" environment variable in the backend and frontend containers +# respectively. API_PORT=8080 APP_PORT=3000 -# Ports on the host Docker will listen on. +# Ports on the host Docker will listen on. Specific to using docker-compose, +# can be ignored for a deployment with managed containers or k8s. HOST_API_PORT=8080 HOST_APP_PORT=3000 -# Set this to the domain and port used by your users to access Marble's frontend. The second one must include the scheme (e.g http://). -MARBLE_APP_HOST="localhost:3000" -MARBLE_APP_DOMAIN="http://localhost:3000" +# Set this to the url used by your users to access Marble's frontend from the browser. +# Must include the scheme (e.g http://), and the port if non-standard. +MARBLE_APP_URL="http://localhost:3000" # Set those values to the URLs the Marble API can be reached at. Both must include the scheme (e.g http://) -# "MARBLE_API_DOMAIN_SERVER" must be reachable from the frontend host. -# "MARBLE_API_DOMAIN_CLIENT" must be reachable from the user's browser. -MARBLE_API_DOMAIN_SERVER="http://api:8080" -MARBLE_API_DOMAIN_CLIENT="http://localhost:8080" - -# Set your license key here if you have one in order to access premium features. -LICENSE_KEY= +# "MARBLE_API_URL_SERVER" must be reachable from the frontend host. With docker-compose, use the api container name as host name. +# "MARBLE_API_URL_CLIENT" must be reachable from the user's browser. +MARBLE_API_URL_SERVER="http://api:8080" +MARBLE_API_URL_CLIENT="http://localhost:8080" # RSA private key, in PEM format, used for for signing authentication tokens. MUST be changed for production. -# We recommend using AUTHENTICATION_JWT_SIGNING_KEY_FILE to point to a private key on disk. +# We recommend using AUTHENTICATION_JWT_SIGNING_KEY_FILE to point to a private key on disk because multi-line +# environment variables are not handled very gracefully. +# +# If using AUTHENTICATION_JWT_SIGNING_KEY, line breaks should be replaced with '\n': +# e.g. AUTHENTICATION_JWT_SIGNING_KEY="-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBA... +# # To generate a private key, use `openssl genrsa -out /path/to/private/key.pem 4096`. -# If left empty, will be regenerated on every app restart. +# If left empty, a key will be regenerated on every app restart, which may cause unexpected logouts. AUTHENTICATION_JWT_SIGNING_KEY= +# AUTHENTICATION_JWT_SIGNING_KEY_FILE=/shared/jwt_key.pem +# Configure your PostgreSQL database connection information, either by providing a DSN using this form: +# PG_CONNECTION_STRING='postgres://postgres:marble@localhost:5432/marble?sslmode=prefer' # Or by setting each piece of information in those variables: PG_HOSTNAME=db -PG_PORT=54321 +PG_PORT=5432 PG_USER=postgres -PG_PASSWORD=marble - -# Change the three settings below to create your initial organization and users. -# The user must also have a matching account in the authentication store used (in Firebase, for example). -# Subsequent users will be able to be created from the application. -CREATE_ORG_NAME=Zorg -CREATE_ORG_ADMIN_EMAIL=jbe@zorg.com +PG_PASSWORD=changeme +# For production, this SHOULD be set to 'require' +PG_SSL_MODE=prefer # Configure the ID of your Firebase project for authentication and the path to the service account's JSON private key file. # - The 'Project ID' can be found in the 'General' section of your Firebase project's settings page. -# - The private key must be generated in the 'Serv -GOOGLE_CLOUD_PROJECT="test-project" -# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account/private/key.json +# - The private key must be generated in the 'Service accounts' tab by clicking 'Generate new private key' +# If you are using GCS as a blob storage backend, the same service account key is used to sign URLs. +GOOGLE_CLOUD_PROJECT= +GOOGLE_APPLICATION_CREDENTIALS=/shared/service_account_key.json # Configure the document blob storage backend. # The example values MUST be changed for anything but a local test environment since the file provider does not support all features. @@ -72,32 +80,67 @@ GOOGLE_CLOUD_PROJECT="test-project" INGESTION_BUCKET_URL="file://./tempFiles/data-ingestion-bucket?create_dir=true" CASE_MANAGER_BUCKET_URL="file://./tempFiles/case-manager-bucket?create_dir=true" -# Configure the connection details to your Metabase instance. -# - To retrieve the JWT signing key, go to your Metabase admin panel, in 'Settings', then 'Embedding', and click 'Manage' under 'Static embedding' -METABASE_SITE_URL="https://your_subdomain.metabaseapp.com" -METABASE_JWT_SIGNING_KEY="dummy" -METABASE_GLOBAL_DASHBOARD_ID=123 - # Configure your Firebase project to allow username and password authentication. -FIREBASE_PROJECT_ID= +# To retrieve this value, go into the settings of your Firebase project, and register +# a new app from the 'General' tab. FIREBASE_API_KEY= -SESSION_SECRET=SESSION_SECRET +# Generate a random string, for instance using `openssl rand -base64 128 | tr -d "\n"` +SESSION_SECRET=changeme SESSION_MAX_AGE=43200 # # OPTIONAL SETTINGS # -# You can adjust the URLs used to access the Firebase emulator. -# - The "FIREBASE_AUTH_EMULATOR_HOST_CLIENT" URL needs to be accessible from your browser. -# - The "FIREBASE_AUTH_EMULATOR_HOST_SERVER" URL needs to be accessible from the API container. -FIREBASE_AUTH_EMULATOR_HOST_CLIENT=localhost:9099 -FIREBASE_AUTH_EMULATOR_HOST_SERVER=firebase-auth:9099 -FIREBASE_AUTH_EMULATOR_PORT=9099 +# Set your license key here if you have one in order to access premium features. +LICENSE_KEY= + +# Change the two settings below to create your initial organization and users. +# The user must also have a matching account in the authentication store used (in Firebase, for example). +# Subsequent users will be able to be created from the application, and the variables +# can be left blank. +CREATE_ORG_NAME= +CREATE_ORG_ADMIN_EMAIL= + +# Configure the connection details to your Metabase instance. +# - To retrieve the JWT signing key, go to your Metabase admin panel, in 'Settings', then 'Embedding', and click 'Manage' under 'Static embedding' +# - The dashboard ID is the ID of the dashboard you want to embed in Marble. You will find it when you publish a dashboard as +# a static embedding on Metabase. +# - The site url can be on a metabase domain, or can use your self-hosted Metabase instance. +METABASE_SITE_URL= +METABASE_JWT_SIGNING_KEY= +METABASE_GLOBAL_DASHBOARD_ID= + +# Set up connection details to Convoy to enable webhooks sending. +# You can get your project ID and API key from your project settings page in Convoy's dashboard, in the "Secrets" section. +# NB: CONVOY_API_URL should be {scheme}://{host}:{port}/api - forgetting the /api will result in unexpected errors. +CONVOY_API_URL= +CONVOY_API_KEY= +CONVOY_PROJECT_ID= + +# Configure the URL of the Elasticsearch instance used by yente for sanction screening. +# Those are not specific to marble, and are only used to configure containers +# when using docker-compose. +YENTE_ELASTICSEARCH_URL=http://marble-es:9200 +YENTE_PORT=8000 +ES_PORT=9200 + +# Configure access to the Open Sanctions API to use sanction checks +# +# If you use the SaaS Open Sanctions API, only uncomment OPENSANCTIONS_API_KEY and provide its values. +# If you self-host the API, uncomment all relevant settings to point to your deployment and configure authentication. +# - OPENSANCTIONS_AUTH_METHOD supports 'bearer' and 'basic' +# If basic, provide the username and password in the form of 'user:password' +# OPENSANCTIONS_API_HOST=http://self.hosted.local +# OPENSANCTIONS_AUTH_METHOD=bearer +# OPENSANCTIONS_API_KEY= + +# Configure various external integrations. +SENTRY_DSN= # If you need to support federated authentication through Firebase, you will need to configure the following settings: -# To retrieve those value, go into the settings of your Firebase project, and register a new app from the 'General' tab. -# The two information below will be provided to you after that. +# To retrieve those values, go into the settings of your Firebase project, and register a new app from the 'General' tab. +# The two pieces of information below will be provided to you after that. # FIREBASE_APP_ID= # FIREBASE_AUTH_DOMAIN= diff --git a/.gitignore b/.gitignore index f788de5..31f82ad 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ .secrets key.json .env +.env.dev diff --git a/api b/api index 2f8e083..f1ef4de 160000 --- a/api +++ b/api @@ -1 +1 @@ -Subproject commit 2f8e083f031f519c51aa915d641098068539bb45 +Subproject commit f1ef4de8f77c37fda5b11251dd1731880280e5d5 diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml new file mode 100644 index 0000000..bfd15cf --- /dev/null +++ b/docker-compose-dev.yaml @@ -0,0 +1,167 @@ +name: marble + +x-backend-image-version: &backend-image-version + image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:v0.38.2 +x-frontend-image-version: &frontend-image-version + image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend:v0.38.0 + +x-shared-environment: &shared-env + ENV: development + MARBLE_APP_URL: http://localhost:${HOST_APP_PORT:-3000} +x-backend-environment: &backend-env + LICENSE_KEY: ${LICENSE_KEY:-LICENSE_KEY} + PORT: 8080 + + PG_HOSTNAME: db + PG_PORT: ${PG_PORT:-5432} + PG_USER: postgres + PG_PASSWORD: changeme + + INGESTION_BUCKET_URL: ${INGESTION_BUCKET_URL:-} + CASE_MANAGER_BUCKET_URL: ${CASE_MANAGER_BUCKET_URL:-} + + FIREBASE_AUTH_EMULATOR_HOST: firebase-auth:9099 + + # default value of GOOGLE_CLOUD_PROJECT must be kept if working with the emulator (and the emulator is running in the docker image below) + GOOGLE_CLOUD_PROJECT: test-project + + CREATE_ORG_NAME: ${CREATE_ORG_NAME:-} + CREATE_ORG_ADMIN_EMAIL: ${CREATE_ORG_ADMIN_EMAIL:-} + + SEGMENT_WRITE_KEY: ${SEGMENT_WRITE_KEY_BACKEND:-UgkImFmHmBZAWh5fxIKBY3QtvlcBrhqQ} + + METABASE_SITE_URL: ${METABASE_SITE_URL:-} + METABASE_JWT_SIGNING_KEY: ${METABASE_JWT_SIGNING_KEY:-} + METABASE_GLOBAL_DASHBOARD_ID: ${METABASE_GLOBAL_DASHBOARD_ID:-} + + CONVOY_API_KEY: ${CONVOY_API_KEY:-} + CONVOY_API_URL: ${CONVOY_API_URL:-} + CONVOY_PROJECT_ID: ${CONVOY_PROJECT_ID:-} +x-frontend-environment: &frontend-env + NODE_ENV: production + SESSION_SECRET: ${SESSION_SECRET:-} + SESSION_MAX_AGE: 43200 + + MARBLE_API_URL_CLIENT: http://localhost:${HOST_API_PORT:-8080} + MARBLE_API_URL_SERVER: http://api:${HOST_API_PORT:-8080} + + FIREBASE_AUTH_EMULATOR_HOST: localhost:9099 + + SEGMENT_WRITE_KEY: ${SEGMENT_WRITE_KEY:-hC8qrY2OLhUpl1Xycw523tbuClxlQR6u} + +services: + db: + container_name: marble-postgres + image: postgres:15 + restart: always + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: changeme + POSTGRES_DB: marble + PGDATA: /data/postgres + PGPORT: ${PG_PORT:-5432} + ports: + - ${PG_PORT:-5432}:${PG_PORT:-5432} + volumes: + - marble-db:/data/postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 2s + timeout: 1s + retries: 5 + api: &backend + <<: *backend-image-version + container_name: marble-api + platform: linux/amd64 + restart: always + depends_on: + - db + - yente + entrypoint: ["./app", "--server", "--migrations"] + healthcheck: + test: wget --no-verbose --tries=1 --spider http://localhost:8080/liveness || exit 1 + interval: 10s + timeout: 60s + retries: 1 + start_period: 30s + ports: + - ${HOST_API_PORT:-8080}:8080 + volumes: + - marble-tempfiles:/tempFiles + environment: + <<: [*shared-env, *backend-env] + cron: + <<: *backend + container_name: marble-api-cron + depends_on: + - api + entrypoint: ["./app", "--worker"] + healthcheck: + disable: true + ports: [] + app: + <<: *frontend-image-version + container_name: marble-app + platform: linux/amd64 + restart: always + depends_on: + - api + - firebase_auth + healthcheck: + test: curl --fail http://localhost:8080/healthcheck || exit 1 + interval: 10s + timeout: 60s + retries: 1 + start_period: 30s + ports: + - ${HOST_APP_PORT:-3000}:8080 + environment: + <<: [*shared-env, *frontend-env] + firebase_auth: + container_name: firebase-auth + image: europe-west1-docker.pkg.dev/marble-infra/marble/firebase-emulator:latest + restart: always + ports: + - 9099:9099 + - 4000:4000 + elasticsearch: + container_name: marble-es + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.3 + ports: + - ${ES_PORT:-9200}:${ES_PORT:-9200} + environment: + - node.name=es + - cluster.name=marble-es + - discovery.type=single-node + - bootstrap.memory_lock=true + - xpack.security.enabled=false + - "ES_JAVA_OPTS=-Xms2g -Xmx2g" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - marble-es:/usr/share/elasticsearch/data + yente: + container_name: marble-yente + image: ghcr.io/opensanctions/yente:4.2.1 + depends_on: + - elasticsearch + ports: + - ${YENTE_PORT:-8000}:${YENTE_PORT:-8000} + volumes: + # This configuration file will load only a reduced into the elastic search index. + # This is only a convenience for development purposes, and should be removed or adapted + # for production environments. + - ./contrib/yente-datasets.yml:/app/manifests/default.yml + environment: + YENTE_INDEX_TYPE: elasticsearch + YENTE_INDEX_URL: ${YENTE_ELASTICSEARCH_HOST:-http://marble-es:9200} + YENTE_UPDATE_TOKEN: "" + +volumes: + marble-db: + driver: local + marble-es: + driver: local + marble-tempfiles: diff --git a/docker-compose.yaml b/docker-compose.yaml index 1dee2bc..cc56539 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,14 +1,80 @@ -version: "3.8" - name: marble + +x-backend-image-version: &backend-image-version + image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:v0.38.2 +x-frontend-image-version: &frontend-image-version + image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend:v0.38.0 + +x-shared-environment: &shared-env + ENV: ${ENV} + MARBLE_APP_URL: ${MARBLE_APP_URL} + SENTRY_DSN: ${SENTRY_DSN} +x-backend-environment: &backend-env + LICENSE_KEY: ${LICENSE_KEY} + + PORT: ${API_PORT:-8080} + + PG_HOSTNAME: ${PG_HOSTNAME} + PG_PORT: ${PG_PORT} + PG_USER: ${PG_USER} + PG_PASSWORD: ${PG_PASSWORD} + PG_SSL_MODE: ${PG_SSL_MODE} + PG_CONNECTION_STRING: ${PG_CONNECTION_STRING} + + AUTHENTICATION_JWT_SIGNING_KEY: ${AUTHENTICATION_JWT_SIGNING_KEY} + + INGESTION_BUCKET_URL: ${INGESTION_BUCKET_URL} + CASE_MANAGER_BUCKET_URL: ${CASE_MANAGER_BUCKET_URL} + + GOOGLE_CLOUD_PROJECT: ${GOOGLE_CLOUD_PROJECT} + GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS} + + REQUEST_LOGGING_LEVEL: ${REQUEST_LOGGING_LEVEL} + LOGGING_FORMAT: ${LOGGING_FORMAT} + + CREATE_ORG_NAME: ${CREATE_ORG_NAME} + CREATE_ORG_ADMIN_EMAIL: ${CREATE_ORG_ADMIN_EMAIL} + + METABASE_SITE_URL: ${METABASE_SITE_URL} + METABASE_JWT_SIGNING_KEY: ${METABASE_JWT_SIGNING_KEY} + METABASE_GLOBAL_DASHBOARD_ID: ${METABASE_GLOBAL_DASHBOARD_ID} + + CONVOY_API_URL: ${CONVOY_API_URL} + CONVOY_API_KEY: ${CONVOY_API_KEY} + CONVOY_PROJECT_ID: ${CONVOY_PROJECT_ID} + + OPENSANCTIONS_API_HOST: ${OPENSANCTIONS_API_HOST} + OPENSANCTIONS_AUTH_METHOD: ${OPENSANCTIONS_AUTH_METHOD} + OPENSANCTIONS_API_KEY: ${OPENSANCTIONS_API_KEY} + + SEGMENT_WRITE_KEY: UgkImFmHmBZAWh5fxIKBY3QtvlcBrhqQ +x-frontend-environment: &frontend-env + NODE_ENV: production + PORT: ${APP_PORT:-8080} + + SESSION_SECRET: ${SESSION_SECRET} + SESSION_MAX_AGE: ${SESSION_MAX_AGE} + + MARBLE_API_URL_CLIENT: ${MARBLE_API_URL_CLIENT} + MARBLE_API_URL_SERVER: ${MARBLE_API_URL_SERVER} + + FIREBASE_PROJECT_ID: ${GOOGLE_CLOUD_PROJECT} + FIREBASE_API_KEY: ${FIREBASE_API_KEY} + FIREBASE_AUTH_DOMAIN: ${FIREBASE_AUTH_DOMAIN} + FIREBASE_APP_ID: ${FIREBASE_APP_ID} + + SEGMENT_WRITE_KEY: AnqbZOIJ8TvCbQUhXLqin1tXXnxHX0CE + services: + # NB: the docker compose file includes a postgres image for the sake of completeness, but we strongly + # suggest using a production-grade database to run Marble in a production environment db: container_name: marble-postgres image: postgres:15 restart: always environment: POSTGRES_USER: postgres - POSTGRES_PASSWORD: ${PG_PASSWORD:-marble} + POSTGRES_PASSWORD: ${PG_PASSWORD:-changeme} POSTGRES_DB: marble PGDATA: /data/postgres PGPORT: ${PG_PORT:-5432} @@ -21,9 +87,9 @@ services: interval: 2s timeout: 1s retries: 5 - api: + api: &backend + <<: *backend-image-version container_name: marble-api - image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:v0.37.0 platform: linux/amd64 restart: always depends_on: @@ -39,60 +105,28 @@ services: ports: - ${HOST_API_PORT:-8080}:${API_PORT:-8080} volumes: - - marble-tempfiles:/tempFiles - # - ./local_shared:/shared # uncomment and adapt this line if you need to inject a service account key into the container + # adapt this line so that the left-hand side is the address of a folder containing: + # - the service account key + # - optionally the JWT signing RSA key in .pem format + - ./local_shared:/shared environment: - ENV: ${ENV:-} - LICENSE_KEY: ${LICENSE_KEY:-LICENSE_KEY} - PORT: ${API_PORT:-} - PG_HOSTNAME: ${PG_HOSTNAME:-} - PG_PORT: ${PG_PORT:-} - PG_USER: postgres - PG_PASSWORD: ${PG_PASSWORD:-} - AUTHENTICATION_JWT_SIGNING_KEY: ${AUTHENTICATION_JWT_SIGNING_KEY:-} - INGESTION_BUCKET_URL: ${INGESTION_BUCKET_URL:-} - CASE_MANAGER_BUCKET_URL: ${CASE_MANAGER_BUCKET_URL:-} - FIREBASE_AUTH_EMULATOR_HOST: ${FIREBASE_AUTH_EMULATOR_HOST_SERVER:-} - MARBLE_APP_HOST: ${MARBLE_APP_HOST:-localhost:${HOST_APP_PORT:-3000}} - # default value of GOOGLE_CLOUD_PROJECT must be kept if working with the emulator (and the emulator is running in the docker image below) - GOOGLE_CLOUD_PROJECT: ${GOOGLE_CLOUD_PROJECT:-test-project} - GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS:-} - REQUEST_LOGGING_LEVEL: ${REQUEST_LOGGING_LEVEL:-} - LOGGING_FORMAT: ${LOGGING_FORMAT:-} - CREATE_GLOBAL_ADMIN_EMAIL: ${CREATE_GLOBAL_ADMIN_EMAIL:-} - CREATE_ORG_NAME: ${CREATE_ORG_NAME:-} - CREATE_ORG_ADMIN_EMAIL: ${CREATE_ORG_ADMIN_EMAIL:-} - SEGMENT_WRITE_KEY: ${SEGMENT_WRITE_KEY_BACKEND:-UgkImFmHmBZAWh5fxIKBY3QtvlcBrhqQ} - METABASE_SITE_URL: ${METABASE_SITE_URL:-} - METABASE_JWT_SIGNING_KEY: ${METABASE_JWT_SIGNING_KEY:-} - METABASE_GLOBAL_DASHBOARD_ID: ${METABASE_GLOBAL_DASHBOARD_ID:-} + <<: [*shared-env, *backend-env] cron: + <<: *backend container_name: marble-api-cron - image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend:v0.37.0 - platform: linux/amd64 - restart: always depends_on: - api entrypoint: ["./app", "--worker"] - volumes: - - marble-tempfiles:/tempFiles - environment: - ENV: ${ENV:-} - PG_HOSTNAME: ${PG_HOSTNAME:-} - PG_PORT: ${PG_PORT:-} - PG_USER: postgres - PG_PASSWORD: ${PG_PASSWORD:-} - INGESTION_BUCKET_URL: ${INGESTION_BUCKET_URL:-} - REQUEST_LOGGING_LEVEL: ${REQUEST_LOGGING_LEVEL:-} - LOGGING_FORMAT: ${LOGGING_FORMAT:-} + healthcheck: + disable: true + ports: [] app: + <<: *frontend-image-version container_name: marble-app - image: europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend:v0.37.0 platform: linux/amd64 restart: always depends_on: - api - - firebase_auth healthcheck: test: curl --fail http://localhost:${APP_PORT:-8080}/healthcheck || exit 1 interval: 10s @@ -100,28 +134,12 @@ services: retries: 1 start_period: 30s ports: - - ${HOST_APP_PORT:-3000}:8080 + - ${HOST_APP_PORT:-3000}:${APP_PORT:-8080} environment: - ENV: ${ENV:-development} - NODE_ENV: ${NODE_ENV:-production} - SESSION_SECRET: ${SESSION_SECRET:-placeholder} - SESSION_MAX_AGE: ${SESSION_MAX_AGE:-43200} - MARBLE_API_DOMAIN_CLIENT: ${MARBLE_API_DOMAIN_CLIENT:-http://localhost:8080} - MARBLE_API_DOMAIN_SERVER: ${MARBLE_API_DOMAIN_SERVER:-http://api:8080} - MARBLE_APP_DOMAIN: ${MARBLE_APP_DOMAIN:-http://localhost:${HOST_APP_PORT:-8080}} - FIREBASE_AUTH_EMULATOR_HOST: ${FIREBASE_AUTH_EMULATOR_HOST_CLIENT:-} - FIREBASE_API_KEY: ${FIREBASE_API_KEY:-placeholder} - FIREBASE_AUTH_DOMAIN: ${FIREBASE_AUTH_DOMAIN:-placeholder} - FIREBASE_PROJECT_ID: ${GOOGLE_CLOUD_PROJECT:-placeholder} - FIREBASE_APP_ID: ${FIREBASE_APP_ID:-placeholder} - SEGMENT_WRITE_KEY: ${SEGMENT_WRITE_KEY:-hC8qrY2OLhUpl1Xycw523tbuClxlQR6u} - firebase_auth: - container_name: firebase-auth - image: europe-west1-docker.pkg.dev/marble-infra/marble/firebase-emulator:latest - restart: always - ports: - - ${FIREBASE_AUTH_EMULATOR_PORT:-9099}:9099 - - 4000:4000 + <<: [*shared-env, *frontend-env] + + # NB: the docker compose file includes an elasticsearch image for the sake of completeness, but we strongly + # suggest using a production-grade instance to run Marble in a production environment elasticsearch: container_name: marble-es image: docker.elastic.co/elasticsearch/elasticsearch:8.14.3 @@ -147,11 +165,9 @@ services: - elasticsearch ports: - ${YENTE_PORT:-8000}:${YENTE_PORT:-8000} - volumes: - - ./contrib/yente-datasets.yml:/app/manifests/default.yml environment: YENTE_INDEX_TYPE: elasticsearch - YENTE_INDEX_URL: "${YENTE_ELASTICSEARCH_HOST:-http://marble-es:9200}" + YENTE_INDEX_URL: "${YENTE_ELASTICSEARCH_URL}" YENTE_UPDATE_TOKEN: "" volumes: @@ -159,4 +175,3 @@ volumes: driver: local marble-es: driver: local - marble-tempfiles: diff --git a/front b/front index aedb83a..3410108 160000 --- a/front +++ b/front @@ -1 +1 @@ -Subproject commit aedb83a60ca89b99481bdfbf16fef11821e1c09c +Subproject commit 34101082e09e701e4324b895398aedf96d1d25ff diff --git a/installation/README.md b/installation/README.md index a75da24..88fe086 100644 --- a/installation/README.md +++ b/installation/README.md @@ -1,45 +1,176 @@ -### πŸ”§ **How to Install Marble** +# Marble Installation Guide ---- +## Overview + +Welcome to Marble! This guide will help you get started with installing and deploying Marble. Choose the path that best matches your needs: + +- πŸš€ [Quick Start](#quick-start) - Get up and running quickly +- πŸ’» [Local Development](./test_run.md) - Set up a development environment +- 🌐 [Production Deployment](./production_run.md) - Deploy for production use +- πŸ—οΈ [Architecture Overview](#architecture-overview) - Understand Marble's components + +## Quick Start + +### Prerequisites + +- Docker and Docker Compose +- Git +- 4GB RAM minimum +- 10GB disk space + +### Basic Installation + +```bash +# 1. Clone the repository +git clone +cd marble + +# 2. Copy example environment file +cp .env.dev.example .env.dev + +# 3. Start Marble +docker compose -f docker-compose-dev.yaml --env-file .env.dev.example up +``` + +## Architecture Overview + +### Core Components + +Marble consists of three main services: + +1. **Backend API** (Go + Gin) + + - REST API server + - Business logic + - Database interactions + +2. **Worker** (Go) + + - Background tasks + - Scheduled jobs + - Data processing + +3. **Frontend** (TypeScript + Remix) + - Web interface + - User interactions + - API integration + +### System Architecture + +The standard Marble deployment architecture: +![Deployment Architecture](https://github.com/user-attachments/assets/80d7a9d3-10d0-4a14-ac14-d6badcc5393d) + +For a detailed view of component interactions: +![Functional Architecture](https://github.com/user-attachments/assets/d9b85e87-532c-4efc-9367-b63eaf93a2da) + +## Deployment Options + +### 1. Local Development + +Best for: -> The following instructions are for a docker-compose setup. You can also take inspiration from the terraform templates provided in the repository to create a serverless GCP deployment of Marble, inspired by Marble's own cloud deployment. +- Testing features +- Quick evaluation + β†’ [Local Setup Guide](./test_run.md) -Simply clone this repository and run `docker compose --env-file .env.example up` (customize the .env-example file or provide your own copy). -It will run out of the box with the firebase auth emulator. If you wish to run Marble open-source in production, you will need to create a firebase auth app. +### 2. Production Server -The first time you run the code, you should enter an organization name and organization admin user email to create using the `CREATE_ORG_NAME` and `CREATE_ORG_ADMIN_EMAIL` environment variables. Unless using the firebase emulator, you must enter an actual email address that you own so that you may verify it and login with firebase. You can always create new organizations later using the same procedure. +Best for: -> `CREATE_GLOBAL_ADMIN_EMAIL` is optional. It is useful if you want to access cross organization data. If you provide it, the user will be created as a global admin. It must be different from `CREATE_ORG_ADMIN_EMAIL`. +- Production deployments +- Custom infrastructure +- Self-hosted installations + β†’ [Production Guide](./production_run.md) -**In a local demo setup:** +### 3. Cloud Deployment -> In a local test setup (meaning if you are running with the firebase auth emulator), the License key is not required. You can leave it empty. The full feature set is available. +Best for: -- just run the docker-compose as it is, it should work -- give the firebase emulator a moment to get started, it's a bit slow when first launched -- create a Firebase user with the email you provided in the `CREATE_ORG_ADMIN_EMAIL` environment variable (you can do this on the Marble login page by using the SSO button or sign up with email) +- Scalable deployments +- Managed services +- Cloud-native architecture + β†’ [Production Guide](./production_run.md) -**In a production setup:** +## Configuration -- set the `FIREBASE_AUTH_EMULATOR_HOST_SERVER` and `FIREBASE_AUTH_EMULATOR_HOST_CLIENT` env variables to empty strings in your .env file, or remove them. -- create a Firebase project and a Firebase app, and set the relevant env variables (`FIREBASE_API_KEY` to `FIREBASE_APP_ID` as well as `GOOGLE_CLOUD_PROJECT`) in your .env file -- create a service account json key in the GCP interface, mount it in the docker container and set the path to the key in the `GOOGLE_APPLICATION_CREDENTIALS` env variable (see below) -- if you plan to use the batch ingestion feature or the case manager with file storing feature, you need to create a pair of storage buckets (GCP cloud storage, AWS S3, Azure storage account), pass the url to the bucket in the `INGESTION_BUCKET_URL` and `CASE_MANAGER_BUCKET_URL` env variables (see details in the .env.example - you can use the same bucket for both), and make sure that any information necessary for the client to automatically authenticate is present (using env variables, credential files, or automatic credentials discovery from a metadata server) -- create a Firebase user with the email you provided in the `CREATE_ORG_ADMIN_EMAIL` environment variable (you can do this on the Marble login page by using the SSO button or sign up with email) -- if you have a license key, set it in the `LICENSE_KEY` env variable in your .env file +### Environment Files -**GCP & Firebase authentication:** +**Simple template for local run** -In a production setup, you need to authenticate to GCP to use Firebase (as well as Cloud Storage if you are using it as a storage bucket). If you are not running the container directly in a GCP environment, here is how you could do this: +- `.env.dev.example` - Development configuration template +- `docker-compose-dev.yaml` - Development container setup -- create a volume attached to the marble-api container -- place the json service account key for GCP in the local shared folder (or otherwise inject it into the docker container, depending on how you run Marble) -- set the `GOOGLE_APPLICATION_CREDENTIALS` variable equal to the path to the service account key +**Complete template for production run** -#### **How to upgrade your Marble version** +- `.env.example` - Production configuration template +- `docker-compose.yaml` - Container orchestration -You upgrade your Marble version by checking out the release of your choice, and simply running it. By running the docker image with the options `"--server", "--migrations"`, you execute the database migrations of the version and then start the server again. +## Version Management -If you are running in a production environment, remember that while the migrations are running, your server will not be responding. You should preferably upgrade your version outside of the busy hours of the week. +### Release schedule + +A new version of the Marble application is released about every week, usuallly in the beginning of the week. + +Every release on the [Marble repository](https://github.com/checkmarble/marble) consists in a pair of versions for the backend and frontend containers. For example, version 0.36.0 of Marble uses the backend container v0.36.1 and frontend container v0.36.1. Refer to the versions used in the `docker-compose.yaml` file to know which versions to run together in a given release. + +Most new features are released as minor versions (according to semantic versioning), while releases that only contain bug fixes or dependency upgrades may be released as patch versions. + +If the release makes important changes to the Marble interface, the configuration options, or the API, this is emphasized in the release note attached to the release. + +We recommend you update your Marble version at least every few weeks. + +### Upgrading Marble + +1. Check out new version +2. Run migrations: + + ```bash + # Option 1: Restart backend container + docker compose restart api + + # Option 2: Run migration only + docker compose run --rm api --migrate + ``` + + The database migrations are designed such that you can safely run the migration for version N+1 while still serving version N, and upgrade the backend and frontend containers after the migration is done. Some migrations may take a while to run, if new indexes need to be created or if some existing data needs to be migrated to a new format. Most migrations are expected to run very quickly. + + The same migration SHOULD NOT be run in several processes/containers in parallel. + +3. Upgrade the backend and frontend containers: + + ```bash + # Stop all containers + docker compose down + + # Start with new versions + docker compose -f docker-compose.yaml --env-file ./env up -d + ``` + +4. Best practices: + + - Test in staging first + - Schedule during low traffic + +### Version Support + +- Latest version: Check releases page +- Support policy: We only support the latest version. You should upgrade regularly to stay on a supported version. +- Migration path: You must upgrade versions one at a time in sequence (e.g. v0.35 -> v0.36 -> v0.37). Skipping versions is not recommended. + +## Getting Help + +- πŸ“– [Product Documentation](https://docs.checkmarble.com/) +- πŸ”§ [Troubleshooting Guides](./test_run.md#troubleshooting) +- πŸ“‹ [Issue tracker](https://github.com/checkmarble/marble/issues) +- πŸ’¬ [Community support](https://join.slack.com/t/marble-communitysiege/shared_invite/zt-2b8iree6b-ZLwCiafKV9rR0O6FO7Jqcw) + +## Additional Resources + +- [Environment Variables Guide](../.env.example) +- [Docker Configuration](../docker-compose.yaml) +- [Version support policy](./version_support_policy.md) +- [Create a first organization and user](./first_connection.md) + +--- -You should always increase the version of Marble one version at a time, giving the time to version N to migrate and deploy before you upgrade to version N+1. +> πŸ’‘ **Tip**: Start with the [Local Development Guide](./test_run.md) to get familiar with Marble before moving to production. diff --git a/installation/deployment.md b/installation/deployment.md deleted file mode 100644 index 659b1f4..0000000 --- a/installation/deployment.md +++ /dev/null @@ -1,69 +0,0 @@ -# Deploying Marble - -## Marble architecture - -#### Components - -The typical architecture of a self-hosted Marble deployment is as below: -Deployment architecture - -The corresponding functional architecture is the following: -Deployment architecture - -The Marble software is composed of 2 docker images and 3 parts: - -1. **[Docker 1]** a back-end API server (go + [gin](https://github.com/gin-gonic/gin)), that serves the internal and public REST APIs -2. **[Docker 1]** a set of cron job scripts (go, run from the same docker image with a different entrypoint as the back-end API server), meant to run periodically -3. **[Docker 2]** a front-end API server (typescript + [Remix](https://remix.run/)), that serves html and exposes actions to the browser - -It relies on the existence of a basic set of infrastructure to work: - -1. A Postgresql database -2. A scheduler for the cron jobs -3. A set of object storage buckets, to store documents uploaded in the case manager and csv files for batch data ingestion (compatible with Google Cloud Storage, AWS S3 and Azure Blob Storage) -4. A load balancer with TLS termination to expose the APIs -5. A configured Firebase app for end user authentication - -#### Marble images - -The docker images for Marble are stored on the registry `europe-west1-docker.pkg.dev/marble-infra/marble/marble-backend` and `europe-west1-docker.pkg.dev/marble-infra/marble/marble-frontend` for the API respectively. - -#### Marble cloud - -For reference, below is a schematic of Marble's cloud offering architecture. It is essentially a variation of the architecture described above, with some infrastructure choices that work well for our cloud offering specifically. -Software architecture (2) - -## Deploy Marble serverless - -Here at Marble, we choose to deploy our code on a serverless infrastructure. We do this to leverage the full ease of use, flexibility and scalability of GCP's Cloud Run offering. -Doing this, we choose to run the back-end API server and the cron jobs (which are run with different flags from the same docker image) separately: - -- the API is a Cloud Run service -- the scripts are run as distinct Cloud Run Jobs, scheduled by a Cloud Scheduler - -_Technically_ we could have the cron jobs scheduled within the same container that is running the API service, but we avoid this for the following reasons: - -- we don’t want a batch job to be stopped/to fail because an api instance is torn down -- we don’t want the api to be impacted by a batch job’s cpu/memory usage -- the cloud run API has limits in how long it can take at most to handle requests (which are shorter than the typical cron job execution time) - -However, running it all together could make sense if Marble is run in a VM, more on this below. - -Moreover, in our cloud deployment, we use Google Secret Manager (integrated with Cloud Run) to inject secrets as environment variables into the containers, and a GCP application load balancer for load balancing, TLS termination, DDoS protection and other security rules. - -In this repository, we provide an example set of terraform files that you can adapt to deploy Marble serverless in your own GCP environment. -We have also received community contributions giving an example set of terraform files for deploying Marble in managed containers on AWS. -It should also be reasonably simple to run an equivalent deployment on Azure Container Instances + scheduler on Azure. - -## Deploy Marble on a VM - -While Marble on a VM is not ideal for our cloud offering, it may make sense for an open-source or on-premise usecase. If you do so, you can run the back-end API and built-in go scheduler together by passing the `--server --cron` arguments to the docker container. - -## Deploy Marble on Kubernetes - -While we do not currently provide support for deploying Marble on Kubernetes, it should work very similarly to a serverless deployment. You can schedule the cron jobs by using Kubernetes' built-in scheduling tool. - -# Deploy Marble outside of GCP - -Marble relies on Firebase Authentication (a part of Google's web app deployment toolbox) for authentication. This means that, even if you are running Marble in another cloud provider's cloud, or in your own infrastructure, you need to create a Firebase app, as well as a service account key to access it from the backend container. -In practice, any usage of Marble should fall under the Firebase Auth free plan, though you may still need to provide a credit card number. diff --git a/installation/first_connection.md b/installation/first_connection.md new file mode 100644 index 0000000..8a0a9de --- /dev/null +++ b/installation/first_connection.md @@ -0,0 +1,34 @@ +# First Connection Guide + +## Initial Setup + +### 1. Environment Configuration + +First, configure the required environment variables as described in `.env.example`: + +- `CREATE_ORG_NAME`: Name of the initial organization +- `CREATE_ORG_ADMIN_EMAIL`: Email address for initial admin user + +### 2. Authentication Setup + +Two authentication methods are available for first login: + +#### Option A: Social Sign-In (Recommended) + +1. Configure Firebase authentication providers in your project (works out of the box with the emulator) +2. Use the configured email (`CREATE_ORG_ADMIN_EMAIL`) to sign in + +#### Option B: Email/Password + +1. Click "Is it your first connection? Sign up" link on the sign in page + +2. Create your account with: + + - Email: Value of `CREATE_ORG_ADMIN_EMAIL` + - Choose a secure password + +3. Verify your email: + - In development: Check Firebase emulator logs for verification link (or set up the password directly in the auth emulator interface, http://localhost:4000 by default) + - In production: Check your email for verification link + +> πŸ’‘ **Note**: These environment variables are only used during first startup. They can be removed after initial setup is complete. diff --git a/installation/production_run.md b/installation/production_run.md new file mode 100644 index 0000000..73ffdbc --- /dev/null +++ b/installation/production_run.md @@ -0,0 +1,236 @@ +# Production Deployment Guide + +## Overview + +This guide explains how to deploy Marble in a production environment. It covers all required dependencies, configuration steps, and deployment options. + +## Prerequisites + +Before starting a production deployment, ensure you have: + +### Required Expertise + +- Docker containerization +- DNS configuration and TLS certificates +- PostgreSQL database administration +- Cloud provider experience (GCP/AWS/Azure/other) + +### Required Infrastructure + +- Production-grade PostgreSQL database (v15+) +- Blob storage bucket (GCS, S3...) +- Firebase project +- Convoy instance if you need webhooks +- Yente+Elasticsearch setup if you need sanction checks + +## Detailed Dependencies Setup + +### 1. PostgreSQL Database + +Requirements: + +- Version: 15 or higher +- Database name: `marble` +- Schema: `marble,public` + +#### Configuration + +```sql +-- If not using default postgres user: +CREATE SCHEMA IF NOT EXISTS marble; +ALTER USER your_user SET search_path TO marble,public; +GRANT ALL ON SCHEMA marble TO your_user; +``` + +> ⚠️ **Note**: While the development docker-compose includes PostgreSQL, do not use this for production. Use a managed service like GCP Cloud SQL or AWS RDS instead. + +> 🚫 **Unsupported**: Marble has not been tested with PostgreSQL-compatible databases (CockroachDB, AlloyDB...). + +### 2. Cloud Storage + +Purpose: + +- Store CSV files for batch ingestion +- Store documents for case management + +Supported Providers: + +- Google Cloud Storage +- Amazon S3 +- Azure Blob Storage +- Any S3-compatible storage, such as Minio + +Configuration details are available in `.env.example`. +The .env file expects two different buckets for the two purposes, but you may use the same bucket for both. + +### 3. Firebase Authentication + +#### Setup Steps + +1. **Create Firebase Project** + + - Visit [Firebase Console](https://console.firebase.google.com/) + - Create a new project or use an existing GCP project + +2. **Configure Authentication** + + - Enable Firebase Authentication + - Go to Project overview β†’ project settings β†’ Service Accounts β†’ Generate new private key + - Create service account + - Download service account key + +3. **Configure Domain** + + - Go to Authentication β†’ Settings β†’ Authorized Domains + - Add your Marble application domain (should use https) + +4. **Optional: Configure sign-in with Google/Microsoft** + + - [Google Sign-in](https://firebase.google.com/docs/auth/web/google-signin) + - [Microsoft Sign-in](https://firebase.google.com/docs/auth/web/microsoft-oauth) + +5. **Environment Setup** + + #### Backend + + - Mount service account key in backend container (see volumes on the api container in `docker-compose.yaml`) + - Set GOOGLE_APPLICATION_CREDENTIALS path + + #### Frontend + + - Configure Firebase variables (see `.env.example`) + +> πŸ’‘ **Cost**: Firebase Auth free tier should be sufficient, but credit card required for project setup. + +### 4. Convoy (Webhook Gateway) + +Options: + +- Self-hosted: Follow [Docker installation guide](https://www.getconvoy.io/docs/deployment/install-convoy/docker) +- Managed: Use [Convoy Cloud](https://www.getconvoy.io/) + +Setup Steps: + +1. Create webhooks project +2. Generate API key +3. Configure in Marble: + ```bash + CONVOY_API_URL=https://your-convoy-instance/api + CONVOY_API_KEY=your-api-key + CONVOY_PROJECT_ID=your-project-id + ``` + +### 5. Yente + Elasticsearch + +Purpose: Sanctions screening and search functionality + +Options: + +- Self-hosted Yente + Elasticsearch +- OpenSanctions managed API + +> ⚠️ **Note**: While development docker-compose includes Elasticsearch, use a production-grade service for deployment. + +## Deployment Architecture + +### Components + +The Marble platform consists of three services: + +1. **Marble API** + + - Main backend service + - Handles REST API requests + - Requires direct database access + +2. **Marble Worker** + + - Background task processor + - Uses same container as API + - Handles scheduled jobs & tasks + +3. **Marble Frontend** + - User interface + - Browser application + - Communicates with API + +### Deployment Options + +1. **Server/VM Deployment** + + - Use nginx/reverse proxy + - Route requests to services + - Manage SSL termination + +2. **Managed Container Service** + + - GCP Cloud Run + - AWS ECS + - Or similar option at other cloud providers + - Configure auto-scaling + +3. **Kubernetes** + - Deploy as containers + - Use service mesh + - Configure ingress + +## App Configuration + +Reference the following files for detailed configuration: + +- `.env.example`: Environment variables +- `docker-compose.yaml`: Service configuration + +## Troubleshooting + +### Common Issues + +1. **CORS Errors** + + - Ensure `MARBLE_APP_URL` environment variable is set correctly + - Must use `https://` protocol in production + - Example: `MARBLE_APP_URL=https://app.yourdomain.com` + - Common symptoms: + - API requests failing in browser + - Console errors about CORS policy + - Authentication issues + +2. **API URL Configuration** + + - Frontend needs two API URLs configured: + - `MARBLE_API_URL_CLIENT`: URL for browser requests (public URL) + - Example: `https://api.yourdomain.com` + - `MARBLE_API_URL_SERVER`: URL for container-to-container requests + - Example: `http://api:8080` (Docker internal network) + - Incorrect configuration leads to: + - Failed API calls + - CORS errors + - Authentication failures + +3. **Network Connectivity** + + - Confirm services can reach each other + - Check firewall rules + - Verify DNS resolution works + - Test internal container networking + +4. **Firebase Configuration** + + - Service Account Key: + + - ⚠️ Backend service will fail to start without accessible Firebase service account key + - Exception: On GCP, automatic service account discovery may work + - Generate key in Firebase Console β†’ Project Settings β†’ Service Accounts + - Mount key file to container and set `GOOGLE_APPLICATION_CREDENTIALS` + - Example: `GOOGLE_APPLICATION_CREDENTIALS=/secrets/firebase-key.json` + + - Required Environment Variables: + - `GOOGLE_CLOUD_PROJECT`: Firebase project ID + - `FIREBASE_API_KEY`: Web API key from Firebase Console + - `FIREBASE_AUTH_DOMAIN`: Auth domain from Firebase settings + - `FIREBASE_APP_ID`: Application ID from Firebase Console + +## Next Steps + +- [Deployment Architecture](./README.md#system-architecture) +- [Version Upgrade Guide](./README.md#version-management) diff --git a/installation/test_run.md b/installation/test_run.md new file mode 100644 index 0000000..a907630 --- /dev/null +++ b/installation/test_run.md @@ -0,0 +1,113 @@ +# Local Development Environment + +## Overview + +This guide explains how to quickly set up a local development environment for Marble. This setup is optimized for fast testing and development, not for production use. + +## Quick Start + +1. **Clone the Repository** + + ```bash + git clone git@github.com:checkmarble/marble.git + cd marble + ``` + +2. **Start the Environment** + ```bash + docker compose -f docker-compose-dev.yaml --env-file .env.dev.example up + ``` + > πŸ’‘ **Tip**: Create your own `.env.dev` file based on `.env.dev.example` for custom configuration. + +## Included Components + +The development environment includes everything needed to run Marble locally: + +### Core Services + +- Marble API (Backend) +- Marble Worker +- Marble Frontend + +### Development Dependencies + +- PostgreSQL database +- Firebase Auth Emulator +- Elasticsearch instance +- Object storage emulation + +## Configuration + +### Basic Setup + +The `.env.dev.example` provides a minimal configuration that works out of the box: + +### Optional Features + +You can enhance your development environment by configuring: + +1. **Webhooks Testing** + + - Configure Convoy settings + - See [Production Deployment Guide](./production_run.md) + - Set webhook endpoints + +2. **Sanctions Screening** + + - Enable Yente integration + - Configure Elasticsearch + +3. **Storage Features** + - Set up local storage + - Or connect to cloud storage + +## Troubleshooting + +### Common Issues + +1. **Port Conflicts** + + ```bash + # Check for port usage + lsof -i :8080 + lsof -i :3000 + ``` + +2. **Database Connection** + + - Ensure PostgreSQL is running + - Check connection settings + - Verify database exists + +3. **Firebase Emulator** + - Wait for complete startup + - Check emulator logs + - Verify port accessibility + +### Logs and Debugging + +Access service logs: + +```bash +# All services +docker compose logs -f + +# Specific service +docker compose logs -f api +``` + +## Environment Variables + +See the following files for configuration options: + +- `.env.dev.example`: Development defaults +- `.env.example`: All available options +- `docker-compose.yaml`: Service configuration + +## Next Steps + +- [Production Deployment Guide](./production_run.md) +- [Deployment Architecture](./deployment.md) +- [Configuration Reference](./.env.example) + +> ⚠️ **Remember**: This setup is for development only. See the [Production Deployment Guide](./production_run.md) for production setup. diff --git a/installation/version_support_policy.md b/installation/version_support_policy.md new file mode 100644 index 0000000..7c2268c --- /dev/null +++ b/installation/version_support_policy.md @@ -0,0 +1,30 @@ +# Version Support Policy + +## Database + +### PostgreSQL + +- Version 15 and above is supported + +## Search & Sanctions Screening + +### Yente + +- Version 4.x is required and supported +- Earlier versions are not compatible + +### Elasticsearch + +- Version 8.14+ is supported and tested +- Specifically tested on Elasticsearch 8.14 +- Newer minor versions (8.x) expected to work but not explicitly tested +- Major version changes may require updates + +## Version Updates + +We recommend: + +- Staying within tested version ranges +- Planning upgrades before end-of-support dates +- Testing version upgrades in staging environments first +- Following our upgrade guides when changing versions diff --git a/kubernetes/.versions b/kubernetes/.versions index e0f5404..d13e93c 100644 --- a/kubernetes/.versions +++ b/kubernetes/.versions @@ -1,5 +1,5 @@ -MARBLE-FRONT-IMG=v0.37.0 -MARBLE-FRONT-VERSION=0.37.0 -MARBLE-BACK-IMG=v0.37.0 -MARBLE-BACK-VERSION=0.37.0 -APPLICATION-VERSION=0.37.0 \ No newline at end of file +MARBLE-FRONT-IMG=v0.38.0 +MARBLE-FRONT-VERSION=0.38.0 +MARBLE-BACK-IMG=v0.38.2 +MARBLE-BACK-VERSION=0.38.2 +APPLICATION-VERSION=0.38.0 \ No newline at end of file diff --git a/kubernetes/marble/marble-backend/templates/configmap.yaml b/kubernetes/marble/marble-backend/templates/configmap.yaml index 358dcc4..9b85612 100644 --- a/kubernetes/marble/marble-backend/templates/configmap.yaml +++ b/kubernetes/marble/marble-backend/templates/configmap.yaml @@ -12,7 +12,7 @@ data: PG_USER: {{ .Values.configMap.pgUser }} PG_PASSWORD: !!str {{ .Values.configMap.pgPassword }} - MARBLE_APP_HOST: {{ .Values.configMap.marbleAppHost }} + MARBLE_APP_URL: {{ .Values.configMap.marbleAppUrl }} AUTHENTICATION_JWT_SIGNING_KEY: {{ .Values.configMap.authJwtSignKey }} INGESTION_BUCKET_URL: !!str {{ .Values.configMap.ingestionBucketUrl }} diff --git a/kubernetes/marble/marble-backend/values.yaml b/kubernetes/marble/marble-backend/values.yaml index 03060f9..467b359 100644 --- a/kubernetes/marble/marble-backend/values.yaml +++ b/kubernetes/marble/marble-backend/values.yaml @@ -36,7 +36,7 @@ configMap: pgPort: 5432 pgUser: "postgres" pgPassword: "marble" - marbleAppHost: "localhost:3000" + marbleAppUrl: "http://localhost:3000" authJwtSignKey: "" ingestionBucketUrl: "file:///tempFiles/data-ingestion-bucket?create_dir=true" caseManagerUrl: "file:///tempFiles/case-manager-bucket?create_dir=true" diff --git a/kubernetes/marble/marble-front/templates/configmap.yaml b/kubernetes/marble/marble-front/templates/configmap.yaml index 6c1cacb..8185839 100644 --- a/kubernetes/marble/marble-front/templates/configmap.yaml +++ b/kubernetes/marble/marble-front/templates/configmap.yaml @@ -10,10 +10,10 @@ data: SESSION_SECRET: !!str {{ .Values.configMap.sessionSecret }} SESSION_MAX_AGE: !!str {{ .Values.configMap.sessionMaxAge }} - MARBLE_API_DOMAIN_CLIENT: !!str {{ .Values.configMap.marbleApiDomainClient }} + MARBLE_API_URL_CLIENT: !!str {{ .Values.configMap.marbleApiUrlClient }} - MARBLE_API_DOMAIN_SERVER: !!str {{ .Values.configMap.marbleApiDomainServer }} - MARBLE_APP_DOMAIN: !!str {{ .Values.configMap.marbleAppDomain }} + MARBLE_API_URL_SERVER: !!str {{ .Values.configMap.marbleApiUrlServer }} + MARBLE_APP_URL: !!str {{ .Values.configMap.marbleAppUrl }} FIREBASE_AUTH_EMULATOR_HOST: !!str {{ .Values.configMap.firebaseAuthEmulatorHost }} diff --git a/kubernetes/marble/marble-front/values.yaml b/kubernetes/marble/marble-front/values.yaml index b827134..ab1c341 100644 --- a/kubernetes/marble/marble-front/values.yaml +++ b/kubernetes/marble/marble-front/values.yaml @@ -35,9 +35,9 @@ configMap: nodeEnv: "production" sessionSecret: "placeholder" sessionMaxAge: 43200 - marbleApiDomainClient: "http://localhost:8080" - marbleApiDomainServer: "http://marble-backend.checkmarble.svc.cluster.local:8090" - marbleAppDomain: "http://localhost:8080" + marbleApiUrlClient: "http://localhost:8080" + marbleApiUrlServer: "http://marble-backend.checkmarble.svc.cluster.local:8090" + marbleAppUrl: "http://localhost:8080" firebaseAuthEmulatorHost: "firebase-auth:9099" firebaseApiKey: "placeholder" firebaseAuthDomain: "placeholder" diff --git a/terraform_templates/AWS/esc_tasks_definition.tf b/terraform_templates/AWS/esc_tasks_definition.tf index d6243e6..8586799 100644 --- a/terraform_templates/AWS/esc_tasks_definition.tf +++ b/terraform_templates/AWS/esc_tasks_definition.tf @@ -19,10 +19,10 @@ resource "aws_ecs_task_definition" "app" { { name = "PORT", value = "3000" }, { name = "ENV", value = "production" }, { name = "NODE_ENV", value = "production" }, - { name = "MARBLE_APP_DOMAIN", value = local.environment.frontend.url }, - { name = "MARBLE_API_DOMAIN", value = local.environment.backend.url }, - { name = "MARBLE_API_DOMAIN_SERVER", value = local.environment.backend.url }, - { name = "MARBLE_API_DOMAIN_CLIENT", value = local.environment.backend.url }, + { name = "MARBLE_APP_URL", value = local.environment.frontend.url }, + { name = "MARBLE_API_URL", value = local.environment.backend.url }, + { name = "MARBLE_API_URL_SERVER", value = local.environment.backend.url }, + { name = "MARBLE_API_URL_CLIENT", value = local.environment.backend.url }, { name = "FIREBASE_API_KEY", value = local.environment.firebase.apiKey }, { name = "FIREBASE_AUTH_DOMAIN", value = local.environment.firebase.authDomain }, { name = "FIREBASE_PROJECT_ID", value = local.environment.firebase.projectId }, @@ -65,7 +65,7 @@ resource "aws_ecs_task_definition" "app" { { name = "CREATE_GLOBAL_ADMIN_EMAIL", value = local.environment.org.global }, { name = "CREATE_ORG_NAME", value = local.environment.org.name }, { name = "CREATE_ORG_ADMIN_EMAIL", value = local.environment.org.admin }, - { name = "MARBLE_APP_HOST", value = local.environment.frontend.domain }, + { name = "MARBLE_APP_URL", value = local.environment.frontend.domain }, { name = "MARBLE_BACKOFFICE_HOST", value = local.environment.backend.domain }, { name = "SESSION_SECRET", value = local.environment.session.secret }, { name = "SESSION_MAX_AGE", value = local.environment.session.max_age }, diff --git a/terraform_templates/GCP/cloud_run_backend_service.tf b/terraform_templates/GCP/cloud_run_backend_service.tf index c72e202..28cdbbb 100644 --- a/terraform_templates/GCP/cloud_run_backend_service.tf +++ b/terraform_templates/GCP/cloud_run_backend_service.tf @@ -106,7 +106,7 @@ resource "google_cloud_run_v2_service" "backend" { } env { - name = "MARBLE_APP_HOST" + name = "MARBLE_APP_URL" value = local.environment.frontend.domain } diff --git a/terraform_templates/GCP/cloud_run_frontend_service.tf b/terraform_templates/GCP/cloud_run_frontend_service.tf index 1b0a971..fa3aa53 100644 --- a/terraform_templates/GCP/cloud_run_frontend_service.tf +++ b/terraform_templates/GCP/cloud_run_frontend_service.tf @@ -51,12 +51,12 @@ resource "google_cloud_run_v2_service" "frontend" { } env { - name = "MARBLE_API_DOMAIN" + name = "MARBLE_API_URL" value = local.environment.backend.url } env { - name = "MARBLE_APP_DOMAIN" + name = "MARBLE_APP_URL" value = local.environment.frontend.url } @@ -91,12 +91,12 @@ resource "google_cloud_run_v2_service" "frontend" { } env { - name = "MARBLE_API_DOMAIN_SERVER" + name = "MARBLE_API_URL_SERVER" value = local.environment.backend.url } env { - name = "MARBLE_API_DOMAIN_CLIENT" + name = "MARBLE_API_URL_CLIENT" value = local.environment.backend.url }