Allow configuration overrides

.gitignore

@@ -3,6 +3,10 @@
 # data/ is intended for database data from the mysql container
 data/
 
+# overrides should not be submitted as it may contain sensitive data
+override.env
+src/config.override.toml
+
 # Generated Logstash configuration
 logstash/config/config/logstash.yml
 logstash/config/config/pipelines.yml

README.md

@@ -106,6 +106,26 @@ mysql> SHOW DATABASES;
 
 Now, you can visit the server from your browser at `localhost:8000/docs`.
 
+### Changing the configuration
+You may need to change the configuration locally, for example if you want different ports to be used.
+Do not change files, instead add overrides.
+
+#### Docker Compose
+For docker compose, the environment variables are defined in the `.env` file. 
+To override variables, for example `AIOD_LOGSTASH_PORT`, add a new file called `override.env`:
+```bash {title='override.env'}
+AIOD_LOGSTASH_PORT=5001
+```
+Then also specify this when you invoke docker compose, e.g.:
+`docker compose --env-file=.env --env-file=override.env up`
+Note that **order is important**, later environment files will override earlier ones.
+
+#### Config.toml
+The main application supports configuration options through a `toml` file.
+The defaults can be found at `src/config.default.toml`.
+To override them, add a `src/config.override.toml` file.
+It follows the same structure as the default file, but you only need to specify the variables to override.
+
 #### Using connectors
 You can specify different connectors using
 

docker-compose.yaml

@@ -8,7 +8,6 @@ services:
       dockerfile: Dockerfile
     image: aiod_metadata_catalogue
     container_name: apiserver
-    env_file: .env
     environment:
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
     ports:
@@ -34,7 +33,6 @@ services:
     profiles: ["examples"]
     image: aiod_metadata_catalogue
     container_name: fill-db-with-examples
-    env_file: .env
     environment:
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
     volumes:
@@ -53,7 +51,6 @@ services:
       dockerfile: Dockerfile
     image: aiod_deletion
     container_name: deletion
-    env_file: .env
     volumes:
       - ./src:/app
       - ${DATA_PATH}/deletion:/opt/deletion/data
@@ -67,7 +64,6 @@ services:
     profiles: ["huggingface-datasets"]
     image: aiod_metadata_catalogue
     container_name: huggingface-dataset-connector
-    env_file: .env
     environment:
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
     volumes:
@@ -87,7 +83,6 @@ services:
       dockerfile: Dockerfile
     image: aiod_openml_connector
     container_name: openml-connector
-    env_file: .env
     environment:
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
     volumes:
@@ -107,7 +102,6 @@ services:
       dockerfile: Dockerfile
     image: aiod_zenodo_connector
     container_name: zenodo-dataset-connector
-    env_file: .env
     environment:
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
     volumes:
@@ -123,7 +117,6 @@ services:
   sqlserver:
     image: mysql:8.3.0
     container_name: sqlserver
-    env_file: .env
     environment:
       - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}
     volumes:
@@ -139,7 +132,6 @@ services:
   keycloak:
     image: quay.io/keycloak/keycloak:24.0.4
     container_name: keycloak
-    env_file: .env
     environment:
       - REDIRECT_URIS=$REDIRECT_URIS
       - POST_LOGOUT_REDIRECT_URIS=$POST_LOGOUT_REDIRECT_URIS
@@ -172,7 +164,6 @@ services:
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.8.2
     container_name: elasticsearch
-    env_file: .env
     environment:
       - ES_JAVA_OPTS=$ES_JAVA_OPTS
       - ELASTIC_USER=$ES_USER
@@ -196,11 +187,11 @@ services:
   es_logstash_setup:
     image: aiod_metadata_catalogue
     container_name:  es_logstash_setup
-    env_file: .env
     environment:
       - MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD
       - ES_USER=$ES_USER
       - ES_PASSWORD=$ES_PASSWORD
+      - PYTHONPATH=$PYTHONPATH
     volumes:
       - ./src:/app
       - ./logstash:/logstash
@@ -218,7 +209,6 @@ services:
       dockerfile: Dockerfile
     image: aiod_logstash
     container_name: logstash
-    env_file: .env
     environment:
       - LS_JAVA_OPTS=$LS_JAVA_OPTS
     ports:

src/config.py

@@ -1,9 +1,35 @@
+import copy
 import pathlib
 import tomllib
+from typing import Any
 
-with open(pathlib.Path(__file__).parent / "config.toml", "rb") as fh:
-    CONFIG = tomllib.load(fh)
+with open(pathlib.Path(__file__).parent / "config.default.toml", "rb") as fh:
+    DEFAULT_CONFIG = tomllib.load(fh)
 
+OVERRIDE_CONFIG_PATH = pathlib.Path(__file__).parent / "config.override.toml"
+if OVERRIDE_CONFIG_PATH.exists():
+    with open(OVERRIDE_CONFIG_PATH, "rb") as fh:
+        OVERRIDE_CONFIG = tomllib.load(fh)
+else:
+    OVERRIDE_CONFIG = {}
+
+
+def _merge_configurations(
+    default: dict[str, Any], override: dict[str, Any], path: str = "root"
+) -> dict[str, Any]:
+    if extra_keys := (set(override) - set(default)):
+        keys = ", ".join(map(repr, extra_keys))
+        raise KeyError(f"The custom configuration has unknown key(s) at {path!r}: {keys}")
+    merged = copy.copy(default)
+    for key, value in override.items():
+        if isinstance(value, dict):
+            merged[key] = _merge_configurations(default[key], value, path=f"{path}.{key}")
+        else:
+            merged[key] = value
+    return merged
+
+
+CONFIG = _merge_configurations(DEFAULT_CONFIG, OVERRIDE_CONFIG)
 DB_CONFIG = CONFIG.get("database", {})
 KEYCLOAK_CONFIG = CONFIG.get("keycloak", {})
 REQUEST_TIMEOUT = CONFIG.get("dev", {}).get("request_timeout", None)

src/tests/test_configuration.py

@@ -0,0 +1,70 @@
+import tomllib
+
+from config import _merge_configurations
+
+DEFAULT_CONFIG = """
+[section1]
+integer = 2
+string = "hello"
+
+[section1.subsection]
+float = 3.2
+array = [1, 2, 3]
+
+[section2]
+integer = 42
+"""
+
+
+def test_merge_configuration_no_override_means_no_change():
+    default = tomllib.loads(DEFAULT_CONFIG)
+    merged = _merge_configurations(default, override={})
+    assert default == merged
+
+
+def test_merge_configuration_nonnested():
+    default = tomllib.loads(DEFAULT_CONFIG)
+    override_text = """
+    [section1]
+    integer = 0
+    """
+    override = tomllib.loads(override_text)
+
+    merged = _merge_configurations(default, override)
+    assert merged["section1"]["integer"] == 0
+
+    del merged["section1"]["integer"]
+    del default["section1"]["integer"]
+    assert default == merged
+
+
+def test_merge_configuration_nested():
+    default = tomllib.loads(DEFAULT_CONFIG)
+    override_text = """
+    [section1.subsection]
+    float = 1.0
+    """
+    override = tomllib.loads(override_text)
+
+    merged = _merge_configurations(default, override)
+    assert merged["section1"]["subsection"]["float"] == 1.0
+
+    del merged["section1"]["subsection"]["float"]
+    del default["section1"]["subsection"]["float"]
+    assert default == merged
+
+
+def test_merge_configuration_array():
+    default = tomllib.loads(DEFAULT_CONFIG)
+    override_text = """
+    [section1.subsection]
+    array = [0,3,7]
+    """
+    override = tomllib.loads(override_text)
+
+    merged = _merge_configurations(default, override)
+    assert merged["section1"]["subsection"]["array"] == [0, 3, 7]
+
+    del merged["section1"]["subsection"]["array"]
+    del default["section1"]["subsection"]["array"]
+    assert default == merged