Add config from fideslib

Makefile

@@ -127,6 +127,7 @@ pytest: compose-build
 		-e ANALYTICS_OPT_OUT \
 		$(IMAGE_NAME) \
 		pytest $(pytestpath) -m "not integration and not integration_external and not integration_saas"
+
 	@make teardown
 
 pytest-integration:

requirements.txt

@@ -10,6 +10,7 @@ fastapi-caching[redis]
 fastapi-pagination[sqlalchemy]~= 0.8.3
 fastapi[all]==0.78.0
 fideslang==1.0.0
+fideslib==2.0.3
 fideslog==1.1.5
 multidimensional_urlencode==0.0.4
 pandas==1.3.3

src/fidesops/core/config.py

@@ -1,82 +1,31 @@
 # pylint: disable=C0115,C0116, E0213
 
-import hashlib
 import logging
 import os
-from typing import Any, Dict, List, MutableMapping, Optional, Tuple, Union
+from typing import Any, Dict, MutableMapping, Optional
 
-import bcrypt
 import toml
+from fideslib.core.config import (
+    DatabaseSettings,
+    FidesSettings,
+    SecuritySettings,
+    get_config,
+    load_file,
+    load_toml,
+)
 from fideslog.sdk.python.utils import FIDESOPS, generate_client_id
-from pydantic import AnyHttpUrl, BaseSettings, PostgresDsn, ValidationError, validator
-from pydantic.env_settings import SettingsSourceCallable
+from pydantic import validator
 
-from fidesops.common_exceptions import MissingConfig
 from fidesops.util.logger import NotPii
 
 logger = logging.getLogger(__name__)
 
 
-class FidesSettings(BaseSettings):
-    """Class used as a base model for configuration subsections."""
-
-    class Config:
-
-        # Set environment variables to take precedence over init values
-        @classmethod
-        def customise_sources(
-            cls,
-            init_settings: SettingsSourceCallable,
-            env_settings: SettingsSourceCallable,
-            file_secret_settings: SettingsSourceCallable,
-        ) -> Tuple[SettingsSourceCallable, ...]:
-            return env_settings, init_settings
-
-
-class DatabaseSettings(FidesSettings):
+class FidesopsDatabaseSettings(DatabaseSettings):
     """Configuration settings for Postgres."""
 
-    SERVER: str
-    USER: str
-    PASSWORD: str
-    DB: str
-    PORT: str = "5432"
-    TEST_DB: str = "test"
     ENABLED: bool = True
 
-    SQLALCHEMY_DATABASE_URI: Optional[PostgresDsn] = None
-    SQLALCHEMY_TEST_DATABASE_URI: Optional[PostgresDsn] = None
-
-    @validator("SQLALCHEMY_DATABASE_URI", pre=True)
-    def assemble_db_connection(cls, v: Optional[str], values: Dict[str, str]) -> str:
-        """Join DB connection credentials into a connection string"""
-        if isinstance(v, str):
-            return v
-        return PostgresDsn.build(
-            scheme="postgresql",
-            user=values["USER"],
-            password=values["PASSWORD"],
-            host=values["SERVER"],
-            port=values.get("PORT"),
-            path=f"/{values.get('DB') or ''}",
-        )
-
-    @validator("SQLALCHEMY_TEST_DATABASE_URI", pre=True)
-    def assemble_test_db_connection(
-        cls, v: Optional[str], values: Dict[str, str]
-    ) -> str:
-        """Join DB connection credentials into a connection string"""
-        if isinstance(v, str):
-            return v
-        return PostgresDsn.build(
-            scheme="postgresql",
-            user=values["USER"],
-            password=values["PASSWORD"],
-            host=values["SERVER"],
-            port=values["PORT"],
-            path=f"/{values.get('TEST_DB') or ''}",
-        )
-
     class Config:
         env_prefix = "FIDESOPS__DATABASE__"
 
@@ -115,64 +64,9 @@ class Config:
         env_prefix = "FIDESOPS__REDIS__"
 
 
-class SecuritySettings(FidesSettings):
+class FidesopsSecuritySettings(SecuritySettings):
     """Configuration settings for Security variables."""
 
-    AES_ENCRYPTION_KEY_LENGTH: int = 16
-    AES_GCM_NONCE_LENGTH: int = 12
-    APP_ENCRYPTION_KEY: str
-    DRP_JWT_SECRET: str
-
-    @validator("APP_ENCRYPTION_KEY")
-    def validate_encryption_key_length(
-        cls, v: Optional[str], values: Dict[str, str]
-    ) -> Optional[str]:
-        """Validate the encryption key is exactly 32 characters"""
-        if v is None:
-            raise ValueError("APP_ENCRYPTION_KEY value not provided!")
-        encryption_key = v.encode(values.get("ENCODING", "UTF-8"))
-        if len(encryption_key) != 32:
-            raise ValueError(
-                f"APP_ENCRYPTION_KEY value must be exactly 32 characters, "
-                f"received {len(encryption_key)} characters!"
-            )
-        return v
-
-    CORS_ORIGINS: List[AnyHttpUrl] = []
-
-    @validator("CORS_ORIGINS", pre=True)
-    def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str]:
-        """Return a list of valid origins for CORS requests"""
-        if isinstance(v, str) and not v.startswith("["):
-            return [i.strip() for i in v.split(",")]
-        if isinstance(v, (list, str)):
-            return v
-        raise ValueError(v)
-
-    ENCODING: str = "UTF-8"
-
-    # OAuth
-    OAUTH_ROOT_CLIENT_ID: str
-    OAUTH_ROOT_CLIENT_SECRET: str
-    OAUTH_ROOT_CLIENT_SECRET_HASH: Optional[Tuple]
-    OAUTH_ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
-    OAUTH_CLIENT_ID_LENGTH_BYTES = 16
-    OAUTH_CLIENT_SECRET_LENGTH_BYTES = 16
-
-    @validator("OAUTH_ROOT_CLIENT_SECRET_HASH", pre=True)
-    def assemble_root_access_token(
-        cls, v: Optional[str], values: Dict[str, str]
-    ) -> Tuple:
-        """Returns a hashed value of the root access key. This is hashed as it is not wise to
-        return a plaintext for of the root credential anywhere in the system"""
-        value = values["OAUTH_ROOT_CLIENT_SECRET"]
-        encoding = values["ENCODING"]
-        assert value is not None
-        assert encoding is not None
-        salt = bcrypt.gensalt()
-        hashed_client_id = hashlib.sha512(value.encode(encoding) + salt).hexdigest()
-        return hashed_client_id, salt
-
     LOG_LEVEL: str = "INFO"
 
     @validator("LOG_LEVEL", pre=True)
@@ -229,9 +123,9 @@ class Config:
 class FidesopsConfig(FidesSettings):
     """Configuration variables for the FastAPI project"""
 
-    database: DatabaseSettings
+    database: FidesopsDatabaseSettings
     redis: RedisSettings
-    security: SecuritySettings
+    security: FidesopsSecuritySettings
     execution: ExecutionSettings
     root_user: RootUserSettings
 
@@ -253,7 +147,7 @@ class Config:  # pylint: disable=C0115
     def log_all_config_values(self) -> None:
         """Output DEBUG logs of all the config values."""
         for settings in [self.database, self.redis, self.security, self.execution]:
-            for key, value in settings.dict().items():
+            for key, value in settings.dict().items():  # type: ignore
                 logger.debug(
                     "Using config: %s%s = %s",
                     NotPii(settings.Config.env_prefix),  # type: ignore
@@ -262,71 +156,6 @@ def log_all_config_values(self) -> None:
                 )
 
 
-def load_file(file_name: str) -> str:
-    """Load a file and from the first matching location.
-
-    In order, will check:
-    - A path set at ENV variable FIDESOPS__CONFIG_PATH
-    - The current directory
-    - The parent directory
-    - users home (~) directory
-
-    raises FileNotFound if none is found
-    """
-    possible_directories = [
-        os.getenv("FIDESOPS__CONFIG_PATH"),
-        os.curdir,
-        os.pardir,
-        os.path.expanduser("~"),
-    ]
-
-    directories: List[str] = [d for d in possible_directories if d]
-
-    for dir_str in directories:
-        possible_location = os.path.join(dir_str, file_name)
-        if possible_location and os.path.isfile(possible_location):
-            logger.info("Loading file %s from %s", NotPii(file_name), NotPii(dir_str))
-            return possible_location
-        logger.debug("%s not found at %s", NotPii(file_name), NotPii(dir_str))
-    raise FileNotFoundError
-
-
-def load_toml(file_name: str) -> MutableMapping[str, Any]:
-    """
-    Load toml file from possible locations specified in load_file.
-
-    Will raise FileNotFoundError or ValidationError on missing or
-    bad file
-    """
-    return toml.load(load_file(file_name))
-
-
-def get_config() -> FidesopsConfig:
-    """
-    Attempt to read config file from:
-    a) env var FIDESOPS__CONFIG_PATH
-    b) local directory
-    c) parent directory
-    d) home directory
-    This will fail on the first encountered bad conf file.
-    """
-    try:
-        return FidesopsConfig.parse_obj(load_toml("fidesops.toml"))
-    except (FileNotFoundError) as e:
-        logger.warning("fidesops.toml could not be loaded: %s", NotPii(e))
-        # If no path is specified Pydantic will attempt to read settings from
-        # the environment. Default values will still be used if the matching
-        # environment variable is not set.
-        try:
-            return FidesopsConfig()
-        except ValidationError as exc:
-            logger.error("Fidesops config could not be loaded: %s", NotPii(exc))
-            # If FidesopsConfig is missing any required values Pydantic will throw
-            # an ImportError. This means the config has not been correctly specified
-            # so we can throw the missing config error.
-            raise MissingConfig(exc.args[0])
-
-
 CONFIG_KEY_ALLOWLIST = {
     "database": [
         "SERVER",
@@ -379,8 +208,8 @@ def update_config_file(updates: Dict[str, Dict[str, Any]]) -> None:
     :param updates: A nested `dict`, where top-level keys correspond to configuration sections and top-level values contain `dict`s whose key/value pairs correspond to the desired option/value updates.
     """
     try:
-        config_path: str = load_file("fidesops.toml")
-        current_config: MutableMapping[str, Any] = load_toml("fidesops.toml")
+        config_path: str = load_file(["fidesops.toml"])
+        current_config: MutableMapping[str, Any] = load_toml(["fidesops.toml"])
     except FileNotFoundError as e:
         logger.warning("fidesops.toml could not be loaded: %s", NotPii(e))
 
@@ -400,7 +229,7 @@ def update_config_file(updates: Dict[str, Dict[str, Any]]) -> None:
             logger.info("\tSet %s.%s = %s", NotPii(key), NotPii(subkey), NotPii(val))
 
 
-config = get_config()
+config = get_config(FidesopsConfig)
 # `censored_config` is included below because it's important we keep the censored
 # config at parity with `config`. This means if we change the path at which fidesops
 # loads `config`, we should also change `censored_config`.

tests/conftest.py

@@ -6,6 +6,7 @@
 
 import pytest
 from fastapi.testclient import TestClient
+from fideslib.core.config import load_toml
 from sqlalchemy_utils.functions import create_database, database_exists, drop_database
 
 from fidesops.api.v1.scope_registry import SCOPE_REGISTRY
@@ -154,7 +155,7 @@ def _build_jwt(webhook: PolicyPreWebhook) -> Dict[str, str]:
 
 @pytest.fixture(scope="session")
 def integration_config() -> MutableMapping[str, Any]:
-    yield load_toml("fidesops-integration.toml")
+    yield load_toml(["fidesops-integration.toml"])
 
 
 @pytest.fixture(autouse=True, scope="session")