fix(config-parsing): add support for variable expansion for in variables in between string

metadata-ingestion/src/datahub/configuration/config_loader.py

@@ -1,5 +1,6 @@
 import io
 import pathlib
+import re
 from typing import Union
 
 from expandvars import UnboundVariable, expandvars
@@ -10,7 +11,7 @@
 
 
 def resolve_element(element: str) -> str:
-    if element.startswith("${") & element.endswith("}"):
+    if re.search("(\$\{).+(\})", element):  # noqa: W605
         return expandvars(element, nounset=True)
     elif element.startswith("$"):
         try:
@@ -28,20 +29,23 @@ def resolve_list(ele_list: list) -> list:
             new_v.append(resolve_element(ele))  # type:ignore
         elif isinstance(ele, list):
             new_v.append(resolve_list(ele))  # type:ignore
+        elif isinstance(ele, dict):
+            resolve_env_variables(ele)
+            new_v.append(resolve_env_variables(ele))  # type:ignore
         else:
-            new_v = ele_list
+            new_v.append(ele)
     return new_v
 
 
-def resolve_env_variables(config: dict) -> None:
+def resolve_env_variables(config: dict) -> dict:
     for k, v in config.items():
         if isinstance(v, dict):
             resolve_env_variables(v)
         elif isinstance(v, list):
             config[k] = resolve_list(v)
         elif isinstance(v, str):
             config[k] = resolve_element(v)
-    return None
+    return config
 
 
 def load_config_file(config_file: Union[pathlib.Path, str]) -> dict:

metadata-ingestion/tests/unit/config/bad_complex_variable_expansion.yml

@@ -0,0 +1,2 @@
+stuff:
+  password: ${SOME_PASSWORD

metadata-ingestion/tests/unit/config/complex_variable_expansion.yml

@@ -0,0 +1,31 @@
+---
+foo: bar
+bar: ${VAR1}
+nested:
+  hi: hello
+  vanilla_val_1: vanillaValue
+  vanilla_val_2: vanilla$Value
+  vanilla_val_3: vanilla${Value
+  vanilla_val_4: vani}lla${Value
+  vanilla_val_5: vanillaValue}
+  vanilla_val_6: vanilla$Value}
+  password: $VAR2
+  password_1: ${VAR3}
+  array:
+    - one
+    - two
+    - ${VAR4}
+    - three$array
+    -   - in_one
+        - in_two
+        - $VAR5
+        - test_url${VAR6}
+        - test_url${VAR7}/action
+    - inner_array:
+        - in_ar_one
+        - ${VAR8}
+        - test_url$vanillavar
+        - test_url${VAR9}vanillaVar
+        - \${VAR10}
+
+

metadata-ingestion/tests/unit/config/test_config_loader.py

@@ -61,6 +61,66 @@
             {},
             ConfigurationError,
         ),
+        (
+            # Variable expansion error
+            "tests/unit/config/bad_complex_variable_expansion.yml",
+            None,
+            {},
+            expandvars.MissingClosingBrace,
+        ),
+        (
+            "tests/unit/config/complex_variable_expansion.yml",
+            {
+                "foo": "bar",
+                "bar": "stuff1",
+                "nested": {
+                    "hi": "hello",
+                    "vanilla_val_1": "vanillaValue",
+                    "vanilla_val_2": "vanilla$Value",
+                    "vanilla_val_3": "vanilla${Value",
+                    "vanilla_val_4": "vani}lla${Value",
+                    "vanilla_val_5": "vanillaValue}",
+                    "vanilla_val_6": "vanilla$Value}",
+                    "password": "stuff2",
+                    "password_1": "stuff3",
+                    "array": [
+                        "one",
+                        "two",
+                        "stuff4",
+                        "three$array",
+                        [
+                            "in_one",
+                            "in_two",
+                            "stuff5",
+                            "test_urlstuff6",
+                            "test_urlstuff7/action",
+                        ],
+                        {
+                            "inner_array": [
+                                "in_ar_one",
+                                "stuff8",
+                                "test_url$vanillavar",
+                                "test_urlstuff9vanillaVar",
+                                "stuff10",
+                            ]
+                        },
+                    ],
+                },
+            },
+            {
+                "VAR1": "stuff1",
+                "VAR2": "stuff2",
+                "VAR3": "stuff3",
+                "VAR4": "stuff4",
+                "VAR5": "stuff5",
+                "VAR6": "stuff6",
+                "VAR7": "stuff7",
+                "VAR8": "stuff8",
+                "VAR9": "stuff9",
+                "VAR10": "stuff10",
+            },
+            None,
+        ),
     ],
 )
 def test_load(pytestconfig, filename, golden_config, env, error_type):