feat: brownie config supports env vars

CHANGELOG.md

@@ -6,6 +6,8 @@ This changelog format is based on [Keep a Changelog](https://keepachangelog.com/
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased](https://github.com/eth-brownie/brownie)
+### Added
+- Support for environment variables in brownie config ([#1012](https://github.com/eth-brownie/brownie/pull/1012))
 
 ## [1.14.3](https://github.com/eth-brownie/brownie/tree/v1.14.3) - 2021-03-27
 ### Added

CONTRIBUTING.md

@@ -44,3 +44,19 @@ It's a good idea to make pull requests early on. A pull request represents the s
 If you are opening a work-in-progress pull request to verify that it passes CI tests, please consider [marking it as a draft](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests#draft-pull-requests).
 
 Join the Brownie [Gitter channel](https://gitter.im/eth-brownie/community) if you have any questions.
+
+## Productivity Tips
+
+### Running Tests
+
+Instead of running the entire test suite each time you make a change, run specific tests and fail fast (`-x`):
+
+```bash
+docker-compose exec sandbox bash -c 'python -m pytest tests/project/test_brownie_config.py::TestFooBar -x'
+```
+
+Drop to a pdb shell upon error with the `--pdb` flag:
+
+```sh
+docker-compose exec sandbox bash -c 'python -m pytest tests/project/test_brownie_config.py -x --pdb'
+```

brownie/_config.py

@@ -11,10 +11,12 @@
 from typing import Any, Dict, List, Optional
 
 import yaml
+from dotenv import dotenv_values
 from hypothesis import Phase
 from hypothesis import settings as hp_settings
 from hypothesis.database import DirectoryBasedExampleDatabase
 
+from brownie._expansion import expand_posix_vars
 from brownie._singleton import _Singleton
 
 __version__ = "1.14.3"
@@ -187,6 +189,17 @@ def _load_project_config(project_path: Path) -> None:
     """Loads configuration settings from a project's brownie-config.yaml"""
     config_path = project_path.joinpath("brownie-config")
     config_data = _load_config(config_path)
+    config_vars = _load_project_envvars(project_path)
+
+    if "dotenv" in config_data:
+        if not isinstance(config_data["dotenv"], str):
+            raise ValueError(f'Invalid value passed to dotenv: {config_data["dotenv"]}')
+        env_path = project_path.joinpath(config_data["dotenv"])
+        if not env_path.is_file():
+            raise ValueError(f"Dotenv specified in config but not found at path: {env_path}")
+        config_vars.update(dotenv_values(dotenv_path=env_path))  # type: ignore
+        config_data = expand_posix_vars(config_data, config_vars)
+
     if not config_data:
         return
 
@@ -217,6 +230,8 @@ def _load_project_config(project_path: Path) -> None:
 
     CONFIG.settings._unlock()
     _recursive_update(CONFIG.settings, config_data)
+    _recursive_update(CONFIG.settings, expand_posix_vars(CONFIG.settings, config_vars))
+
     CONFIG.settings._lock()
     if "hypothesis" in config_data:
         _modify_hypothesis_settings(config_data["hypothesis"], "brownie", "brownie-base")
@@ -233,6 +248,19 @@ def _load_project_compiler_config(project_path: Optional[Path]) -> Dict:
     return compiler_data
 
 
+def _load_project_envvars(project_path: Path) -> Dict:
+    config_vars = dict(os.environ)
+    if CONFIG.settings.get("dotenv"):
+        dotenv_path = CONFIG.settings["dotenv"]
+        if not isinstance(dotenv_path, str):
+            raise ValueError(f"Invalid value passed to dotenv: {dotenv_path}")
+        env_path = project_path.joinpath(dotenv_path)
+        if not env_path.is_file():
+            raise ValueError(f"Dotenv specified in config but not found at path: {env_path}")
+        config_vars.update(dotenv_values(dotenv_path=env_path))  # type: ignore
+    return config_vars
+
+
 def _load_project_structure_config(project_path):
     structure = CONFIG.settings["project_structure"]._copy()
 

brownie/_expansion.py

@@ -0,0 +1,58 @@
+import re
+from typing import Any, Mapping, Optional, Text
+
+from dotenv.variables import parse_variables
+
+
+def expand_posix_vars(obj: Any, variables: Mapping[Text, Optional[Any]]) -> Any:
+    """expand_posix_vars recursively expands POSIX values in an object.
+
+    Args:
+        obj (any): object in which to interpolate variables.
+        variables (dict): dictionary that maps variable names to their value
+    """
+    if isinstance(obj, (dict,)):
+        for key, val in obj.items():
+            obj[key] = expand_posix_vars(val, variables)
+    elif isinstance(obj, (list,)):
+        for index in range(len(obj)):
+            obj[index] = expand_posix_vars(obj[index], variables)
+    elif isinstance(obj, (str,)):
+        obj = _str_to_python_value(_expand(obj, variables))
+    return obj
+
+
+def _expand(value, variables={}):
+    """_expand does POSIX-style variable expansion
+
+    This is adapted from python-dotenv, specifically here:
+
+    https://github.com/theskumar/python-dotenv/commit/17dba65244c1d4d10f591fe37c924bd2c6fd1cfc
+
+    We need this layer here so we can explicitly pass in variables;
+    python-dotenv assumes you want to use os.environ.
+    """
+
+    if not isinstance(value, (str,)):
+        return value
+    atoms = parse_variables(value)
+    return "".join([str(atom.resolve(variables)) for atom in atoms])
+
+
+INT_REGEX = re.compile(r"^[-+]?[0-9]+$")
+
+
+def _str_to_python_value(val):
+    """_str_to_python_value infers the data type from a string.
+
+    This could eventually use PyYAML's parsing logic.
+    """
+    if not isinstance(val, (str,)):
+        return val
+    elif val == "true" or val == "True" or val == "on":
+        return True
+    elif val == "false" or val == "False" or val == "off":
+        return False
+    elif INT_REGEX.match(val):
+        return int(val)
+    return val

brownie/project/main.py

@@ -29,8 +29,10 @@
     _load_project_compiler_config,
     _load_project_config,
     _load_project_dependencies,
+    _load_project_envvars,
     _load_project_structure_config,
 )
+from brownie._expansion import expand_posix_vars
 from brownie.exceptions import (
     BrownieEnvironmentWarning,
     InvalidPackage,
@@ -167,7 +169,10 @@ class Project(_ProjectBase):
 
     def __init__(self, name: str, project_path: Path) -> None:
         self._path: Path = project_path
-        self._structure = _load_project_structure_config(project_path)
+        self._envvars = _load_project_envvars(project_path)
+        self._structure = expand_posix_vars(
+            _load_project_structure_config(project_path), self._envvars
+        )
         self._build_path: Path = project_path.joinpath(self._structure["build"])
 
         self._name = name
@@ -221,7 +226,9 @@ def load(self) -> None:
             self._build._add_interface(build_json)
             interface_hashes[path.stem] = build_json["sha1"]
 
-        self._compiler_config = _load_project_compiler_config(self._path)
+        self._compiler_config = expand_posix_vars(
+            _load_project_compiler_config(self._path), self._envvars
+        )
 
         # compile updated sources, update build
         changed = self._get_changed_contracts(interface_hashes)

docs/config.rst

@@ -10,6 +10,8 @@ The configuration file must be saved as ``brownie-config.yaml``. If saved in the
 
 All configuration fields are optional. You can copy from the examples below and modify the settings as required.
 
+Configuration values can also be set using environment variables, as well as by specifying the `dotenv` top-level key.
+
 Default Configuration
 =====================
 
@@ -20,6 +22,23 @@ The following example shows all configuration settings and their default values:
     :lines: 8-
     :language: yaml
 
+Variable Expansion
+==================
+
+Brownie supports POSIX-style variable expansion for environment variables.
+
+.. code-block:: yaml
+
+    networks:
+        default: ${DEFAULT_NETWORK}
+
+You can also provide defaults.
+
+    .. code-block:: yaml
+
+    networks:
+        default: ${DEFAULT_NETWORK:-mainnet}
+
 Settings
 ========
 
@@ -325,3 +344,11 @@ Other Settings
     This is useful if another application, such as a front end framework, needs access to deployment artifacts while you are on a development network.
 
     default value: ``false``
+
+.. py:attribute:: dotenv
+
+    If present, Brownie will load the .env file, resolving the file relative to the project root. Will fail loudly if .env file is missing.
+
+    .. code-block:: yaml
+
+        dotenv: .env

requirements.in

@@ -8,13 +8,14 @@ hexbytes<1
 hypothesis<6
 prompt-toolkit<4
 psutil>=5.7.3,<6
-py>=1.5.0,<2
 py-solc-ast>=1.2.8,<2
 py-solc-x>=1.1.0,<2
-pygments<3
+py>=1.5.0,<2
 pygments-lexer-solidity<1
-pytest<7
+pygments<3
 pytest-xdist<2
+pytest<7
+python-dotenv>=0.16.0,<0.17.0
 pythx<2
 pyyaml>=5.3.0,<6
 requests>=2.25.0,<3

requirements.txt

@@ -88,6 +88,13 @@ hypothesis==5.41.3
     # via -r requirements.in
 idna==2.10
     # via requests
+importlib-metadata==3.10.0
+    # via
+    #   jsonschema
+    #   pluggy
+    #   pytest
+importlib-resources==5.1.2
+    # via netaddr
 inflection==0.5.0
     # via
     #   mythx-models
@@ -163,6 +170,8 @@ python-dateutil==2.8.1
     # via
     #   mythx-models
     #   pythx
+python-dotenv==0.16.0
+    # via -r requirements.in
 pythx==1.6.1
     # via -r requirements.in
 pyyaml==5.4.1
@@ -209,6 +218,10 @@ tqdm==4.53.0
     # via -r requirements.in
 typed-ast==1.4.2
     # via black
+typing-extensions==3.7.4.3
+    # via
+    #   importlib-metadata
+    #   web3
 urllib3==1.26.4
     # via requests
 varint==1.0.2
@@ -223,6 +236,10 @@ web3==5.11.1
     # via -r requirements.in
 websockets==8.1
     # via web3
+zipp==3.4.1
+    # via
+    #   importlib-metadata
+    #   importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools