diff --git a/CHANGELOG.md b/CHANGELOG.md index 52eafa5a5..ff315dc5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ This changelog format is based on [Keep a Changelog](https://keepachangelog.com/ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased](https://github.com/eth-brownie/brownie) +### Added +- Support for environment variables in brownie config ([#1012](https://github.com/eth-brownie/brownie/pull/1012)) ## [1.14.3](https://github.com/eth-brownie/brownie/tree/v1.14.3) - 2021-03-27 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4b1fa519..122a21cf5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,3 +44,19 @@ It's a good idea to make pull requests early on. A pull request represents the s If you are opening a work-in-progress pull request to verify that it passes CI tests, please consider [marking it as a draft](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests#draft-pull-requests). Join the Brownie [Gitter channel](https://gitter.im/eth-brownie/community) if you have any questions. + +## Productivity Tips + +### Running Tests + +Instead of running the entire test suite each time you make a change, run specific tests and fail fast (`-x`): + +```bash +docker-compose exec sandbox bash -c 'python -m pytest tests/project/test_brownie_config.py::TestFooBar -x' +``` + +Drop to a pdb shell upon error with the `--pdb` flag: + +```sh +docker-compose exec sandbox bash -c 'python -m pytest tests/project/test_brownie_config.py -x --pdb' +``` diff --git a/brownie/_config.py b/brownie/_config.py index 34cf04bb6..83d152091 100644 --- a/brownie/_config.py +++ b/brownie/_config.py @@ -11,10 +11,12 @@ from typing import Any, Dict, List, Optional import yaml +from dotenv import dotenv_values from hypothesis import Phase from hypothesis import settings as hp_settings from hypothesis.database import DirectoryBasedExampleDatabase +from brownie._expansion import expand_posix_vars from brownie._singleton import _Singleton __version__ = "1.14.3" @@ -187,6 +189,17 @@ def _load_project_config(project_path: Path) -> None: """Loads configuration settings from a project's brownie-config.yaml""" config_path = project_path.joinpath("brownie-config") config_data = _load_config(config_path) + config_vars = _load_project_envvars(project_path) + + if "dotenv" in config_data: + if not isinstance(config_data["dotenv"], str): + raise ValueError(f'Invalid value passed to dotenv: {config_data["dotenv"]}') + env_path = project_path.joinpath(config_data["dotenv"]) + if not env_path.is_file(): + raise ValueError(f"Dotenv specified in config but not found at path: {env_path}") + config_vars.update(dotenv_values(dotenv_path=env_path)) # type: ignore + config_data = expand_posix_vars(config_data, config_vars) + if not config_data: return @@ -217,6 +230,8 @@ def _load_project_config(project_path: Path) -> None: CONFIG.settings._unlock() _recursive_update(CONFIG.settings, config_data) + _recursive_update(CONFIG.settings, expand_posix_vars(CONFIG.settings, config_vars)) + CONFIG.settings._lock() if "hypothesis" in config_data: _modify_hypothesis_settings(config_data["hypothesis"], "brownie", "brownie-base") @@ -233,6 +248,19 @@ def _load_project_compiler_config(project_path: Optional[Path]) -> Dict: return compiler_data +def _load_project_envvars(project_path: Path) -> Dict: + config_vars = dict(os.environ) + if CONFIG.settings.get("dotenv"): + dotenv_path = CONFIG.settings["dotenv"] + if not isinstance(dotenv_path, str): + raise ValueError(f"Invalid value passed to dotenv: {dotenv_path}") + env_path = project_path.joinpath(dotenv_path) + if not env_path.is_file(): + raise ValueError(f"Dotenv specified in config but not found at path: {env_path}") + config_vars.update(dotenv_values(dotenv_path=env_path)) # type: ignore + return config_vars + + def _load_project_structure_config(project_path): structure = CONFIG.settings["project_structure"]._copy() diff --git a/brownie/_expansion.py b/brownie/_expansion.py new file mode 100644 index 000000000..0a803d6c5 --- /dev/null +++ b/brownie/_expansion.py @@ -0,0 +1,58 @@ +import re +from typing import Any, Mapping, Optional, Text + +from dotenv.variables import parse_variables + + +def expand_posix_vars(obj: Any, variables: Mapping[Text, Optional[Any]]) -> Any: + """expand_posix_vars recursively expands POSIX values in an object. + + Args: + obj (any): object in which to interpolate variables. + variables (dict): dictionary that maps variable names to their value + """ + if isinstance(obj, (dict,)): + for key, val in obj.items(): + obj[key] = expand_posix_vars(val, variables) + elif isinstance(obj, (list,)): + for index in range(len(obj)): + obj[index] = expand_posix_vars(obj[index], variables) + elif isinstance(obj, (str,)): + obj = _str_to_python_value(_expand(obj, variables)) + return obj + + +def _expand(value, variables={}): + """_expand does POSIX-style variable expansion + + This is adapted from python-dotenv, specifically here: + + https://github.com/theskumar/python-dotenv/commit/17dba65244c1d4d10f591fe37c924bd2c6fd1cfc + + We need this layer here so we can explicitly pass in variables; + python-dotenv assumes you want to use os.environ. + """ + + if not isinstance(value, (str,)): + return value + atoms = parse_variables(value) + return "".join([str(atom.resolve(variables)) for atom in atoms]) + + +INT_REGEX = re.compile(r"^[-+]?[0-9]+$") + + +def _str_to_python_value(val): + """_str_to_python_value infers the data type from a string. + + This could eventually use PyYAML's parsing logic. + """ + if not isinstance(val, (str,)): + return val + elif val == "true" or val == "True" or val == "on": + return True + elif val == "false" or val == "False" or val == "off": + return False + elif INT_REGEX.match(val): + return int(val) + return val diff --git a/brownie/project/main.py b/brownie/project/main.py index 68a1ec699..e143cb897 100644 --- a/brownie/project/main.py +++ b/brownie/project/main.py @@ -29,8 +29,10 @@ _load_project_compiler_config, _load_project_config, _load_project_dependencies, + _load_project_envvars, _load_project_structure_config, ) +from brownie._expansion import expand_posix_vars from brownie.exceptions import ( BrownieEnvironmentWarning, InvalidPackage, @@ -167,7 +169,10 @@ class Project(_ProjectBase): def __init__(self, name: str, project_path: Path) -> None: self._path: Path = project_path - self._structure = _load_project_structure_config(project_path) + self._envvars = _load_project_envvars(project_path) + self._structure = expand_posix_vars( + _load_project_structure_config(project_path), self._envvars + ) self._build_path: Path = project_path.joinpath(self._structure["build"]) self._name = name @@ -221,7 +226,9 @@ def load(self) -> None: self._build._add_interface(build_json) interface_hashes[path.stem] = build_json["sha1"] - self._compiler_config = _load_project_compiler_config(self._path) + self._compiler_config = expand_posix_vars( + _load_project_compiler_config(self._path), self._envvars + ) # compile updated sources, update build changed = self._get_changed_contracts(interface_hashes) diff --git a/docs/config.rst b/docs/config.rst index 3dc937c59..9374052b4 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -10,6 +10,8 @@ The configuration file must be saved as ``brownie-config.yaml``. If saved in the All configuration fields are optional. You can copy from the examples below and modify the settings as required. +Configuration values can also be set using environment variables, as well as by specifying the `dotenv` top-level key. + Default Configuration ===================== @@ -20,6 +22,23 @@ The following example shows all configuration settings and their default values: :lines: 8- :language: yaml +Variable Expansion +================== + +Brownie supports POSIX-style variable expansion for environment variables. + +.. code-block:: yaml + + networks: + default: ${DEFAULT_NETWORK} + +You can also provide defaults. + + .. code-block:: yaml + + networks: + default: ${DEFAULT_NETWORK:-mainnet} + Settings ======== @@ -325,3 +344,11 @@ Other Settings This is useful if another application, such as a front end framework, needs access to deployment artifacts while you are on a development network. default value: ``false`` + +.. py:attribute:: dotenv + + If present, Brownie will load the .env file, resolving the file relative to the project root. Will fail loudly if .env file is missing. + + .. code-block:: yaml + + dotenv: .env \ No newline at end of file diff --git a/requirements.in b/requirements.in index 654221885..235abd1df 100644 --- a/requirements.in +++ b/requirements.in @@ -8,13 +8,14 @@ hexbytes<1 hypothesis<6 prompt-toolkit<4 psutil>=5.7.3,<6 -py>=1.5.0,<2 py-solc-ast>=1.2.8,<2 py-solc-x>=1.1.0,<2 -pygments<3 +py>=1.5.0,<2 pygments-lexer-solidity<1 -pytest<7 +pygments<3 pytest-xdist<2 +pytest<7 +python-dotenv>=0.16.0,<0.17.0 pythx<2 pyyaml>=5.3.0,<6 requests>=2.25.0,<3 diff --git a/requirements.txt b/requirements.txt index c70db6089..60ed9bec8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -88,6 +88,13 @@ hypothesis==5.41.3 # via -r requirements.in idna==2.10 # via requests +importlib-metadata==3.10.0 + # via + # jsonschema + # pluggy + # pytest +importlib-resources==5.1.2 + # via netaddr inflection==0.5.0 # via # mythx-models @@ -163,6 +170,8 @@ python-dateutil==2.8.1 # via # mythx-models # pythx +python-dotenv==0.16.0 + # via -r requirements.in pythx==1.6.1 # via -r requirements.in pyyaml==5.4.1 @@ -209,6 +218,10 @@ tqdm==4.53.0 # via -r requirements.in typed-ast==1.4.2 # via black +typing-extensions==3.7.4.3 + # via + # importlib-metadata + # web3 urllib3==1.26.4 # via requests varint==1.0.2 @@ -223,6 +236,10 @@ web3==5.11.1 # via -r requirements.in websockets==8.1 # via web3 +zipp==3.4.1 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/tests/project/test_brownie_config.py b/tests/project/test_brownie_config.py index 80625569c..993919ef7 100644 --- a/tests/project/test_brownie_config.py +++ b/tests/project/test_brownie_config.py @@ -1,5 +1,7 @@ #!/usr/bin/python3 +import copy + import pytest import yaml @@ -7,36 +9,38 @@ from brownie.network import web3 from brownie.network.rpc.ganache import _validate_cmd_settings +BASE_PROJECT_CONFIG = yaml.safe_load( + """ +networks: + default: development + development: + gas_limit: 6543210 + gas_price: 1000 + reverting_tx_gas_limit: 8765432 + default_contract_owner: false + cmd_settings: + network_id: 777 + chain_id: 666 + gas_limit: 7654321 + block_time: 5 + default_balance: 15 milliether + time: 2019-04-05T14:30:11Z + accounts: 15 + evm_version: byzantium + mnemonic: brownie2 + unlock: + - 0x16Fb96a5fa0427Af0C8F7cF1eB4870231c8154B6 + - "0x81431b69B1e0E334d4161A13C2955e0f3599381e" +""" +) + @pytest.fixture -def settings_proj(testproject): +def project_settings(testproject): """Creates a config file in the testproject root folder and loads it manually.""" - - # Save the following config as "brownie-config.yaml" in the testproject root - test_brownie_config = """ - networks: - default: development - development: - gas_limit: 6543210 - gas_price: 1000 - reverting_tx_gas_limit: 8765432 - default_contract_owner: false - cmd_settings: - network_id: 777 - chain_id: 666 - gas_limit: 7654321 - block_time: 5 - default_balance: 15 milliether - time: 2019-04-05T14:30:11Z - accounts: 15 - evm_version: byzantium - mnemonic: brownie2 - unlock: - - 0x16Fb96a5fa0427Af0C8F7cF1eB4870231c8154B6 - - "0x81431b69B1e0E334d4161A13C2955e0f3599381e" - """ + # Save the project config as "brownie-config.yaml" in the testproject root with testproject._path.joinpath("brownie-config.yaml").open("w") as fp: - yaml.dump(yaml.safe_load(test_brownie_config), fp) + yaml.dump(BASE_PROJECT_CONFIG, fp) # Load the networks.development config from the created file and yield it with testproject._path.joinpath("brownie-config.yaml").open() as fp: @@ -45,7 +49,7 @@ def settings_proj(testproject): yield conf -def test_load_project_cmd_settings(config, testproject, settings_proj): +def test_load_project_cmd_settings(config, testproject, project_settings): """Tests if project specific cmd_settings update the network config when a project is loaded""" # get raw cmd_setting config data from the network-config.yaml file config_path_network = _get_data_folder().joinpath("network-config.yaml") @@ -60,26 +64,26 @@ def test_load_project_cmd_settings(config, testproject, settings_proj): # Load the project with its project specific settings and assert that the CONFIG was updated testproject.load_config() cmd_settings_config = config.networks["development"]["cmd_settings"] - for k, v in settings_proj["cmd_settings"].items(): + for k, v in project_settings["cmd_settings"].items(): if k != "port": assert cmd_settings_config[k] == v -def test_rpc_project_cmd_settings(devnetwork, testproject, config, settings_proj): +def test_rpc_project_cmd_settings(devnetwork, testproject, config, project_settings): """Test if project specific settings are properly passed on to the RPC.""" if devnetwork.rpc.is_active(): devnetwork.rpc.kill() - cmd_settings_proj = settings_proj["cmd_settings"] + cmd_project_settings = project_settings["cmd_settings"] testproject.load_config() devnetwork.connect("development") # Check if rpc time is roughly the start time in the config file # Use diff < 25h to dodge potential timezone differences - assert cmd_settings_proj["time"].timestamp() - devnetwork.chain.time() < 60 * 60 * 25 + assert cmd_project_settings["time"].timestamp() - devnetwork.chain.time() < 60 * 60 * 25 accounts = devnetwork.accounts - assert cmd_settings_proj["accounts"] + len(cmd_settings_proj["unlock"]) == len(accounts) - assert cmd_settings_proj["default_balance"] == accounts[0].balance() + assert cmd_project_settings["accounts"] + len(cmd_project_settings["unlock"]) == len(accounts) + assert cmd_project_settings["default_balance"] == accounts[0].balance() # Test if mnemonic was updated to "brownie2" assert "0x816200940a049ff1DEAB864d67a71ae6Dd1ebc3e" == accounts[0].address @@ -90,8 +94,8 @@ def test_rpc_project_cmd_settings(devnetwork, testproject, config, settings_proj # Test if gas limit and price are loaded from the config tx = accounts[0].transfer(accounts[1], 0) - assert tx.gas_limit == settings_proj["gas_limit"] - assert tx.gas_price == settings_proj["gas_price"] + assert tx.gas_limit == project_settings["gas_limit"] + assert tx.gas_price == project_settings["gas_price"] # Test if chain ID and network ID can be properly queried assert web3.isConnected() @@ -128,3 +132,47 @@ def test_validate_cmd_settings(): def test_raise_validate_cmd_settings(invalid_setting): with pytest.raises(TypeError): _validate_cmd_settings(invalid_setting) + + +DOTENV_CONTNENTS = """ +DEFAULT_BALANCE="42 miliether" +SHOW_COLORS=false +""".strip() + + +@pytest.fixture +def env_file(testproject): + env_file_path = testproject._path.joinpath(".env") + with env_file_path.open("w") as fp: + fp.write(DOTENV_CONTNENTS) + yield env_file_path + + +@pytest.fixture +def project_settings_with_dotenv(testproject, env_file): + """Creates a config file in the testproject root folder and loads it manually.""" + + project_config = copy.deepcopy(BASE_PROJECT_CONFIG) + project_config["dotenv"] = str(env_file) + project_config["networks"]["development"]["default_balance"] = "${DEFAULT_BALANCE}" + if "console" not in project_config: + project_config["console"] = {} + project_config["console"]["show_colors"] = "${SHOW_COLORS}" + + # Save the project config as "brownie-config.yaml" in the testproject root + with testproject._path.joinpath("brownie-config.yaml").open("w") as fp: + yaml.dump(project_config, fp) + + # Load the networks.development config from the created file and yield it + with testproject._path.joinpath("brownie-config.yaml").open("r") as fp: + conf = yaml.safe_load(fp) + + yield conf + + +def test_dotenv_imports(config, testproject, env_file, project_settings_with_dotenv): + config_path = _get_data_folder().joinpath("brownie-config.yaml") + _load_config(config_path) + testproject.load_config() + assert config.settings["console"]["show_colors"] == False # noqa: E712 + assert config.settings["networks"]["development"]["default_balance"] == "42 miliether" diff --git a/tests/test_expansion.py b/tests/test_expansion.py new file mode 100644 index 000000000..e7d2632d8 --- /dev/null +++ b/tests/test_expansion.py @@ -0,0 +1,76 @@ +import unittest +import uuid + +from brownie._expansion import expand_posix_vars + + +class TestExpandDict(unittest.TestCase): + def setUp(self): + self.v = str(uuid.uuid4()) + self.input = { + "non": "b", + "simple": "${FOO}", + "partial": "the ${FOO}", + "number": 1, + "bool": True, + "nested": { + "one": "nest ${FOO}", + "super_nested": {"two": "real ${FOO}", "three": "not"}, + }, + "${A}": "abc", + "default_envvar_present": "${FOO:-xyz}", + "default_envvar_missing": "${ABC:-bar}", + "default_int_present": "${NUM:-42}", + "default_int_missing": "${ABC:-42}", + "arr": [{"a": False, "b": False}, {"a": True, "b": "${FOO}"}], + } + variables = {"FOO": self.v, "NUM": 314} + self.res = expand_posix_vars(self.input, variables,) + + def test_basic_string(self): + assert self.res["non"] == "b" + + def test_simple_expansion(self): + assert self.res["simple"] == self.v + + def test_partial_string_expansion(self): + assert self.res["partial"] == f"the {self.v}" + + def test_number(self): + assert self.res["number"] == 1 + + def test_bool(self): + assert self.res["bool"] == True # noqa: E712 + + def test_nested_partial_string(self): + assert self.res["nested"]["one"] == f"nest {self.v}" + + def test_double_nested_partial_string(self): + assert self.res["nested"]["super_nested"]["two"] == f"real {self.v}" + + def test_double_nested_plain(self): + assert self.res["nested"]["super_nested"]["three"] == "not" + + def test_variable_name_not_expanded(self): + assert self.res["${A}"] == "abc" + + def test_list_basic(self): + assert self.res["arr"][0]["a"] == False # noqa: E712 + + def test_list_bool(self): + assert self.res["arr"][1]["a"] == True # noqa: E712 + + def test_arr_expanded(self): + assert self.res["arr"][1]["b"] == self.v + + def test_envvar_with_default_value_present(self): + assert self.res["default_envvar_present"] == self.v + + def test_envvar_with_default_value_missing(self): + assert self.res["default_envvar_missing"] == "bar" + + def test_envvar_with_default_int_value_present(self): + assert self.res["default_int_present"] == 314 + + def test_envvar_with_default_int_value_missing(self): + assert self.res["default_int_missing"] == 42