From c83467ecdaddc3826923f42c300b714f9019e190 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Sun, 29 Nov 2020 13:10:42 +0400 Subject: [PATCH 1/7] feat: generate build json for dependencies --- brownie/project/compiler/__init__.py | 39 ++++++++++++++++++---------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/brownie/project/compiler/__init__.py b/brownie/project/compiler/__init__.py index 7385f157f..1afeeba17 100644 --- a/brownie/project/compiler/__init__.py +++ b/brownie/project/compiler/__init__.py @@ -138,12 +138,8 @@ def compile_and_format( ) output_json = compile_from_input_json(input_json, silent, allow_paths) - - output_json["contracts"] = { - k: v for k, v in output_json["contracts"].items() if k in path_list - } - build_json.update(generate_build_json(input_json, output_json, compiler_data, silent)) + return build_json @@ -280,18 +276,33 @@ def generate_build_json( compiler_data = {} compiler_data["evm_version"] = input_json["settings"]["evmVersion"] build_json: Dict = {} - path_list = list(input_json["sources"]) if input_json["language"] == "Solidity": compiler_data["optimizer"] = input_json["settings"]["optimizer"] source_nodes, statement_nodes, branch_nodes = solidity._get_nodes(output_json) for path_str, contract_name in [ - (k, v) for k in path_list for v in output_json["contracts"].get(k, {}) + (k, x) for k, v in output_json["contracts"].items() for x in v.keys() ]: + contract_alias = contract_name + + if path_str in input_json["sources"]: + source = input_json["sources"][path_str]["content"] + else: + # If the source is not present in `input_json`, this is likely a + # dependency from an installed package. We alias the contract as + # [PACKAGE]/[NAME] to avoid namespace collisions. + with Path(path_str).open() as fp: + source = fp.read() + + data_path = _get_data_folder().parts + path_parts = Path(path_str).parts + if path_parts[: len(data_path)] == data_path: + idx = len(data_path) + 1 + contract_alias = f"{path_parts[idx]}/{path_parts[idx+1]}/{contract_name}" if not silent: - print(f" - {contract_name}...") + print(f" - {contract_alias}") abi = output_json["contracts"][path_str][contract_name]["abi"] natspec = merge_natspec( @@ -299,14 +310,14 @@ def generate_build_json( output_json["contracts"][path_str][contract_name].get("userdoc", {}), ) output_evm = output_json["contracts"][path_str][contract_name]["evm"] - if contract_name in build_json and not output_evm["deployedBytecode"]["object"]: + if contract_alias in build_json and not output_evm["deployedBytecode"]["object"]: continue if input_json["language"] == "Solidity": contract_node = next( i[contract_name] for i in source_nodes if i.absolutePath == path_str ) - build_json[contract_name] = solidity._get_unique_build_json( + build_json[contract_alias] = solidity._get_unique_build_json( output_evm, contract_node, statement_nodes, @@ -322,10 +333,10 @@ def generate_build_json( path_str, contract_name, output_json["sources"][path_str]["ast"], - (0, len(input_json["sources"][path_str]["content"])), + (0, len(source)), ) - build_json[contract_name].update( + build_json[contract_alias].update( { "abi": abi, "ast": output_json["sources"][path_str]["ast"], @@ -336,8 +347,8 @@ def generate_build_json( "language": input_json["language"], "natspec": natspec, "opcodes": output_evm["deployedBytecode"]["opcodes"], - "sha1": sha1(input_json["sources"][path_str]["content"].encode()).hexdigest(), - "source": input_json["sources"][path_str]["content"], + "sha1": sha1(source.encode()).hexdigest(), + "source": source, "sourceMap": output_evm["bytecode"].get("sourceMap", ""), "sourcePath": path_str, } From 8aa38610231b2ace08b4b7b75aa88af7b0f9b2fd Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Sun, 29 Nov 2020 13:12:05 +0400 Subject: [PATCH 2/7] feat: store dependency build artifacts at `build/contracts/dependencies/` --- brownie/project/main.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/brownie/project/main.py b/brownie/project/main.py index 24e28b6da..7132268aa 100644 --- a/brownie/project/main.py +++ b/brownie/project/main.py @@ -102,12 +102,22 @@ def _compile(self, contract_sources: Dict, compiler_config: Dict, silent: bool) finally: os.chdir(cwd) - for data in build_json.values(): + for alias, data in build_json.items(): if self._build_path is not None: - path = self._build_path.joinpath(f"contracts/{data['contractName']}.json") + if alias == data["contractName"]: + # if the alias == contract name, this is a part of the core project + path = self._build_path.joinpath(f"contracts/{alias}.json") + else: + # otherwise, this is an artifact from an external dependency + path = self._build_path.joinpath(f"contracts/dependencies/{alias}.json") + for parent in list(path.parents)[::-1]: + parent.mkdir(exist_ok=True) with path.open("w") as fp: json.dump(data, fp, sort_keys=True, indent=2, default=sorted) - self._build._add_contract(data) + + if alias == data["contractName"]: + # only add artifacts from the core project for now + self._build._add_contract(data) def _create_containers(self) -> None: # create container objects From fa9f57721afea6c0766ab750a1372bfee6f1eaa8 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 4 Dec 2020 17:18:13 +0200 Subject: [PATCH 3/7] fix: use aliases in contract dependencies --- brownie/project/compiler/__init__.py | 12 ++---------- brownie/project/compiler/solidity.py | 15 +++++++++++---- brownie/project/compiler/utils.py | 17 +++++++++++++++++ 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/brownie/project/compiler/__init__.py b/brownie/project/compiler/__init__.py index 1afeeba17..0518a3034 100644 --- a/brownie/project/compiler/__init__.py +++ b/brownie/project/compiler/__init__.py @@ -18,7 +18,7 @@ install_solc, set_solc_version, ) -from brownie.project.compiler.utils import merge_natspec +from brownie.project.compiler.utils import _get_alias, merge_natspec from brownie.project.compiler.vyper import find_vyper_versions, set_vyper_version from brownie.utils import notify @@ -289,17 +289,9 @@ def generate_build_json( if path_str in input_json["sources"]: source = input_json["sources"][path_str]["content"] else: - # If the source is not present in `input_json`, this is likely a - # dependency from an installed package. We alias the contract as - # [PACKAGE]/[NAME] to avoid namespace collisions. with Path(path_str).open() as fp: source = fp.read() - - data_path = _get_data_folder().parts - path_parts = Path(path_str).parts - if path_parts[: len(data_path)] == data_path: - idx = len(data_path) + 1 - contract_alias = f"{path_parts[idx]}/{path_parts[idx+1]}/{contract_name}" + contract_alias = _get_alias(contract_name, path_str) if not silent: print(f" - {contract_alias}") diff --git a/brownie/project/compiler/solidity.py b/brownie/project/compiler/solidity.py index 5ba0d4709..bf5fca947 100644 --- a/brownie/project/compiler/solidity.py +++ b/brownie/project/compiler/solidity.py @@ -13,7 +13,7 @@ from brownie._config import EVM_EQUIVALENTS from brownie.exceptions import CompilerError, IncompatibleSolcVersion -from brownie.project.compiler.utils import expand_source_map +from brownie.project.compiler.utils import _get_alias, expand_source_map from . import sources @@ -252,14 +252,21 @@ def _get_unique_build_json( has_fallback, instruction_count, ) + + dependencies = [] + for node in [i for i in contract_node.dependencies if i.nodeType == "ContractDefinition"]: + # use contract aliases when recording dependencies, to avoid + # potential namespace collisions when importing across projects + name = node.name + path_str = node.parent().absolutePath + dependencies.append(_get_alias(name, path_str)) + return { "allSourcePaths": paths, "bytecode": bytecode, "bytecodeSha1": sha1(_remove_metadata(bytecode).encode()).hexdigest(), "coverageMap": {"statements": statement_map, "branches": branch_map}, - "dependencies": [ - i.name for i in contract_node.dependencies if i.nodeType == "ContractDefinition" - ], + "dependencies": dependencies, "offset": contract_node.offset, "pcMap": pc_map, "type": contract_node.contractKind, diff --git a/brownie/project/compiler/utils.py b/brownie/project/compiler/utils.py index 473d53ace..7c0627094 100644 --- a/brownie/project/compiler/utils.py +++ b/brownie/project/compiler/utils.py @@ -1,7 +1,10 @@ #!/usr/bin/python3 +from pathlib import Path from typing import Dict, List +from brownie._config import _get_data_folder + def expand_source_map(source_map_str: str) -> List: # Expands the compressed sourceMap supplied by solc into a list of lists @@ -52,3 +55,17 @@ def merge_natspec(devdoc: Dict, userdoc: Dict) -> Dict: # sometimes Solidity has inconsistent NatSpec formatting ¯\_(ツ)_/¯ pass return natspec + + +def _get_alias(contract_name: str, path_str: str) -> str: + # Generate an alias for a contract, used when tracking dependencies. + # For a contract within the project, the alias == the name. For contracts + # imported from a dependency, the alias is set as [PACKAGE]/[NAME] + # to avoid namespace collisions. + data_path = _get_data_folder().parts + path_parts = Path(path_str).parts + if path_parts[: len(data_path)] == data_path: + idx = len(data_path) + 1 + return f"{path_parts[idx]}/{path_parts[idx+1]}/{contract_name}" + else: + return contract_name From ec4fde5aaacaaf7e8407badf0a1903bd57c91ac4 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 4 Dec 2020 18:32:20 +0200 Subject: [PATCH 4/7] feat: include source and offset when compiling interfaces --- brownie/project/compiler/__init__.py | 49 +++++++++++++++------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/brownie/project/compiler/__init__.py b/brownie/project/compiler/__init__.py index 0518a3034..307cc4b10 100644 --- a/brownie/project/compiler/__init__.py +++ b/brownie/project/compiler/__init__.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Dict, Optional +import solcast from eth_utils import remove_0x_prefix from semantic_version import Version @@ -401,6 +402,8 @@ def get_abi( "abi": json.loads(v), "contractName": Path(k).stem, "type": "interface", + "source": None, + "offset": None, "sha1": sha1(v.encode()).hexdigest(), } for k, v in contract_sources.items() @@ -421,36 +424,38 @@ def get_abi( "abi": output_json["contracts"][path][name]["abi"], "contractName": name, "type": "interface", + "source": source, + "offset": [0, len(source)], "sha1": sha1(contract_sources[path].encode()).hexdigest(), } solc_sources = {k: v for k, v in contract_sources.items() if Path(k).suffix == ".sol"} - if solc_sources: + if not solc_sources: + return final_output - compiler_targets = find_solc_versions(solc_sources, install_needed=True, silent=silent) + compiler_targets = find_solc_versions(solc_sources, install_needed=True, silent=silent) - for version, path_list in compiler_targets.items(): - to_compile = {k: v for k, v in contract_sources.items() if k in path_list} + for version, path_list in compiler_targets.items(): + to_compile = {k: v for k, v in contract_sources.items() if k in path_list} - set_solc_version(version) - input_json = generate_input_json(to_compile, language="Solidity", remappings=remappings) - input_json["settings"]["outputSelection"]["*"] = {"*": ["abi"]} + set_solc_version(version) + input_json = generate_input_json(to_compile, language="Solidity", remappings=remappings) + input_json["settings"]["outputSelection"]["*"] = {"*": ["abi"], "": ["ast"]} - output_json = compile_from_input_json(input_json, silent, allow_paths) - output_json = {k: v for k, v in output_json["contracts"].items() if k in path_list} - - final_output.update( - { - name: { - "abi": data["abi"], - "contractName": name, - "type": "interface", - "sha1": sha1(contract_sources[path].encode()).hexdigest(), - } - for path, v in output_json.items() - for name, data in v.items() - } - ) + output_json = compile_from_input_json(input_json, silent, allow_paths) + source_nodes = solcast.from_standard_output(output_json) + output_json = {k: v for k, v in output_json["contracts"].items() if k in path_list} + + for path, name, data in [(k, x, y) for k, v in output_json.items() for x, y in v.items()]: + contract_node = next(i[name] for i in source_nodes if i.absolutePath == path) + final_output[name] = { + "abi": data["abi"], + "contractName": name, + "type": "interface", + "source": contract_sources[path], + "offset": contract_node.offset, + "sha1": sha1(contract_sources[path].encode()).hexdigest(), + } return final_output From 225e1486c61b80dc884914f3558bf9d70ce5f140 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 4 Dec 2020 18:33:40 +0200 Subject: [PATCH 5/7] feat: include dependencies in build artifacts --- brownie/project/build.py | 9 ++++++--- brownie/project/main.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/brownie/project/build.py b/brownie/project/build.py index 1e50c5e18..c3a0ab24a 100644 --- a/brownie/project/build.py +++ b/brownie/project/build.py @@ -45,8 +45,8 @@ def __init__(self, sources: Sources) -> None: self._contracts: Dict = {} self._interfaces: Dict = {} - def _add_contract(self, build_json: Dict) -> None: - contract_name = build_json["contractName"] + def _add_contract(self, build_json: Dict, alias: str = None) -> None: + contract_name = alias or build_json["contractName"] if contract_name in self._contracts and build_json["type"] == "interface": return self._contracts[contract_name] = build_json @@ -115,7 +115,10 @@ def _remove_interface(self, contract_name: str) -> None: def get(self, contract_name: str) -> Dict: """Returns build data for the given contract name.""" - return self._contracts[self._stem(contract_name)] + key = self._stem(contract_name) + if key in self._contracts: + return self._contracts[key] + return self._interfaces[key] def items(self, path: Optional[str] = None) -> Union[ItemsView, List]: """Provides an list of tuples as (key,value), similar to calling dict.items. diff --git a/brownie/project/main.py b/brownie/project/main.py index 7132268aa..e04a6c70f 100644 --- a/brownie/project/main.py +++ b/brownie/project/main.py @@ -226,6 +226,8 @@ def load(self) -> None: changed = self._get_changed_contracts(interface_hashes) self._compile(changed, self._compiler_config, False) self._compile_interfaces(interface_hashes) + self._load_dependency_artifacts() + self._create_containers() self._load_deployments() @@ -315,6 +317,17 @@ def _compile_interfaces(self, compiled_hashes: Dict) -> None: json.dump(abi, fp, sort_keys=True, indent=2, default=sorted) self._build._add_interface(abi) + def _load_dependency_artifacts(self) -> None: + dep_build_path = self._build_path.joinpath("contracts/dependencies/") + for path in list(dep_build_path.glob("**/*.json")): + contract_alias = path.relative_to(dep_build_path).with_suffix("").as_posix() + if self._build.get_dependents(contract_alias): + with path.open() as fp: + build_json = json.load(fp) + self._build._add_contract(build_json, contract_alias) + else: + path.unlink() + def _load_deployments(self) -> None: if CONFIG.network_type != "live" and not CONFIG.settings["dev_deployment_artifacts"]: return From 6c0ca937fd69cbcbac596b1161fe97171eb87117 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 4 Dec 2020 23:00:13 +0200 Subject: [PATCH 6/7] fix: with contract alias during compiling --- brownie/project/compiler/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/brownie/project/compiler/__init__.py b/brownie/project/compiler/__init__.py index 307cc4b10..6dae87247 100644 --- a/brownie/project/compiler/__init__.py +++ b/brownie/project/compiler/__init__.py @@ -320,11 +320,11 @@ def generate_build_json( else: if contract_name == "": - contract_name = "Vyper" - build_json[contract_name] = vyper._get_unique_build_json( + contract_name = contract_alias = "Vyper" + build_json[contract_alias] = vyper._get_unique_build_json( output_evm, path_str, - contract_name, + contract_alias, output_json["sources"][path_str]["ast"], (0, len(source)), ) From 615e5382b124fcc9dcc1ca4e6ab0617e965ce5ac Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Sat, 5 Dec 2020 01:20:51 +0200 Subject: [PATCH 7/7] test: fix failing test --- tests/project/main/test_interfaces_solc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/project/main/test_interfaces_solc.py b/tests/project/main/test_interfaces_solc.py index 6e0b8fbc3..5aabe9e4a 100644 --- a/tests/project/main/test_interfaces_solc.py +++ b/tests/project/main/test_interfaces_solc.py @@ -48,7 +48,7 @@ def test_contract_requires_interface(newproject, contract_type, import_path): with newproject._path.joinpath("contracts/Bar.sol").open("w") as fp: fp.write(CONTRACT.format(import_path)) newproject.load() - assert not newproject._path.joinpath("build/contracts/Foo.json").exists() + assert newproject._path.joinpath("build/contracts/Foo.json").exists() assert not hasattr(newproject, "Foo")