From ffe59daf10edc19ee5f05227b25bac8d090e8aa4 Mon Sep 17 00:00:00 2001 From: pacrob Date: Thu, 20 Jan 2022 15:19:29 -0700 Subject: [PATCH 01/48] move default_account and default_block properties and setters to BaseEth so Eth and AsyncEth can access --- docs/providers.rst | 2 + docs/web3.eth.rst | 4 +- newsfragments/2315.feature.rst | 1 + web3/_utils/module_testing/eth_module.py | 73 ++++++++++++++++++++++ web3/eth.py | 78 +++++++++++------------- 5 files changed, 114 insertions(+), 44 deletions(-) create mode 100644 newsfragments/2315.feature.rst diff --git a/docs/providers.rst b/docs/providers.rst index 861baab9e4..ee94169787 100644 --- a/docs/providers.rst +++ b/docs/providers.rst @@ -408,6 +408,8 @@ Eth - :meth:`web3.eth.block_number ` - :meth:`web3.eth.chain_id ` - :meth:`web3.eth.coinbase ` +- :meth:`web3.eth.default_account ` +- :meth:`web3.eth.default_block ` - :meth:`web3.eth.gas_price ` - :meth:`web3.eth.hashrate ` - :meth:`web3.eth.max_priority_fee ` diff --git a/docs/web3.eth.rst b/docs/web3.eth.rst index 569373a05f..eaa1653782 100644 --- a/docs/web3.eth.rst +++ b/docs/web3.eth.rst @@ -49,7 +49,7 @@ The following properties are available on the ``web3.eth`` namespace. .. py:attribute:: Eth.default_account The ethereum address that will be used as the default ``from`` address for - all transactions. + all transactions. Defaults to empty. .. py:attribute:: Eth.defaultAccount @@ -61,7 +61,7 @@ The following properties are available on the ``web3.eth`` namespace. .. py:attribute:: Eth.default_block The default block number that will be used for any RPC methods that accept - a block identifier. Defaults to ``'latest'``. + a block identifier. Defaults to ``'latest'``. .. py:attribute:: Eth.defaultBlock diff --git a/newsfragments/2315.feature.rst b/newsfragments/2315.feature.rst new file mode 100644 index 0000000000..1fa55c3ac3 --- /dev/null +++ b/newsfragments/2315.feature.rst @@ -0,0 +1 @@ +add Async access to `default_account` and `default_block` diff --git a/web3/_utils/module_testing/eth_module.py b/web3/_utils/module_testing/eth_module.py index 321eebb595..6a35067f50 100644 --- a/web3/_utils/module_testing/eth_module.py +++ b/web3/_utils/module_testing/eth_module.py @@ -35,6 +35,9 @@ HexBytes, ) +from web3._utils.empty import ( + empty, +) from web3._utils.ens import ( ens_addresses, ) @@ -845,6 +848,41 @@ async def test_async_eth_accounts(self, async_w3: "Web3") -> None: )) assert await async_w3.eth.coinbase in accounts # type: ignore + def test_async_provider_default_account( + self, + async_w3: "Web3", + unlocked_account_dual_type: ChecksumAddress + ) -> None: + + # check defaults to empty + default_account = async_w3.eth.default_account + assert default_account is empty + + # check setter + async_w3.eth.default_account = unlocked_account_dual_type + default_account = async_w3.eth.default_account + assert default_account == unlocked_account_dual_type + + # reset to default + async_w3.eth.default_account = empty + + def test_async_provider_default_block( + self, + async_w3: "Web3", + ) -> None: + + # check defaults to 'latest' + default_block = async_w3.eth.default_block + assert default_block == 'latest' + + # check setter + async_w3.eth.default_block = BlockNumber(12345) + default_block = async_w3.eth.default_block + assert default_block == BlockNumber(12345) + + # reset to default + async_w3.eth.default_block = 'latest' + class EthModuleTest: def test_eth_protocol_version(self, web3: "Web3") -> None: @@ -2919,3 +2957,38 @@ def test_eth_get_raw_transaction_by_block_raises_error_block_identifier( ) ): web3.eth.get_raw_transaction_by_block(unknown_identifier, 0) # type: ignore + + def test_default_account( + self, + web3: "Web3", + unlocked_account_dual_type: ChecksumAddress + ) -> None: + + # check defaults to empty + default_account = web3.eth.default_account + assert default_account is empty + + # check setter + web3.eth.default_account = unlocked_account_dual_type + default_account = web3.eth.default_account + assert default_account == unlocked_account_dual_type + + # reset to default + web3.eth.default_account = empty + + def test_default_block( + self, + web3: "Web3", + ) -> None: + + # check defaults to 'latest' + default_block = web3.eth.default_block + assert default_block == 'latest' + + # check setter + web3.eth.default_block = BlockNumber(12345) + default_block = web3.eth.default_block + assert default_block == BlockNumber(12345) + + # reset to default + web3.eth.default_block = 'latest' diff --git a/web3/eth.py b/web3/eth.py index 7287903119..00f6930ef7 100644 --- a/web3/eth.py +++ b/web3/eth.py @@ -117,7 +117,6 @@ class BaseEth(Module): mungers=None, ) - """ property default_block """ @property def default_block(self) -> BlockIdentifier: return self._default_block @@ -126,10 +125,46 @@ def default_block(self) -> BlockIdentifier: def default_block(self, value: BlockIdentifier) -> None: self._default_block = value + @property + def defaultBlock(self) -> BlockIdentifier: + warnings.warn( + 'defaultBlock is deprecated in favor of default_block', + category=DeprecationWarning, + ) + return self._default_block + + @defaultBlock.setter + def defaultBlock(self, value: BlockIdentifier) -> None: + warnings.warn( + 'defaultBlock is deprecated in favor of default_block', + category=DeprecationWarning, + ) + self._default_block = value + @property def default_account(self) -> Union[ChecksumAddress, Empty]: return self._default_account + @default_account.setter + def default_account(self, account: Union[ChecksumAddress, Empty]) -> None: + self._default_account = account + + @property + def defaultAccount(self) -> Union[ChecksumAddress, Empty]: + warnings.warn( + 'defaultAccount is deprecated in favor of default_account', + category=DeprecationWarning, + ) + return self._default_account + + @defaultAccount.setter + def defaultAccount(self, account: Union[ChecksumAddress, Empty]) -> None: + warnings.warn( + 'defaultAccount is deprecated in favor of default_account', + category=DeprecationWarning, + ) + self._default_account = account + def send_transaction_munger(self, transaction: TxParams) -> Tuple[TxParams]: if 'from' not in transaction and is_checksum_address(self.default_account): transaction = assoc(transaction, 'from', self.default_account) @@ -551,52 +586,11 @@ def chainId(self) -> int: ) return self.chain_id - """ property default_account """ - @property - def default_account(self) -> Union[ChecksumAddress, Empty]: - return self._default_account - - @default_account.setter - def default_account(self, account: Union[ChecksumAddress, Empty]) -> None: - self._default_account = account - - @property - def defaultAccount(self) -> Union[ChecksumAddress, Empty]: - warnings.warn( - 'defaultAccount is deprecated in favor of default_account', - category=DeprecationWarning, - ) - return self._default_account - - @defaultAccount.setter - def defaultAccount(self, account: Union[ChecksumAddress, Empty]) -> None: - warnings.warn( - 'defaultAccount is deprecated in favor of default_account', - category=DeprecationWarning, - ) - self._default_account = account - get_balance: Method[Callable[..., Wei]] = Method( RPC.eth_getBalance, mungers=[BaseEth.block_id_munger], ) - @property - def defaultBlock(self) -> BlockIdentifier: - warnings.warn( - 'defaultBlock is deprecated in favor of default_block', - category=DeprecationWarning, - ) - return self._default_block - - @defaultBlock.setter - def defaultBlock(self, value: BlockIdentifier) -> None: - warnings.warn( - 'defaultBlock is deprecated in favor of default_block', - category=DeprecationWarning, - ) - self._default_block = value - @property def max_priority_fee(self) -> Wei: return self._max_priority_fee() From ec1c9fee1e625bc50d4981645209a2282268842b Mon Sep 17 00:00:00 2001 From: AlwaysData Date: Wed, 26 Jan 2022 11:48:51 -0500 Subject: [PATCH 02/48] Feature/ens request (#2319) * fixed ens contract function called twice * newsfragment --- ens/main.py | 2 +- newsfragments/2318.bugfix.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/2318.bugfix.rst diff --git a/ens/main.py b/ens/main.py index 0e22d32075..4d62e35690 100644 --- a/ens/main.py +++ b/ens/main.py @@ -245,7 +245,7 @@ def resolve(self, name: str, get: str = 'addr') -> Optional[Union[ChecksumAddres address = lookup_function(namehash).call() if is_none_or_zero_address(address): return None - return lookup_function(namehash).call() + return address else: return None diff --git a/newsfragments/2318.bugfix.rst b/newsfragments/2318.bugfix.rst new file mode 100644 index 0000000000..7b69a5600c --- /dev/null +++ b/newsfragments/2318.bugfix.rst @@ -0,0 +1 @@ +In ENS the contract function to resolve an ENS address was being called twice in error. One of those calls was removed. \ No newline at end of file From 3a1b892747f65a3080feb1cb65c9af3764296ba9 Mon Sep 17 00:00:00 2001 From: alex <67626131+alexpvpmindustry@users.noreply.github.com> Date: Fri, 20 Aug 2021 14:50:25 +0800 Subject: [PATCH 03/48] small typo in documentation --- docs/web3.eth.account.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/web3.eth.account.rst b/docs/web3.eth.account.rst index 51996fc83e..60a25839f2 100644 --- a/docs/web3.eth.account.rst +++ b/docs/web3.eth.account.rst @@ -34,7 +34,7 @@ Hosted Private Key .. WARNING:: - It is unnacceptable for a hosted node to offer hosted private keys. It + It is unacceptable for a hosted node to offer hosted private keys. It gives other people complete control over your account. "Not your keys, not your Ether" in the wise words of Andreas Antonopoulos. From ac8703c88fd9c3cfd72b86bfe90b69aa509c06c2 Mon Sep 17 00:00:00 2001 From: pacrob Date: Wed, 26 Jan 2022 11:34:36 -0700 Subject: [PATCH 04/48] add newsfragment --- newsfragments/2111.doc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2111.doc.rst diff --git a/newsfragments/2111.doc.rst b/newsfragments/2111.doc.rst new file mode 100644 index 0000000000..5828417126 --- /dev/null +++ b/newsfragments/2111.doc.rst @@ -0,0 +1 @@ +fix typo in `eth.account` docs From c85b7d91f915fd81c72bf3244c3845b83d6521f2 Mon Sep 17 00:00:00 2001 From: Felipe Selmo Date: Wed, 26 Jan 2022 10:05:39 -0700 Subject: [PATCH 05/48] Only apply ``to_hexbytes`` formatter if value is not null --- newsfragments/2321.bugfix.rst | 1 + tests/core/eth-module/test_block_api.py | 64 +++++++++++++++++++ .../core/eth-module/test_default_block_api.py | 25 -------- web3/_utils/method_formatters.py | 10 +-- 4 files changed, 70 insertions(+), 30 deletions(-) create mode 100644 newsfragments/2321.bugfix.rst create mode 100644 tests/core/eth-module/test_block_api.py delete mode 100644 tests/core/eth-module/test_default_block_api.py diff --git a/newsfragments/2321.bugfix.rst b/newsfragments/2321.bugfix.rst new file mode 100644 index 0000000000..61c942b0c9 --- /dev/null +++ b/newsfragments/2321.bugfix.rst @@ -0,0 +1 @@ +``to_hexbytes`` block formatters no longer throw when value is ``None`` \ No newline at end of file diff --git a/tests/core/eth-module/test_block_api.py b/tests/core/eth-module/test_block_api.py new file mode 100644 index 0000000000..2ea7d0a8dc --- /dev/null +++ b/tests/core/eth-module/test_block_api.py @@ -0,0 +1,64 @@ +import pytest + +from web3._utils.rpc_abi import ( + RPC, +) +from web3.middleware import ( + construct_result_generator_middleware, +) + + +@pytest.fixture(autouse=True) +def wait_for_first_block(web3, wait_for_block): + wait_for_block(web3) + + +def test_uses_default_block(web3, extra_accounts, + wait_for_transaction): + assert(web3.eth.default_block == 'latest') + web3.eth.default_block = web3.eth.block_number + assert(web3.eth.default_block == web3.eth.block_number) + + +def test_uses_defaultBlock_with_warning(web3, extra_accounts, + wait_for_transaction): + with pytest.warns(DeprecationWarning): + assert web3.eth.defaultBlock == 'latest' + + with pytest.warns(DeprecationWarning): + web3.eth.defaultBlock = web3.eth.block_number + + with pytest.warns(DeprecationWarning): + assert(web3.eth.defaultBlock == web3.eth.block_number) + + +def test_get_block_formatters_with_null_values(web3): + null_values_block = { + 'baseFeePerGas': None, + 'extraData': None, + 'gasLimit': None, + 'gasUsed': None, + 'size': None, + 'timestamp': None, + 'hash': None, + 'logsBloom': None, + 'miner': None, + 'mixHash': None, + 'nonce': None, + 'number': None, + 'parentHash': None, + 'sha3Uncles': None, + 'difficulty': None, + 'receiptsRoot': None, + 'statesRoot': None, + 'totalDifficulty': None, + 'transactionsRoot': None, + } + result_middleware = construct_result_generator_middleware({ + RPC.eth_getBlockByNumber: lambda *_: null_values_block, + }) + + web3.middleware_onion.inject(result_middleware, 'result_middleware', layer=0) + + received_block = web3.eth.get_block('pending') + assert received_block == null_values_block diff --git a/tests/core/eth-module/test_default_block_api.py b/tests/core/eth-module/test_default_block_api.py deleted file mode 100644 index e2c2279c6e..0000000000 --- a/tests/core/eth-module/test_default_block_api.py +++ /dev/null @@ -1,25 +0,0 @@ -import pytest - - -@pytest.fixture(autouse=True) -def wait_for_first_block(web3, wait_for_block): - wait_for_block(web3) - - -def test_uses_default_block(web3, extra_accounts, - wait_for_transaction): - assert(web3.eth.default_block == 'latest') - web3.eth.default_block = web3.eth.block_number - assert(web3.eth.default_block == web3.eth.block_number) - - -def test_uses_defaultBlock_with_warning(web3, extra_accounts, - wait_for_transaction): - with pytest.warns(DeprecationWarning): - assert web3.eth.defaultBlock == 'latest' - - with pytest.warns(DeprecationWarning): - web3.eth.defaultBlock = web3.eth.block_number - - with pytest.warns(DeprecationWarning): - assert(web3.eth.defaultBlock == web3.eth.block_number) diff --git a/web3/_utils/method_formatters.py b/web3/_utils/method_formatters.py index 0828b54ac7..5105b5f0e0 100644 --- a/web3/_utils/method_formatters.py +++ b/web3/_utils/method_formatters.py @@ -210,7 +210,7 @@ def apply_list_to_array_formatter(formatter: Any) -> Callable[..., Any]: BLOCK_FORMATTERS = { 'baseFeePerGas': to_integer_if_hex, - 'extraData': to_hexbytes(32, variable_length=True), + 'extraData': apply_formatter_if(is_not_null, to_hexbytes(32, variable_length=True)), 'gasLimit': to_integer_if_hex, 'gasUsed': to_integer_if_hex, 'size': to_integer_if_hex, @@ -218,21 +218,21 @@ def apply_list_to_array_formatter(formatter: Any) -> Callable[..., Any]: 'hash': apply_formatter_if(is_not_null, to_hexbytes(32)), 'logsBloom': apply_formatter_if(is_not_null, to_hexbytes(256)), 'miner': apply_formatter_if(is_not_null, to_checksum_address), - 'mixHash': to_hexbytes(32), + 'mixHash': apply_formatter_if(is_not_null, to_hexbytes(32)), 'nonce': apply_formatter_if(is_not_null, to_hexbytes(8, variable_length=True)), 'number': apply_formatter_if(is_not_null, to_integer_if_hex), 'parentHash': apply_formatter_if(is_not_null, to_hexbytes(32)), 'sha3Uncles': apply_formatter_if(is_not_null, to_hexbytes(32)), 'uncles': apply_list_to_array_formatter(to_hexbytes(32)), 'difficulty': to_integer_if_hex, - 'receiptsRoot': to_hexbytes(32), - 'stateRoot': to_hexbytes(32), + 'receiptsRoot': apply_formatter_if(is_not_null, to_hexbytes(32)), + 'stateRoot': apply_formatter_if(is_not_null, to_hexbytes(32)), 'totalDifficulty': to_integer_if_hex, 'transactions': apply_one_of_formatters(( (is_array_of_dicts, apply_list_to_array_formatter(transaction_result_formatter)), (is_array_of_strings, apply_list_to_array_formatter(to_hexbytes(32))), )), - 'transactionsRoot': to_hexbytes(32), + 'transactionsRoot': apply_formatter_if(is_not_null, to_hexbytes(32)), } From 7723030f489eb17276042f7f07a042860ecf3fe4 Mon Sep 17 00:00:00 2001 From: DefiDebauchery <75273961+DefiDebauchery@users.noreply.github.com> Date: Wed, 26 Jan 2022 16:12:02 -0500 Subject: [PATCH 06/48] asyncify eth.get_logs (#2310) * asyncify eth.get_logs * factor out `assert_contains_log` function Co-authored-by: Paul Robinson --- docs/providers.rst | 1 + newsfragments/2310.feature.rst | 1 + web3/_utils/module_testing/eth_module.py | 199 ++++++++++++++++++++--- web3/eth.py | 11 ++ 4 files changed, 190 insertions(+), 22 deletions(-) create mode 100644 newsfragments/2310.feature.rst diff --git a/docs/providers.rst b/docs/providers.rst index ee94169787..fe0a5be23f 100644 --- a/docs/providers.rst +++ b/docs/providers.rst @@ -420,6 +420,7 @@ Eth - :meth:`web3.eth.get_balance() ` - :meth:`web3.eth.get_block() ` - :meth:`web3.eth.get_code() ` +- :meth:`web3.eth.get_logs() ` - :meth:`web3.eth.get_raw_transaction() ` - :meth:`web3.eth.get_raw_transaction_by_block() ` - :meth:`web3.eth.get_transaction() ` diff --git a/newsfragments/2310.feature.rst b/newsfragments/2310.feature.rst new file mode 100644 index 0000000000..d5b359f3e8 --- /dev/null +++ b/newsfragments/2310.feature.rst @@ -0,0 +1 @@ +Add async `eth.get_logs` method diff --git a/web3/_utils/module_testing/eth_module.py b/web3/_utils/module_testing/eth_module.py index 6a35067f50..80cd38ef5a 100644 --- a/web3/_utils/module_testing/eth_module.py +++ b/web3/_utils/module_testing/eth_module.py @@ -83,6 +83,22 @@ def mine_pending_block(web3: "Web3") -> None: web3.geth.miner.stop() # type: ignore +def _assert_contains_log( + result: Sequence[LogReceipt], + block_with_txn_with_log: BlockData, + emitter_contract_address: ChecksumAddress, + txn_hash_with_log: HexStr, +) -> None: + assert len(result) == 1 + log_entry = result[0] + assert log_entry['blockNumber'] == block_with_txn_with_log['number'] + assert log_entry['blockHash'] == block_with_txn_with_log['hash'] + assert log_entry['logIndex'] == 0 + assert is_same_address(log_entry['address'], emitter_contract_address) + assert log_entry['transactionIndex'] == 0 + assert log_entry['transactionHash'] == HexBytes(txn_hash_with_log) + + class AsyncEthModuleTest: @pytest.mark.asyncio async def test_eth_gas_price(self, async_w3: "Web3") -> None: @@ -848,6 +864,143 @@ async def test_async_eth_accounts(self, async_w3: "Web3") -> None: )) assert await async_w3.eth.coinbase in accounts # type: ignore + @pytest.mark.asyncio + async def test_async_eth_get_logs_without_logs( + self, async_w3: "Web3", block_with_txn_with_log: BlockData + ) -> None: + # Test with block range + + filter_params: FilterParams = { + "fromBlock": BlockNumber(0), + "toBlock": BlockNumber(block_with_txn_with_log['number'] - 1), + } + result = await async_w3.eth.get_logs(filter_params) # type: ignore + assert len(result) == 0 + + # the range is wrong + filter_params = { + "fromBlock": block_with_txn_with_log['number'], + "toBlock": BlockNumber(block_with_txn_with_log['number'] - 1), + } + result = await async_w3.eth.get_logs(filter_params) # type: ignore + assert len(result) == 0 + + # Test with `address` + + # filter with other address + filter_params = { + "fromBlock": BlockNumber(0), + "address": UNKNOWN_ADDRESS, + } + result = await async_w3.eth.get_logs(filter_params) # type: ignore + assert len(result) == 0 + + # Test with multiple `address` + + # filter with other address + filter_params = { + "fromBlock": BlockNumber(0), + "address": [UNKNOWN_ADDRESS, UNKNOWN_ADDRESS], + } + result = await async_w3.eth.get_logs(filter_params) # type: ignore + assert len(result) == 0 + + @pytest.mark.asyncio + async def test_async_eth_get_logs_with_logs( + self, + async_w3: "Web3", + block_with_txn_with_log: BlockData, + emitter_contract_address: ChecksumAddress, + txn_hash_with_log: HexStr, + ) -> None: + + # Test with block range + + # the range includes the block where the log resides in + filter_params: FilterParams = { + "fromBlock": block_with_txn_with_log['number'], + "toBlock": block_with_txn_with_log['number'], + } + result = await async_w3.eth.get_logs(filter_params) # type: ignore + _assert_contains_log( + result, + block_with_txn_with_log, + emitter_contract_address, + txn_hash_with_log + ) + + # specify only `from_block`. by default `to_block` should be 'latest' + filter_params = { + "fromBlock": BlockNumber(0), + } + result = await async_w3.eth.get_logs(filter_params) # type: ignore + _assert_contains_log( + result, + block_with_txn_with_log, + emitter_contract_address, + txn_hash_with_log + ) + + # Test with `address` + + # filter with emitter_contract.address + filter_params = { + "fromBlock": BlockNumber(0), + "address": emitter_contract_address, + } + + @pytest.mark.asyncio + async def test_async_eth_get_logs_with_logs_topic_args( + self, + async_w3: "Web3", + block_with_txn_with_log: BlockData, + emitter_contract_address: ChecksumAddress, + txn_hash_with_log: HexStr, + ) -> None: + + # Test with None event sig + + filter_params: FilterParams = { + "fromBlock": BlockNumber(0), + "topics": [ + None, + HexStr('0x000000000000000000000000000000000000000000000000000000000000d431')], + } + + result = await async_w3.eth.get_logs(filter_params) # type: ignore + _assert_contains_log( + result, + block_with_txn_with_log, + emitter_contract_address, + txn_hash_with_log + ) + + # Test with None indexed arg + filter_params = { + "fromBlock": BlockNumber(0), + "topics": [ + HexStr('0x057bc32826fbe161da1c110afcdcae7c109a8b69149f727fc37a603c60ef94ca'), + None], + } + result = await async_w3.eth.get_logs(filter_params) # type: ignore + _assert_contains_log( + result, + block_with_txn_with_log, + emitter_contract_address, + txn_hash_with_log + ) + + @pytest.mark.asyncio + async def test_async_eth_get_logs_with_logs_none_topic_args(self, async_w3: "Web3") -> None: + # Test with None overflowing + filter_params: FilterParams = { + "fromBlock": BlockNumber(0), + "topics": [None, None, None], + } + + result = await async_w3.eth.get_logs(filter_params) # type: ignore + assert len(result) == 0 + def test_async_provider_default_account( self, async_w3: "Web3", @@ -2718,15 +2871,6 @@ def test_eth_get_logs_with_logs( emitter_contract_address: ChecksumAddress, txn_hash_with_log: HexStr, ) -> None: - def assert_contains_log(result: Sequence[LogReceipt]) -> None: - assert len(result) == 1 - log_entry = result[0] - assert log_entry['blockNumber'] == block_with_txn_with_log['number'] - assert log_entry['blockHash'] == block_with_txn_with_log['hash'] - assert log_entry['logIndex'] == 0 - assert is_same_address(log_entry['address'], emitter_contract_address) - assert log_entry['transactionIndex'] == 0 - assert log_entry['transactionHash'] == HexBytes(txn_hash_with_log) # Test with block range @@ -2736,14 +2880,24 @@ def assert_contains_log(result: Sequence[LogReceipt]) -> None: "toBlock": block_with_txn_with_log['number'], } result = web3.eth.get_logs(filter_params) - assert_contains_log(result) + _assert_contains_log( + result, + block_with_txn_with_log, + emitter_contract_address, + txn_hash_with_log + ) # specify only `from_block`. by default `to_block` should be 'latest' filter_params = { "fromBlock": BlockNumber(0), } result = web3.eth.get_logs(filter_params) - assert_contains_log(result) + _assert_contains_log( + result, + block_with_txn_with_log, + emitter_contract_address, + txn_hash_with_log + ) # Test with `address` @@ -2760,15 +2914,6 @@ def test_eth_get_logs_with_logs_topic_args( emitter_contract_address: ChecksumAddress, txn_hash_with_log: HexStr, ) -> None: - def assert_contains_log(result: Sequence[LogReceipt]) -> None: - assert len(result) == 1 - log_entry = result[0] - assert log_entry['blockNumber'] == block_with_txn_with_log['number'] - assert log_entry['blockHash'] == block_with_txn_with_log['hash'] - assert log_entry['logIndex'] == 0 - assert is_same_address(log_entry['address'], emitter_contract_address) - assert log_entry['transactionIndex'] == 0 - assert log_entry['transactionHash'] == HexBytes(txn_hash_with_log) # Test with None event sig @@ -2780,7 +2925,12 @@ def assert_contains_log(result: Sequence[LogReceipt]) -> None: } result = web3.eth.get_logs(filter_params) - assert_contains_log(result) + _assert_contains_log( + result, + block_with_txn_with_log, + emitter_contract_address, + txn_hash_with_log + ) # Test with None indexed arg filter_params = { @@ -2790,7 +2940,12 @@ def assert_contains_log(result: Sequence[LogReceipt]) -> None: None], } result = web3.eth.get_logs(filter_params) - assert_contains_log(result) + _assert_contains_log( + result, + block_with_txn_with_log, + emitter_contract_address, + txn_hash_with_log + ) def test_eth_get_logs_with_logs_none_topic_args(self, web3: "Web3") -> None: # Test with None overflowing diff --git a/web3/eth.py b/web3/eth.py index 00f6930ef7..2dc05952ad 100644 --- a/web3/eth.py +++ b/web3/eth.py @@ -436,6 +436,17 @@ async def get_code( ) -> HexBytes: return await self._get_code(account, block_identifier) + _get_logs: Method[Callable[[FilterParams], Awaitable[List[LogReceipt]]]] = Method( + RPC.eth_getLogs, + mungers=[default_root_munger] + ) + + async def get_logs( + self, + filter_params: FilterParams, + ) -> List[LogReceipt]: + return await self._get_logs(filter_params) + _get_transaction_count: Method[Callable[..., Awaitable[Nonce]]] = Method( RPC.eth_getTransactionCount, mungers=[BaseEth.block_id_munger], From 8fc2387a16ba137191ecc531d36c680d2790a69d Mon Sep 17 00:00:00 2001 From: Mikko Ohtamaa Date: Thu, 20 Jan 2022 09:38:09 +0100 Subject: [PATCH 07/48] Add Github link to the main doc landing Because Github link is extremely useful --- docs/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 21faacf45d..f3c59c70e9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ Your next steps depend on where you're standing: - Need help debugging? → `StackExchange`_ - Like to give back? → :ref:`Contribute ` - Want to chat? → `Discord`_ +- Read the source? → `Github`_ Table of Contents ----------------- @@ -91,3 +92,4 @@ Indices and tables .. _ethereum.org/python: https://ethereum.org/python/ .. _StackExchange: https://ethereum.stackexchange.com/questions/tagged/web3.py .. _Discord: https://discord.gg/GHryRvPB84 +.. _Github: https://github.com/ethereum/web3.py From beeb052628fb3185c72c6a715951f88d97cb18de Mon Sep 17 00:00:00 2001 From: Felipe Selmo Date: Thu, 27 Jan 2022 13:13:06 -0700 Subject: [PATCH 08/48] Newsfragment for github link to docs --- newsfragments/2313.doc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2313.doc.rst diff --git a/newsfragments/2313.doc.rst b/newsfragments/2313.doc.rst new file mode 100644 index 0000000000..202a7c1cfc --- /dev/null +++ b/newsfragments/2313.doc.rst @@ -0,0 +1 @@ +add github link to the main docs landing page \ No newline at end of file From cb6530db61275f9317be8270612dcc81b379222d Mon Sep 17 00:00:00 2001 From: kclowes Date: Thu, 27 Jan 2022 15:27:48 -0700 Subject: [PATCH 09/48] Update typing extensions to allow v4 (#2217) * Update typing extensions to allow v4 * Loosen typing-extensions version * Add newsfragment for typing-extensions bump --- newsfragments/2217.misc.rst | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/2217.misc.rst diff --git a/newsfragments/2217.misc.rst b/newsfragments/2217.misc.rst new file mode 100644 index 0000000000..2c74d97bdd --- /dev/null +++ b/newsfragments/2217.misc.rst @@ -0,0 +1 @@ +Update ``typing-extensions`` dependency to allow for v4 diff --git a/setup.py b/setup.py index c37be6916f..4641f372a7 100644 --- a/setup.py +++ b/setup.py @@ -89,7 +89,7 @@ "pywin32>=223;platform_system=='Windows'", "requests>=2.16.0,<3.0.0", # remove typing_extensions after python_requires>=3.8, see web3._utils.compat - "typing-extensions>=3.7.4.1,<4;python_version<'3.8'", + "typing-extensions>=3.7.4.1,<5;python_version<'3.8'", "websockets>=9.1,<10", ], python_requires='>=3.6,<4', From a3937942209ae69bfa5c28889739f1a145c34bf1 Mon Sep 17 00:00:00 2001 From: kclowes Date: Wed, 26 Jan 2022 10:07:55 -0700 Subject: [PATCH 10/48] Try out new py-evm requirements in eth-tester --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4641f372a7..a8eb245248 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,8 @@ extras_require = { 'tester': [ - "eth-tester[py-evm]==v0.6.0-beta.4", + # "eth-tester[py-evm]==v0.6.0-beta.4", + 'eth-tester[py-evm]@git+ssh://git@github.com/kclowes/eth-tester@revert-deps#egg=some-pkg', "py-geth>=3.7.0,<4", ], 'linter': [ From bfef9b3d8450b879459bc7cd8469dcd865b7f4d0 Mon Sep 17 00:00:00 2001 From: kclowes Date: Wed, 26 Jan 2022 12:04:55 -0700 Subject: [PATCH 11/48] Remove xfails for newly passing eth-tester tests --- setup.py | 3 +-- tests/integration/test_ethereum_tester.py | 14 -------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/setup.py b/setup.py index a8eb245248..b1c01031ab 100644 --- a/setup.py +++ b/setup.py @@ -7,8 +7,7 @@ extras_require = { 'tester': [ - # "eth-tester[py-evm]==v0.6.0-beta.4", - 'eth-tester[py-evm]@git+ssh://git@github.com/kclowes/eth-tester@revert-deps#egg=some-pkg', + "eth-tester[py-evm]==v0.6.0-beta.6", "py-geth>=3.7.0,<4", ], 'linter': [ diff --git a/tests/integration/test_ethereum_tester.py b/tests/integration/test_ethereum_tester.py index a477e61d48..42dc4ab98c 100644 --- a/tests/integration/test_ethereum_tester.py +++ b/tests/integration/test_ethereum_tester.py @@ -415,20 +415,6 @@ def test_eth_chainId(self, web3): assert is_integer(chain_id) assert chain_id == 61 - @pytest.mark.xfail(raises=KeyError, reason="ethereum tester doesn't return 'to' key") - def test_eth_get_transaction_receipt_mined(self, web3, block_with_txn, mined_txn_hash): - super().test_eth_get_transaction_receipt_mined(web3, block_with_txn, mined_txn_hash) - - @pytest.mark.xfail(raises=KeyError, reason="ethereum tester doesn't return 'to' key") - def test_eth_getTransactionReceipt_mined_deprecated(self, web3, block_with_txn, mined_txn_hash): - super().test_eth_getTransactionReceipt_mined_deprecated(web3, - block_with_txn, - mined_txn_hash) - - @pytest.mark.xfail(raises=KeyError, reason="ethereum tester doesn't return 'to' key") - def test_eth_wait_for_transaction_receipt_mined(self, web3, block_with_txn, mined_txn_hash): - super().test_eth_wait_for_transaction_receipt_mined(web3, block_with_txn, mined_txn_hash) - @disable_auto_mine def test_eth_wait_for_transaction_receipt_unmined(self, eth_tester, From 8e498edd81fca5be389eebc1df4fe158d421ab51 Mon Sep 17 00:00:00 2001 From: kclowes Date: Thu, 27 Jan 2022 16:23:11 -0700 Subject: [PATCH 12/48] Upgrade eth-account requirement --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b1c01031ab..414a0f10d6 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,7 @@ install_requires=[ "aiohttp>=3.7.4.post0,<4", "eth-abi>=2.0.0b6,<3.0.0", - "eth-account>=0.5.6,<0.6.0", + "eth-account>=0.5.7,<0.6.0", "eth-hash[pycryptodome]>=0.2.0,<1.0.0", "eth-typing>=2.0.0,<3.0.0", "eth-utils>=1.9.5,<2.0.0", From fc742b633a7f449b2dc1ba4813ed94e9f69946a7 Mon Sep 17 00:00:00 2001 From: kclowes Date: Fri, 28 Jan 2022 12:09:42 -0700 Subject: [PATCH 13/48] Add newsfragment for eth-tester bump --- newsfragments/2320.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2320.feature.rst diff --git a/newsfragments/2320.feature.rst b/newsfragments/2320.feature.rst new file mode 100644 index 0000000000..35906a85d3 --- /dev/null +++ b/newsfragments/2320.feature.rst @@ -0,0 +1 @@ +Update eth-tester and eth-account dependencies to pull in bugfix from eth-keys From 560da6681295ff8cd9a30dc197c2e51df0798d09 Mon Sep 17 00:00:00 2001 From: pacrob Date: Fri, 28 Jan 2022 12:35:40 -0700 Subject: [PATCH 14/48] correct misspellings and update referenced geth version --- docs/web3.geth.rst | 6 +++--- newsfragments/2326.doc.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 newsfragments/2326.doc.rst diff --git a/docs/web3.geth.rst b/docs/web3.geth.rst index 9cea1f77cd..ac9186d0c4 100644 --- a/docs/web3.geth.rst +++ b/docs/web3.geth.rst @@ -309,7 +309,7 @@ The following methods are available on the ``web3.geth.personal`` namespace. Unlocks the given ``account`` for ``duration`` seconds. If ``duration`` is ``None``, then the account will remain unlocked - for 300 seconds (which is current default by Geth v1.9.5); + for 300 seconds (which is current default by Geth v1.10.15); if ``duration`` is set to ``0``, the account will remain unlocked indefinitely. Returns boolean as to whether the account was successfully unlocked. @@ -356,7 +356,7 @@ The following methods are available on the ``web3.geth.txpool`` namespace. * Delegates to ``txpool_inspect`` RPC Method Returns a textual summary of all transactions currently pending for - inclusing in the next block(s) as will as ones that are scheduled for + inclusion in the next block(s) as well as ones that are scheduled for future execution. .. code-block:: python @@ -418,7 +418,7 @@ The following methods are available on the ``web3.geth.txpool`` namespace. * Delegates to ``txpool_status`` RPC Method Returns a textual summary of all transactions currently pending for - inclusing in the next block(s) as will as ones that are scheduled for + inclusion in the next block(s) as well as ones that are scheduled for future execution. .. code-block:: python diff --git a/newsfragments/2326.doc.rst b/newsfragments/2326.doc.rst new file mode 100644 index 0000000000..3f96ed14f2 --- /dev/null +++ b/newsfragments/2326.doc.rst @@ -0,0 +1 @@ +fix typos and update referenced `geth` version From 553d9bfc692d2677078dba29248e5e69c73cf1f9 Mon Sep 17 00:00:00 2001 From: kclowes Date: Mon, 31 Jan 2022 11:36:56 -0700 Subject: [PATCH 15/48] Compile release notes --- docs/releases.rst | 54 ++++++++++++++++++++++++++++++++++ newsfragments/1413.feature.rst | 1 - newsfragments/2111.doc.rst | 1 - newsfragments/2211.bugfix.rst | 1 - newsfragments/2217.misc.rst | 1 - newsfragments/2293.doc.rst | 1 - newsfragments/2302.doc.rst | 1 - newsfragments/2303.doc.rst | 1 - newsfragments/2304.feature.rst | 1 - newsfragments/2310.feature.rst | 1 - newsfragments/2313.doc.rst | 1 - newsfragments/2315.feature.rst | 1 - newsfragments/2318.bugfix.rst | 1 - newsfragments/2320.feature.rst | 1 - newsfragments/2321.bugfix.rst | 1 - newsfragments/2326.doc.rst | 1 - 16 files changed, 54 insertions(+), 15 deletions(-) delete mode 100644 newsfragments/1413.feature.rst delete mode 100644 newsfragments/2111.doc.rst delete mode 100644 newsfragments/2211.bugfix.rst delete mode 100644 newsfragments/2217.misc.rst delete mode 100644 newsfragments/2293.doc.rst delete mode 100644 newsfragments/2302.doc.rst delete mode 100644 newsfragments/2303.doc.rst delete mode 100644 newsfragments/2304.feature.rst delete mode 100644 newsfragments/2310.feature.rst delete mode 100644 newsfragments/2313.doc.rst delete mode 100644 newsfragments/2315.feature.rst delete mode 100644 newsfragments/2318.bugfix.rst delete mode 100644 newsfragments/2320.feature.rst delete mode 100644 newsfragments/2321.bugfix.rst delete mode 100644 newsfragments/2326.doc.rst diff --git a/docs/releases.rst b/docs/releases.rst index 204e61aae1..3b45188245 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -3,6 +3,60 @@ Release Notes .. towncrier release notes start +v5.27.0 (2022-01-31) +-------------------- + +Features +~~~~~~~~ + +- Added Async functions for Geth TxPool (`#1413 + `__) +- external modules are no longer required to inherit from the + ``web3.module.Module`` class (`#2304 + `__) +- Add async `eth.get_logs` method (`#2310 + `__) +- add Async access to `default_account` and `default_block` (`#2315 + `__) +- Update eth-tester and eth-account dependencies to pull in bugfix from + eth-keys (`#2320 `__) + + +Bugfixes +~~~~~~~~ + +- Fixed issues with parsing tuples and nested tuples in event logs (`#2211 + `__) +- In ENS the contract function to resolve an ENS address was being called twice + in error. One of those calls was removed. (`#2318 + `__) +- ``to_hexbytes`` block formatters no longer throw when value is ``None`` + (`#2321 `__) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- fix typo in `eth.account` docs (`#2111 + `__) +- explicitly add `output_values` to contracts example (`#2293 + `__) +- update imports for `AsyncHTTPProvider` sample code (`#2302 + `__) +- fixed broken link to filter schema (`#2303 + `__) +- add github link to the main docs landing page (`#2313 + `__) +- fix typos and update referenced `geth` version (`#2326 + `__) + + +Misc +~~~~ + +- `#2217 `__ + + v5.26.0 (2022-01-06) -------------------- diff --git a/newsfragments/1413.feature.rst b/newsfragments/1413.feature.rst deleted file mode 100644 index 52835c3d0f..0000000000 --- a/newsfragments/1413.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added Async functions for Geth TxPool \ No newline at end of file diff --git a/newsfragments/2111.doc.rst b/newsfragments/2111.doc.rst deleted file mode 100644 index 5828417126..0000000000 --- a/newsfragments/2111.doc.rst +++ /dev/null @@ -1 +0,0 @@ -fix typo in `eth.account` docs diff --git a/newsfragments/2211.bugfix.rst b/newsfragments/2211.bugfix.rst deleted file mode 100644 index b5b3212734..0000000000 --- a/newsfragments/2211.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed issues with parsing tuples and nested tuples in event logs \ No newline at end of file diff --git a/newsfragments/2217.misc.rst b/newsfragments/2217.misc.rst deleted file mode 100644 index 2c74d97bdd..0000000000 --- a/newsfragments/2217.misc.rst +++ /dev/null @@ -1 +0,0 @@ -Update ``typing-extensions`` dependency to allow for v4 diff --git a/newsfragments/2293.doc.rst b/newsfragments/2293.doc.rst deleted file mode 100644 index e869c4644d..0000000000 --- a/newsfragments/2293.doc.rst +++ /dev/null @@ -1 +0,0 @@ -explicitly add `output_values` to contracts example diff --git a/newsfragments/2302.doc.rst b/newsfragments/2302.doc.rst deleted file mode 100644 index 7a8c7ce9dd..0000000000 --- a/newsfragments/2302.doc.rst +++ /dev/null @@ -1 +0,0 @@ -update imports for `AsyncHTTPProvider` sample code diff --git a/newsfragments/2303.doc.rst b/newsfragments/2303.doc.rst deleted file mode 100644 index 301cd9e65a..0000000000 --- a/newsfragments/2303.doc.rst +++ /dev/null @@ -1 +0,0 @@ -fixed broken link to filter schema \ No newline at end of file diff --git a/newsfragments/2304.feature.rst b/newsfragments/2304.feature.rst deleted file mode 100644 index b0c995e6bf..0000000000 --- a/newsfragments/2304.feature.rst +++ /dev/null @@ -1 +0,0 @@ -external modules are no longer required to inherit from the ``web3.module.Module`` class \ No newline at end of file diff --git a/newsfragments/2310.feature.rst b/newsfragments/2310.feature.rst deleted file mode 100644 index d5b359f3e8..0000000000 --- a/newsfragments/2310.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add async `eth.get_logs` method diff --git a/newsfragments/2313.doc.rst b/newsfragments/2313.doc.rst deleted file mode 100644 index 202a7c1cfc..0000000000 --- a/newsfragments/2313.doc.rst +++ /dev/null @@ -1 +0,0 @@ -add github link to the main docs landing page \ No newline at end of file diff --git a/newsfragments/2315.feature.rst b/newsfragments/2315.feature.rst deleted file mode 100644 index 1fa55c3ac3..0000000000 --- a/newsfragments/2315.feature.rst +++ /dev/null @@ -1 +0,0 @@ -add Async access to `default_account` and `default_block` diff --git a/newsfragments/2318.bugfix.rst b/newsfragments/2318.bugfix.rst deleted file mode 100644 index 7b69a5600c..0000000000 --- a/newsfragments/2318.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -In ENS the contract function to resolve an ENS address was being called twice in error. One of those calls was removed. \ No newline at end of file diff --git a/newsfragments/2320.feature.rst b/newsfragments/2320.feature.rst deleted file mode 100644 index 35906a85d3..0000000000 --- a/newsfragments/2320.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Update eth-tester and eth-account dependencies to pull in bugfix from eth-keys diff --git a/newsfragments/2321.bugfix.rst b/newsfragments/2321.bugfix.rst deleted file mode 100644 index 61c942b0c9..0000000000 --- a/newsfragments/2321.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -``to_hexbytes`` block formatters no longer throw when value is ``None`` \ No newline at end of file diff --git a/newsfragments/2326.doc.rst b/newsfragments/2326.doc.rst deleted file mode 100644 index 3f96ed14f2..0000000000 --- a/newsfragments/2326.doc.rst +++ /dev/null @@ -1 +0,0 @@ -fix typos and update referenced `geth` version From eb1051ee9db60c20bef764c680af040516901275 Mon Sep 17 00:00:00 2001 From: kclowes Date: Mon, 31 Jan 2022 11:44:55 -0700 Subject: [PATCH 16/48] =?UTF-8?q?Bump=20version:=205.26.0=20=E2=86=92=205.?= =?UTF-8?q?27.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0711fd90b2..b716075c39 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.26.0 +current_version = 5.27.0 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-(?P[^.]*)\.(?P\d+))? diff --git a/setup.py b/setup.py index 414a0f10d6..10eb738a03 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ setup( name='web3', # *IMPORTANT*: Don't manually change the version here. Use the 'bumpversion' utility. - version='5.26.0', + version='5.27.0', description="""Web3.py""", long_description_content_type='text/markdown', long_description=long_description, From e89243424ef67043b5a4ee0dcae3c95149827a82 Mon Sep 17 00:00:00 2001 From: AlwaysData Date: Mon, 31 Jan 2022 17:57:18 -0500 Subject: [PATCH 17/48] Add Async Geth Personal module (#2299) --- docs/providers.rst | 12 +- newsfragments/2299.feature.rst | 1 + .../go_ethereum/test_goethereum_http.py | 15 +- web3/_utils/module_testing/__init__.py | 2 +- ...dule.py => go_ethereum_personal_module.py} | 81 +++++++++ web3/geth.py | 166 ++++++++++++++++-- 6 files changed, 253 insertions(+), 24 deletions(-) create mode 100644 newsfragments/2299.feature.rst rename web3/_utils/module_testing/{personal_module.py => go_ethereum_personal_module.py} (76%) diff --git a/docs/providers.rst b/docs/providers.rst index fe0a5be23f..736e748f0d 100644 --- a/docs/providers.rst +++ b/docs/providers.rst @@ -392,7 +392,8 @@ AsyncHTTPProvider ... modules={'eth': (AsyncEth,), ... 'net': (AsyncNet,), ... 'geth': (Geth, - ... {'txpool': (AsyncGethTxPool,)}) + ... {'txpool': (AsyncGethTxPool,), + ... 'personal': (AsyncGethPersonal,)}) ... }, ... middlewares=[]) # See supported middleware section below for middleware options @@ -438,6 +439,15 @@ Net Geth **** +- :meth:`web3.geth.personal.ec_recover()` +- :meth:`web3.geth.personal.import_raw_key() ` +- :meth:`web3.geth.personal.list_accounts() ` +- :meth:`web3.geth.personal.list_wallets() ` +- :meth:`web3.geth.personal.lock_account() ` +- :meth:`web3.geth.personal.new_account() ` +- :meth:`web3.geth.personal.send_transaction() ` +- :meth:`web3.geth.personal.sign()` +- :meth:`web3.geth.personal.unlock_account() ` - :meth:`web3.geth.txpool.inspect() ` - :meth:`web3.geth.txpool.content() ` - :meth:`web3.geth.txpool.status() ` diff --git a/newsfragments/2299.feature.rst b/newsfragments/2299.feature.rst new file mode 100644 index 0000000000..025051f606 --- /dev/null +++ b/newsfragments/2299.feature.rst @@ -0,0 +1 @@ +Added Async functions for Geth Personal module diff --git a/tests/integration/go_ethereum/test_goethereum_http.py b/tests/integration/go_ethereum/test_goethereum_http.py index 6c14cf5787..90e0b7313e 100644 --- a/tests/integration/go_ethereum/test_goethereum_http.py +++ b/tests/integration/go_ethereum/test_goethereum_http.py @@ -4,10 +4,14 @@ get_open_port, ) from web3 import Web3 +from web3._utils.module_testing.go_ethereum_personal_module import ( + GoEthereumAsyncPersonalModuleTest, +) from web3.eth import ( AsyncEth, ) from web3.geth import ( + AsyncGethPersonal, AsyncGethTxPool, Geth, ) @@ -94,10 +98,11 @@ async def async_w3(geth_process, endpoint_uri): async_gas_price_strategy_middleware, async_buffered_gas_estimate_middleware ], - modules={'eth': (AsyncEth,), - 'async_net': (AsyncNet,), + modules={'eth': AsyncEth, + 'async_net': AsyncNet, 'geth': (Geth, - {'txpool': (AsyncGethTxPool,)} + {'txpool': (AsyncGethTxPool,), + 'personal': (AsyncGethPersonal,)} ) } ) @@ -144,6 +149,10 @@ class TestGoEthereumPersonalModuleTest(GoEthereumPersonalModuleTest): pass +class TestGoEthereumAsyncPersonalModuleTest(GoEthereumAsyncPersonalModuleTest): + pass + + class TestGoEthereumAsyncEthModuleTest(GoEthereumAsyncEthModuleTest): pass diff --git a/web3/_utils/module_testing/__init__.py b/web3/_utils/module_testing/__init__.py index 57b52501a1..6d5cdfc5a4 100644 --- a/web3/_utils/module_testing/__init__.py +++ b/web3/_utils/module_testing/__init__.py @@ -13,7 +13,7 @@ AsyncNetModuleTest, NetModuleTest, ) -from .personal_module import ( # noqa: F401 +from .go_ethereum_personal_module import ( # noqa: F401 GoEthereumPersonalModuleTest, ) from .version_module import ( # noqa: F401 diff --git a/web3/_utils/module_testing/personal_module.py b/web3/_utils/module_testing/go_ethereum_personal_module.py similarity index 76% rename from web3/_utils/module_testing/personal_module.py rename to web3/_utils/module_testing/go_ethereum_personal_module.py index b21b5a839f..e39a1b177c 100644 --- a/web3/_utils/module_testing/personal_module.py +++ b/web3/_utils/module_testing/go_ethereum_personal_module.py @@ -21,6 +21,9 @@ from web3 import ( constants, ) +from web3.datastructures import ( + AttributeDict, +) from web3.types import ( # noqa: F401 TxParams, Wei, @@ -31,6 +34,7 @@ PRIVATE_KEY_HEX = '0x56ebb41875ceedd42e395f730e03b5c44989393c9f0484ee6bc05f933673458f' SECOND_PRIVATE_KEY_HEX = '0x56ebb41875ceedd42e395f730e03b5c44989393c9f0484ee6bc05f9336712345' +THIRD_PRIVATE_KEY_HEX = '0x56ebb41875ceedd42e395f730e03b5c44989393c9f0484ee6bc05f9336754321' PASSWORD = 'web3-testing' ADDRESS = '0x844B417c0C58B02c2224306047B9fb0D3264fE8c' SECOND_ADDRESS = '0xB96b6B21053e67BA59907E252D990C71742c41B8' @@ -343,3 +347,80 @@ def test_personal_sign_typed_data_deprecated( ) assert signature == expected_signature assert len(signature) == 32 + 32 + 1 + + +class GoEthereumAsyncPersonalModuleTest: + + @pytest.mark.asyncio + async def test_async_sign_and_ec_recover(self, + async_w3: "Web3", + unlockable_account_dual_type: ChecksumAddress, + unlockable_account_pw: str) -> None: + message = "This is a test" + signature = await async_w3.geth.personal.sign(message, # type: ignore + unlockable_account_dual_type, + unlockable_account_pw) + address = await async_w3.geth.personal.ec_recover(message, signature) # type: ignore + assert is_same_address(unlockable_account_dual_type, address) + + @pytest.mark.asyncio + async def test_async_import_key(self, async_w3: "Web3") -> None: + address = await async_w3.geth.personal.import_raw_key(THIRD_PRIVATE_KEY_HEX, # type: ignore + "Testing") + assert address is not None + + @pytest.mark.asyncio + async def test_async_list_accounts(self, async_w3: "Web3") -> None: + accounts = await async_w3.geth.personal.list_accounts() # type: ignore + assert len(accounts) > 0 + + @pytest.mark.asyncio + async def test_async_list_wallets(self, async_w3: "Web3") -> None: + wallets = await async_w3.geth.personal.list_wallets() # type: ignore + assert isinstance(wallets[0], AttributeDict) + + @pytest.mark.asyncio + async def test_async_new_account(self, async_w3: "Web3") -> None: + passphrase = "Create New Account" + account = await async_w3.geth.personal.new_account(passphrase) # type: ignore + assert is_checksum_address(account) + + @pytest.mark.asyncio + async def test_async_unlock_lock_account(self, + async_w3: "Web3", + unlockable_account_dual_type: ChecksumAddress, + unlockable_account_pw: str) -> None: + unlocked = await async_w3.geth.personal.unlock_account( # type: ignore + unlockable_account_dual_type, + unlockable_account_pw) + assert unlocked is True + locked = await async_w3.geth.personal.lock_account( # type: ignore + unlockable_account_dual_type) + assert locked is True + + @pytest.mark.asyncio + async def test_async_send_transaction(self, + async_w3: "Web3", + unlockable_account_dual_type: ChecksumAddress, + unlockable_account_pw: str) -> None: + tx_params = TxParams() + tx_params["to"] = unlockable_account_dual_type + tx_params["from"] = unlockable_account_dual_type + tx_params["value"] = Wei(123) + response = await async_w3.geth.personal.send_transaction( # type: ignore + tx_params, + unlockable_account_pw) + assert response is not None + + @pytest.mark.xfail(reason="personal_signTypedData JSON RPC call has not been released in geth") + @pytest.mark.asyncio + async def test_async_sign_typed_data(self, + async_w3: "Web3", + unlockable_account_dual_type: ChecksumAddress, + unlockable_account_pw: str) -> None: + message = {"message": "This is a test"} + signature = await async_w3.geth.personal.sign_typed_data(message, # type: ignore + unlockable_account_dual_type, + unlockable_account_pw) + address = await async_w3.geth.personal.ec_recover(message, signature) # type: ignore + assert is_same_address(unlockable_account_dual_type, address) diff --git a/web3/geth.py b/web3/geth.py index 478e0a141b..2320607078 100644 --- a/web3/geth.py +++ b/web3/geth.py @@ -1,6 +1,19 @@ from typing import ( Any, Awaitable, + Dict, + List, + Optional, +) + +from eth_typing.encoding import ( + HexStr, +) +from eth_typing.evm import ( + ChecksumAddress, +) +from hexbytes.main import ( + HexBytes, ) from web3._utils.admin import ( @@ -64,35 +77,150 @@ Module, ) from web3.types import ( + GethWallet, + TxParams, TxPoolContent, TxPoolInspect, TxPoolStatus, ) -class GethPersonal(Module): +class BaseGethPersonal(Module): """ https://github.com/ethereum/go-ethereum/wiki/management-apis#personal """ - ec_recover = ec_recover - import_raw_key = import_raw_key - list_accounts = list_accounts - list_wallets = list_wallets - lock_account = lock_account - new_account = new_account - send_transaction = send_transaction - sign = sign - sign_typed_data = sign_typed_data - unlock_account = unlock_account + _ec_recover = ec_recover + _import_raw_key = import_raw_key + _list_accounts = list_accounts + _list_wallets = list_wallets + _lock_account = lock_account + _new_account = new_account + _send_transaction = send_transaction + _sign = sign + _sign_typed_data = sign_typed_data + _unlock_account = unlock_account # deprecated - ecRecover = ecRecover - importRawKey = importRawKey - listAccounts = listAccounts - lockAccount = lockAccount - newAccount = newAccount - sendTransaction = sendTransaction - signTypedData = signTypedData - unlockAccount = unlockAccount + _ecRecover = ecRecover + _importRawKey = importRawKey + _listAccounts = listAccounts + _lockAccount = lockAccount + _newAccount = newAccount + _sendTransaction = sendTransaction + _signTypedData = signTypedData + _unlockAccount = unlockAccount + + +class GethPersonal(BaseGethPersonal): + is_async = False + + def ec_recover(self, message: str, signature: HexStr) -> ChecksumAddress: + return self._ec_recover(message, signature) + + def import_raw_key(self, private_key: str, passphrase: str) -> ChecksumAddress: + return self._import_raw_key(private_key, passphrase) + + def list_accounts(self) -> List[ChecksumAddress]: + return self._list_accounts() + + def list_wallets(self) -> List[GethWallet]: + return self._list_wallets() + + def lock_account(self, account: ChecksumAddress) -> bool: + return self._lock_account(account) + + def new_account(self, passphrase: str) -> ChecksumAddress: + return self._new_account(passphrase) + + def send_transaction(self, transaction: TxParams, passphrase: str) -> HexBytes: + return self._send_transaction(transaction, passphrase) + + def sign(self, message: str, account: ChecksumAddress, password: Optional[str]) -> HexStr: + return self._sign(message, account, password) + + def sign_typed_data(self, + message: Dict[str, Any], + account: ChecksumAddress, + password: Optional[str]) -> HexStr: + return self._sign_typed_data(message, account, password) + + def unlock_account(self, + account: ChecksumAddress, + passphrase: str, + duration: Optional[int] = None) -> bool: + return self._unlock_account(account, passphrase, duration) + + def ecRecover(self, message: str, signature: HexStr) -> ChecksumAddress: + return self._ecRecover(message, signature) + + def importRawKey(self, private_key: str, passphrase: str) -> ChecksumAddress: + return self._importRawKey(private_key, passphrase) + + def listAccounts(self) -> List[ChecksumAddress]: + return self._listAccounts() + + def lockAccount(self, account: ChecksumAddress) -> bool: + return self._lockAccount(account) + + def newAccount(self, passphrase: str) -> ChecksumAddress: + return self._newAccount(passphrase) + + def sendTransaction(self, transaction: TxParams, passphrase: str) -> HexBytes: + return self._sendTransaction(transaction, passphrase) + + def signTypedData(self, + message: Dict[str, Any], + account: ChecksumAddress, + password: Optional[str] = None) -> HexStr: + return self._signTypedData(message, account, password) + + def unlockAccount(self, + account: ChecksumAddress, + passphrase: str, + duration: Optional[int] = None) -> bool: + return self._unlockAccount(account, passphrase, duration) + + +class AsyncGethPersonal(BaseGethPersonal): + is_async = True + + async def ec_recover(self, message: str, signature: HexStr) -> Awaitable[ChecksumAddress]: + return await self._ec_recover(message, signature) # type: ignore + + async def import_raw_key(self, private_key: str, passphrase: str) -> Awaitable[ChecksumAddress]: + return await self._import_raw_key(private_key, passphrase) # type: ignore + + async def list_accounts(self) -> Awaitable[List[ChecksumAddress]]: + return await self._list_accounts() # type: ignore + + async def list_wallets(self) -> Awaitable[List[GethWallet]]: + return await self._list_wallets() # type: ignore + + async def lock_account(self, account: ChecksumAddress) -> Awaitable[bool]: + return await self._lock_account(account) # type: ignore + + async def new_account(self, passphrase: str) -> Awaitable[ChecksumAddress]: + return await self._new_account(passphrase) # type: ignore + + async def send_transaction(self, transaction: TxParams, passphrase: str) -> Awaitable[HexBytes]: + return await self._send_transaction(transaction, passphrase) # type: ignore + + async def sign(self, + message: str, + account: ChecksumAddress, + password: Optional[str]) -> Awaitable[HexStr]: + return await self._sign(message, account, password) # type: ignore + + async def sign_typed_data(self, + message: Dict[str, Any], + account: ChecksumAddress, + password: Optional[str]) -> Awaitable[HexStr]: + return await self._sign_typed_data(message, account, password) # type: ignore + + async def unlock_account(self, + account: ChecksumAddress, + passphrase: str, + duration: Optional[int] = None) -> Awaitable[bool]: + return await self._unlock_account(account, passphrase, duration) # type: ignore class BaseTxPool(Module): From 2f2ca36d13888e60b16681e0366c9c849fc8d5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20=C5=A0uppa?= Date: Tue, 1 Feb 2022 00:12:20 +0100 Subject: [PATCH 18/48] fix: Missing commas (#2327) * fix: Missing commas * Add newsfragment for exception retry middleware whitelist Co-authored-by: kclowes --- newsfragments/2327.bugfix.rst | 1 + web3/middleware/exception_retry_request.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/2327.bugfix.rst diff --git a/newsfragments/2327.bugfix.rst b/newsfragments/2327.bugfix.rst new file mode 100644 index 0000000000..bac3de984b --- /dev/null +++ b/newsfragments/2327.bugfix.rst @@ -0,0 +1 @@ +The exception retry middleware whitelist was missing a comma between ``txpool`` and ``testing`` diff --git a/web3/middleware/exception_retry_request.py b/web3/middleware/exception_retry_request.py index 3a7192abf7..3289189d5a 100644 --- a/web3/middleware/exception_retry_request.py +++ b/web3/middleware/exception_retry_request.py @@ -25,7 +25,7 @@ 'admin', 'miner', 'net', - 'txpool' + 'txpool', 'testing', 'evm', 'eth_protocolVersion', From 84fd27a3f7d2f27ba2f2c2d1c94243c52877e6f1 Mon Sep 17 00:00:00 2001 From: broper2 Date: Thu, 23 Dec 2021 20:42:35 -0600 Subject: [PATCH 19/48] Fixes #2259, remove dependency on eth_maxPriorityFeePerGas --- .../test_contract_buildTransaction.py | 28 +++++++++---------- web3/eth.py | 12 ++++++-- web3/providers/eth_tester/defaults.py | 4 ++- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/tests/core/contracts/test_contract_buildTransaction.py b/tests/core/contracts/test_contract_buildTransaction.py index 85a8ffe324..637c4ddb84 100644 --- a/tests/core/contracts/test_contract_buildTransaction.py +++ b/tests/core/contracts/test_contract_buildTransaction.py @@ -54,8 +54,8 @@ def test_build_transaction_not_paying_to_nonpayable_function( 'to': payable_tester_contract.address, 'data': '0xe4cb8f5c', 'value': 0, - 'maxFeePerGas': 2750000000, - 'maxPriorityFeePerGas': 10 ** 9, + 'maxFeePerGas': 3750000000, + 'maxPriorityFeePerGas': 2 * (10 ** 9), 'chainId': 61, } @@ -76,8 +76,8 @@ def test_build_transaction_with_contract_no_arguments(web3, math_contract, build 'to': math_contract.address, 'data': '0xd09de08a', 'value': 0, - 'maxFeePerGas': 2750000000, - 'maxPriorityFeePerGas': 10 ** 9, + 'maxFeePerGas': 3750000000, + 'maxPriorityFeePerGas': 2 * (10 ** 9), 'chainId': 61, } @@ -88,8 +88,8 @@ def test_build_transaction_with_contract_fallback_function(web3, fallback_functi 'to': fallback_function_contract.address, 'data': '0x', 'value': 0, - 'maxFeePerGas': 2750000000, - 'maxPriorityFeePerGas': 10 ** 9, + 'maxFeePerGas': 3750000000, + 'maxPriorityFeePerGas': 2 * (10 ** 9), 'chainId': 61, } @@ -108,8 +108,8 @@ def test_build_transaction_with_contract_class_method( 'to': math_contract.address, 'data': '0xd09de08a', 'value': 0, - 'maxFeePerGas': 2750000000, - 'maxPriorityFeePerGas': 10 ** 9, + 'maxFeePerGas': 3750000000, + 'maxPriorityFeePerGas': 2 * (10 ** 9), 'chainId': 61, } @@ -123,8 +123,8 @@ def test_build_transaction_with_contract_default_account_is_set( 'to': math_contract.address, 'data': '0xd09de08a', 'value': 0, - 'maxFeePerGas': 2750000000, - 'maxPriorityFeePerGas': 10 ** 9, + 'maxFeePerGas': 3750000000, + 'maxPriorityFeePerGas': 2 * (10 ** 9), 'chainId': 61, } @@ -167,14 +167,14 @@ def test_build_transaction_with_contract_to_address_supplied_errors(web3, ( {}, (5,), {}, { 'data': '0x7cf5dab00000000000000000000000000000000000000000000000000000000000000005', # noqa: E501 - 'value': 0, 'maxFeePerGas': 2750000000, 'maxPriorityFeePerGas': 1000000000, + 'value': 0, 'maxFeePerGas': 3750000000, 'maxPriorityFeePerGas': 2 * (10 ** 9), 'chainId': 61, }, False ), ( {'gas': 800000}, (5,), {}, { 'data': '0x7cf5dab00000000000000000000000000000000000000000000000000000000000000005', # noqa: E501 - 'value': 0, 'maxFeePerGas': 2750000000, 'maxPriorityFeePerGas': 1000000000, + 'value': 0, 'maxFeePerGas': 3750000000, 'maxPriorityFeePerGas': 2 * (10 ** 9), 'chainId': 61, }, False ), @@ -194,14 +194,14 @@ def test_build_transaction_with_contract_to_address_supplied_errors(web3, ( {'nonce': 7}, (5,), {}, { 'data': '0x7cf5dab00000000000000000000000000000000000000000000000000000000000000005', # noqa: E501 - 'value': 0, 'maxFeePerGas': 2750000000, 'maxPriorityFeePerGas': 1000000000, + 'value': 0, 'maxFeePerGas': 3750000000, 'maxPriorityFeePerGas': 2 * (10 ** 9), 'nonce': 7, 'chainId': 61, }, True ), ( {'value': 20000}, (5,), {}, { 'data': '0x7cf5dab00000000000000000000000000000000000000000000000000000000000000005', # noqa: E501 - 'value': 20000, 'maxFeePerGas': 2750000000, 'maxPriorityFeePerGas': 1000000000, + 'value': 20000, 'maxFeePerGas': 3750000000, 'maxPriorityFeePerGas': 2 * (10 ** 9), 'chainId': 61, }, False ), diff --git a/web3/eth.py b/web3/eth.py index 2dc05952ad..35c3fe27bf 100644 --- a/web3/eth.py +++ b/web3/eth.py @@ -356,7 +356,11 @@ async def hashrate(self) -> int: @property async def max_priority_fee(self) -> Wei: - return await self._max_priority_fee() # type: ignore + fee_history = await self.fee_history(20, 'pending', [float(1)]) + priority_fees_per_gas = fee_history['reward'] + fees_sum = sum([fee[0] for fee in priority_fees_per_gas]) + max_priority_fee_estimate = round(fees_sum/len(priority_fees_per_gas)) + return Wei(max_priority_fee_estimate) @property async def mining(self) -> bool: @@ -604,7 +608,11 @@ def chainId(self) -> int: @property def max_priority_fee(self) -> Wei: - return self._max_priority_fee() + fee_history = self.fee_history(20, 'pending', [float(1)]) + priority_fees_per_gas = fee_history['reward'] + fees_sum = sum([fee[0] for fee in priority_fees_per_gas]) + max_priority_fee_estimate = round(fees_sum/len(priority_fees_per_gas)) + return Wei(max_priority_fee_estimate) def get_storage_at_munger( self, diff --git a/web3/providers/eth_tester/defaults.py b/web3/providers/eth_tester/defaults.py index a33d337ef6..532b03ffae 100644 --- a/web3/providers/eth_tester/defaults.py +++ b/web3/providers/eth_tester/defaults.py @@ -213,7 +213,9 @@ def personal_send_transaction(eth_tester: "EthereumTester", params: Any) -> HexS 'mining': static_return(False), 'hashrate': static_return(0), 'chainId': static_return('0x3d'), - 'feeHistory': not_implemented, + 'feeHistory': static_return( + {'baseFeePerGas': [134919017071, 134775902021, 117928914269], 'gasUsedRatio': [0.4957570088140204, 0.0], + 'oldestBlock': 13865084, 'reward': [[2500000000], [1500000000]]}), 'maxPriorityFeePerGas': static_return(10 ** 9), 'gasPrice': static_return(10 ** 9), # must be >= base fee post-London 'accounts': call_eth_tester('get_accounts'), From e1c3e29b5843e87f0ada1904b0312c9248f5a906 Mon Sep 17 00:00:00 2001 From: broper2 Date: Thu, 23 Dec 2021 21:08:27 -0600 Subject: [PATCH 20/48] fix lint and integration tests --- tests/integration/test_ethereum_tester.py | 8 ++++---- web3/eth.py | 4 ++-- web3/providers/eth_tester/defaults.py | 7 +++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_ethereum_tester.py b/tests/integration/test_ethereum_tester.py index 42dc4ab98c..ce134917cd 100644 --- a/tests/integration/test_ethereum_tester.py +++ b/tests/integration/test_ethereum_tester.py @@ -7,6 +7,9 @@ from eth_tester.exceptions import ( TransactionFailed, ) +from eth_typing import ( + BlockNumber, +) from eth_utils import ( is_checksum_address, is_dict, @@ -292,15 +295,12 @@ def test_eth_getBlockByHash_pending( block = web3.eth.get_block('pending') assert block['hash'] is not None - @pytest.mark.xfail(reason='eth_feeHistory is not implemented on eth-tester') def test_eth_fee_history(self, web3: "Web3"): super().test_eth_fee_history(web3) - @pytest.mark.xfail(reason='eth_feeHistory is not implemented on eth-tester') def test_eth_fee_history_with_integer(self, web3: "Web3"): - super().test_eth_fee_history_with_integer(web3) + super().test_eth_fee_history_with_integer(web3, BlockData(number=BlockNumber(1))) - @pytest.mark.xfail(reason='eth_feeHistory is not implemented on eth-tester') def test_eth_fee_history_no_reward_percentiles(self, web3: "Web3"): super().test_eth_fee_history_no_reward_percentiles(web3) diff --git a/web3/eth.py b/web3/eth.py index 35c3fe27bf..017d43fba2 100644 --- a/web3/eth.py +++ b/web3/eth.py @@ -359,7 +359,7 @@ async def max_priority_fee(self) -> Wei: fee_history = await self.fee_history(20, 'pending', [float(1)]) priority_fees_per_gas = fee_history['reward'] fees_sum = sum([fee[0] for fee in priority_fees_per_gas]) - max_priority_fee_estimate = round(fees_sum/len(priority_fees_per_gas)) + max_priority_fee_estimate = round(fees_sum / len(priority_fees_per_gas)) return Wei(max_priority_fee_estimate) @property @@ -611,7 +611,7 @@ def max_priority_fee(self) -> Wei: fee_history = self.fee_history(20, 'pending', [float(1)]) priority_fees_per_gas = fee_history['reward'] fees_sum = sum([fee[0] for fee in priority_fees_per_gas]) - max_priority_fee_estimate = round(fees_sum/len(priority_fees_per_gas)) + max_priority_fee_estimate = round(fees_sum / len(priority_fees_per_gas)) return Wei(max_priority_fee_estimate) def get_storage_at_munger( diff --git a/web3/providers/eth_tester/defaults.py b/web3/providers/eth_tester/defaults.py index 532b03ffae..4a39d5551e 100644 --- a/web3/providers/eth_tester/defaults.py +++ b/web3/providers/eth_tester/defaults.py @@ -214,8 +214,11 @@ def personal_send_transaction(eth_tester: "EthereumTester", params: Any) -> HexS 'hashrate': static_return(0), 'chainId': static_return('0x3d'), 'feeHistory': static_return( - {'baseFeePerGas': [134919017071, 134775902021, 117928914269], 'gasUsedRatio': [0.4957570088140204, 0.0], - 'oldestBlock': 13865084, 'reward': [[2500000000], [1500000000]]}), + {'baseFeePerGas': [134919017071, 134775902021, 117928914269], + 'gasUsedRatio': [0.4957570088140204, 0.0], + 'oldestBlock': 13865084, + 'reward': [[2500000000], [1500000000]]} + ), 'maxPriorityFeePerGas': static_return(10 ** 9), 'gasPrice': static_return(10 ** 9), # must be >= base fee post-London 'accounts': call_eth_tester('get_accounts'), From 634c89877f0a28d40333782db62e109f80c659d9 Mon Sep 17 00:00:00 2001 From: Felipe Selmo Date: Mon, 31 Jan 2022 11:36:04 -0700 Subject: [PATCH 21/48] refactor: utility for estimating maxPriorityFeePerGas via eth_feeHistory Refactor idea from PR #2259 into sync and async fee utility methods. Change params passed into eth_feeHistory to values that allowed for better results when we tested locally. Add a min and max to the estimated fee history so that we don't allow unsuspecting users to contribute to fee bloating. Max and min values keep the priority fee within a range that healthy blocks should accept, so these transactions would be accepted when fee prices settle from high-fee periods. --- newsfragments/2259.feature.rst | 1 + tests/integration/test_ethereum_tester.py | 8 ++-- web3/_utils/fee_utils.py | 53 +++++++++++++++++++++++ web3/eth.py | 38 +++++++++++----- web3/providers/eth_tester/defaults.py | 7 +-- 5 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 newsfragments/2259.feature.rst create mode 100644 web3/_utils/fee_utils.py diff --git a/newsfragments/2259.feature.rst b/newsfragments/2259.feature.rst new file mode 100644 index 0000000000..d320c46090 --- /dev/null +++ b/newsfragments/2259.feature.rst @@ -0,0 +1 @@ +Calculate a default ``maxPriorityFeePerGas`` using ``eth_feeHistory`` when ``eth_maxPriorityFeePerGas`` is not available, since the latter is not a part of the Ethereum JSON-RPC specs and only supported by certain clients. \ No newline at end of file diff --git a/tests/integration/test_ethereum_tester.py b/tests/integration/test_ethereum_tester.py index ce134917cd..42dc4ab98c 100644 --- a/tests/integration/test_ethereum_tester.py +++ b/tests/integration/test_ethereum_tester.py @@ -7,9 +7,6 @@ from eth_tester.exceptions import ( TransactionFailed, ) -from eth_typing import ( - BlockNumber, -) from eth_utils import ( is_checksum_address, is_dict, @@ -295,12 +292,15 @@ def test_eth_getBlockByHash_pending( block = web3.eth.get_block('pending') assert block['hash'] is not None + @pytest.mark.xfail(reason='eth_feeHistory is not implemented on eth-tester') def test_eth_fee_history(self, web3: "Web3"): super().test_eth_fee_history(web3) + @pytest.mark.xfail(reason='eth_feeHistory is not implemented on eth-tester') def test_eth_fee_history_with_integer(self, web3: "Web3"): - super().test_eth_fee_history_with_integer(web3, BlockData(number=BlockNumber(1))) + super().test_eth_fee_history_with_integer(web3) + @pytest.mark.xfail(reason='eth_feeHistory is not implemented on eth-tester') def test_eth_fee_history_no_reward_percentiles(self, web3: "Web3"): super().test_eth_fee_history_no_reward_percentiles(web3) diff --git a/web3/_utils/fee_utils.py b/web3/_utils/fee_utils.py new file mode 100644 index 0000000000..8ba6758f23 --- /dev/null +++ b/web3/_utils/fee_utils.py @@ -0,0 +1,53 @@ +from typing import ( + TYPE_CHECKING, +) + +from web3.types import ( + FeeHistory, + Wei, +) + +if TYPE_CHECKING: + from web3.eth import ( + AsyncEth, # noqa: F401 + Eth, # noqa: F401 + ) + +PRIORITY_FEE_MAX = Wei(1500000000) # 1.5 gwei +PRIORITY_FEE_MIN = Wei(1000000000) # 1 gwei + +# 5th percentile fee history from the last 10 blocks +PRIORITY_FEE_HISTORY_PARAMS = (10, 'pending', [5.0]) + + +def _fee_history_priority_fee_estimate(fee_history: FeeHistory) -> Wei: + # grab only non-zero fees and average against only that list + non_empty_block_fees = [fee[0] for fee in fee_history['reward'] if fee[0] != 0] + + # prevent division by zero in the extremely unlikely case that all fees within the polled fee + # history range for the specified percentile are 0 + divisor = len(non_empty_block_fees) if len(non_empty_block_fees) != 0 else 1 + + priority_fee_average_for_percentile = Wei( + round(sum(non_empty_block_fees) / divisor) + ) + + return ( # keep estimated priority fee within a max / min range + PRIORITY_FEE_MAX if priority_fee_average_for_percentile > PRIORITY_FEE_MAX else + PRIORITY_FEE_MIN if priority_fee_average_for_percentile < PRIORITY_FEE_MIN else + priority_fee_average_for_percentile + ) + + +def fee_history_priority_fee(eth: "Eth") -> Wei: + # This is a tested internal call so no need for type hinting. We can keep better consistency + # between the sync and async calls by unpacking PRIORITY_FEE_HISTORY_PARAMS as constants here. + fee_history = eth.fee_history(*PRIORITY_FEE_HISTORY_PARAMS) # type: ignore + return _fee_history_priority_fee_estimate(fee_history) + + +async def async_fee_history_priority_fee(async_eth: "AsyncEth") -> Wei: + # This is a tested internal call so no need for type hinting. We can keep better consistency + # between the sync and async calls by unpacking PRIORITY_FEE_HISTORY_PARAMS as constants here. + fee_history = await async_eth.fee_history(*PRIORITY_FEE_HISTORY_PARAMS) # type: ignore + return _fee_history_priority_fee_estimate(fee_history) diff --git a/web3/eth.py b/web3/eth.py index 017d43fba2..8edd74f9da 100644 --- a/web3/eth.py +++ b/web3/eth.py @@ -49,6 +49,10 @@ from web3._utils.encoding import ( to_hex, ) +from web3._utils.fee_utils import ( + async_fee_history_priority_fee, + fee_history_priority_fee, +) from web3._utils.filters import ( select_filter_method, ) @@ -356,11 +360,18 @@ async def hashrate(self) -> int: @property async def max_priority_fee(self) -> Wei: - fee_history = await self.fee_history(20, 'pending', [float(1)]) - priority_fees_per_gas = fee_history['reward'] - fees_sum = sum([fee[0] for fee in priority_fees_per_gas]) - max_priority_fee_estimate = round(fees_sum / len(priority_fees_per_gas)) - return Wei(max_priority_fee_estimate) + """ + Try to use eth_maxPriorityFeePerGas but, since this is not part of the spec and is only + supported by some clients, fall back to an eth_feeHistory calculation with min and max caps. + """ + try: + return await self._max_priority_fee() # type: ignore + except ValueError: + warnings.warn( + "There was an issue with the method eth_maxPriorityFeePerGas. Calculating using " + "eth_feeHistory." + ) + return await async_fee_history_priority_fee(self) @property async def mining(self) -> bool: @@ -608,11 +619,18 @@ def chainId(self) -> int: @property def max_priority_fee(self) -> Wei: - fee_history = self.fee_history(20, 'pending', [float(1)]) - priority_fees_per_gas = fee_history['reward'] - fees_sum = sum([fee[0] for fee in priority_fees_per_gas]) - max_priority_fee_estimate = round(fees_sum / len(priority_fees_per_gas)) - return Wei(max_priority_fee_estimate) + """ + Try to use eth_maxPriorityFeePerGas but, since this is not part of the spec and is only + supported by some clients, fall back to an eth_feeHistory calculation with min and max caps. + """ + try: + return self._max_priority_fee() + except ValueError: + warnings.warn( + "There was an issue with the method eth_maxPriorityFeePerGas. Calculating using " + "eth_feeHistory." + ) + return fee_history_priority_fee(self) def get_storage_at_munger( self, diff --git a/web3/providers/eth_tester/defaults.py b/web3/providers/eth_tester/defaults.py index 4a39d5551e..a33d337ef6 100644 --- a/web3/providers/eth_tester/defaults.py +++ b/web3/providers/eth_tester/defaults.py @@ -213,12 +213,7 @@ def personal_send_transaction(eth_tester: "EthereumTester", params: Any) -> HexS 'mining': static_return(False), 'hashrate': static_return(0), 'chainId': static_return('0x3d'), - 'feeHistory': static_return( - {'baseFeePerGas': [134919017071, 134775902021, 117928914269], - 'gasUsedRatio': [0.4957570088140204, 0.0], - 'oldestBlock': 13865084, - 'reward': [[2500000000], [1500000000]]} - ), + 'feeHistory': not_implemented, 'maxPriorityFeePerGas': static_return(10 ** 9), 'gasPrice': static_return(10 ** 9), # must be >= base fee post-London 'accounts': call_eth_tester('get_accounts'), From df45de4d4301072cd3c60eb35bc446b84e2aac3a Mon Sep 17 00:00:00 2001 From: Felipe Selmo Date: Tue, 1 Feb 2022 10:21:29 -0700 Subject: [PATCH 22/48] add tests for max_priority_fee when eth_maxPriorityFeePerGas is not available --- .../test_contract_buildTransaction.py | 28 ++++---- tests/core/utilities/test_fee_utils.py | 65 +++++++++++++++++++ tests/integration/test_ethereum_tester.py | 4 ++ web3/_utils/module_testing/eth_module.py | 40 ++++++++++++ web3/middleware/fixture.py | 29 ++++++++- 5 files changed, 149 insertions(+), 17 deletions(-) create mode 100644 tests/core/utilities/test_fee_utils.py diff --git a/tests/core/contracts/test_contract_buildTransaction.py b/tests/core/contracts/test_contract_buildTransaction.py index 637c4ddb84..85a8ffe324 100644 --- a/tests/core/contracts/test_contract_buildTransaction.py +++ b/tests/core/contracts/test_contract_buildTransaction.py @@ -54,8 +54,8 @@ def test_build_transaction_not_paying_to_nonpayable_function( 'to': payable_tester_contract.address, 'data': '0xe4cb8f5c', 'value': 0, - 'maxFeePerGas': 3750000000, - 'maxPriorityFeePerGas': 2 * (10 ** 9), + 'maxFeePerGas': 2750000000, + 'maxPriorityFeePerGas': 10 ** 9, 'chainId': 61, } @@ -76,8 +76,8 @@ def test_build_transaction_with_contract_no_arguments(web3, math_contract, build 'to': math_contract.address, 'data': '0xd09de08a', 'value': 0, - 'maxFeePerGas': 3750000000, - 'maxPriorityFeePerGas': 2 * (10 ** 9), + 'maxFeePerGas': 2750000000, + 'maxPriorityFeePerGas': 10 ** 9, 'chainId': 61, } @@ -88,8 +88,8 @@ def test_build_transaction_with_contract_fallback_function(web3, fallback_functi 'to': fallback_function_contract.address, 'data': '0x', 'value': 0, - 'maxFeePerGas': 3750000000, - 'maxPriorityFeePerGas': 2 * (10 ** 9), + 'maxFeePerGas': 2750000000, + 'maxPriorityFeePerGas': 10 ** 9, 'chainId': 61, } @@ -108,8 +108,8 @@ def test_build_transaction_with_contract_class_method( 'to': math_contract.address, 'data': '0xd09de08a', 'value': 0, - 'maxFeePerGas': 3750000000, - 'maxPriorityFeePerGas': 2 * (10 ** 9), + 'maxFeePerGas': 2750000000, + 'maxPriorityFeePerGas': 10 ** 9, 'chainId': 61, } @@ -123,8 +123,8 @@ def test_build_transaction_with_contract_default_account_is_set( 'to': math_contract.address, 'data': '0xd09de08a', 'value': 0, - 'maxFeePerGas': 3750000000, - 'maxPriorityFeePerGas': 2 * (10 ** 9), + 'maxFeePerGas': 2750000000, + 'maxPriorityFeePerGas': 10 ** 9, 'chainId': 61, } @@ -167,14 +167,14 @@ def test_build_transaction_with_contract_to_address_supplied_errors(web3, ( {}, (5,), {}, { 'data': '0x7cf5dab00000000000000000000000000000000000000000000000000000000000000005', # noqa: E501 - 'value': 0, 'maxFeePerGas': 3750000000, 'maxPriorityFeePerGas': 2 * (10 ** 9), + 'value': 0, 'maxFeePerGas': 2750000000, 'maxPriorityFeePerGas': 1000000000, 'chainId': 61, }, False ), ( {'gas': 800000}, (5,), {}, { 'data': '0x7cf5dab00000000000000000000000000000000000000000000000000000000000000005', # noqa: E501 - 'value': 0, 'maxFeePerGas': 3750000000, 'maxPriorityFeePerGas': 2 * (10 ** 9), + 'value': 0, 'maxFeePerGas': 2750000000, 'maxPriorityFeePerGas': 1000000000, 'chainId': 61, }, False ), @@ -194,14 +194,14 @@ def test_build_transaction_with_contract_to_address_supplied_errors(web3, ( {'nonce': 7}, (5,), {}, { 'data': '0x7cf5dab00000000000000000000000000000000000000000000000000000000000000005', # noqa: E501 - 'value': 0, 'maxFeePerGas': 3750000000, 'maxPriorityFeePerGas': 2 * (10 ** 9), + 'value': 0, 'maxFeePerGas': 2750000000, 'maxPriorityFeePerGas': 1000000000, 'nonce': 7, 'chainId': 61, }, True ), ( {'value': 20000}, (5,), {}, { 'data': '0x7cf5dab00000000000000000000000000000000000000000000000000000000000000005', # noqa: E501 - 'value': 20000, 'maxFeePerGas': 3750000000, 'maxPriorityFeePerGas': 2 * (10 ** 9), + 'value': 20000, 'maxFeePerGas': 2750000000, 'maxPriorityFeePerGas': 1000000000, 'chainId': 61, }, False ), diff --git a/tests/core/utilities/test_fee_utils.py b/tests/core/utilities/test_fee_utils.py new file mode 100644 index 0000000000..106b247a24 --- /dev/null +++ b/tests/core/utilities/test_fee_utils.py @@ -0,0 +1,65 @@ +import pytest + +from eth_utils import ( + is_integer, +) + +from web3.middleware import ( + construct_error_generator_middleware, + construct_result_generator_middleware, +) +from web3.types import ( + RPCEndpoint, +) + + +@pytest.mark.parametrize( + 'fee_history_rewards,expected_max_prio_calc', + ( + ( + [[10 ** 20], [10 ** 20], [10 ** 20], [10 ** 20]], + 15 * (10 ** 8), + ), + ( + [[10 ** 2], [10 ** 2], [10 ** 2], [10 ** 2], [10 ** 2]], + 10 ** 9, + ), + ( + [[0], [0], [0], [0], [0]], + 10 ** 9, + ), + ( + [[1223344455], [1111111111], [1222777777], [0], [1000222111], [0], [0]], + round(sum([1223344455, 1111111111, 1222777777, 1000222111]) / 4), + ), + ), + ids=[ + 'test-max', 'test-min', 'test-min-all-zero-fees', 'test-non-zero-average' + ] +) +# Test fee_utils indirectly by mocking eth_feeHistory results and checking against expected output +def test_fee_utils_indirectly( + web3, fee_history_rewards, expected_max_prio_calc +) -> None: + fail_max_prio_middleware = construct_error_generator_middleware( + {RPCEndpoint("eth_maxPriorityFeePerGas"): lambda *_: ''} + ) + fee_history_result_middleware = construct_result_generator_middleware( + {RPCEndpoint('eth_feeHistory'): lambda *_: {'reward': fee_history_rewards}} + ) + + web3.middleware_onion.add(fail_max_prio_middleware, 'fail_max_prio') + web3.middleware_onion.inject(fee_history_result_middleware, 'fee_history_result', layer=0) + + with pytest.warns( + UserWarning, + match="There was an issue with the method eth_maxPriorityFeePerGas. Calculating using " + "eth_feeHistory." + ): + max_priority_fee = web3.eth.max_priority_fee + assert is_integer(max_priority_fee) + assert max_priority_fee == expected_max_prio_calc + + # clean up + web3.middleware_onion.remove('fail_max_prio') + web3.middleware_onion.remove('fee_history_result') diff --git a/tests/integration/test_ethereum_tester.py b/tests/integration/test_ethereum_tester.py index 42dc4ab98c..5d9297e723 100644 --- a/tests/integration/test_ethereum_tester.py +++ b/tests/integration/test_ethereum_tester.py @@ -239,6 +239,10 @@ def func_wrapper(self, eth_tester, *args, **kwargs): class TestEthereumTesterEthModule(EthModuleTest): + test_eth_max_priority_fee_with_fee_history_calculation = not_implemented( + EthModuleTest.test_eth_max_priority_fee_with_fee_history_calculation, + ValueError + ) test_eth_sign = not_implemented(EthModuleTest.test_eth_sign, ValueError) test_eth_sign_ens_names = not_implemented( EthModuleTest.test_eth_sign_ens_names, ValueError diff --git a/web3/_utils/module_testing/eth_module.py b/web3/_utils/module_testing/eth_module.py index 80cd38ef5a..7097d71f03 100644 --- a/web3/_utils/module_testing/eth_module.py +++ b/web3/_utils/module_testing/eth_module.py @@ -54,11 +54,16 @@ TransactionNotFound, TransactionTypeMismatch, ) +from web3.middleware.fixture import ( + async_construct_error_generator_middleware, + construct_error_generator_middleware, +) from web3.types import ( # noqa: F401 BlockData, FilterParams, LogReceipt, Nonce, + RPCEndpoint, SyncStatus, TxParams, Wei, @@ -423,6 +428,25 @@ async def test_eth_max_priority_fee(self, async_w3: "Web3") -> None: max_priority_fee = await async_w3.eth.max_priority_fee # type: ignore assert is_integer(max_priority_fee) + @pytest.mark.asyncio + async def test_eth_max_priority_fee_with_fee_history_calculation( + self, async_w3: "Web3" + ) -> None: + fail_max_prio_middleware = await async_construct_error_generator_middleware( + {RPCEndpoint("eth_maxPriorityFeePerGas"): lambda *_: ''} + ) + async_w3.middleware_onion.add(fail_max_prio_middleware, name='fail_max_prio_middleware') + + with pytest.warns( + UserWarning, + match="There was an issue with the method eth_maxPriorityFeePerGas. Calculating using " + "eth_feeHistory." + ): + max_priority_fee = await async_w3.eth.max_priority_fee # type: ignore + assert is_integer(max_priority_fee) + + async_w3.middleware_onion.remove('fail_max_prio_middleware') # clean up + @pytest.mark.asyncio async def test_eth_getBlockByHash( self, async_w3: "Web3", empty_block: BlockData @@ -1136,6 +1160,22 @@ def test_eth_max_priority_fee(self, web3: "Web3") -> None: max_priority_fee = web3.eth.max_priority_fee assert is_integer(max_priority_fee) + def test_eth_max_priority_fee_with_fee_history_calculation(self, web3: "Web3") -> None: + fail_max_prio_middleware = construct_error_generator_middleware( + {RPCEndpoint("eth_maxPriorityFeePerGas"): lambda *_: ''} + ) + web3.middleware_onion.add(fail_max_prio_middleware, name='fail_max_prio_middleware') + + with pytest.warns( + UserWarning, + match="There was an issue with the method eth_maxPriorityFeePerGas. Calculating using " + "eth_feeHistory." + ): + max_priority_fee = web3.eth.max_priority_fee + assert is_integer(max_priority_fee) + + web3.middleware_onion.remove('fail_max_prio_middleware') # clean up + def test_eth_accounts(self, web3: "Web3") -> None: accounts = web3.eth.accounts assert is_list_like(accounts) diff --git a/web3/middleware/fixture.py b/web3/middleware/fixture.py index b54d0902a8..1565ab50a7 100644 --- a/web3/middleware/fixture.py +++ b/web3/middleware/fixture.py @@ -2,6 +2,7 @@ TYPE_CHECKING, Any, Callable, + Coroutine, Dict, ) @@ -21,7 +22,7 @@ def construct_fixture_middleware(fixtures: Dict[RPCEndpoint, Any]) -> Middleware which is found in the provided fixtures. """ def fixture_middleware( - make_request: Callable[[RPCEndpoint, Any], Any], web3: "Web3" + make_request: Callable[[RPCEndpoint, Any], Any], _: "Web3" ) -> Callable[[RPCEndpoint, Any], RPCResponse]: def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: if method in fixtures: @@ -43,7 +44,7 @@ def construct_result_generator_middleware( functions with the signature `fn(method, params)`. """ def result_generator_middleware( - make_request: Callable[[RPCEndpoint, Any], Any], web3: "Web3" + make_request: Callable[[RPCEndpoint, Any], Any], _: "Web3" ) -> Callable[[RPCEndpoint, Any], RPCResponse]: def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: if method in result_generators: @@ -65,7 +66,7 @@ def construct_error_generator_middleware( functions with the signature `fn(method, params)`. """ def error_generator_middleware( - make_request: Callable[[RPCEndpoint, Any], Any], web3: "Web3" + make_request: Callable[[RPCEndpoint, Any], Any], _: "Web3" ) -> Callable[[RPCEndpoint, Any], RPCResponse]: def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: if method in error_generators: @@ -75,3 +76,25 @@ def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: return make_request(method, params) return middleware return error_generator_middleware + + +async def async_construct_error_generator_middleware( + error_generators: Dict[RPCEndpoint, Any] +) -> Middleware: + """ + Constructs a middleware which intercepts requests for any method found in + the provided mapping of endpoints to generator functions, returning + whatever error message the generator function returns. Callbacks must be + functions with the signature `fn(method, params)`. + """ + async def error_generator_middleware( + make_request: Callable[[RPCEndpoint, Any], Any], _: "Web3" + ) -> Callable[[RPCEndpoint, Any], Coroutine[Any, Any, RPCResponse]]: + async def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: + if method in error_generators: + error_msg = error_generators[method](method, params) + return {'error': error_msg} + else: + return await make_request(method, params) + return middleware + return error_generator_middleware From 99b21f463683326ac314a2a2a09617c3e335fe31 Mon Sep 17 00:00:00 2001 From: pacrob Date: Wed, 2 Feb 2022 13:38:09 -0700 Subject: [PATCH 23/48] asyncify eth.syncing --- docs/providers.rst | 1 + newsfragments/2331.feature.rst | 1 + web3/_utils/module_testing/eth_module.py | 18 ++++++++++++++++++ web3/eth.py | 16 ++++++++++------ 4 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 newsfragments/2331.feature.rst diff --git a/docs/providers.rst b/docs/providers.rst index 736e748f0d..e966d83984 100644 --- a/docs/providers.rst +++ b/docs/providers.rst @@ -415,6 +415,7 @@ Eth - :meth:`web3.eth.hashrate ` - :meth:`web3.eth.max_priority_fee ` - :meth:`web3.eth.mining ` +- :meth:`web3.eth.syncing ` - :meth:`web3.eth.call() ` - :meth:`web3.eth.estimate_gas() ` - :meth:`web3.eth.generate_gas_price() ` diff --git a/newsfragments/2331.feature.rst b/newsfragments/2331.feature.rst new file mode 100644 index 0000000000..8e69fd1ee0 --- /dev/null +++ b/newsfragments/2331.feature.rst @@ -0,0 +1 @@ +Add async `eth.syncing` method diff --git a/web3/_utils/module_testing/eth_module.py b/web3/_utils/module_testing/eth_module.py index 7097d71f03..887201fa39 100644 --- a/web3/_utils/module_testing/eth_module.py +++ b/web3/_utils/module_testing/eth_module.py @@ -1025,6 +1025,24 @@ async def test_async_eth_get_logs_with_logs_none_topic_args(self, async_w3: "Web result = await async_w3.eth.get_logs(filter_params) # type: ignore assert len(result) == 0 + @pytest.mark.asyncio + async def test_async_eth_syncing(self, async_w3: "Web3") -> None: + syncing = await async_w3.eth.syncing # type: ignore + + assert is_boolean(syncing) or is_dict(syncing) + + if is_boolean(syncing): + assert syncing is False + elif is_dict(syncing): + sync_dict = cast(SyncStatus, syncing) + assert 'startingBlock' in sync_dict + assert 'currentBlock' in sync_dict + assert 'highestBlock' in sync_dict + + assert is_integer(sync_dict['startingBlock']) + assert is_integer(sync_dict['currentBlock']) + assert is_integer(sync_dict['highestBlock']) + def test_async_provider_default_account( self, async_w3: "Web3", diff --git a/web3/eth.py b/web3/eth.py index 8edd74f9da..1e644609e6 100644 --- a/web3/eth.py +++ b/web3/eth.py @@ -322,6 +322,11 @@ def call_munger( mungers=None, ) + _is_syncing: Method[Callable[[], Union[SyncStatus, bool]]] = Method( + RPC.eth_syncing, + mungers=None, + ) + _get_transaction_receipt: Method[Callable[[_Hash32], TxReceipt]] = Method( RPC.eth_getTransactionReceipt, mungers=[default_root_munger] @@ -377,6 +382,10 @@ async def max_priority_fee(self) -> Wei: async def mining(self) -> bool: return await self._is_mining() # type: ignore + @property + async def syncing(self) -> Union[SyncStatus, bool]: + return await self._is_syncing() # type: ignore + async def fee_history( self, block_count: int, @@ -551,14 +560,9 @@ def protocolVersion(self) -> str: ) return self.protocol_version - is_syncing: Method[Callable[[], Union[SyncStatus, bool]]] = Method( - RPC.eth_syncing, - mungers=None, - ) - @property def syncing(self) -> Union[SyncStatus, bool]: - return self.is_syncing() + return self._is_syncing() @property def coinbase(self) -> ChecksumAddress: From b915dcfc3528bdc196c681b02333b88e02480aa2 Mon Sep 17 00:00:00 2001 From: Felipe Selmo Date: Fri, 16 Jul 2021 13:04:25 -0600 Subject: [PATCH 24/48] formatting and validation middleware async support --- newsfragments/2098.feature.rst | 1 + .../go_ethereum/test_goethereum_http.py | 4 +- web3/_utils/module_testing/eth_module.py | 69 +++++++++ web3/manager.py | 6 +- web3/middleware/__init__.py | 2 + web3/middleware/buffered_gas_estimate.py | 4 +- web3/middleware/fixture.py | 26 +++- web3/middleware/formatting.py | 136 ++++++++++++------ web3/middleware/gas_price_strategy.py | 4 +- web3/middleware/geth_poa.py | 27 ++++ web3/middleware/validation.py | 86 +++++++---- web3/types.py | 8 +- 12 files changed, 291 insertions(+), 82 deletions(-) create mode 100644 newsfragments/2098.feature.rst diff --git a/newsfragments/2098.feature.rst b/newsfragments/2098.feature.rst new file mode 100644 index 0000000000..b1dffae787 --- /dev/null +++ b/newsfragments/2098.feature.rst @@ -0,0 +1 @@ +async support for formatting, validation, and geth poa middlewares \ No newline at end of file diff --git a/tests/integration/go_ethereum/test_goethereum_http.py b/tests/integration/go_ethereum/test_goethereum_http.py index 90e0b7313e..c4c4db63a4 100644 --- a/tests/integration/go_ethereum/test_goethereum_http.py +++ b/tests/integration/go_ethereum/test_goethereum_http.py @@ -18,6 +18,7 @@ from web3.middleware import ( async_buffered_gas_estimate_middleware, async_gas_price_strategy_middleware, + async_validation_middleware, ) from web3.net import ( AsyncNet, @@ -95,8 +96,9 @@ async def async_w3(geth_process, endpoint_uri): _web3 = Web3( AsyncHTTPProvider(endpoint_uri), middlewares=[ + async_buffered_gas_estimate_middleware, async_gas_price_strategy_middleware, - async_buffered_gas_estimate_middleware + async_validation_middleware, ], modules={'eth': AsyncEth, 'async_net': AsyncNet, diff --git a/web3/_utils/module_testing/eth_module.py b/web3/_utils/module_testing/eth_module.py index 887201fa39..6d238b1f63 100644 --- a/web3/_utils/module_testing/eth_module.py +++ b/web3/_utils/module_testing/eth_module.py @@ -53,9 +53,14 @@ TimeExhausted, TransactionNotFound, TransactionTypeMismatch, + ValidationError, +) +from web3.middleware import ( + async_geth_poa_middleware, ) from web3.middleware.fixture import ( async_construct_error_generator_middleware, + async_construct_result_generator_middleware, construct_error_generator_middleware, ) from web3.types import ( # noqa: F401 @@ -290,6 +295,47 @@ async def test_eth_send_transaction_max_fee_less_than_tip( ): await async_w3.eth.send_transaction(txn_params) # type: ignore + @pytest.mark.asyncio + async def test_validation_middleware_chain_id_mismatch( + self, async_w3: "Web3", unlocked_account_dual_type: ChecksumAddress + ) -> None: + wrong_chain_id = 1234567890 + actual_chain_id = await async_w3.eth.chain_id # type: ignore + + txn_params: TxParams = { + 'from': unlocked_account_dual_type, + 'to': unlocked_account_dual_type, + 'value': Wei(1), + 'gas': Wei(21000), + 'maxFeePerGas': async_w3.toWei(2, 'gwei'), + 'maxPriorityFeePerGas': async_w3.toWei(1, 'gwei'), + 'chainId': wrong_chain_id, + + } + with pytest.raises( + ValidationError, + match=f'The transaction declared chain ID {wrong_chain_id}, ' + f'but the connected node is on {actual_chain_id}' + ): + await async_w3.eth.send_transaction(txn_params) # type: ignore + + @pytest.mark.asyncio + async def test_geth_poa_middleware(self, async_w3: "Web3") -> None: + return_block_with_long_extra_data = await async_construct_result_generator_middleware( + { + RPCEndpoint('eth_getBlockByNumber'): lambda *_: {'extraData': '0x' + 'ff' * 33}, + } + ) + async_w3.middleware_onion.inject(async_geth_poa_middleware, 'poa', layer=0) + async_w3.middleware_onion.inject(return_block_with_long_extra_data, 'extradata', layer=0) + block = await async_w3.eth.get_block('latest') # type: ignore + assert 'extraData' not in block + assert block.proofOfAuthorityData == b'\xff' * 33 + + # clean up + async_w3.middleware_onion.remove('poa') + async_w3.middleware_onion.remove('extradata') + @pytest.mark.asyncio async def test_eth_send_raw_transaction(self, async_w3: "Web3") -> None: # private key 0x3c2ab4e8f17a7dea191b8c991522660126d681039509dc3bb31af7c9bdb63518 @@ -1997,6 +2043,29 @@ def test_eth_send_transaction_max_fee_less_than_tip( ): web3.eth.send_transaction(txn_params) + def test_validation_middleware_chain_id_mismatch( + self, web3: "Web3", unlocked_account_dual_type: ChecksumAddress + ) -> None: + wrong_chain_id = 1234567890 + actual_chain_id = web3.eth.chain_id + + txn_params: TxParams = { + 'from': unlocked_account_dual_type, + 'to': unlocked_account_dual_type, + 'value': Wei(1), + 'gas': Wei(21000), + 'maxFeePerGas': web3.toWei(2, 'gwei'), + 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), + 'chainId': wrong_chain_id, + + } + with pytest.raises( + ValidationError, + match=f'The transaction declared chain ID {wrong_chain_id}, ' + f'but the connected node is on {actual_chain_id}' + ): + web3.eth.send_transaction(txn_params) + @pytest.mark.parametrize( "max_fee", (1000000000, None), diff --git a/web3/manager.py b/web3/manager.py index b731ee7a81..1cd4e1c241 100644 --- a/web3/manager.py +++ b/web3/manager.py @@ -128,11 +128,11 @@ def default_middlewares( """ return [ (request_parameter_normalizer, 'request_param_normalizer'), # Delete - (gas_price_strategy_middleware, 'gas_price_strategy'), # Add Async + (gas_price_strategy_middleware, 'gas_price_strategy'), (name_to_address_middleware(web3), 'name_to_address'), # Add Async (attrdict_middleware, 'attrdict'), # Delete (pythonic_middleware, 'pythonic'), # Delete - (validation_middleware, 'validation'), # Add async + (validation_middleware, 'validation'), (abi_middleware, 'abi'), # Delete (buffered_gas_estimate_middleware, 'gas_estimate'), ] @@ -159,8 +159,8 @@ async def _coro_make_request( self.logger.debug("Making request. Method: %s", method) return await request_func(method, params) + @staticmethod def formatted_response( - self, response: RPCResponse, params: Any, error_formatters: Optional[Callable[..., Any]] = None, diff --git a/web3/middleware/__init__.py b/web3/middleware/__init__.py index baad91b1b6..ea2f00c6ed 100644 --- a/web3/middleware/__init__.py +++ b/web3/middleware/__init__.py @@ -51,6 +51,7 @@ gas_price_strategy_middleware, ) from .geth_poa import ( # noqa: F401 + async_geth_poa_middleware, geth_poa_middleware, ) from .names import ( # noqa: F401 @@ -69,6 +70,7 @@ make_stalecheck_middleware, ) from .validation import ( # noqa: F401 + async_validation_middleware, validation_middleware, ) diff --git a/web3/middleware/buffered_gas_estimate.py b/web3/middleware/buffered_gas_estimate.py index 8f75f8ae86..194c4178e1 100644 --- a/web3/middleware/buffered_gas_estimate.py +++ b/web3/middleware/buffered_gas_estimate.py @@ -2,7 +2,6 @@ TYPE_CHECKING, Any, Callable, - Coroutine, ) from eth_utils.toolz import ( @@ -16,6 +15,7 @@ get_buffered_gas_estimate, ) from web3.types import ( + AsyncMiddleware, RPCEndpoint, RPCResponse, ) @@ -43,7 +43,7 @@ def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: async def async_buffered_gas_estimate_middleware( make_request: Callable[[RPCEndpoint, Any], Any], web3: "Web3" -) -> Callable[[RPCEndpoint, Any], Coroutine[Any, Any, RPCResponse]]: +) -> AsyncMiddleware: async def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: if method == 'eth_sendTransaction': transaction = params[0] diff --git a/web3/middleware/fixture.py b/web3/middleware/fixture.py index 1565ab50a7..cba6372f57 100644 --- a/web3/middleware/fixture.py +++ b/web3/middleware/fixture.py @@ -2,11 +2,11 @@ TYPE_CHECKING, Any, Callable, - Coroutine, Dict, ) from web3.types import ( + AsyncMiddleware, Middleware, RPCEndpoint, RPCResponse, @@ -78,6 +78,28 @@ def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: return error_generator_middleware +# --- async --- # + +async def async_construct_result_generator_middleware( + result_generators: Dict[RPCEndpoint, Any] +) -> Middleware: + """ + Constructs a middleware which returns a static response for any method + which is found in the provided fixtures. + """ + async def result_generator_middleware( + make_request: Callable[[RPCEndpoint, Any], Any], _: "Web3" + ) -> AsyncMiddleware: + async def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: + if method in result_generators: + result = result_generators[method](method, params) + return {'result': result} + else: + return await make_request(method, params) + return middleware + return result_generator_middleware + + async def async_construct_error_generator_middleware( error_generators: Dict[RPCEndpoint, Any] ) -> Middleware: @@ -89,7 +111,7 @@ async def async_construct_error_generator_middleware( """ async def error_generator_middleware( make_request: Callable[[RPCEndpoint, Any], Any], _: "Web3" - ) -> Callable[[RPCEndpoint, Any], Coroutine[Any, Any, RPCResponse]]: + ) -> AsyncMiddleware: async def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: if method in error_generators: error_msg = error_generators[method](method, params) diff --git a/web3/middleware/formatting.py b/web3/middleware/formatting.py index 3542583a9a..04602be0ae 100644 --- a/web3/middleware/formatting.py +++ b/web3/middleware/formatting.py @@ -2,18 +2,20 @@ TYPE_CHECKING, Any, Callable, + Coroutine, Optional, ) from eth_utils.toolz import ( assoc, - curry, merge, ) from web3.types import ( + AsyncMiddleware, Formatters, FormattersDict, + Literal, Middleware, RPCEndpoint, RPCResponse, @@ -22,6 +24,38 @@ if TYPE_CHECKING: from web3 import Web3 # noqa: F401 +FORMATTER_DEFAULTS: FormattersDict = { + "request_formatters": {}, + "result_formatters": {}, + "error_formatters": {}, +} + + +def _apply_response_formatters( + method: RPCEndpoint, + response: RPCResponse, + result_formatters: Formatters, + error_formatters: Formatters, +) -> RPCResponse: + + def _format_response( + response_type: Literal["result", "error"], + method_response_formatter: Callable[..., Any] + ) -> RPCResponse: + appropriate_response = response[response_type] + return assoc( + response, response_type, method_response_formatter(appropriate_response) + ) + + if "result" in response and method in result_formatters: + return _format_response("result", result_formatters[method]) + elif "error" in response and method in error_formatters: + return _format_response("error", error_formatters[method]) + else: + return response + + +# --- sync -- # def construct_formatting_middleware( request_formatters: Optional[Formatters] = None, @@ -29,7 +63,7 @@ def construct_formatting_middleware( error_formatters: Optional[Formatters] = None ) -> Middleware: def ignore_web3_in_standard_formatters( - w3: "Web3", + _w3: "Web3", _method: RPCEndpoint, ) -> FormattersDict: return dict( request_formatters=request_formatters or {}, @@ -41,55 +75,67 @@ def ignore_web3_in_standard_formatters( def construct_web3_formatting_middleware( - web3_formatters_builder: Callable[["Web3"], FormattersDict] + web3_formatters_builder: Callable[["Web3", RPCEndpoint], FormattersDict], ) -> Middleware: def formatter_middleware( - make_request: Callable[[RPCEndpoint, Any], Any], w3: "Web3" + make_request: Callable[[RPCEndpoint, Any], Any], + w3: "Web3", ) -> Callable[[RPCEndpoint, Any], RPCResponse]: - formatters = merge( - { - "request_formatters": {}, - "result_formatters": {}, - "error_formatters": {}, - }, - web3_formatters_builder(w3), - ) - return apply_formatters(make_request=make_request, **formatters) + def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: + formatters = merge( + FORMATTER_DEFAULTS, + web3_formatters_builder(w3, method), + ) + request_formatters = formatters.pop('request_formatters') + + if method in request_formatters: + formatter = request_formatters[method] + params = formatter(params) + response = make_request(method, params) + return _apply_response_formatters(method=method, response=response, **formatters) + return middleware return formatter_middleware -@curry -def apply_formatters( - method: RPCEndpoint, - params: Any, - make_request: Callable[[RPCEndpoint, Any], RPCResponse], - request_formatters: Formatters, - result_formatters: Formatters, - error_formatters: Formatters, -) -> RPCResponse: - if method in request_formatters: - formatter = request_formatters[method] - formatted_params = formatter(params) - response = make_request(method, formatted_params) - else: - response = make_request(method, params) +# --- async --- # - if "result" in response and method in result_formatters: - formatter = result_formatters[method] - formatted_response = assoc( - response, - "result", - formatter(response["result"]), - ) - return formatted_response - elif "error" in response and method in error_formatters: - formatter = error_formatters[method] - formatted_response = assoc( - response, - "error", - formatter(response["error"]), +async def async_construct_formatting_middleware( + request_formatters: Optional[Formatters] = None, + result_formatters: Optional[Formatters] = None, + error_formatters: Optional[Formatters] = None +) -> Middleware: + async def ignore_web3_in_standard_formatters( + _w3: "Web3", _method: RPCEndpoint, + ) -> FormattersDict: + return dict( + request_formatters=request_formatters or {}, + result_formatters=result_formatters or {}, + error_formatters=error_formatters or {}, ) - return formatted_response - else: - return response + return await async_construct_web3_formatting_middleware(ignore_web3_in_standard_formatters) + + +async def async_construct_web3_formatting_middleware( + async_web3_formatters_builder: + Callable[["Web3", RPCEndpoint], Coroutine[Any, Any, FormattersDict]] +) -> Callable[[Callable[[RPCEndpoint, Any], Any], "Web3"], Coroutine[Any, Any, AsyncMiddleware]]: + async def formatter_middleware( + make_request: Callable[[RPCEndpoint, Any], Any], + async_w3: "Web3", + ) -> AsyncMiddleware: + async def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: + formatters = merge( + FORMATTER_DEFAULTS, + await async_web3_formatters_builder(async_w3, method), + ) + request_formatters = formatters.pop('request_formatters') + + if method in request_formatters: + formatter = request_formatters[method] + params = formatter(params) + response = await make_request(method, params) + + return _apply_response_formatters(method=method, response=response, **formatters) + return middleware + return formatter_middleware diff --git a/web3/middleware/gas_price_strategy.py b/web3/middleware/gas_price_strategy.py index f0a7254fcc..9d3fcacc6d 100644 --- a/web3/middleware/gas_price_strategy.py +++ b/web3/middleware/gas_price_strategy.py @@ -2,7 +2,6 @@ TYPE_CHECKING, Any, Callable, - Coroutine, ) from eth_utils.toolz import ( @@ -22,6 +21,7 @@ TransactionTypeMismatch, ) from web3.types import ( + AsyncMiddleware, BlockData, RPCEndpoint, RPCResponse, @@ -94,7 +94,7 @@ def middleware(method: RPCEndpoint, params: Any) -> RPCResponse: async def async_gas_price_strategy_middleware( make_request: Callable[[RPCEndpoint, Any], Any], web3: "Web3" -) -> Callable[[RPCEndpoint, Any], Coroutine[Any, Any, RPCResponse]]: +) -> AsyncMiddleware: """ - Uses a gas price strategy if one is set. This is only supported for legacy transactions. It is recommended to send dynamic fee transactions (EIP-1559) whenever possible. diff --git a/web3/middleware/geth_poa.py b/web3/middleware/geth_poa.py index f941b69a01..6048cc70f0 100644 --- a/web3/middleware/geth_poa.py +++ b/web3/middleware/geth_poa.py @@ -1,3 +1,9 @@ +from typing import ( + TYPE_CHECKING, + Any, + Callable, +) + from eth_utils.curried import ( apply_formatter_if, apply_formatters_to_dict, @@ -16,8 +22,16 @@ RPC, ) from web3.middleware.formatting import ( + async_construct_formatting_middleware, construct_formatting_middleware, ) +from web3.types import ( + AsyncMiddleware, + RPCEndpoint, +) + +if TYPE_CHECKING: + from web3 import Web3 # noqa: F401 is_not_null = complement(is_null) @@ -31,9 +45,22 @@ geth_poa_cleanup = compose(pythonic_geth_poa, remap_geth_poa_fields) + geth_poa_middleware = construct_formatting_middleware( result_formatters={ RPC.eth_getBlockByHash: apply_formatter_if(is_not_null, geth_poa_cleanup), RPC.eth_getBlockByNumber: apply_formatter_if(is_not_null, geth_poa_cleanup), }, ) + + +async def async_geth_poa_middleware( + make_request: Callable[[RPCEndpoint, Any], Any], web3: "Web3" +) -> AsyncMiddleware: + middleware = await async_construct_formatting_middleware( + result_formatters={ + RPC.eth_getBlockByHash: apply_formatter_if(is_not_null, geth_poa_cleanup), + RPC.eth_getBlockByNumber: apply_formatter_if(is_not_null, geth_poa_cleanup), + }, + ) + return await middleware(make_request, web3) diff --git a/web3/middleware/validation.py b/web3/middleware/validation.py index 2e956655f4..44d79825de 100644 --- a/web3/middleware/validation.py +++ b/web3/middleware/validation.py @@ -2,6 +2,7 @@ TYPE_CHECKING, Any, Callable, + Dict, ) from eth_utils.curried import ( @@ -32,10 +33,14 @@ ValidationError, ) from web3.middleware.formatting import ( + async_construct_web3_formatting_middleware, construct_web3_formatting_middleware, ) from web3.types import ( + AsyncMiddleware, + Formatters, FormattersDict, + RPCEndpoint, TxParams, ) @@ -45,25 +50,24 @@ MAX_EXTRADATA_LENGTH = 32 is_not_null = complement(is_null) - to_integer_if_hex = apply_formatter_if(is_string, hex_to_integer) @curry -def validate_chain_id(web3: "Web3", chain_id: int) -> int: - if to_integer_if_hex(chain_id) == web3.eth.chain_id: +def _validate_chain_id(web3_chain_id: int, chain_id: int) -> int: + if to_integer_if_hex(chain_id) == web3_chain_id: return chain_id else: raise ValidationError( "The transaction declared chain ID %r, " "but the connected node is on %r" % ( chain_id, - web3.eth.chain_id, + web3_chain_id, ) ) -def check_extradata_length(val: Any) -> Any: +def _check_extradata_length(val: Any) -> Any: if not isinstance(val, (str, int, bytes)): return val result = HexBytes(val) @@ -80,16 +84,16 @@ def check_extradata_length(val: Any) -> Any: return val -def transaction_normalizer(transaction: TxParams) -> TxParams: +def _transaction_normalizer(transaction: TxParams) -> TxParams: return dissoc(transaction, 'chainId') -def transaction_param_validator(web3: "Web3") -> Callable[..., Any]: +def _transaction_param_validator(web3_chain_id: int) -> Callable[..., Any]: transactions_params_validators = { "chainId": apply_formatter_if( # Bypass `validate_chain_id` if chainId can't be determined - lambda _: is_not_null(web3.eth.chain_id), - validate_chain_id(web3), + lambda _: is_not_null(web3_chain_id), + _validate_chain_id(web3_chain_id), ), } return apply_formatter_at_index( @@ -99,36 +103,70 @@ def transaction_param_validator(web3: "Web3") -> Callable[..., Any]: BLOCK_VALIDATORS = { - 'extraData': check_extradata_length, + 'extraData': _check_extradata_length, } - - block_validator = apply_formatter_if( is_not_null, apply_formatters_to_dict(BLOCK_VALIDATORS) ) +METHODS_TO_VALIDATE = [ + RPC.eth_sendTransaction, + RPC.eth_estimateGas, + RPC.eth_call +] -@curry -def chain_id_validator(web3: "Web3") -> Callable[..., Any]: + +def _chain_id_validator(web3_chain_id: int) -> Callable[..., Any]: return compose( - apply_formatter_at_index(transaction_normalizer, 0), - transaction_param_validator(web3) + apply_formatter_at_index(_transaction_normalizer, 0), + _transaction_param_validator(web3_chain_id) ) -def build_validators_with_web3(w3: "Web3") -> FormattersDict: +def _build_formatters_dict(request_formatters: Dict[RPCEndpoint, Any]) -> FormattersDict: return dict( - request_formatters={ - RPC.eth_sendTransaction: chain_id_validator(w3), - RPC.eth_estimateGas: chain_id_validator(w3), - RPC.eth_call: chain_id_validator(w3), - }, + request_formatters=request_formatters, result_formatters={ RPC.eth_getBlockByHash: block_validator, RPC.eth_getBlockByNumber: block_validator, - }, + } ) +# -- sync -- # + + +def build_method_validators(w3: "Web3", method: RPCEndpoint) -> FormattersDict: + request_formatters = {} + if RPCEndpoint(method) in METHODS_TO_VALIDATE: + w3_chain_id = w3.eth.chain_id + for method in METHODS_TO_VALIDATE: + request_formatters[method] = _chain_id_validator(w3_chain_id) + + return _build_formatters_dict(request_formatters) + -validation_middleware = construct_web3_formatting_middleware(build_validators_with_web3) +validation_middleware = construct_web3_formatting_middleware( + build_method_validators +) + + +# -- async --- # + +async def async_build_method_validators(async_w3: "Web3", method: RPCEndpoint) -> FormattersDict: + request_formatters: Formatters = {} + if RPCEndpoint(method) in METHODS_TO_VALIDATE: + w3_chain_id = await async_w3.eth.chain_id # type: ignore + for method in METHODS_TO_VALIDATE: + request_formatters[method] = _chain_id_validator(w3_chain_id) + + return _build_formatters_dict(request_formatters) + + +async def async_validation_middleware( + make_request: Callable[[RPCEndpoint, Any], Any], web3: "Web3" +) -> AsyncMiddleware: + middleware = await async_construct_web3_formatting_middleware( + async_build_method_validators + ) + return await middleware(make_request, web3) diff --git a/web3/types.py b/web3/types.py index f475fc14d3..fa9ddac284 100644 --- a/web3/types.py +++ b/web3/types.py @@ -2,6 +2,7 @@ TYPE_CHECKING, Any, Callable, + Coroutine, Dict, List, NewType, @@ -135,13 +136,14 @@ class RPCResponse(TypedDict, total=False): Middleware = Callable[[Callable[[RPCEndpoint, Any], RPCResponse], "Web3"], Any] +AsyncMiddleware = Callable[[RPCEndpoint, Any], Coroutine[Any, Any, RPCResponse]] MiddlewareOnion = NamedElementOnion[str, Middleware] class FormattersDict(TypedDict, total=False): - error_formatters: Formatters - request_formatters: Formatters - result_formatters: Formatters + error_formatters: Optional[Formatters] + request_formatters: Optional[Formatters] + result_formatters: Optional[Formatters] class FilterParams(TypedDict, total=False): From 36adb16c68f570c343d01ecc8d0096cbac814172 Mon Sep 17 00:00:00 2001 From: Felipe Selmo Date: Thu, 3 Feb 2022 16:23:29 -0700 Subject: [PATCH 25/48] Properly test unused code in test --- web3/_utils/module_testing/eth_module.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web3/_utils/module_testing/eth_module.py b/web3/_utils/module_testing/eth_module.py index 6d238b1f63..1b88175b6f 100644 --- a/web3/_utils/module_testing/eth_module.py +++ b/web3/_utils/module_testing/eth_module.py @@ -1018,6 +1018,13 @@ async def test_async_eth_get_logs_with_logs( "fromBlock": BlockNumber(0), "address": emitter_contract_address, } + result = await async_w3.eth.get_logs(filter_params) # type: ignore + _assert_contains_log( + result, + block_with_txn_with_log, + emitter_contract_address, + txn_hash_with_log + ) @pytest.mark.asyncio async def test_async_eth_get_logs_with_logs_topic_args( From 32564fc7e9a24343716d959d5dc7720f49a99850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C4=83lina=20Cenan?= Date: Sat, 5 Feb 2022 00:24:36 +0200 Subject: [PATCH 26/48] Align NamedTuples (#2312) * Align NamedTuples * Add NamedTuple alignment test. * Add newsfragment for NamedTuple change Co-authored-by: kclowes --- newsfragments/2312.feature.rst | 1 + tests/core/utilities/test_abi.py | 21 +++++++++++++++++++++ web3/_utils/abi.py | 5 ++++- 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 newsfragments/2312.feature.rst diff --git a/newsfragments/2312.feature.rst b/newsfragments/2312.feature.rst new file mode 100644 index 0000000000..902b51b134 --- /dev/null +++ b/newsfragments/2312.feature.rst @@ -0,0 +1 @@ +Allow NamedTuples in ABI inputs diff --git a/tests/core/utilities/test_abi.py b/tests/core/utilities/test_abi.py index 571585006f..b861ee83ab 100644 --- a/tests/core/utilities/test_abi.py +++ b/tests/core/utilities/test_abi.py @@ -1,5 +1,8 @@ import json import pytest +from typing import ( + NamedTuple, +) from web3._utils.abi import ( abi_data_tree, @@ -41,6 +44,15 @@ def test_get_tuple_type_str_parts(input, expected): assert get_tuple_type_str_parts(input) == expected +MyXYTuple = NamedTuple( + "MyXYTuple", + [ + ("x", int), + ("y", int), + ] +) + + TEST_FUNCTION_ABI_JSON = """ { "constant": false, @@ -165,6 +177,15 @@ def test_get_tuple_type_str_parts(input, expected): ), GET_ABI_INPUTS_OUTPUT, ), + ( + TEST_FUNCTION_ABI, + { + 's': {'a': 1, 'b': [2, 3, 4], 'c': [(5, 6), (7, 8), MyXYTuple(x=9, y=10)]}, + 't': MyXYTuple(x=11, y=12), + 'a': 13, + }, + GET_ABI_INPUTS_OUTPUT, + ), ( {}, (), diff --git a/web3/_utils/abi.py b/web3/_utils/abi.py index 28f1d0c72a..698f4151cd 100644 --- a/web3/_utils/abi.py +++ b/web3/_utils/abi.py @@ -551,7 +551,10 @@ def _align_abi_input(arg_abi: ABIFunctionParams, arg: Any) -> Tuple[Any, ...]: ), ) - return type(aligned_arg)( + # convert NamedTuple to regular tuple + typing = tuple if isinstance(aligned_arg, tuple) else type(aligned_arg) + + return typing( _align_abi_input(sub_abi, sub_arg) for sub_abi, sub_arg in zip(sub_abis, aligned_arg) ) From 0cf51a81dfe4d721e0db5440c4e37b335eb49934 Mon Sep 17 00:00:00 2001 From: Harmouch101 Date: Tue, 18 May 2021 00:49:36 +0300 Subject: [PATCH 27/48] rm ens.utils.dict_copy Signed-off-by: Harmouch101 --- ens/main.py | 47 +++++++++++++++++++++++++++++++++-------------- ens/utils.py | 16 ---------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/ens/main.py b/ens/main.py index 4d62e35690..27388a631e 100644 --- a/ens/main.py +++ b/ens/main.py @@ -1,6 +1,8 @@ from functools import ( wraps, ) +from copy import deepcopy + from typing import ( TYPE_CHECKING, Optional, @@ -39,7 +41,6 @@ address_in, address_to_reverse_domain, default, - dict_copy, init_web3, is_none_or_zero_address, is_valid_name, @@ -149,12 +150,11 @@ def name(self, address: ChecksumAddress) -> Optional[str]: reversed_domain = address_to_reverse_domain(address) return self.resolve(reversed_domain, get='name') - @dict_copy def setup_address( self, name: str, address: Union[Address, ChecksumAddress, HexAddress] = cast(ChecksumAddress, default), - transact: "TxParams" = {} + transact: Optional["TxParams"] = None ) -> HexBytes: """ Set up the name to point to the supplied address. @@ -173,6 +173,9 @@ def setup_address( :raises InvalidName: if ``name`` has invalid syntax :raises UnauthorizedError: if ``'from'`` in `transact` does not own `name` """ + if not transact: + transact = {} + transact = deepcopy(transact) owner = self.setup_owner(name, transact=transact) self._assert_control(owner, name) if is_none_or_zero_address(address): @@ -191,9 +194,11 @@ def setup_address( resolver: 'Contract' = self._set_resolver(name, transact=transact) return resolver.functions.setAddr(raw_name_to_hash(name), address).transact(transact) - @dict_copy def setup_name( - self, name: str, address: ChecksumAddress = None, transact: "TxParams" = {} + self, + name: str, + address: Optional[ChecksumAddress] = None, + transact: Optional["TxParams"] = None ) -> HexBytes: """ Set up the address for reverse lookup, aka "caller ID". @@ -209,6 +214,9 @@ def setup_name( :raises UnauthorizedError: if ``'from'`` in `transact` does not own `name` :raises UnownedName: if no one owns `name` """ + if not transact: + transact = {} + transact = deepcopy(transact) if not name: self._assert_control(address, 'the reverse record') return self._setup_reverse(None, address, transact=transact) @@ -273,12 +281,11 @@ def owner(self, name: str) -> ChecksumAddress: node = raw_name_to_hash(name) return self.ens.caller.owner(node) - @dict_copy def setup_owner( self, name: str, new_owner: ChecksumAddress = cast(ChecksumAddress, default), - transact: "TxParams" = {} + transact: Optional["TxParams"] = None ) -> ChecksumAddress: """ Set the owner of the supplied name to `new_owner`. @@ -303,6 +310,9 @@ def setup_owner( :raises UnauthorizedError: if ``'from'`` in `transact` does not own `name` :returns: the new owner's address """ + if not transact: + transact = {} + transact = deepcopy(transact) (super_owner, unowned, owned) = self._first_owner(name) if new_owner is default: new_owner = super_owner @@ -345,15 +355,17 @@ def _first_owner(self, name: str) -> Tuple[Optional[ChecksumAddress], Sequence[s unowned.append(pieces.pop(0)) return (owner, unowned, name) - @dict_copy def _claim_ownership( self, owner: ChecksumAddress, unowned: Sequence[str], owned: str, - old_owner: ChecksumAddress = None, - transact: "TxParams" = {} + old_owner: Optional[ChecksumAddress] = None, + transact: Optional["TxParams"] = None ) -> None: + if not transact: + transact = {} + transact = deepcopy(transact) transact['from'] = old_owner or owner for label in reversed(unowned): self.ens.functions.setSubnodeOwner( @@ -363,10 +375,15 @@ def _claim_ownership( ).transact(transact) owned = "%s.%s" % (label, owned) - @dict_copy def _set_resolver( - self, name: str, resolver_addr: ChecksumAddress = None, transact: "TxParams" = {} + self, + name: str, + resolver_addr: Optional[ChecksumAddress] = None, + transact: Optional["TxParams"] = None ) -> 'Contract': + if not transact: + transact = {} + transact = deepcopy(transact) if is_none_or_zero_address(resolver_addr): resolver_addr = self.address('resolver.eth') namehash = raw_name_to_hash(name) @@ -377,10 +394,12 @@ def _set_resolver( ).transact(transact) return self._resolverContract(address=resolver_addr) - @dict_copy def _setup_reverse( - self, name: str, address: ChecksumAddress, transact: "TxParams" = {} + self, name: str, address: ChecksumAddress, transact: Optional["TxParams"] = None ) -> HexBytes: + if not transact: + transact = {} + transact = deepcopy(transact) if name: name = normalize_name(name) else: diff --git a/ens/utils.py b/ens/utils.py index bef9c31914..43c223b5e2 100644 --- a/ens/utils.py +++ b/ens/utils.py @@ -1,16 +1,12 @@ -import copy import datetime -import functools from typing import ( TYPE_CHECKING, Any, - Callable, Collection, Optional, Sequence, Tuple, Type, - TypeVar, Union, cast, ) @@ -61,18 +57,6 @@ def Web3() -> Type['_Web3']: return Web3Main -TFunc = TypeVar("TFunc", bound=Callable[..., Any]) - - -def dict_copy(func: TFunc) -> TFunc: - "copy dict keyword args, to avoid modifying caller's copy" - @functools.wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> TFunc: - copied_kwargs = copy.deepcopy(kwargs) - return func(*args, **copied_kwargs) - return cast(TFunc, wrapper) - - def init_web3( provider: 'BaseProvider' = cast('BaseProvider', default), middlewares: Optional[Sequence[Tuple['Middleware', str]]] = None, From 6199febfd7da45af7a4a75692e1b9440a684bf40 Mon Sep 17 00:00:00 2001 From: Harmouch101 Date: Tue, 18 May 2021 00:56:45 +0300 Subject: [PATCH 28/48] fixed lint error Signed-off-by: Harmouch101 --- ens/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ens/main.py b/ens/main.py index 27388a631e..7ee0786c0b 100644 --- a/ens/main.py +++ b/ens/main.py @@ -1,8 +1,10 @@ from functools import ( wraps, ) -from copy import deepcopy +from copy import ( + deepcopy, +) from typing import ( TYPE_CHECKING, Optional, From 8078f6558c4758dcde454bc293059de275328bbe Mon Sep 17 00:00:00 2001 From: Mahmoud Harmouch Date: Tue, 18 May 2021 01:12:41 +0300 Subject: [PATCH 29/48] Update main.py --- ens/main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ens/main.py b/ens/main.py index 7ee0786c0b..2c8360682a 100644 --- a/ens/main.py +++ b/ens/main.py @@ -1,10 +1,9 @@ -from functools import ( - wraps, -) - from copy import ( deepcopy, ) +from functools import ( + wraps, +) from typing import ( TYPE_CHECKING, Optional, From e316a1978618a287b8e9893e5d19ed23e9b32905 Mon Sep 17 00:00:00 2001 From: pacrob Date: Fri, 4 Feb 2022 15:40:27 -0700 Subject: [PATCH 30/48] add newsfragment --- newsfragments/1423.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/1423.bugfix.rst diff --git a/newsfragments/1423.bugfix.rst b/newsfragments/1423.bugfix.rst new file mode 100644 index 0000000000..5c465e869c --- /dev/null +++ b/newsfragments/1423.bugfix.rst @@ -0,0 +1 @@ +remove `ens.utils.dict_copy` decorator From 0a2cc6b5af7210449fb0af9a4d9f2cacb5b974ca Mon Sep 17 00:00:00 2001 From: AlwaysData Date: Mon, 7 Feb 2022 14:01:52 -0500 Subject: [PATCH 31/48] Feature/async geth admin (#2329) * Added BaseGethPersonal to geth.py * Added AsyncGethPersonal and test * Added Docs * lint fixes * news fragment update * removed import_raw_key test for now * mypy typing issues * typo * Added AsyncGethAdmin and BaseGethAdmin. Also, fixed test due to typing * made suggested changes * made suggested changes * fixed spelling errors * added test and docs * newsfragment * merge conflict * remove setSolc * copy in suggested test * forgot to check liniting before commit * linting * linting --- docs/providers.rst | 11 +- docs/web3.geth.rst | 4 - newsfragments/1413.feature.rst | 1 + newsfragments/2299.feature.rst | 1 - .../go_ethereum/test_goethereum_http.py | 26 +++- .../go_ethereum_admin_module.py | 44 +++++- web3/geth.py | 131 ++++++++++++++++-- 7 files changed, 195 insertions(+), 23 deletions(-) create mode 100644 newsfragments/1413.feature.rst delete mode 100644 newsfragments/2299.feature.rst diff --git a/docs/providers.rst b/docs/providers.rst index e966d83984..6b13f88aa3 100644 --- a/docs/providers.rst +++ b/docs/providers.rst @@ -393,7 +393,8 @@ AsyncHTTPProvider ... 'net': (AsyncNet,), ... 'geth': (Geth, ... {'txpool': (AsyncGethTxPool,), - ... 'personal': (AsyncGethPersonal,)}) + ... 'personal': (AsyncGethPersonal,), + ... 'admin' : (AsyncGethAdmin,)}) ... }, ... middlewares=[]) # See supported middleware section below for middleware options @@ -440,6 +441,14 @@ Net Geth **** +- :meth:`web3.geth.admin.add_peer() ` +- :meth:`web3.geth.admin.datadir() ` +- :meth:`web3.geth.admin.node_info() ` +- :meth:`web3.geth.admin.peers() ` +- :meth:`web3.geth.admin.start_rpc() ` +- :meth:`web3.geth.admin.start_ws() ` +- :meth:`web3.geth.admin.stop_rpc() ` +- :meth:`web3.geth.admin.stop_ws() ` - :meth:`web3.geth.personal.ec_recover()` - :meth:`web3.geth.personal.import_raw_key() ` - :meth:`web3.geth.personal.list_accounts() ` diff --git a/docs/web3.geth.rst b/docs/web3.geth.rst index ac9186d0c4..f448e35c7a 100644 --- a/docs/web3.geth.rst +++ b/docs/web3.geth.rst @@ -122,10 +122,6 @@ The ``web3.geth.admin`` object exposes methods to interact with the RPC APIs und .. warning:: Deprecated: This method is deprecated in favor of :meth:`~web3.geth.admin.add_peer()` -.. py:method:: setSolc(solc_path) - - .. Warning:: This method has been removed from Geth - .. py:method:: start_rpc(host='localhost', port=8545, cors="", apis="eth,net,web3") * Delegates to ``admin_startRPC`` RPC Method diff --git a/newsfragments/1413.feature.rst b/newsfragments/1413.feature.rst new file mode 100644 index 0000000000..f6607f7fba --- /dev/null +++ b/newsfragments/1413.feature.rst @@ -0,0 +1 @@ +Added Async functions for Geth Personal and Admin modules diff --git a/newsfragments/2299.feature.rst b/newsfragments/2299.feature.rst deleted file mode 100644 index 025051f606..0000000000 --- a/newsfragments/2299.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added Async functions for Geth Personal module diff --git a/tests/integration/go_ethereum/test_goethereum_http.py b/tests/integration/go_ethereum/test_goethereum_http.py index c4c4db63a4..a94dc8b7f7 100644 --- a/tests/integration/go_ethereum/test_goethereum_http.py +++ b/tests/integration/go_ethereum/test_goethereum_http.py @@ -4,6 +4,9 @@ get_open_port, ) from web3 import Web3 +from web3._utils.module_testing.go_ethereum_admin_module import ( + GoEthereumAsyncAdminModuleTest, +) from web3._utils.module_testing.go_ethereum_personal_module import ( GoEthereumAsyncPersonalModuleTest, ) @@ -11,6 +14,7 @@ AsyncEth, ) from web3.geth import ( + AsyncGethAdmin, AsyncGethPersonal, AsyncGethTxPool, Geth, @@ -104,7 +108,8 @@ async def async_w3(geth_process, endpoint_uri): 'async_net': AsyncNet, 'geth': (Geth, {'txpool': (AsyncGethTxPool,), - 'personal': (AsyncGethPersonal,)} + 'personal': (AsyncGethPersonal,), + 'admin': (AsyncGethAdmin,)} ) } ) @@ -131,6 +136,25 @@ def test_admin_start_stop_ws(self, web3: "Web3") -> None: super().test_admin_start_stop_ws(web3) +class TestGoEthereumAsyncAdminModuleTest(GoEthereumAsyncAdminModuleTest): + @pytest.mark.asyncio + @pytest.mark.xfail(reason="running geth with the --nodiscover flag doesn't allow peer addition") + async def test_admin_peers(self, web3: "Web3") -> None: + await super().test_admin_peers(web3) + + @pytest.mark.asyncio + async def test_admin_start_stop_rpc(self, web3: "Web3") -> None: + # This test causes all tests after it to fail on CI if it's allowed to run + pytest.xfail(reason='Only one RPC endpoint is allowed to be active at any time') + await super().test_admin_start_stop_rpc(web3) + + @pytest.mark.asyncio + async def test_admin_start_stop_ws(self, web3: "Web3") -> None: + # This test causes all tests after it to fail on CI if it's allowed to run + pytest.xfail(reason='Only one WS endpoint is allowed to be active at any time') + await super().test_admin_start_stop_ws(web3) + + class TestGoEthereumEthModuleTest(GoEthereumEthModuleTest): pass diff --git a/web3/_utils/module_testing/go_ethereum_admin_module.py b/web3/_utils/module_testing/go_ethereum_admin_module.py index bdfaba9600..9a88f0bafe 100644 --- a/web3/_utils/module_testing/go_ethereum_admin_module.py +++ b/web3/_utils/module_testing/go_ethereum_admin_module.py @@ -1,6 +1,7 @@ import pytest from typing import ( TYPE_CHECKING, + List, ) from web3.datastructures import ( @@ -80,7 +81,7 @@ def test_admin_start_stop_ws(self, web3: "Web3") -> None: def test_admin_addPeer(self, web3: "Web3") -> None: with pytest.warns(DeprecationWarning): result = web3.geth.admin.addPeer( - 'enode://f1a6b0bdbf014355587c3018454d070ac57801f05d3b39fe85da574f002a32e929f683d72aa5a8318382e4d3c7a05c9b91687b0d997a39619fb8a6e7ad88e512@1.1.1.1:30303', # noqa: E501 + EnodeURI('enode://f1a6b0bdbf014355587c3018454d070ac57801f05d3b39fe85da574f002a32e929f683d72aa5a8318382e4d3c7a05c9b91687b0d997a39619fb8a6e7ad88e512@1.1.1.1:30303'), # noqa: E501 ) assert result is True @@ -98,3 +99,44 @@ def test_admin_nodeInfo(self, web3: "Web3") -> None: }) # Test that result gives at least the keys that are listed in `expected` assert not set(expected.keys()).difference(result.keys()) + + +class GoEthereumAsyncAdminModuleTest: + + @pytest.mark.asyncio + async def test_async_datadir(self, async_w3: "Web3") -> None: + datadir = await async_w3.geth.admin.datadir() # type: ignore + assert isinstance(datadir, str) + + @pytest.mark.asyncio + async def test_async_nodeinfo(self, async_w3: "Web3") -> None: + node_info = await async_w3.geth.admin.node_info() # type: ignore + assert "Geth" in node_info["name"] + + @pytest.mark.asyncio + async def test_async_nodes(self, async_w3: "Web3") -> None: + nodes = await async_w3.geth.admin.peers() # type: ignore + assert isinstance(nodes, List) + + @pytest.mark.asyncio + async def test_admin_peers(self, web3: "Web3") -> None: + enode = await web3.geth.admin.node_info()['enode'] # type: ignore + web3.geth.admin.add_peer(enode) + result = await web3.geth.admin.peers() # type: ignore + assert len(result) == 1 + + @pytest.mark.asyncio + async def test_admin_start_stop_rpc(self, web3: "Web3") -> None: + stop = await web3.geth.admin.stop_rpc() # type: ignore + assert stop is True + + start = await web3.geth.admin.start_rpc() # type: ignore + assert start is True + + @pytest.mark.asyncio + async def test_admin_start_stop_ws(self, web3: "Web3") -> None: + stop = await web3.geth.admin.stop_ws() # type: ignore + assert stop is True + + start = await web3.geth.admin.start_ws() # type: ignore + assert start is True diff --git a/web3/geth.py b/web3/geth.py index 2320607078..5f1399a51f 100644 --- a/web3/geth.py +++ b/web3/geth.py @@ -77,7 +77,10 @@ Module, ) from web3.types import ( + EnodeURI, GethWallet, + NodeInfo, + Peer, TxParams, TxPoolContent, TxPoolInspect, @@ -258,25 +261,123 @@ async def status(self) -> Awaitable[Any]: return await self._status() # type: ignore -class GethAdmin(Module): +class BaseGethAdmin(Module): """ https://github.com/ethereum/go-ethereum/wiki/Management-APIs#admin """ - add_peer = add_peer - node_info = node_info - start_rpc = start_rpc - start_ws = start_ws - stop_ws = stop_ws - stop_rpc = stop_rpc + _add_peer = add_peer + _datadir = datadir + _node_info = node_info + _peers = peers + _start_rpc = start_rpc + _start_ws = start_ws + _stop_ws = stop_ws + _stop_rpc = stop_rpc # deprecated - addPeer = addPeer - datadir = datadir - nodeInfo = nodeInfo - peers = peers - startRPC = startRPC - startWS = startWS - stopRPC = stopRPC - stopWS = stopWS + _addPeer = addPeer + _nodeInfo = nodeInfo + _startRPC = startRPC + _startWS = startWS + _stopRPC = stopRPC + _stopWS = stopWS + + +class GethAdmin(BaseGethAdmin): + is_async = False + + def add_peer(self, node_url: EnodeURI) -> bool: + return self._add_peer(node_url) + + def datadir(self) -> str: + return self._datadir() + + def node_info(self) -> NodeInfo: + return self._node_info() + + def peers(self) -> List[Peer]: + return self._peers() + + def start_rpc(self, + host: str = "localhost", + port: int = 8546, + cors: str = "", + apis: str = "eth,net,web3") -> bool: + return self._start_rpc(host, port, cors, apis) + + def start_ws(self, + host: str = "localhost", + port: int = 8546, + cors: str = "", + apis: str = "eth,net,web3") -> bool: + return self._start_ws(host, port, cors, apis) + + def stop_rpc(self) -> bool: + return self._stop_rpc() + + def stop_ws(self) -> bool: + return self._stop_ws() + + def addPeer(self, node_url: EnodeURI) -> bool: + return self._addPeer(node_url) + + def nodeInfo(self) -> NodeInfo: + return self._nodeInfo() + + def startRPC(self, + host: str = "localhost", + port: int = 8546, + cors: str = "", + apis: str = "eth,net,web3") -> bool: + return self._startRPC(host, port, cors, apis) + + def startWS(self, + host: str = "localhost", + port: int = 8546, + cors: str = "", + apis: str = "eth,net,web3") -> bool: + return self._startWS(host, port, cors, apis) + + def stopRPC(self) -> bool: + return self._stopRPC() + + def stopWS(self) -> bool: + return self._stopWS() + + +class AsyncGethAdmin(BaseGethAdmin): + is_async = True + + async def add_peer(self, node_url: EnodeURI) -> Awaitable[bool]: + return await self._add_peer(node_url) # type: ignore + + async def datadir(self) -> Awaitable[str]: + return await self._datadir() # type: ignore + + async def node_info(self) -> Awaitable[NodeInfo]: + return await self._node_info() # type: ignore + + async def peers(self) -> Awaitable[List[Peer]]: + return await self._peers() # type: ignore + + async def start_rpc(self, + host: str = "localhost", + port: int = 8546, + cors: str = "", + apis: str = "eth,net,web3") -> Awaitable[bool]: + return await self._start_rpc(host, port, cors, apis) # type: ignore + + async def start_ws(self, + host: str = "localhost", + port: int = 8546, + cors: str = "", + apis: str = "eth,net,web3") -> Awaitable[bool]: + return await self._start_ws(host, port, cors, apis) # type: ignore + + async def stop_rpc(self) -> Awaitable[bool]: + return await self._stop_rpc() # type: ignore + + async def stop_ws(self) -> Awaitable[bool]: + return await self._stop_ws() # type: ignore class GethMiner(Module): From 41f154fb800f802f06385b7a899f9ca9bca4f112 Mon Sep 17 00:00:00 2001 From: Felipe Selmo Date: Tue, 1 Feb 2022 17:33:35 -0700 Subject: [PATCH 32/48] Properly initialize external modules Properly initialize modules that do not inherit from the `web3.module.Module` class. We weren't properly testing self-referential, non-static methods with this functionality and so a test was also added for this. --- newsfragments/2328.bugfix.rst | 1 + tests/core/conftest.py | 16 +++++++++++----- tests/core/utilities/test_attach_modules.py | 9 +++++---- tests/core/web3-module/test_attach_modules.py | 9 +++++---- web3/_utils/module.py | 2 +- 5 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 newsfragments/2328.bugfix.rst diff --git a/newsfragments/2328.bugfix.rst b/newsfragments/2328.bugfix.rst new file mode 100644 index 0000000000..ef7bbad1d1 --- /dev/null +++ b/newsfragments/2328.bugfix.rst @@ -0,0 +1 @@ +Properly initialize external modules that do not inherit from the ``web3.module.Module`` class \ No newline at end of file diff --git a/tests/core/conftest.py b/tests/core/conftest.py index cdaa362fce..cdc01b255a 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -50,29 +50,35 @@ class Module4(Module): def module1_unique(): class Module1: a = 'a' + + def __init__(self): + self._b = "b" + + def b(self): + return self._b return Module1 @pytest.fixture(scope='module') def module2_unique(): class Module2: - b = 'b' + c = 'c' @staticmethod - def c(): - return 'c' + def d(): + return 'd' return Module2 @pytest.fixture(scope='module') def module3_unique(): class Module3: - d = 'd' + e = 'e' return Module3 @pytest.fixture(scope='module') def module4_unique(): class Module4: - e = 'e' + f = 'f' return Module4 diff --git a/tests/core/utilities/test_attach_modules.py b/tests/core/utilities/test_attach_modules.py index 60914dc21f..db7c01728f 100644 --- a/tests/core/utilities/test_attach_modules.py +++ b/tests/core/utilities/test_attach_modules.py @@ -147,16 +147,17 @@ def test_attach_external_modules_that_do_not_inherit_from_module_class( # assert module1 attached assert hasattr(w3, 'module1') assert w3.module1.a == 'a' + assert w3.module1.b() == 'b' # assert module2 + submodules attached assert hasattr(w3, 'module2') - assert w3.module2.b == 'b' - assert w3.module2.c() == 'c' + assert w3.module2.c == 'c' + assert w3.module2.d() == 'd' assert hasattr(w3.module2, 'submodule1') - assert w3.module2.submodule1.d == 'd' + assert w3.module2.submodule1.e == 'e' assert hasattr(w3.module2.submodule1, 'submodule2') - assert w3.module2.submodule1.submodule2.e == 'e' + assert w3.module2.submodule1.submodule2.f == 'f' # assert default modules intact assert hasattr(w3, 'geth') diff --git a/tests/core/web3-module/test_attach_modules.py b/tests/core/web3-module/test_attach_modules.py index 58b3d5aca3..a5e7f3f44b 100644 --- a/tests/core/web3-module/test_attach_modules.py +++ b/tests/core/web3-module/test_attach_modules.py @@ -51,16 +51,17 @@ def test_attach_modules_that_do_not_inherit_from_module_class( # assert module1 attached assert hasattr(web3, 'module1') assert web3.module1.a == 'a' + assert web3.module1.b() == 'b' # assert module2 + submodules attached assert hasattr(web3, 'module2') - assert web3.module2.b == 'b' - assert web3.module2.c() == 'c' + assert web3.module2.c == 'c' + assert web3.module2.d() == 'd' assert hasattr(web3.module2, 'submodule1') - assert web3.module2.submodule1.d == 'd' + assert web3.module2.submodule1.e == 'e' assert hasattr(web3.module2.submodule1, 'submodule2') - assert web3.module2.submodule1.submodule2.e == 'e' + assert web3.module2.submodule1.submodule2.f == 'f' # assert default modules intact assert hasattr(web3, 'geth') diff --git a/web3/_utils/module.py b/web3/_utils/module.py index 89142583de..0b76882bd0 100644 --- a/web3/_utils/module.py +++ b/web3/_utils/module.py @@ -45,7 +45,7 @@ def attach_modules( setattr(parent_module, module_name, module_class(w3)) else: # An external `module_class` need not inherit from the `web3.module.Module` class. - setattr(parent_module, module_name, module_class) + setattr(parent_module, module_name, module_class()) if module_info_is_list_like: if len(module_info) == 2: From bdd89234ea3b91b3cdebd78f89cf80758025865b Mon Sep 17 00:00:00 2001 From: Marc Garreau Date: Wed, 2 Feb 2022 11:54:41 -0700 Subject: [PATCH 33/48] correct docs for external modules --- docs/web3.main.rst | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/docs/web3.main.rst b/docs/web3.main.rst index 3feeefb684..d50c0b7b92 100644 --- a/docs/web3.main.rst +++ b/docs/web3.main.rst @@ -419,17 +419,15 @@ web3.py library. External Modules ~~~~~~~~~~~~~~~~ -External modules can be used to introduce custom or third-party APIs to your ``Web3`` instance. Adding external modules -can occur either at instantiation of the ``Web3`` instance or by making use of the ``attach_modules()`` method. - -Unlike the native modules, external modules need not inherit from the ``web3.module.Module`` class. The only requirement -is that a Module must be a class and, if you'd like to make use of the parent ``Web3`` instance, it must be passed into -the ``__init__`` function. For example: +External modules can be used to introduce custom or third-party APIs to your ``Web3`` instance. They are not required to +utilize the parent ``Web3`` instance; if no reference is required, the external module need only be a standard class. +If, however, you do want to reference the parent ``Web3`` object, the external module must inherit from the +``web3.module.Module`` class and handle the instance as an argument within the ``__init__`` function: .. code-block:: python - >>> class ExampleModule(): - ... + >>> from web3.module import Module + >>> class ExampleModule(Module): ... def __init__(self, w3): ... self.w3 = w3 ... @@ -440,7 +438,8 @@ the ``__init__`` function. For example: .. warning:: Given the flexibility of external modules, use caution and only import modules from trusted third parties and open source code you've vetted! -To instantiate the ``Web3`` instance with external modules: +Configuring external modules can occur either at instantiation of the ``Web3`` instance or by making use of the +``attach_modules()`` method. To instantiate the ``Web3`` instance with external modules: .. code-block:: python @@ -466,11 +465,11 @@ To instantiate the ``Web3`` instance with external modules: ... ) # `return_zero`, in this case, is an example attribute of the `ModuleClass1` object - >>> w3.module1.return_zero + >>> w3.module1.return_zero() 0 - >>> w3.module2.submodule1.return_one + >>> w3.module2.submodule1.return_one() 1 - >>> w3.module2.submodule2.submodule2a.return_two + >>> w3.module2.submodule2.submodule2a.return_two() 2 @@ -504,9 +503,9 @@ To instantiate the ``Web3`` instance with external modules: ... }) ... }) ... }) - >>> w3.module1.return_zero + >>> w3.module1.return_zero() 0 - >>> w3.module2.submodule1.return_one + >>> w3.module2.submodule1.return_one() 1 - >>> w3.module2.submodule2.submodule2a.return_two + >>> w3.module2.submodule2.submodule2a.return_two() 2 From 3bcf7de51af77110647001294e36cf28a7317ab3 Mon Sep 17 00:00:00 2001 From: Felipe Selmo Date: Wed, 2 Feb 2022 16:37:53 -0700 Subject: [PATCH 34/48] Refactor attach_module logic * Allow for accepting the ``Web3`` instance as the first argument in any module's ``__init()`` method, rather than requiring a module to inherit from ``web3.module.Module`` if it needs to make use of the ``Web3`` instance. * Update tests to test the above change. * Add a more friendly error message if the module has more than one __init__() argument. Add test for this error message / case. --- tests/core/conftest.py | 17 +++++++- tests/core/utilities/test_attach_modules.py | 18 ++++++++ tests/core/web3-module/test_attach_modules.py | 17 ++++++++ web3/_utils/module.py | 43 ++++++++++++++----- 4 files changed, 84 insertions(+), 11 deletions(-) diff --git a/tests/core/conftest.py b/tests/core/conftest.py index cdc01b255a..6947f8d334 100644 --- a/tests/core/conftest.py +++ b/tests/core/conftest.py @@ -48,14 +48,20 @@ class Module4(Module): @pytest.fixture(scope='module') def module1_unique(): + # uses ``Web3`` instance by accepting it as first arg in the ``__init__()`` method class Module1: a = 'a' - def __init__(self): + def __init__(self, w3): self._b = "b" + self.w3 = w3 def b(self): return self._b + + @property + def return_eth_chain_id(self): + return self.w3.eth.chain_id return Module1 @@ -82,3 +88,12 @@ def module4_unique(): class Module4: f = 'f' return Module4 + + +@pytest.fixture(scope='module') +def module_many_init_args(): + class ModuleManyArgs: + def __init__(self, a, b): + self.a = a + self.b = b + return ModuleManyArgs diff --git a/tests/core/utilities/test_attach_modules.py b/tests/core/utilities/test_attach_modules.py index db7c01728f..4d151cc73a 100644 --- a/tests/core/utilities/test_attach_modules.py +++ b/tests/core/utilities/test_attach_modules.py @@ -1,3 +1,6 @@ +from io import ( + UnsupportedOperation, +) import pytest from eth_utils import ( @@ -148,6 +151,7 @@ def test_attach_external_modules_that_do_not_inherit_from_module_class( assert hasattr(w3, 'module1') assert w3.module1.a == 'a' assert w3.module1.b() == 'b' + assert w3.module1.return_eth_chain_id == w3.eth.chain_id # assert module2 + submodules attached assert hasattr(w3, 'module2') @@ -163,3 +167,17 @@ def test_attach_external_modules_that_do_not_inherit_from_module_class( assert hasattr(w3, 'geth') assert hasattr(w3, 'eth') assert is_integer(w3.eth.chain_id) + + +def test_attach_modules_for_module_with_more_than_one_init_argument(web3, module_many_init_args): + with pytest.raises( + UnsupportedOperation, + match=( + "A module class may accept a single `Web3` instance as the first argument of its " + "__init__\\(\\) method. More than one argument found for ModuleManyArgs: \\['a', 'b']" + ) + ): + Web3( + EthereumTesterProvider(), + external_modules={'module_should_fail': module_many_init_args} + ) diff --git a/tests/core/web3-module/test_attach_modules.py b/tests/core/web3-module/test_attach_modules.py index a5e7f3f44b..35f575cb06 100644 --- a/tests/core/web3-module/test_attach_modules.py +++ b/tests/core/web3-module/test_attach_modules.py @@ -1,3 +1,8 @@ +from io import ( + UnsupportedOperation, +) +import pytest + from eth_utils import ( is_integer, ) @@ -52,6 +57,7 @@ def test_attach_modules_that_do_not_inherit_from_module_class( assert hasattr(web3, 'module1') assert web3.module1.a == 'a' assert web3.module1.b() == 'b' + assert web3.module1.return_eth_chain_id == web3.eth.chain_id # assert module2 + submodules attached assert hasattr(web3, 'module2') @@ -67,3 +73,14 @@ def test_attach_modules_that_do_not_inherit_from_module_class( assert hasattr(web3, 'geth') assert hasattr(web3, 'eth') assert is_integer(web3.eth.chain_id) + + +def test_attach_modules_for_module_with_more_than_one_init_argument(web3, module_many_init_args): + with pytest.raises( + UnsupportedOperation, + match=( + "A module class may accept a single `Web3` instance as the first argument of its " + "__init__\\(\\) method. More than one argument found for ModuleManyArgs: \\['a', 'b']" + ) + ): + web3.attach_modules({'module_should_fail': module_many_init_args}) diff --git a/web3/_utils/module.py b/web3/_utils/module.py index 0b76882bd0..7bba1a86db 100644 --- a/web3/_utils/module.py +++ b/web3/_utils/module.py @@ -1,7 +1,12 @@ +import inspect +from io import ( + UnsupportedOperation, +) from typing import ( TYPE_CHECKING, Any, Dict, + List, Optional, Sequence, Union, @@ -18,6 +23,22 @@ from web3 import Web3 # noqa: F401 +def _validate_init_params_and_return_if_found(module_class: Any) -> List[str]: + init_params_raw = list(inspect.signature(module_class.__init__).parameters) + module_init_params = [ + param for param in init_params_raw if param not in ['self', 'args', 'kwargs'] + ] + + if len(module_init_params) > 1: + raise UnsupportedOperation( + "A module class may accept a single `Web3` instance as the first argument of its " + f"__init__() method. More than one argument found for {module_class.__name__}: " + f"{module_init_params}" + ) + + return module_init_params + + def attach_modules( parent_module: Union["Web3", "Module"], module_definitions: Dict[str, Any], @@ -34,17 +55,19 @@ def attach_modules( "already has an attribute with that name" ) - if issubclass(module_class, Module): - # If the `module_class` inherits from the `web3.module.Module` class, it has access to - # caller functions internal to the web3.py library and sets up a proper codec. This - # is likely important for all modules internal to the library. - if w3 is None: - setattr(parent_module, module_name, module_class(parent_module)) - w3 = parent_module - else: - setattr(parent_module, module_name, module_class(w3)) + # The parent module is the ``Web3`` instance on first run of the loop + if type(parent_module).__name__ == 'Web3': + w3 = parent_module + + module_init_params = _validate_init_params_and_return_if_found(module_class) + if len(module_init_params) == 1: + # Modules that need access to the ``Web3`` instance may accept the instance as the first + # arg in their ``__init__()`` method. This is the case for any module that inherits from + # ``web3.module.Module``. + # e.g. def __init__(self, w3): + setattr(parent_module, module_name, module_class(w3)) else: - # An external `module_class` need not inherit from the `web3.module.Module` class. + # Modules need not take in a ``Web3`` instance in their ``__init__()`` if not needed setattr(parent_module, module_name, module_class()) if module_info_is_list_like: From e4331ead97d320d374e49316462b780e3ac21a1e Mon Sep 17 00:00:00 2001 From: Marc Garreau Date: Mon, 7 Feb 2022 12:06:07 -0700 Subject: [PATCH 35/48] recorrect docs for external modules --- docs/web3.main.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/web3.main.rst b/docs/web3.main.rst index d50c0b7b92..f85840c62d 100644 --- a/docs/web3.main.rst +++ b/docs/web3.main.rst @@ -419,15 +419,13 @@ web3.py library. External Modules ~~~~~~~~~~~~~~~~ -External modules can be used to introduce custom or third-party APIs to your ``Web3`` instance. They are not required to -utilize the parent ``Web3`` instance; if no reference is required, the external module need only be a standard class. -If, however, you do want to reference the parent ``Web3`` object, the external module must inherit from the -``web3.module.Module`` class and handle the instance as an argument within the ``__init__`` function: +External modules can be used to introduce custom or third-party APIs to your ``Web3`` instance. External modules are simply +classes whose methods and properties can be made available within the ``Web3`` instance. Optionally, the external module may +make use of the parent ``Web3`` instance by accepting it as the first argument within the ``__init__`` function: .. code-block:: python - >>> from web3.module import Module - >>> class ExampleModule(Module): + >>> class ExampleModule: ... def __init__(self, w3): ... self.w3 = w3 ... @@ -439,7 +437,8 @@ If, however, you do want to reference the parent ``Web3`` object, the external m and open source code you've vetted! Configuring external modules can occur either at instantiation of the ``Web3`` instance or by making use of the -``attach_modules()`` method. To instantiate the ``Web3`` instance with external modules: +``attach_modules()`` method. To instantiate the ``Web3`` instance with external modules use the ``external_modules`` +keyword argument: .. code-block:: python From 05ea17fbec7b635ffa881912728af2628cfb9093 Mon Sep 17 00:00:00 2001 From: kclowes Date: Wed, 9 Feb 2022 11:39:16 -0700 Subject: [PATCH 36/48] Compile release notes --- docs/releases.rst | 33 +++++++++++++++++++++++++++++++++ newsfragments/1413.feature.rst | 1 - newsfragments/1423.bugfix.rst | 1 - newsfragments/2098.feature.rst | 1 - newsfragments/2259.feature.rst | 1 - newsfragments/2312.feature.rst | 1 - newsfragments/2327.bugfix.rst | 1 - newsfragments/2328.bugfix.rst | 1 - newsfragments/2331.feature.rst | 1 - 9 files changed, 33 insertions(+), 8 deletions(-) delete mode 100644 newsfragments/1413.feature.rst delete mode 100644 newsfragments/1423.bugfix.rst delete mode 100644 newsfragments/2098.feature.rst delete mode 100644 newsfragments/2259.feature.rst delete mode 100644 newsfragments/2312.feature.rst delete mode 100644 newsfragments/2327.bugfix.rst delete mode 100644 newsfragments/2328.bugfix.rst delete mode 100644 newsfragments/2331.feature.rst diff --git a/docs/releases.rst b/docs/releases.rst index 3b45188245..d65a9a9a7d 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -3,6 +3,39 @@ Release Notes .. towncrier release notes start +v5.28.0 (2022-02-09) +-------------------- + +Features +~~~~~~~~ + +- Added Async functions for Geth Personal and Admin modules (`#1413 + `__) +- async support for formatting, validation, and geth poa middlewares (`#2098 + `__) +- Calculate a default ``maxPriorityFeePerGas`` using ``eth_feeHistory`` when + ``eth_maxPriorityFeePerGas`` is not available, since the latter is not a part + of the Ethereum JSON-RPC specs and only supported by certain clients. (`#2259 + `__) +- Allow NamedTuples in ABI inputs (`#2312 + `__) +- Add async `eth.syncing` method (`#2331 + `__) + + +Bugfixes +~~~~~~~~ + +- remove `ens.utils.dict_copy` decorator (`#1423 + `__) +- The exception retry middleware whitelist was missing a comma between + ``txpool`` and ``testing`` (`#2327 + `__) +- Properly initialize external modules that do not inherit from the + ``web3.module.Module`` class (`#2328 + `__) + + v5.27.0 (2022-01-31) -------------------- diff --git a/newsfragments/1413.feature.rst b/newsfragments/1413.feature.rst deleted file mode 100644 index f6607f7fba..0000000000 --- a/newsfragments/1413.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added Async functions for Geth Personal and Admin modules diff --git a/newsfragments/1423.bugfix.rst b/newsfragments/1423.bugfix.rst deleted file mode 100644 index 5c465e869c..0000000000 --- a/newsfragments/1423.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -remove `ens.utils.dict_copy` decorator diff --git a/newsfragments/2098.feature.rst b/newsfragments/2098.feature.rst deleted file mode 100644 index b1dffae787..0000000000 --- a/newsfragments/2098.feature.rst +++ /dev/null @@ -1 +0,0 @@ -async support for formatting, validation, and geth poa middlewares \ No newline at end of file diff --git a/newsfragments/2259.feature.rst b/newsfragments/2259.feature.rst deleted file mode 100644 index d320c46090..0000000000 --- a/newsfragments/2259.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Calculate a default ``maxPriorityFeePerGas`` using ``eth_feeHistory`` when ``eth_maxPriorityFeePerGas`` is not available, since the latter is not a part of the Ethereum JSON-RPC specs and only supported by certain clients. \ No newline at end of file diff --git a/newsfragments/2312.feature.rst b/newsfragments/2312.feature.rst deleted file mode 100644 index 902b51b134..0000000000 --- a/newsfragments/2312.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Allow NamedTuples in ABI inputs diff --git a/newsfragments/2327.bugfix.rst b/newsfragments/2327.bugfix.rst deleted file mode 100644 index bac3de984b..0000000000 --- a/newsfragments/2327.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -The exception retry middleware whitelist was missing a comma between ``txpool`` and ``testing`` diff --git a/newsfragments/2328.bugfix.rst b/newsfragments/2328.bugfix.rst deleted file mode 100644 index ef7bbad1d1..0000000000 --- a/newsfragments/2328.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Properly initialize external modules that do not inherit from the ``web3.module.Module`` class \ No newline at end of file diff --git a/newsfragments/2331.feature.rst b/newsfragments/2331.feature.rst deleted file mode 100644 index 8e69fd1ee0..0000000000 --- a/newsfragments/2331.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add async `eth.syncing` method From 78e01cf264753b8a20d9c1cba113d893c0ff4e63 Mon Sep 17 00:00:00 2001 From: kclowes Date: Wed, 9 Feb 2022 11:39:57 -0700 Subject: [PATCH 37/48] =?UTF-8?q?Bump=20version:=205.27.0=20=E2=86=92=205.?= =?UTF-8?q?28.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b716075c39..fea9a4d2fe 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.27.0 +current_version = 5.28.0 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-(?P[^.]*)\.(?P\d+))? diff --git a/setup.py b/setup.py index 10eb738a03..72b10700bb 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ setup( name='web3', # *IMPORTANT*: Don't manually change the version here. Use the 'bumpversion' utility. - version='5.27.0', + version='5.28.0', description="""Web3.py""", long_description_content_type='text/markdown', long_description=long_description, From f93336a054a62fceb16748ad4177af0d4dee1590 Mon Sep 17 00:00:00 2001 From: kclowes Date: Thu, 10 Feb 2022 10:42:52 -0700 Subject: [PATCH 38/48] Add 'Breaking Changes' and 'Deprecation' to our valid newsfragment types (#2340) * Add 'Breaking Change' and 'Deprecation' to our valid newsfragment types * Add newsfragment for new newsfragment categories * Remove removal section of release notes --- newsfragments/2340.feature.rst | 1 + newsfragments/validate_files.py | 3 ++- pyproject.toml | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 newsfragments/2340.feature.rst diff --git a/newsfragments/2340.feature.rst b/newsfragments/2340.feature.rst new file mode 100644 index 0000000000..208a28c40d --- /dev/null +++ b/newsfragments/2340.feature.rst @@ -0,0 +1 @@ +Added 'Breaking Changes' and 'Deprecations' categories to our release notes diff --git a/newsfragments/validate_files.py b/newsfragments/validate_files.py index ba3bae96b2..391483249c 100755 --- a/newsfragments/validate_files.py +++ b/newsfragments/validate_files.py @@ -11,7 +11,8 @@ '.doc.rst', '.feature.rst', '.misc.rst', - '.removal.rst', + '.breaking-change.rst', + '.deprecation.rst', } ALLOWED_FILES = { diff --git a/pyproject.toml b/pyproject.toml index 97533d0116..804cc73713 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,13 @@ underlines = ["-", "~", "^"] issue_format = "`#{issue} `__" title_format = "v{version} ({project_date})" + +[[tool.towncrier.type]] +directory = "breaking-change" +name="Breaking Changes" +showcontent=true + +[[tool.towncrier.type]] +directory = "deprecation" +name = "Deprecations" +showcontent = true From 1284329406d8ae34766de65561229598129a1d19 Mon Sep 17 00:00:00 2001 From: kclowes Date: Fri, 11 Feb 2022 11:38:03 -0700 Subject: [PATCH 39/48] Drop python 3.6 (#2343) * Drop python 3.6 * Remove parity tests * Add newsfragment for py36 drop --- .circleci/config.yml | 231 +------------------------ newsfragments/2343.breaking-change.rst | 2 + setup.py | 3 +- tox.ini | 19 +- 4 files changed, 11 insertions(+), 244 deletions(-) create mode 100644 newsfragments/2343.breaking-change.rst diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f730c3539..c76ed67ab7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,44 +57,6 @@ docs_steps: &docs_steps - ~/.py-geth key: cache-docs-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }} -# parity_steps: &parity_steps - # working_directory: ~/repo - # steps: - # - checkout - # - restore_cache: - # keys: - # - cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }} - # - run: - # name: install dependencies - # command: pip install --user tox - # - run: - # name: install parity if needed - # command: | - # pip install --user requests eth_utils tqdm eth-hash[pycryptodome] - # echo "Job specifies Parity version $PARITY_VERSION" - # python tests/integration/parity/install_parity.py $PARITY_VERSION - # - run: - # name: update parity deps from testing repo if needed - # command: | - # [ "`cat /etc/*release | grep jessie`" != "" ] && echo "On Jessie - installing missing deps..." || echo "Not on Jessie - doing nothing." - # [ "`cat /etc/*release | grep jessie`" != "" ] || exit 0 - # echo "deb http://ftp.debian.org/debian testing main" > testing.list && sudo mv testing.list /etc/apt/sources.list.d/ - # sudo apt update - # sudo apt-get -t testing install libstdc++6 - # - run: - # name: run tox - # command: ~/.local/bin/tox -r - # - save_cache: - # paths: - # - .tox - # - ~/.cache/pip - # - ~/.local - # - ./eggs - # - ~/.ethash - # - ~/.py-geth - # - ~/.parity-bin - # key: cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }} - geth_steps: &geth_steps working_directory: ~/repo steps: @@ -247,105 +209,17 @@ jobs: lint: <<: *common docker: - - image: circleci/python:3.6 + - image: circleci/python:3.9 environment: TOXENV: lint docs: <<: *docs_steps docker: - - image: circleci/python:3.6 + - image: circleci/python:3.9 environment: TOXENV: docs - py36-core: - <<: *common - docker: - - image: circleci/python:3.6 - environment: - TOXENV: py36-core - - py36-ens: - <<: *common - docker: - - image: circleci/python:3.6 - environment: - TOXENV: py36-ens - - py36-ethpm: - <<: *ethpm_steps - docker: - - image: circleci/python:3.6 - environment: - TOXENV: py36-ethpm - # Please don't use this key for any shenanigans - WEB3_INFURA_PROJECT_ID: 7707850c2fb7465ebe6f150d67182e22 - - py36-integration-goethereum-ipc: - <<: *geth_steps - docker: - - image: circleci/python:3.6 - environment: - TOXENV: py36-integration-goethereum-ipc - GETH_VERSION: v1.10.13 - - py36-integration-goethereum-http: - <<: *geth_steps - docker: - - image: circleci/python:3.6 - environment: - TOXENV: py36-integration-goethereum-http - GETH_VERSION: v1.10.13 - - py36-integration-goethereum-ws: - <<: *geth_steps - docker: - - image: circleci/python:3.6 - environment: - TOXENV: py36-integration-goethereum-ws - GETH_VERSION: v1.10.13 - - # py36-integration-parity-ipc: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.6-stretch - # environment: - # TOXENV: py36-integration-parity-ipc - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - - # py36-integration-parity-http: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.6-stretch - # environment: - # TOXENV: py36-integration-parity-http - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - - # py36-integration-parity-ws: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.6-stretch - # environment: - # TOXENV: py36-integration-parity-ws - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - - py36-integration-ethtester-pyevm: - <<: *common - docker: - - image: circleci/python:3.6 - environment: - TOXENV: py36-integration-ethtester - ETHEREUM_TESTER_CHAIN_BACKEND: eth_tester.backends.PyEVMBackend - - py36-wheel-cli: - <<: *common - docker: - - image: circleci/python:3.6 - environment: - TOXENV: py36-wheel-cli # # Python 3.7 @@ -397,33 +271,6 @@ jobs: TOXENV: py37-integration-goethereum-ws GETH_VERSION: v1.10.13 - # py37-integration-parity-ipc: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.7-stretch - # environment: - # TOXENV: py37-integration-parity-ipc - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - - # py37-integration-parity-http: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.7-stretch - # environment: - # TOXENV: py37-integration-parity-http - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - - # py37-integration-parity-ws: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.7-stretch - # environment: - # TOXENV: py37-integration-parity-ws - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - py37-integration-ethtester-pyevm: <<: *common docker: @@ -494,33 +341,6 @@ jobs: TOXENV: py38-integration-goethereum-ws GETH_VERSION: v1.10.13 - # py38-integration-parity-ipc: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.8 - # environment: - # TOXENV: py38-integration-parity-ipc - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - - # py38-integration-parity-http: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.8 - # environment: - # TOXENV: py38-integration-parity-http - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - - # py38-integration-parity-ws: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.8 - # environment: - # TOXENV: py38-integration-parity-ws - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - py38-integration-ethtester-pyevm: <<: *common docker: @@ -586,33 +406,6 @@ jobs: TOXENV: py39-integration-goethereum-ws GETH_VERSION: v1.10.13 - # py39-integration-parity-ipc: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.9 - # environment: - # TOXENV: py39-integration-parity-ipc - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - - # py39-integration-parity-http: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.9 - # environment: - # TOXENV: py39-integration-parity-http - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - - # py39-integration-parity-ws: - # <<: *parity_steps - # docker: - # - image: circleci/python:3.9 - # environment: - # TOXENV: py39-integration-parity-ws - # PARITY_VERSION: v2.3.5 - # PARITY_OS: linux - py39-integration-ethtester-pyevm: <<: *common docker: @@ -641,31 +434,17 @@ workflows: test: jobs: # These are the longest running tests, start them first - - py36-core - py37-core - py38-core - py39-core - lint - docs - benchmark - - py36-ens - - py36-ethpm - - py36-integration-goethereum-ipc - - py36-integration-goethereum-http - - py36-integration-goethereum-ws - # - py36-integration-parity-ipc - # - py36-integration-parity-http - # - py36-integration-parity-ws - - py36-integration-ethtester-pyevm - - py36-wheel-cli - py37-ens - py37-ethpm - py37-integration-goethereum-ipc - py37-integration-goethereum-http - py37-integration-goethereum-ws - # - py37-integration-parity-ipc - # - py37-integration-parity-http - # - py37-integration-parity-ws - py37-integration-ethtester-pyevm - py37-wheel-cli - py37-wheel-cli-windows @@ -674,9 +453,6 @@ workflows: - py38-integration-goethereum-ipc - py38-integration-goethereum-http - py38-integration-goethereum-ws - # - py38-integration-parity-ipc - # - py38-integration-parity-http - # - py38-integration-parity-ws - py38-integration-ethtester-pyevm - py38-wheel-cli - py39-ens @@ -684,8 +460,5 @@ workflows: - py39-integration-goethereum-ipc - py39-integration-goethereum-http - py39-integration-goethereum-ws - # - py39-integration-parity-ipc - # - py39-integration-parity-http - # - py39-integration-parity-ws - py39-integration-ethtester-pyevm - py39-wheel-cli diff --git a/newsfragments/2343.breaking-change.rst b/newsfragments/2343.breaking-change.rst new file mode 100644 index 0000000000..4a4c5fcb12 --- /dev/null +++ b/newsfragments/2343.breaking-change.rst @@ -0,0 +1,2 @@ +Remove support for the unsupported Python 3.6 +Also removes outdated Parity tests diff --git a/setup.py b/setup.py index 72b10700bb..2df5580452 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ "typing-extensions>=3.7.4.1,<5;python_version<'3.8'", "websockets>=9.1,<10", ], - python_requires='>=3.6,<4', + python_requires='>=3.7,<3.10', extras_require=extras_require, py_modules=['web3', 'ens', 'ethpm'], entry_points={"pytest11": ["pytest_ethereum = web3.tools.pytest_ethereum.plugins"]}, @@ -107,7 +107,6 @@ 'License :: OSI Approved :: MIT License', 'Natural Language :: English', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', diff --git a/tox.ini b/tox.ini index 75b1637e78..f91ecfe023 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,13 @@ [tox] envlist= - py{36,37,38,39}-ens - py{36,37,38,39}-ethpm - py{36,37,38,39}-core - py{36,37,38,39}-integration-{goethereum,ethtester} + py{37,38,39}-ens + py{37,38,39}-ethpm + py{37,38,39}-core + py{37,38,39}-integration-{goethereum,ethtester} lint docs benchmark - py{36,37,38,39}-wheel-cli + py{37,38,39}-wheel-cli [isort] combine_as_imports=True @@ -48,8 +48,7 @@ passenv = WEB3_INFURA_PROJECT_ID WEB3_INFURA_API_SECRET basepython = - docs: python3.6 - py36: python3.6 + docs: python3.9 py37: python3.7 py38: python3.8 py39: python3.9 @@ -80,12 +79,6 @@ commands= /bin/bash -c 'pip install --upgrade "$(ls dist/web3-*-py3-none-any.whl)" --progress-bar off' python -c "from web3.auto import w3" -[testenv:py36-wheel-cli] -deps={[common-wheel-cli]deps} -whitelist_externals={[common-wheel-cli]whitelist_externals} -commands={[common-wheel-cli]commands} -skip_install=true - [testenv:py37-wheel-cli] deps={[common-wheel-cli]deps} whitelist_externals={[common-wheel-cli]whitelist_externals} From e2ffabf2e3f1928d0a17b2798cf7362f254cf0d9 Mon Sep 17 00:00:00 2001 From: coccoinomane Date: Fri, 11 Feb 2022 20:02:51 +0100 Subject: [PATCH 40/48] Fix gas types (#2330) * fix: correct type for effectiveGasPrice (Wei, not int) * fix: correct type for gas and gas_limit (int, not Wei) * lint: removed unused type imports of Wei * Add newsfragment for Wei/int typing fixes Co-authored-by: kclowes --- docs/middleware.rst | 2 +- newsfragments/2330.bugfix.rst | 2 + web3/_utils/async_transactions.py | 9 +- web3/_utils/module_testing/eth_module.py | 94 +++++++++---------- .../go_ethereum_personal_module.py | 4 +- web3/_utils/transactions.py | 9 +- web3/eth.py | 6 +- web3/types.py | 12 +-- 8 files changed, 69 insertions(+), 69 deletions(-) create mode 100644 newsfragments/2330.bugfix.rst diff --git a/docs/middleware.rst b/docs/middleware.rst index 64fabf150e..6448c2e745 100644 --- a/docs/middleware.rst +++ b/docs/middleware.rst @@ -78,7 +78,7 @@ Buffered Gas Estimate This adds a gas estimate to transactions if ``gas`` is not present in the transaction parameters. Sets gas to: ``min(w3.eth.estimate_gas + gas_buffer, gas_limit)`` - where the gas_buffer default is 100,000 Wei + where the gas_buffer default is 100,000 HTTPRequestRetry ~~~~~~~~~~~~~~~~~~ diff --git a/newsfragments/2330.bugfix.rst b/newsfragments/2330.bugfix.rst new file mode 100644 index 0000000000..f4588ea93e --- /dev/null +++ b/newsfragments/2330.bugfix.rst @@ -0,0 +1,2 @@ +- Fix types for ``gas``, and ``gasLimit`` (``Wei`` to ``int``) +- Fix types for ``effectiveGasPrice``, (``int`` to ``Wei``) diff --git a/web3/_utils/async_transactions.py b/web3/_utils/async_transactions.py index 2b3124a18b..be5fa6c391 100644 --- a/web3/_utils/async_transactions.py +++ b/web3/_utils/async_transactions.py @@ -7,7 +7,6 @@ from web3.types import ( BlockIdentifier, TxParams, - Wei, ) if TYPE_CHECKING: @@ -17,7 +16,7 @@ async def get_block_gas_limit( web3_eth: "AsyncEth", block_identifier: Optional[BlockIdentifier] = None -) -> Wei: +) -> int: if block_identifier is None: block_identifier = await web3_eth.block_number block = await web3_eth.get_block(block_identifier) @@ -25,8 +24,8 @@ async def get_block_gas_limit( async def get_buffered_gas_estimate( - web3: "Web3", transaction: TxParams, gas_buffer: Wei = Wei(100000) -) -> Wei: + web3: "Web3", transaction: TxParams, gas_buffer: int = 100000 +) -> int: gas_estimate_transaction = cast(TxParams, dict(**transaction)) gas_estimate = await web3.eth.estimate_gas(gas_estimate_transaction) # type: ignore @@ -40,4 +39,4 @@ async def get_buffered_gas_estimate( "limit: {1}".format(gas_estimate, gas_limit) ) - return Wei(min(gas_limit, gas_estimate + gas_buffer)) + return min(gas_limit, gas_estimate + gas_buffer) diff --git a/web3/_utils/module_testing/eth_module.py b/web3/_utils/module_testing/eth_module.py index 1b88175b6f..b3f9b788f1 100644 --- a/web3/_utils/module_testing/eth_module.py +++ b/web3/_utils/module_testing/eth_module.py @@ -128,7 +128,7 @@ async def test_eth_send_transaction_legacy( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': await async_w3.eth.gas_price, # type: ignore } txn_hash = await async_w3.eth.send_transaction(txn_params) # type: ignore @@ -148,7 +148,7 @@ async def test_eth_send_transaction( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': async_w3.toWei(3, 'gwei'), 'maxPriorityFeePerGas': async_w3.toWei(1, 'gwei'), } @@ -171,7 +171,7 @@ async def test_eth_send_transaction_default_fees( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, } txn_hash = await async_w3.eth.send_transaction(txn_params) # type: ignore txn = await async_w3.eth.get_transaction(txn_hash) # type: ignore @@ -192,7 +192,7 @@ async def test_eth_send_transaction_hex_fees( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': hex(250 * 10**9), 'maxPriorityFeePerGas': hex(2 * 10**9), } @@ -233,7 +233,7 @@ async def test_eth_send_transaction_with_gas_price( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': Wei(1), 'maxFeePerGas': Wei(250 * 10**9), 'maxPriorityFeePerGas': Wei(2 * 10**9), @@ -249,7 +249,7 @@ async def test_eth_send_transaction_no_priority_fee( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': Wei(250 * 10**9), } with pytest.raises(InvalidTransaction, match='maxPriorityFeePerGas must be defined'): @@ -264,7 +264,7 @@ async def test_eth_send_transaction_no_max_fee( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxPriorityFeePerGas': maxPriorityFeePerGas, } txn_hash = await async_w3.eth.send_transaction(txn_params) # type: ignore @@ -286,7 +286,7 @@ async def test_eth_send_transaction_max_fee_less_than_tip( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': Wei(1 * 10**9), 'maxPriorityFeePerGas': Wei(2 * 10**9), } @@ -356,7 +356,7 @@ async def test_gas_price_strategy_middleware( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, } two_gwei_in_wei = async_w3.toWei(2, 'gwei') @@ -385,7 +385,7 @@ async def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxPriorityFeePerGas': max_priority_fee, } if max_fee is not None: @@ -414,7 +414,7 @@ async def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn_no_tip( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': Wei(1000000000), } @@ -824,7 +824,7 @@ async def test_async_eth_get_transaction_receipt_unmined( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': async_w3.toWei(3, 'gwei'), 'maxPriorityFeePerGas': async_w3.toWei(1, 'gwei') }) @@ -885,7 +885,7 @@ async def test_async_eth_wait_for_transaction_receipt_unmined( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': async_w3.toWei(3, 'gwei'), 'maxPriorityFeePerGas': async_w3.toWei(1, 'gwei') }) @@ -1704,7 +1704,7 @@ def test_eth_sign_transaction_legacy( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': web3.eth.gas_price, 'nonce': Nonce(0), } @@ -1726,7 +1726,7 @@ def test_eth_sign_transaction( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(2, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), 'nonce': Nonce(0), @@ -1750,7 +1750,7 @@ def test_eth_sign_transaction_hex_fees( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': hex(web3.toWei(2, 'gwei')), 'maxPriorityFeePerGas': hex(web3.toWei(1, 'gwei')), 'nonce': Nonce(0), @@ -1774,7 +1774,7 @@ def test_eth_signTransaction_deprecated(self, 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': web3.eth.gas_price, 'nonce': Nonce(0), } @@ -1797,7 +1797,7 @@ def test_eth_sign_transaction_ens_names( 'from': 'unlocked-account.eth', 'to': 'unlocked-account.eth', 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(2, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), 'nonce': Nonce(0), @@ -1820,7 +1820,7 @@ def test_eth_send_transaction_addr_checksum_required( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(2, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), } @@ -1840,7 +1840,7 @@ def test_eth_send_transaction_legacy( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': web3.toWei(1, 'gwei'), # post-london needs to be more than the base fee } txn_hash = web3.eth.send_transaction(txn_params) @@ -1859,7 +1859,7 @@ def test_eth_send_transaction( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(3, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), } @@ -1881,7 +1881,7 @@ def test_eth_sendTransaction_deprecated( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(3, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), } @@ -1907,7 +1907,7 @@ def test_eth_send_transaction_with_nonce( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, # unique maxFeePerGas to ensure transaction hash different from other tests 'maxFeePerGas': web3.toWei(4.321, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), @@ -1932,7 +1932,7 @@ def test_eth_send_transaction_default_fees( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, } txn_hash = web3.eth.send_transaction(txn_params) txn = web3.eth.get_transaction(txn_hash) @@ -1952,7 +1952,7 @@ def test_eth_send_transaction_hex_fees( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': hex(250 * 10**9), 'maxPriorityFeePerGas': hex(2 * 10**9), } @@ -1991,7 +1991,7 @@ def test_eth_send_transaction_with_gas_price( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': Wei(1), 'maxFeePerGas': Wei(250 * 10**9), 'maxPriorityFeePerGas': Wei(2 * 10**9), @@ -2006,7 +2006,7 @@ def test_eth_send_transaction_no_priority_fee( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': Wei(250 * 10**9), } with pytest.raises(InvalidTransaction, match='maxPriorityFeePerGas must be defined'): @@ -2020,7 +2020,7 @@ def test_eth_send_transaction_no_max_fee( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxPriorityFeePerGas': maxPriorityFeePerGas, } txn_hash = web3.eth.send_transaction(txn_params) @@ -2041,7 +2041,7 @@ def test_eth_send_transaction_max_fee_less_than_tip( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': Wei(1 * 10**9), 'maxPriorityFeePerGas': Wei(2 * 10**9), } @@ -2086,7 +2086,7 @@ def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxPriorityFeePerGas': max_priority_fee, } if max_fee is not None: @@ -2114,7 +2114,7 @@ def test_gas_price_from_strategy_bypassed_for_dynamic_fee_txn_no_tip( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': Wei(1000000000), } @@ -2134,7 +2134,7 @@ def test_eth_replace_transaction_legacy( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': web3.toWei(1, 'gwei'), # must be greater than base_fee post London } txn_hash = web3.eth.send_transaction(txn_params) @@ -2159,7 +2159,7 @@ def test_eth_replace_transaction( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': two_gwei_in_wei, 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), } @@ -2185,7 +2185,7 @@ def test_eth_replace_transaction_underpriced( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(3, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(2, 'gwei'), } @@ -2208,7 +2208,7 @@ def test_eth_replaceTransaction_deprecated( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': two_gwei_in_wei, 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), } @@ -2237,7 +2237,7 @@ def test_eth_replace_transaction_non_existing_transaction( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(3, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), } @@ -2254,7 +2254,7 @@ def test_eth_replace_transaction_already_mined( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(2, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), } @@ -2277,7 +2277,7 @@ def test_eth_replace_transaction_incorrect_nonce( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(2, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), } @@ -2297,7 +2297,7 @@ def test_eth_replace_transaction_gas_price_too_low( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': web3.toWei(2, 'gwei'), } txn_hash = web3.eth.send_transaction(txn_params) @@ -2315,7 +2315,7 @@ def test_eth_replace_transaction_gas_price_defaulting_minimum( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': gas_price, } txn_hash = web3.eth.send_transaction(txn_params) @@ -2333,7 +2333,7 @@ def test_eth_replace_transaction_gas_price_defaulting_strategy_higher( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': web3.toWei(1, 'gwei'), } txn_hash = web3.eth.send_transaction(txn_params) @@ -2360,7 +2360,7 @@ def test_eth_replace_transaction_gas_price_defaulting_strategy_lower( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': gas_price, } txn_hash = web3.eth.send_transaction(txn_params) @@ -2384,7 +2384,7 @@ def test_eth_modify_transaction_legacy( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': web3.toWei(1, 'gwei'), # must be greater than base_fee post London } txn_hash = web3.eth.send_transaction(txn_params) @@ -2407,7 +2407,7 @@ def test_eth_modify_transaction( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxPriorityFeePerGas': web3.toWei(1, 'gwei'), 'maxFeePerGas': web3.toWei(2, 'gwei'), } @@ -2436,7 +2436,7 @@ def test_eth_modifyTransaction_deprecated( 'from': unlocked_account, 'to': unlocked_account, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'gasPrice': web3.toWei(1, 'gwei'), } txn_hash = web3.eth.send_transaction(txn_params) @@ -2795,7 +2795,7 @@ def test_eth_get_transaction_receipt_unmined( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(3, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei') }) @@ -2853,7 +2853,7 @@ def test_eth_wait_for_transaction_receipt_unmined( 'from': unlocked_account_dual_type, 'to': unlocked_account_dual_type, 'value': Wei(1), - 'gas': Wei(21000), + 'gas': 21000, 'maxFeePerGas': web3.toWei(3, 'gwei'), 'maxPriorityFeePerGas': web3.toWei(1, 'gwei') }) diff --git a/web3/_utils/module_testing/go_ethereum_personal_module.py b/web3/_utils/module_testing/go_ethereum_personal_module.py index e39a1b177c..1e7fb35f06 100644 --- a/web3/_utils/module_testing/go_ethereum_personal_module.py +++ b/web3/_utils/module_testing/go_ethereum_personal_module.py @@ -154,7 +154,7 @@ def test_personal_send_transaction( txn_params: TxParams = { 'from': unlockable_account_dual_type, 'to': unlockable_account_dual_type, - 'gas': Wei(21000), + 'gas': 21000, 'value': Wei(1), 'gasPrice': web3.toWei(1, 'gwei'), } @@ -178,7 +178,7 @@ def test_personal_sendTransaction_deprecated( txn_params: TxParams = { 'from': unlockable_account_dual_type, 'to': unlockable_account_dual_type, - 'gas': Wei(21000), + 'gas': 21000, 'value': Wei(1), 'gasPrice': web3.toWei(1, 'gwei'), } diff --git a/web3/_utils/transactions.py b/web3/_utils/transactions.py index 27a285182c..13d43ffea7 100644 --- a/web3/_utils/transactions.py +++ b/web3/_utils/transactions.py @@ -32,7 +32,6 @@ BlockIdentifier, TxData, TxParams, - Wei, _Hash32, ) @@ -119,7 +118,7 @@ def fill_transaction_defaults(web3: "Web3", transaction: TxParams) -> TxParams: return merge(defaults, transaction) -def get_block_gas_limit(web3: "Web3", block_identifier: Optional[BlockIdentifier] = None) -> Wei: +def get_block_gas_limit(web3: "Web3", block_identifier: Optional[BlockIdentifier] = None) -> int: if block_identifier is None: block_identifier = web3.eth.block_number block = web3.eth.get_block(block_identifier) @@ -127,8 +126,8 @@ def get_block_gas_limit(web3: "Web3", block_identifier: Optional[BlockIdentifier def get_buffered_gas_estimate( - web3: "Web3", transaction: TxParams, gas_buffer: Wei = Wei(100000) -) -> Wei: + web3: "Web3", transaction: TxParams, gas_buffer: int = 100000 +) -> int: gas_estimate_transaction = cast(TxParams, dict(**transaction)) gas_estimate = web3.eth.estimate_gas(gas_estimate_transaction) @@ -142,7 +141,7 @@ def get_buffered_gas_estimate( "limit: {1}".format(gas_estimate, gas_limit) ) - return Wei(min(gas_limit, gas_estimate + gas_buffer)) + return min(gas_limit, gas_estimate + gas_buffer) def get_required_transaction(web3: "Web3", transaction_hash: _Hash32) -> TxData: diff --git a/web3/eth.py b/web3/eth.py index 1e644609e6..48eb0c86c3 100644 --- a/web3/eth.py +++ b/web3/eth.py @@ -231,7 +231,7 @@ def estimate_gas_munger( return params - _estimate_gas: Method[Callable[..., Wei]] = Method( + _estimate_gas: Method[Callable[..., int]] = Method( RPC.eth_estimateGas, mungers=[estimate_gas_munger] ) @@ -426,7 +426,7 @@ async def estimate_gas( self, transaction: TxParams, block_identifier: Optional[BlockIdentifier] = None - ) -> Wei: + ) -> int: # types ignored b/c mypy conflict with BlockingEth properties return await self._estimate_gas(transaction, block_identifier) # type: ignore @@ -851,7 +851,7 @@ def estimate_gas( self, transaction: TxParams, block_identifier: Optional[BlockIdentifier] = None - ) -> Wei: + ) -> int: return self._estimate_gas(transaction, block_identifier) def fee_history( diff --git a/web3/types.py b/web3/types.py index fa9ddac284..ffdda4fe58 100644 --- a/web3/types.py +++ b/web3/types.py @@ -183,7 +183,7 @@ class LogReceipt(TypedDict): "chainId": int, "data": Union[bytes, HexStr], "from": ChecksumAddress, - "gas": Wei, + "gas": int, "gasPrice": Wei, "maxFeePerGas": Wei, "maxPriorityFeePerGas": Wei, @@ -206,7 +206,7 @@ class LogReceipt(TypedDict): "data": Union[bytes, HexStr], # addr or ens "from": Union[Address, ChecksumAddress, str], - "gas": Wei, + "gas": int, # legacy pricing "gasPrice": Wei, # dynamic fee pricing @@ -237,8 +237,8 @@ class CallOverrideParams(TypedDict): "blockNumber": BlockNumber, "contractAddress": Optional[ChecksumAddress], "cumulativeGasUsed": int, - "effectiveGasPrice": int, - "gasUsed": Wei, + "effectiveGasPrice": Wei, + "gasUsed": int, "from": ChecksumAddress, "logs": List[LogReceipt], "logsBloom": HexBytes, @@ -308,8 +308,8 @@ class BlockData(TypedDict, total=False): baseFeePerGas: Wei difficulty: int extraData: HexBytes - gasLimit: Wei - gasUsed: Wei + gasLimit: int + gasUsed: int hash: HexBytes logsBloom: HexBytes miner: ChecksumAddress From 26c6d8fdcadcf65e6e8ba0984f4b41c89febed34 Mon Sep 17 00:00:00 2001 From: kclowes Date: Fri, 11 Feb 2022 13:29:13 -0700 Subject: [PATCH 41/48] Upgrade websockets dependency to v10+ (#2324) * Require websockets v10+ - Remove event loop parameter * Add newsfragment for websockets upgrade --- newsfragments/2324.breaking-change.rst | 1 + setup.py | 2 +- tests/core/providers/test_websocket_provider.py | 6 ++++-- tests/integration/go_ethereum/test_goethereum_ws.py | 6 ++++-- tests/utils.py | 4 ++-- web3/providers/websocket.py | 7 +++---- 6 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 newsfragments/2324.breaking-change.rst diff --git a/newsfragments/2324.breaking-change.rst b/newsfragments/2324.breaking-change.rst new file mode 100644 index 0000000000..9bcbb72602 --- /dev/null +++ b/newsfragments/2324.breaking-change.rst @@ -0,0 +1 @@ +Update ``websockets`` dependency to v10+ diff --git a/setup.py b/setup.py index 2df5580452..dc1c816c54 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ "requests>=2.16.0,<3.0.0", # remove typing_extensions after python_requires>=3.8, see web3._utils.compat "typing-extensions>=3.7.4.1,<5;python_version<'3.8'", - "websockets>=9.1,<10", + "websockets>=10.0.0,<11", ], python_requires='>=3.7,<3.10', extras_require=extras_require, diff --git a/tests/core/providers/test_websocket_provider.py b/tests/core/providers/test_websocket_provider.py index 1036b38fe1..562225fa1a 100644 --- a/tests/core/providers/test_websocket_provider.py +++ b/tests/core/providers/test_websocket_provider.py @@ -37,7 +37,9 @@ async def empty_server(websocket, path): data = await websocket.recv() await asyncio.sleep(0.02) await websocket.send(data) - server = websockets.serve(empty_server, '127.0.0.1', open_port, loop=event_loop) + + asyncio.set_event_loop(event_loop) + server = websockets.serve(empty_server, '127.0.0.1', open_port) event_loop.run_until_complete(server) event_loop.run_forever() @@ -54,7 +56,7 @@ def w3(open_port, start_websocket_server): # need new event loop as the one used by server is already running event_loop = asyncio.new_event_loop() endpoint_uri = 'ws://127.0.0.1:{}'.format(open_port) - event_loop.run_until_complete(wait_for_ws(endpoint_uri, event_loop)) + event_loop.run_until_complete(wait_for_ws(endpoint_uri)) provider = WebsocketProvider(endpoint_uri, websocket_timeout=0.01) return Web3(provider) diff --git a/tests/integration/go_ethereum/test_goethereum_ws.py b/tests/integration/go_ethereum/test_goethereum_ws.py index 7de95ba63c..9014f4fc1d 100644 --- a/tests/integration/go_ethereum/test_goethereum_ws.py +++ b/tests/integration/go_ethereum/test_goethereum_ws.py @@ -1,3 +1,4 @@ +import asyncio import pytest from tests.integration.common import ( @@ -63,8 +64,9 @@ def geth_command_arguments(geth_binary, @pytest.fixture(scope="module") -def web3(geth_process, endpoint_uri, event_loop): - event_loop.run_until_complete(wait_for_ws(endpoint_uri, event_loop)) +def web3(geth_process, endpoint_uri): + event_loop = asyncio.new_event_loop() + event_loop.run_until_complete(wait_for_ws(endpoint_uri)) _web3 = Web3(Web3.WebsocketProvider(endpoint_uri, websocket_timeout=30)) return _web3 diff --git a/tests/utils.py b/tests/utils.py index 05f2899621..268d33d6be 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -13,11 +13,11 @@ def get_open_port(): return str(port) -async def wait_for_ws(endpoint_uri, event_loop, timeout=60): +async def wait_for_ws(endpoint_uri, timeout=10): start = time.time() while time.time() < start + timeout: try: - async with websockets.connect(uri=endpoint_uri, loop=event_loop): + async with websockets.connect(uri=endpoint_uri): pass except (ConnectionRefusedError, OSError): await asyncio.sleep(0.01) diff --git a/web3/providers/websocket.py b/web3/providers/websocket.py index 3e4f7e036c..e999d56d3f 100644 --- a/web3/providers/websocket.py +++ b/web3/providers/websocket.py @@ -60,17 +60,16 @@ def get_default_endpoint() -> URI: class PersistentWebSocket: def __init__( - self, endpoint_uri: URI, loop: asyncio.AbstractEventLoop, websocket_kwargs: Any + self, endpoint_uri: URI, websocket_kwargs: Any ) -> None: self.ws: WebSocketClientProtocol = None self.endpoint_uri = endpoint_uri - self.loop = loop self.websocket_kwargs = websocket_kwargs async def __aenter__(self) -> WebSocketClientProtocol: if self.ws is None: self.ws = await connect( - uri=self.endpoint_uri, loop=self.loop, **self.websocket_kwargs + uri=self.endpoint_uri, **self.websocket_kwargs ) return self.ws @@ -113,7 +112,7 @@ def __init__( 'found: {1}'.format(RESTRICTED_WEBSOCKET_KWARGS, found_restricted_keys) ) self.conn = PersistentWebSocket( - self.endpoint_uri, WebsocketProvider._loop, websocket_kwargs + self.endpoint_uri, websocket_kwargs ) super().__init__() From 16aff7e80a6bc089051c154b5079213317c27da8 Mon Sep 17 00:00:00 2001 From: Mahmoud Harmouch Date: Mon, 14 Feb 2022 20:05:02 +0200 Subject: [PATCH 42/48] :heavy_plus_sign: Add Python 3.10 support (#2175) * :heavy_plus_sign: Add Python 3.10 support to CI * Dropped support for all parities * Change docker image to use 3.10 * Update pytest-asyncio plugin * Mark async fixture as such, clean up pytest DeprecationWarnings Signed-off-by: Harmouch101 Co-authored-by: Felipe Selmo Co-authored-by: kclowes --- .circleci/config.yml | 75 ++++++++++++++++++- newsfragments/2175.feature.rst | 1 + pytest.ini | 1 + setup.py | 9 ++- .../core/providers/test_websocket_provider.py | 4 +- tests/integration/conftest.py | 2 +- tests/integration/go_ethereum/conftest.py | 2 +- .../go_ethereum/test_goethereum_http.py | 4 +- tox.ini | 17 +++-- web3/_utils/module_testing/web3_module.py | 2 +- 10 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 newsfragments/2175.feature.rst diff --git a/.circleci/config.yml b/.circleci/config.yml index c76ed67ab7..08c9b05884 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -421,10 +421,75 @@ jobs: environment: TOXENV: py39-wheel-cli + # + # Python 3.10 + # + py310-core: + <<: *common + docker: + - image: circleci/python:3.10 + environment: + TOXENV: py310-core + + py310-ens: + <<: *common + docker: + - image: circleci/python:3.10 + environment: + TOXENV: py310-ens + + py310-ethpm: + <<: *ethpm_steps + docker: + - image: circleci/python:3.10 + environment: + TOXENV: py310-ethpm + # Please don't use this key for any shenanigans + WEB3_INFURA_PROJECT_ID: 7707850c2fb7465ebe6f150d67182e22 + + py310-integration-goethereum-ipc: + <<: *geth_steps + docker: + - image: circleci/python:3.10 + environment: + TOXENV: py310-integration-goethereum-ipc + GETH_VERSION: v1.10.11 + + py310-integration-goethereum-http: + <<: *geth_steps + docker: + - image: circleci/python:3.10 + environment: + TOXENV: py310-integration-goethereum-http + GETH_VERSION: v1.10.11 + + py310-integration-goethereum-ws: + <<: *geth_steps + docker: + - image: circleci/python:3.10 + environment: + TOXENV: py310-integration-goethereum-ws + GETH_VERSION: v1.10.11 + + py310-integration-ethtester-pyevm: + <<: *common + docker: + - image: circleci/python:3.10 + environment: + TOXENV: py310-integration-ethtester + ETHEREUM_TESTER_CHAIN_BACKEND: eth_tester.backends.PyEVMBackend + + py310-wheel-cli: + <<: *common + docker: + - image: circleci/python:3.10 + environment: + TOXENV: py310-wheel-cli + benchmark: <<: *geth_steps docker: - - image: circleci/python:3.9 + - image: circleci/python:3.10 environment: TOXENV: benchmark GETH_VERSION: v1.10.13 @@ -437,6 +502,7 @@ workflows: - py37-core - py38-core - py39-core + - py310-core - lint - docs - benchmark @@ -462,3 +528,10 @@ workflows: - py39-integration-goethereum-ws - py39-integration-ethtester-pyevm - py39-wheel-cli + - py310-ens + - py310-ethpm + - py310-integration-goethereum-ipc + - py310-integration-goethereum-http + - py310-integration-goethereum-ws + - py310-integration-ethtester-pyevm + - py310-wheel-cli diff --git a/newsfragments/2175.feature.rst b/newsfragments/2175.feature.rst new file mode 100644 index 0000000000..dd5d94db98 --- /dev/null +++ b/newsfragments/2175.feature.rst @@ -0,0 +1 @@ +Add support for Python 3.10 diff --git a/pytest.ini b/pytest.ini index f5fdc0ec93..f4b46a1453 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,6 +2,7 @@ addopts= -v --showlocals --durations 10 python_paths= . xfail_strict=true +asyncio_mode=strict [pytest-watch] runner= pytest --failed-first --maxfail=1 --no-success-flaky-report diff --git a/setup.py b/setup.py index dc1c816c54..e4b28448d5 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ "contextlib2>=0.5.4", "py-geth>=3.6.0,<4", "py-solc>=0.4.0", - "pytest>=4.4.0,<5.0.0", + "pytest>=6.2.5,<7", "sphinx>=3.0,<4", "sphinx_rtd_theme>=0.1.9", "toposort>=1.4", @@ -38,8 +38,8 @@ "bumpversion", "flaky>=3.7.0,<4", "hypothesis>=3.31.2,<6", - "pytest>=4.4.0,<5.0.0", - "pytest-asyncio>=0.10.0,<0.11", + "pytest>=6.2.5,<7", + "pytest-asyncio>=0.18.1,<0.19", "pytest-mock>=1.10,<2", "pytest-pythonpath>=0.3", "pytest-watch>=4.2,<5", @@ -92,7 +92,7 @@ "typing-extensions>=3.7.4.1,<5;python_version<'3.8'", "websockets>=10.0.0,<11", ], - python_requires='>=3.7,<3.10', + python_requires='>=3.7,<3.11', extras_require=extras_require, py_modules=['web3', 'ens', 'ethpm'], entry_points={"pytest11": ["pytest_ethereum = web3.tools.pytest_ethereum.plugins"]}, @@ -110,5 +110,6 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], ) diff --git a/tests/core/providers/test_websocket_provider.py b/tests/core/providers/test_websocket_provider.py index 562225fa1a..0cf475d31e 100644 --- a/tests/core/providers/test_websocket_provider.py +++ b/tests/core/providers/test_websocket_provider.py @@ -28,7 +28,7 @@ ) -@pytest.yield_fixture +@pytest.fixture def start_websocket_server(open_port): event_loop = asyncio.new_event_loop() @@ -51,7 +51,7 @@ async def empty_server(websocket, path): event_loop.call_soon_threadsafe(event_loop.stop) -@pytest.fixture() +@pytest.fixture def w3(open_port, start_websocket_server): # need new event loop as the one used by server is already running event_loop = asyncio.new_event_loop() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a9ddbbd19e..dd254c6876 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -37,7 +37,7 @@ def revert_contract_factory(web3): return contract_factory -@pytest.yield_fixture(scope="module") +@pytest.fixture(scope="module") def event_loop(request): loop = asyncio.get_event_loop_policy().new_event_loop() yield loop diff --git a/tests/integration/go_ethereum/conftest.py b/tests/integration/go_ethereum/conftest.py index 7851b2e4a5..59934890c7 100644 --- a/tests/integration/go_ethereum/conftest.py +++ b/tests/integration/go_ethereum/conftest.py @@ -181,7 +181,7 @@ def unlockable_account_dual_type(unlockable_account, address_conversion_func): return address_conversion_func(unlockable_account) -@pytest.yield_fixture +@pytest.fixture def unlocked_account_dual_type(web3, unlockable_account_dual_type, unlockable_account_pw): web3.geth.personal.unlock_account(unlockable_account_dual_type, unlockable_account_pw) yield unlockable_account_dual_type diff --git a/tests/integration/go_ethereum/test_goethereum_http.py b/tests/integration/go_ethereum/test_goethereum_http.py index a94dc8b7f7..a3ea078cac 100644 --- a/tests/integration/go_ethereum/test_goethereum_http.py +++ b/tests/integration/go_ethereum/test_goethereum_http.py @@ -1,5 +1,7 @@ import pytest +import pytest_asyncio + from tests.utils import ( get_open_port, ) @@ -94,7 +96,7 @@ def web3(geth_process, endpoint_uri): return _web3 -@pytest.fixture(scope="module") +@pytest_asyncio.fixture(scope="module") async def async_w3(geth_process, endpoint_uri): await wait_for_aiohttp(endpoint_uri) _web3 = Web3( diff --git a/tox.ini b/tox.ini index f91ecfe023..ada9f14247 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,13 @@ [tox] envlist= - py{37,38,39}-ens - py{37,38,39}-ethpm - py{37,38,39}-core - py{37,38,39}-integration-{goethereum,ethtester} + py{37,38,39,310}-ens + py{37,38,39,310}-ethpm + py{37,38,39,310}-core + py{37,38,39,310}-integration-{goethereum,ethtester} lint docs benchmark - py{37,38,39}-wheel-cli + py{37,38,39,310}-wheel-cli [isort] combine_as_imports=True @@ -52,6 +52,7 @@ basepython = py37: python3.7 py38: python3.8 py39: python3.9 + py310: python3.10 [testenv:lint] basepython=python @@ -97,6 +98,12 @@ whitelist_externals={[common-wheel-cli]whitelist_externals} commands={[common-wheel-cli]commands} skip_install=true +[testenv:py310-wheel-cli] +deps={[common-wheel-cli]deps} +whitelist_externals={[common-wheel-cli]whitelist_externals} +commands={[common-wheel-cli]commands} +skip_install=true + [common-wheel-cli-windows] deps=wheel whitelist_externals= diff --git a/web3/_utils/module_testing/web3_module.py b/web3/_utils/module_testing/web3_module.py index a8eb73f332..17e25e1ff7 100644 --- a/web3/_utils/module_testing/web3_module.py +++ b/web3/_utils/module_testing/web3_module.py @@ -179,7 +179,7 @@ def test_solidityKeccak( self, web3: "Web3", types: Sequence[TypeStr], values: Sequence[Any], expected: HexBytes ) -> None: if isinstance(expected, type) and issubclass(expected, Exception): - with pytest.raises(expected): + with pytest.raises(expected): # type: ignore web3.solidityKeccak(types, values) return From 703b8f6b8874a9acce80025b68583925d9845333 Mon Sep 17 00:00:00 2001 From: dsahdr <72198703+dsahdr@users.noreply.github.com> Date: Fri, 21 Jan 2022 16:58:55 +0300 Subject: [PATCH 43/48] add fork description --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 19b242a6d2..39775cdd73 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,23 @@ A Python library for interacting with Ethereum, inspired by [web3.js](https://gi --- +## Rock'N'Block Fork +Added support of multiple rpc nodes for HTTPProvider. +Usage: +``` +from web3 import Web3 + +web3 = Web3( + Web3.HTTPProvider( + [endpoint1, endpoint2, ...] + ) +) +``` +For a backwards compatibility you can still pass a single string rpc provider as usual, i.e. ```Web3.HTTPProvider('endpoint')``` + +For any blockchain call using web3 (simple requests like "gas_price", or contract interactions and filters,), web3 instance will try to make a call using one provider at once in cycle, and will change provider if any RequestException is thrown. If all endpoints are invalid, built-in "CannotHandleRequest" Exception is thrown. + + ## Quickstart [Get started in 5 minutes](https://web3py.readthedocs.io/en/latest/quickstart.html) or From 8c45122db3955217c5980bbf3db32dce21ed23d9 Mon Sep 17 00:00:00 2001 From: dsahdr <72198703+dsahdr@users.noreply.github.com> Date: Fri, 21 Jan 2022 16:59:28 +0300 Subject: [PATCH 44/48] [NBA-39] add multiple nodes for web3 HTTPProvider (#1) --- web3/providers/rpc.py | 93 +++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 0be744f71d..9cf425dcdd 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -1,4 +1,14 @@ +from eth_typing import ( + URI, +) +from eth_utils import ( + to_dict, +) import logging +import random +from requests import ( + RequestException, +) from typing import ( Any, Dict, @@ -8,13 +18,6 @@ Union, ) -from eth_typing import ( - URI, -) -from eth_utils import ( - to_dict, -) - from web3._utils.http import ( construct_user_agent, ) @@ -26,9 +29,15 @@ from web3.datastructures import ( NamedElementOnion, ) +from web3.exceptions import ( + CannotHandleRequest, +) from web3.middleware import ( http_retry_request_middleware, ) +from web3.providers import ( + BaseProvider, +) from web3.types import ( Middleware, RPCEndpoint, @@ -42,56 +51,70 @@ class HTTPProvider(JSONBaseProvider): logger = logging.getLogger("web3.providers.HTTPProvider") - endpoint_uri = None + providers = None _request_args = None _request_kwargs = None # type ignored b/c conflict with _middlewares attr on BaseProvider - _middlewares: Tuple[Middleware, ...] = NamedElementOnion([(http_retry_request_middleware, 'http_retry_request')]) # type: ignore # noqa: E501 + _middlewares: Tuple[Middleware, ...] = NamedElementOnion([(http_retry_request_middleware, "http_retry_request")]) # type: ignore # noqa: E501 def __init__( - self, endpoint_uri: Optional[Union[URI, str]] = None, - request_kwargs: Optional[Any] = None, - session: Optional[Any] = None + self, + providers: Union[list, str], + randomize: Optional[bool] = False, + request_kwargs: Optional[Any] = None, + session: Optional[Any] = None, ) -> None: - if endpoint_uri is None: - self.endpoint_uri = get_default_http_endpoint() - else: - self.endpoint_uri = URI(endpoint_uri) - + if isinstance(providers, str): + providers = [ + providers, + ] + self.randomize = randomize + self.providers = providers self._request_kwargs = request_kwargs or {} if session: - cache_session(self.endpoint_uri, session) + cache_session(self.providers[0], session) super().__init__() def __str__(self) -> str: - return "RPC connection {0}".format(self.endpoint_uri) + return "RPC connection {0}".format(self.providers) @to_dict def get_request_kwargs(self) -> Iterable[Tuple[str, Any]]: - if 'headers' not in self._request_kwargs: - yield 'headers', self.get_request_headers() + if "headers" not in self._request_kwargs: + yield "headers", self.get_request_headers() for key, value in self._request_kwargs.items(): yield key, value def get_request_headers(self) -> Dict[str, str]: return { - 'Content-Type': 'application/json', - 'User-Agent': construct_user_agent(str(type(self))), + "Content-Type": "application/json", + "User-Agent": construct_user_agent(str(type(self))), } def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: - self.logger.debug("Making request HTTP. URI: %s, Method: %s", - self.endpoint_uri, method) request_data = self.encode_rpc_request(method, params) - raw_response = make_post_request( - self.endpoint_uri, - request_data, - **self.get_request_kwargs() - ) - response = self.decode_rpc_response(raw_response) - self.logger.debug("Getting response HTTP. URI: %s, " - "Method: %s, Response: %s", - self.endpoint_uri, method, response) - return response + if self.randomize: + random.shuffle(self.providers) + for provider in self.providers: + provider_uri = URI(provider) + self.logger.debug( + "Making request HTTP. URI: %s, Method: %s", provider_uri, method + ) + try: + raw_response = make_post_request( + provider_uri, request_data, **self.get_request_kwargs() + ) + response = self.decode_rpc_response(raw_response) + self.logger.debug( + "Getting response HTTP. URI: %s, " "Method: %s, Response: %s", + provider_uri, + method, + response, + ) + return response + except RequestException: + pass + else: + raise CannotHandleRequest From dc674a39ee05c027dae6bfa10a1016ffdf9273c4 Mon Sep 17 00:00:00 2001 From: dsahdr <72198703+dsahdr@users.noreply.github.com> Date: Fri, 21 Jan 2022 17:08:53 +0300 Subject: [PATCH 45/48] Update README.md --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 39775cdd73..5ef453da32 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ A Python library for interacting with Ethereum, inspired by [web3.js](https://gi ## Rock'N'Block Fork Added support of multiple rpc nodes for HTTPProvider. Usage: -``` +```python from web3 import Web3 web3 = Web3( @@ -22,10 +22,14 @@ web3 = Web3( ) ) ``` -For a backwards compatibility you can still pass a single string rpc provider as usual, i.e. ```Web3.HTTPProvider('endpoint')``` +For a backwards compatibility you can still pass a single string rpc provider as usual, i.e. +```python +Web3.HTTPProvider('endpoint') +``` -For any blockchain call using web3 (simple requests like "gas_price", or contract interactions and filters,), web3 instance will try to make a call using one provider at once in cycle, and will change provider if any RequestException is thrown. If all endpoints are invalid, built-in "CannotHandleRequest" Exception is thrown. +For any blockchain call using web3 (simple requests like **"gas_price"**, or contract interactions and filters,), web3 instance will try to make a call using one provider at once in cycle, and will change provider if any RequestException is thrown. If all endpoints are invalid, built-in **"CannotHandleRequest"** Exception is thrown. +Additionally, you can use some kind of node balancing, passing **"randomize=True"** in HTTPProvider initialization. In this case, every time you make a call, nodes are shuffled randomly, meaning that requests will be shared between them (instead of logic "call first infura unless it passes out, after that go to the second one"). ## Quickstart From 11d08baa0f3976de975552ce1e574976f1c86b23 Mon Sep 17 00:00:00 2001 From: dsahdr <72198703+dsahdr@users.noreply.github.com> Date: Fri, 21 Jan 2022 17:14:03 +0300 Subject: [PATCH 46/48] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 5ef453da32..3a6c790bbb 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,11 @@ A Python library for interacting with Ethereum, inspired by [web3.js](https://gi ## Rock'N'Block Fork Added support of multiple rpc nodes for HTTPProvider. Usage: +choose custom source for downloading library, i.e (for installing via requirements.txt): +``` +web3 @ git+https://github.com/Rock-n-Block/web3.py.git +``` + ```python from web3 import Web3 From 6894c962a834cb6160e0f9e72464a16cd9342953 Mon Sep 17 00:00:00 2001 From: dsahdr <72198703+dsahdr@users.noreply.github.com> Date: Fri, 21 Jan 2022 17:14:43 +0300 Subject: [PATCH 47/48] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a6c790bbb..1208c6f564 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ choose custom source for downloading library, i.e (for installing via requiremen ``` web3 @ git+https://github.com/Rock-n-Block/web3.py.git ``` - +initialize web3 instance in code: ```python from web3 import Web3 From 4a45b25818f16f6be0b6f4bf02822a8f71df1899 Mon Sep 17 00:00:00 2001 From: dsahdr <72198703+dsahdr@users.noreply.github.com> Date: Fri, 21 Jan 2022 17:17:41 +0300 Subject: [PATCH 48/48] typo fix --- web3/providers/rpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 9cf425dcdd..820808385b 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -52,6 +52,7 @@ class HTTPProvider(JSONBaseProvider): logger = logging.getLogger("web3.providers.HTTPProvider") providers = None + randomize = False _request_args = None _request_kwargs = None # type ignored b/c conflict with _middlewares attr on BaseProvider