From 96b457d6f79626cb0bfbfadb6a70a73c70dc9df3 Mon Sep 17 00:00:00 2001 From: Marc Garreau <3621728+marcgarreau@users.noreply.github.com> Date: Wed, 19 May 2021 11:13:38 -0600 Subject: [PATCH] benchmark async vs. sync http --- .circleci/config.yml | 9 ++ Makefile | 3 + newsfragments/2002.misc.rst | 1 + setup.py | 2 +- tox.ini | 14 ++- web3/tools/benchmark/__init__.py | 0 web3/tools/benchmark/main.py | 140 ++++++++++++++++++++++++++++++ web3/tools/benchmark/node.py | 115 ++++++++++++++++++++++++ web3/tools/benchmark/reporting.py | 36 ++++++++ web3/tools/benchmark/utils.py | 68 +++++++++++++++ 10 files changed, 384 insertions(+), 4 deletions(-) create mode 100644 newsfragments/2002.misc.rst create mode 100644 web3/tools/benchmark/__init__.py create mode 100644 web3/tools/benchmark/main.py create mode 100644 web3/tools/benchmark/node.py create mode 100644 web3/tools/benchmark/reporting.py create mode 100644 web3/tools/benchmark/utils.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 142dff050b..65908f1438 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -560,6 +560,14 @@ jobs: environment: TOXENV: py39-wheel-cli + benchmark: + <<: *geth_steps + docker: + - image: circleci/python:3.9 + environment: + TOXENV: benchmark + GETH_VERSION: v1.10.1 + workflows: version: 2.1 test: @@ -571,6 +579,7 @@ workflows: - py39-core - lint - docs + - benchmark - py36-ens - py36-ethpm - py36-integration-goethereum-ipc diff --git a/Makefile b/Makefile index 94a0be5e2c..1d6428b94c 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,9 @@ test: test-all: tox +benchmark: + tox -e benchmark + build-docs: sphinx-apidoc -o docs/ . setup.py "web3/utils/*" "*conftest*" "tests" "ethpm" $(MAKE) -C docs clean diff --git a/newsfragments/2002.misc.rst b/newsfragments/2002.misc.rst new file mode 100644 index 0000000000..d1854df436 --- /dev/null +++ b/newsfragments/2002.misc.rst @@ -0,0 +1 @@ +Add basic provider benchmarking infrastructure diff --git a/setup.py b/setup.py index 788acc73d0..e84db7a928 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ "click>=5.1", "configparser==3.5.0", "contextlib2>=0.5.4", - "py-geth>=2.4.0,<3", + "py-geth>=3.0.0,<4", "py-solc>=0.4.0", "pytest>=4.4.0,<5.0.0", "sphinx>=3.0,<4", diff --git a/tox.ini b/tox.ini index a85ae94f4a..9e7da41f9e 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ envlist= py{36,37,38,39}-integration-{goethereum,ethtester,parity} lint docs + benchmark py{36,37,38,39}-wheel-cli [isort] @@ -63,9 +64,16 @@ basepython = basepython=python extras=linter commands= - flake8 {toxinidir}/web3 {toxinidir}/ens {toxinidir}/ethpm {toxinidir}/tests --exclude {toxinidir}/ethpm/ethpm-spec - isort --recursive --check-only --diff {toxinidir}/web3/ {toxinidir}/ens/ {toxinidir}/ethpm/ {toxinidir}/tests/ - mypy -p web3 -p ethpm -p ens --config-file {toxinidir}/mypy.ini + flake8 {toxinidir}/web3 {toxinidir}/ens {toxinidir}/ethpm {toxinidir}/tests --exclude {toxinidir}/ethpm/ethpm-spec + isort --recursive --check-only --diff {toxinidir}/web3/ {toxinidir}/ens/ {toxinidir}/ethpm/ {toxinidir}/tests/ + mypy -p web3 -p ethpm -p ens --config-file {toxinidir}/mypy.ini + +[testenv:benchmark] +basepython=python +commands= + python {toxinidir}/web3/tools/benchmark/main.py --num-calls 5 + python {toxinidir}/web3/tools/benchmark/main.py --num-calls 50 + python {toxinidir}/web3/tools/benchmark/main.py --num-calls 100 [common-wheel-cli] deps=wheel diff --git a/web3/tools/benchmark/__init__.py b/web3/tools/benchmark/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/web3/tools/benchmark/main.py b/web3/tools/benchmark/main.py new file mode 100644 index 0000000000..b3ca83ac2a --- /dev/null +++ b/web3/tools/benchmark/main.py @@ -0,0 +1,140 @@ +import argparse +import asyncio +from collections import ( + defaultdict, +) +import logging +import sys +import timeit +from typing import ( + Any, + Callable, + Dict, + Union, +) + +from web3 import ( + AsyncHTTPProvider, + HTTPProvider, + Web3, +) +from web3.eth import ( + AsyncEth, +) +from web3.tools.benchmark.node import ( + GethBenchmarkFixture, +) +from web3.tools.benchmark.reporting import ( + print_entry, + print_footer, + print_header, +) +from web3.tools.benchmark.utils import ( + wait_for_aiohttp, + wait_for_http, +) + +parser = argparse.ArgumentParser() +parser.add_argument( + "--num-calls", type=int, default=10, help="The number of RPC calls to make", +) + +# TODO - layers to test: +# contract.functions.method(...).call() +# w3.eth.call(...) +# HTTPProvider.make_request(...) + + +def build_web3_http(endpoint_uri: str) -> Web3: + wait_for_http(endpoint_uri) + _web3 = Web3(HTTPProvider(endpoint_uri), middlewares=[]) + return _web3 + + +async def build_async_w3_http(endpoint_uri: str) -> Web3: + await wait_for_aiohttp(endpoint_uri) + _web3 = Web3( + AsyncHTTPProvider(endpoint_uri), # type: ignore + middlewares=[], + modules={"async_eth": (AsyncEth,)}, + ) + return _web3 + + +def sync_benchmark(func: Callable[..., Any], n: int) -> Union[float, str]: + try: + starttime = timeit.default_timer() + for _ in range(n): + func() + endtime = timeit.default_timer() + execution_time = endtime - starttime + return execution_time + except Exception: + return "N/A" + + +async def async_benchmark(func: Callable[..., Any], n: int) -> Union[float, str]: + try: + starttime = timeit.default_timer() + for result in asyncio.as_completed([func() for _ in range(n)]): + await result + execution_time = timeit.default_timer() - starttime + return execution_time + except Exception: + return "N/A" + + +def main(logger: logging.Logger, num_calls: int) -> None: + fixture = GethBenchmarkFixture() + for built_fixture in fixture.build(): + for process in built_fixture: + w3_http = build_web3_http(fixture.endpoint_uri) + loop = asyncio.get_event_loop() + async_w3_http = loop.run_until_complete(build_async_w3_http(fixture.endpoint_uri)) + + methods = [ + { + "name": "eth_gasPrice", + "params": {}, + "exec": lambda: w3_http.eth.gas_price, + "async_exec": lambda: async_w3_http.async_eth.gas_price, + }, + { + "name": "eth_blockNumber", + "params": {}, + "exec": lambda: w3_http.eth.block_number, + "async_exec": lambda: (_ for _ in ()).throw(Exception("not implemented yet")), + }, + { + "name": "eth_getBlock", + "params": {}, + "exec": lambda: w3_http.eth.get_block("1"), + "async_exec": lambda: (_ for _ in ()).throw(Exception("not implemented yet")), + }, + ] + + def benchmark(method: Dict[str, Any]) -> None: + outcomes: Dict[str, Union[str, float]] = defaultdict(lambda: "N/A") + outcomes["name"] = method["name"] + outcomes["HTTPProvider"] = sync_benchmark(method["exec"], num_calls,) + outcomes["AsyncHTTPProvider"] = loop.run_until_complete( + async_benchmark(method["async_exec"], num_calls) + ) + print_entry(logger, outcomes) + + print_header(logger, num_calls) + + for method in methods: + benchmark(method) + + print_footer(logger) + + +if __name__ == "__main__": + args = parser.parse_args() + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + logger.addHandler(logging.StreamHandler(sys.stdout)) + + main(logger, args.num_calls) diff --git a/web3/tools/benchmark/node.py b/web3/tools/benchmark/node.py new file mode 100644 index 0000000000..fc4fae1b11 --- /dev/null +++ b/web3/tools/benchmark/node.py @@ -0,0 +1,115 @@ +import os +import socket +from subprocess import ( + PIPE, + Popen, + check_output, +) +from tempfile import ( + TemporaryDirectory, +) +from typing import ( + Any, + Generator, + Sequence, +) +import zipfile + +from geth.install import ( + get_executable_path, + install_geth, +) + +from web3.tools.benchmark.utils import ( + kill_proc_gracefully, +) + +GETH_FIXTURE_ZIP = "geth-1.10.1-fixture.zip" + + +class GethBenchmarkFixture: + def __init__(self) -> None: + self.rpc_port = self._rpc_port() + self.endpoint_uri = self._endpoint_uri() + self.geth_binary = self._geth_binary() + + def build(self) -> Generator[Any, None, None]: + with TemporaryDirectory() as base_dir: + zipfile_path = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "../../../tests/integration/", + GETH_FIXTURE_ZIP, + ) + ) + tmp_datadir = os.path.join(str(base_dir), "datadir") + with zipfile.ZipFile(zipfile_path, "r") as zip_ref: + zip_ref.extractall(tmp_datadir) + self.datadir = tmp_datadir + + genesis_file = os.path.join(self.datadir, "genesis.json") + + yield self._geth_process(self.datadir, genesis_file, self.rpc_port) + + def _rpc_port(self) -> str: + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + return str(port) + + def _endpoint_uri(self) -> str: + return "http://localhost:{0}".format(self.rpc_port) + + def _geth_binary(self) -> str: + if "GETH_BINARY" in os.environ: + return os.environ["GETH_BINARY"] + elif "GETH_VERSION" in os.environ: + geth_version = os.environ["GETH_VERSION"] + _geth_binary = get_executable_path(geth_version) + if not os.path.exists(_geth_binary): + install_geth(geth_version) + assert os.path.exists(_geth_binary) + return _geth_binary + else: + return "geth" + + def _geth_command_arguments(self, datadir: str) -> Sequence[str]: + return ( + self.geth_binary, + "--datadir", + str(datadir), + "--nodiscover", + "--fakepow", + "--http", + "--http.port", + self.rpc_port, + "--http.api", + "admin,eth,net,web3,personal,miner", + "--ipcdisable", + "--allow-insecure-unlock", + ) + + def _geth_process( + self, datadir: str, genesis_file: str, rpc_port: str + ) -> Generator[Any, None, None]: + init_datadir_command = ( + self.geth_binary, + "--datadir", + str(datadir), + "init", + str(genesis_file), + ) + check_output( + init_datadir_command, stdin=PIPE, stderr=PIPE, + ) + proc = Popen( + self._geth_command_arguments(datadir), + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + ) + try: + yield proc + finally: + kill_proc_gracefully(proc) diff --git a/web3/tools/benchmark/reporting.py b/web3/tools/benchmark/reporting.py new file mode 100644 index 0000000000..3dd0203ad9 --- /dev/null +++ b/web3/tools/benchmark/reporting.py @@ -0,0 +1,36 @@ +from logging import ( + Logger, +) +from typing import ( + Any, + Dict, +) + + +def print_header(logger: Logger, num_calls: int) -> None: + logger.info( + "|{:^26}|{:^20}|{:^20}|{:^20}|{:^20}|".format( + f"Method ({num_calls} calls)", + "HTTPProvider", + "AsyncHTTProvider", + "IPCProvider", + "WebsocketProvider", + ) + ) + logger.info("-" * 112) + + +def print_entry(logger: Logger, method_benchmarks: Dict[str, Any],) -> None: + logger.info( + "|{:^26}|{:^20.10}|{:^20.10}|{:^20.10}|{:^20.10}|".format( + method_benchmarks["name"], + method_benchmarks["HTTPProvider"], + method_benchmarks["AsyncHTTPProvider"], + method_benchmarks["IPCProvider"], + method_benchmarks["WebsocketProvider"], + ) + ) + + +def print_footer(logger: Logger) -> None: + logger.info("-" * 112) diff --git a/web3/tools/benchmark/utils.py b/web3/tools/benchmark/utils.py new file mode 100644 index 0000000000..d1b99d8ede --- /dev/null +++ b/web3/tools/benchmark/utils.py @@ -0,0 +1,68 @@ +import signal +import socket +import time +from typing import ( + Any, +) + +import aiohttp +import requests + + +def wait_for_socket(ipc_path: str, timeout: int = 30) -> None: + start = time.time() + while time.time() < start + timeout: + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(ipc_path) + sock.settimeout(timeout) + except (FileNotFoundError, socket.error): + time.sleep(0.01) + else: + break + + +def wait_for_http(endpoint_uri: str, timeout: int = 60) -> None: + start = time.time() + while time.time() < start + timeout: + try: + requests.get(endpoint_uri) + except requests.ConnectionError: + time.sleep(0.01) + else: + break + + +async def wait_for_aiohttp(endpoint_uri: str, timeout: int = 60) -> None: + start = time.time() + while time.time() < start + timeout: + try: + async with aiohttp.ClientSession() as session: + await session.get(endpoint_uri) + except aiohttp.client_exceptions.ClientConnectorError: + time.sleep(0.01) + else: + break + + +def wait_for_popen(proc: Any, timeout: int) -> None: + start = time.time() + while time.time() < start + timeout: + if proc.poll() is None: + time.sleep(0.01) + else: + break + + +def kill_proc_gracefully(proc: Any) -> None: + if proc.poll() is None: + proc.send_signal(signal.SIGINT) + wait_for_popen(proc, 13) + + if proc.poll() is None: + proc.terminate() + wait_for_popen(proc, 5) + + if proc.poll() is None: + proc.kill() + wait_for_popen(proc, 2)