benchmark async vs. sync http (#2002)

Author: wolovim

.circleci/config.yml

@@ -560,6 +560,14 @@ jobs:
         environment:
           TOXENV: py39-wheel-cli
 
+  benchmark:
+    <<: *geth_steps
+    docker:
+      - image: circleci/python:3.9
+    environment:
+      TOXENV: benchmark
+      GETH_VERSION: v1.10.1
+
 workflows:
   version: 2.1
   test:
@@ -571,6 +579,7 @@ workflows:
       - py39-core
       - lint
       - docs
+      - benchmark
       - py36-ens
       - py36-ethpm
       - py36-integration-goethereum-ipc

Makefile

@@ -34,6 +34,9 @@ test:
 test-all:
 	tox
 
+benchmark:
+	tox -e benchmark
+
 build-docs:
 	sphinx-apidoc -o docs/ . setup.py "web3/utils/*" "*conftest*" "tests" "ethpm"
 	$(MAKE) -C docs clean

newsfragments/2002.misc.rst

@@ -0,0 +1 @@
+Add basic provider benchmarking infrastructure

setup.py

@@ -21,7 +21,7 @@
         "click>=5.1",
         "configparser==3.5.0",
         "contextlib2>=0.5.4",
-        "py-geth>=2.4.0,<3",
+        "py-geth>=3.0.0,<4",
         "py-solc>=0.4.0",
         "pytest>=4.4.0,<5.0.0",
         "sphinx>=3.0,<4",

tox.ini

@@ -6,6 +6,7 @@ envlist=
     py{36,37,38,39}-integration-{goethereum,ethtester,parity}
     lint
     docs
+    benchmark
     py{36,37,38,39}-wheel-cli
 
 [isort]
@@ -63,9 +64,16 @@ basepython =
 basepython=python
 extras=linter
 commands=
-  flake8 {toxinidir}/web3 {toxinidir}/ens {toxinidir}/ethpm {toxinidir}/tests --exclude {toxinidir}/ethpm/ethpm-spec
-  isort --recursive --check-only --diff {toxinidir}/web3/ {toxinidir}/ens/ {toxinidir}/ethpm/ {toxinidir}/tests/
-  mypy -p web3 -p ethpm -p ens --config-file {toxinidir}/mypy.ini
+    flake8 {toxinidir}/web3 {toxinidir}/ens {toxinidir}/ethpm {toxinidir}/tests --exclude {toxinidir}/ethpm/ethpm-spec
+    isort --recursive --check-only --diff {toxinidir}/web3/ {toxinidir}/ens/ {toxinidir}/ethpm/ {toxinidir}/tests/
+    mypy -p web3 -p ethpm -p ens --config-file {toxinidir}/mypy.ini
+
+[testenv:benchmark]
+basepython=python
+commands=
+    python {toxinidir}/web3/tools/benchmark/main.py --num-calls 5
+    python {toxinidir}/web3/tools/benchmark/main.py --num-calls 50
+    python {toxinidir}/web3/tools/benchmark/main.py --num-calls 100
 
 [common-wheel-cli]
 deps=wheel

web3/tools/benchmark/__init__.py

@@ -0,0 +1,140 @@
+import argparse
+import asyncio
+from collections import (
+    defaultdict,
+)
+import logging
+import sys
+import timeit
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Union,
+)
+
+from web3 import (
+    AsyncHTTPProvider,
+    HTTPProvider,
+    Web3,
+)
+from web3.eth import (
+    AsyncEth,
+)
+from web3.tools.benchmark.node import (
+    GethBenchmarkFixture,
+)
+from web3.tools.benchmark.reporting import (
+    print_entry,
+    print_footer,
+    print_header,
+)
+from web3.tools.benchmark.utils import (
+    wait_for_aiohttp,
+    wait_for_http,
+)
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--num-calls", type=int, default=10, help="The number of RPC calls to make",
+)
+
+# TODO - layers to test:
+# contract.functions.method(...).call()
+# w3.eth.call(...)
+# HTTPProvider.make_request(...)
+
+
+def build_web3_http(endpoint_uri: str) -> Web3:
+    wait_for_http(endpoint_uri)
+    _web3 = Web3(HTTPProvider(endpoint_uri), middlewares=[])
+    return _web3
+
+
+async def build_async_w3_http(endpoint_uri: str) -> Web3:
+    await wait_for_aiohttp(endpoint_uri)
+    _web3 = Web3(
+        AsyncHTTPProvider(endpoint_uri),  # type: ignore
+        middlewares=[],
+        modules={"async_eth": (AsyncEth,)},
+    )
+    return _web3
+
+
+def sync_benchmark(func: Callable[..., Any], n: int) -> Union[float, str]:
+    try:
+        starttime = timeit.default_timer()
+        for _ in range(n):
+            func()
+        endtime = timeit.default_timer()
+        execution_time = endtime - starttime
+        return execution_time
+    except Exception:
+        return "N/A"
+
+
+async def async_benchmark(func: Callable[..., Any], n: int) -> Union[float, str]:
+    try:
+        starttime = timeit.default_timer()
+        for result in asyncio.as_completed([func() for _ in range(n)]):
+            await result
+        execution_time = timeit.default_timer() - starttime
+        return execution_time
+    except Exception:
+        return "N/A"
+
+
+def main(logger: logging.Logger, num_calls: int) -> None:
+    fixture = GethBenchmarkFixture()
+    for built_fixture in fixture.build():
+        for process in built_fixture:
+            w3_http = build_web3_http(fixture.endpoint_uri)
+            loop = asyncio.get_event_loop()
+            async_w3_http = loop.run_until_complete(build_async_w3_http(fixture.endpoint_uri))
+
+            methods = [
+                {
+                    "name": "eth_gasPrice",
+                    "params": {},
+                    "exec": lambda: w3_http.eth.gas_price,
+                    "async_exec": lambda: async_w3_http.async_eth.gas_price,
+                },
+                {
+                    "name": "eth_blockNumber",
+                    "params": {},
+                    "exec": lambda: w3_http.eth.block_number,
+                    "async_exec": lambda: (_ for _ in ()).throw(Exception("not implemented yet")),
+                },
+                {
+                    "name": "eth_getBlock",
+                    "params": {},
+                    "exec": lambda: w3_http.eth.get_block("1"),
+                    "async_exec": lambda: (_ for _ in ()).throw(Exception("not implemented yet")),
+                },
+            ]
+
+            def benchmark(method: Dict[str, Any]) -> None:
+                outcomes: Dict[str, Union[str, float]] = defaultdict(lambda: "N/A")
+                outcomes["name"] = method["name"]
+                outcomes["HTTPProvider"] = sync_benchmark(method["exec"], num_calls,)
+                outcomes["AsyncHTTPProvider"] = loop.run_until_complete(
+                    async_benchmark(method["async_exec"], num_calls)
+                )
+                print_entry(logger, outcomes)
+
+            print_header(logger, num_calls)
+
+            for method in methods:
+                benchmark(method)
+
+            print_footer(logger)
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    logger.addHandler(logging.StreamHandler(sys.stdout))
+
+    main(logger, args.num_calls)

web3/tools/benchmark/main.py

@@ -0,0 +1,115 @@
+import os
+import socket
+from subprocess import (
+    PIPE,
+    Popen,
+    check_output,
+)
+from tempfile import (
+    TemporaryDirectory,
+)
+from typing import (
+    Any,
+    Generator,
+    Sequence,
+)
+import zipfile
+
+from geth.install import (
+    get_executable_path,
+    install_geth,
+)
+
+from web3.tools.benchmark.utils import (
+    kill_proc_gracefully,
+)
+
+GETH_FIXTURE_ZIP = "geth-1.10.1-fixture.zip"
+
+
+class GethBenchmarkFixture:
+    def __init__(self) -> None:
+        self.rpc_port = self._rpc_port()
+        self.endpoint_uri = self._endpoint_uri()
+        self.geth_binary = self._geth_binary()
+
+    def build(self) -> Generator[Any, None, None]:
+        with TemporaryDirectory() as base_dir:
+            zipfile_path = os.path.abspath(
+                os.path.join(
+                    os.path.dirname(__file__),
+                    "../../../tests/integration/",
+                    GETH_FIXTURE_ZIP,
+                )
+            )
+            tmp_datadir = os.path.join(str(base_dir), "datadir")
+            with zipfile.ZipFile(zipfile_path, "r") as zip_ref:
+                zip_ref.extractall(tmp_datadir)
+            self.datadir = tmp_datadir
+
+            genesis_file = os.path.join(self.datadir, "genesis.json")
+
+            yield self._geth_process(self.datadir, genesis_file, self.rpc_port)
+
+    def _rpc_port(self) -> str:
+        sock = socket.socket()
+        sock.bind(("127.0.0.1", 0))
+        port = sock.getsockname()[1]
+        sock.close()
+        return str(port)
+
+    def _endpoint_uri(self) -> str:
+        return "http://localhost:{0}".format(self.rpc_port)
+
+    def _geth_binary(self) -> str:
+        if "GETH_BINARY" in os.environ:
+            return os.environ["GETH_BINARY"]
+        elif "GETH_VERSION" in os.environ:
+            geth_version = os.environ["GETH_VERSION"]
+            _geth_binary = get_executable_path(geth_version)
+            if not os.path.exists(_geth_binary):
+                install_geth(geth_version)
+            assert os.path.exists(_geth_binary)
+            return _geth_binary
+        else:
+            return "geth"
+
+    def _geth_command_arguments(self, datadir: str) -> Sequence[str]:
+        return (
+            self.geth_binary,
+            "--datadir",
+            str(datadir),
+            "--nodiscover",
+            "--fakepow",
+            "--http",
+            "--http.port",
+            self.rpc_port,
+            "--http.api",
+            "admin,eth,net,web3,personal,miner",
+            "--ipcdisable",
+            "--allow-insecure-unlock",
+        )
+
+    def _geth_process(
+        self, datadir: str, genesis_file: str, rpc_port: str
+    ) -> Generator[Any, None, None]:
+        init_datadir_command = (
+            self.geth_binary,
+            "--datadir",
+            str(datadir),
+            "init",
+            str(genesis_file),
+        )
+        check_output(
+            init_datadir_command, stdin=PIPE, stderr=PIPE,
+        )
+        proc = Popen(
+            self._geth_command_arguments(datadir),
+            stdin=PIPE,
+            stdout=PIPE,
+            stderr=PIPE,
+        )
+        try:
+            yield proc
+        finally:
+            kill_proc_gracefully(proc)

web3/tools/benchmark/node.py

@@ -0,0 +1,36 @@
+from logging import (
+    Logger,
+)
+from typing import (
+    Any,
+    Dict,
+)
+
+
+def print_header(logger: Logger, num_calls: int) -> None:
+    logger.info(
+        "|{:^26}|{:^20}|{:^20}|{:^20}|{:^20}|".format(
+            f"Method ({num_calls} calls)",
+            "HTTPProvider",
+            "AsyncHTTProvider",
+            "IPCProvider",
+            "WebsocketProvider",
+        )
+    )
+    logger.info("-" * 112)
+
+
+def print_entry(logger: Logger, method_benchmarks: Dict[str, Any],) -> None:
+    logger.info(
+        "|{:^26}|{:^20.10}|{:^20.10}|{:^20.10}|{:^20.10}|".format(
+            method_benchmarks["name"],
+            method_benchmarks["HTTPProvider"],
+            method_benchmarks["AsyncHTTPProvider"],
+            method_benchmarks["IPCProvider"],
+            method_benchmarks["WebsocketProvider"],
+        )
+    )
+
+
+def print_footer(logger: Logger) -> None:
+    logger.info("-" * 112)