Improved hypothesis outputs

brownie/_config.py

@@ -246,7 +246,6 @@ def _modify_hypothesis_settings(settings, name, parent):
         name,
         parent=hp_settings.get_profile(parent),
         database=DirectoryBasedExampleDatabase(_get_data_folder().joinpath("hypothesis")),
-        report_multiple_bugs=False,
         **settings,
     )
     hp_settings.load_profile(name)

brownie/data/default-config.yaml

@@ -45,6 +45,7 @@ console:
 hypothesis:
     deadline: null
     max_examples: 50
+    report_multiple_bugs: False
     stateful_step_count: 10
 
 autofetch_sources: false

brownie/test/__init__.py

@@ -44,6 +44,9 @@ def isolation_wrapper(*args, **kwargs):
         hy_given = _hypothesis_given(*given_args, **given_kwargs)
         hy_wrapped = hy_given(test)
 
+        # modify the wrapper name so it shows correctly in test report
+        isolation_wrapper.__name__ = test.__name__
+
         if hasattr(hy_wrapped, "hypothesis"):
             hy_wrapped.hypothesis.inner_test = isolation_wrapper
         return hy_wrapped

brownie/test/managers/base.py

@@ -2,9 +2,12 @@
 
 import json
 from hashlib import sha1
+from pathlib import Path
 
+from hypothesis.reporting import reporter as hy_reporter
 from py.path import local
 
+import brownie
 from brownie._config import CONFIG
 from brownie.project.scripts import _get_ast_hash
 from brownie.test import _apply_given_wrapper, coverage, output
@@ -69,6 +72,15 @@ def __init__(self, config, project):
             for txhash, coverage_eval in hashes["tx"].items():
                 coverage._add_cached_transaction(txhash, coverage_eval)
 
+    def _reduce_path_strings(self, text):
+        # convert absolute path strings to relative ones, prior to outputting to console
+        base_path = f"{Path(brownie.__file__).parent.as_posix()}"
+        project_path = f"{self.project_path.as_posix()}/"
+
+        text = text.replace(base_path, "brownie")
+        text = text.replace(project_path, "")
+        return text
+
     def _path(self, path):
         return self.project_path.joinpath(path).relative_to(self.project_path).as_posix()
 
@@ -104,15 +116,28 @@ def pytest_sessionstart(self):
         Called after the `Session` object has been created and before performing
         collection and entering the run test loop.
 
-        Removes `PytestAssertRewriteWarning` warnings from the terminalreporter.
-        This prevents warnings that "the `brownie` library was already imported and
-        so related assertions cannot be rewritten". The warning is not relevant
-        for end users who are performing tests with brownie, not on brownie,
-        so we suppress it to avoid confusion.
+        * Replaces the default hypothesis reporter with a one that applies source
+          highlights and increased vertical space between results. The effect is
+          seen in output for `hypothesis.errors.MultipleFailures`.
+
+        * Removes `PytestAssertRewriteWarning` warnings from the terminalreporter.
+          This prevents warnings that "the `brownie` library was already imported and
+          so related assertions cannot be rewritten". The warning is not relevant
+          for end users who are performing tests with brownie, not on brownie,
+          so we suppress it to avoid confusion.
 
         Removal of pytest warnings must be handled in this hook because session
         information is passed between xdist workers and master prior to test execution.
         """
+
+        def _hypothesis_reporter(text):
+            text = self._reduce_path_strings(text)
+            if text.startswith("Falsifying example") or text.startswith("Traceback"):
+                reporter.write("\n")
+            reporter._tw._write_source(text.split("\n"))
+
+        hy_reporter.default = _hypothesis_reporter
+
         reporter = self.config.pluginmanager.get_plugin("terminalreporter")
         warnings = reporter.stats.pop("warnings", [])
         warnings = [i for i in warnings if "PytestAssertRewriteWarning" not in i.message]
@@ -152,6 +177,33 @@ def pytest_report_teststatus(self, report):
                 return "xpassed", "X", "XPASS"
         return report.outcome, convert_outcome(report.outcome), report.outcome.upper()
 
+    def pytest_runtest_makereport(self, item):
+        """
+        Return a _pytest.runner.TestReport object for the given pytest.Item and
+        _pytest.runner.CallInfo.
+
+        Applies source highlighting to hypothesis output that is not related to
+        `hypothesis.errors.MultipleFailures`.
+
+        Attributes
+        ----------
+        item : pytest.Item
+            Object representing the currently active test
+        """
+        if not hasattr(item, "hypothesis_report_information"):
+            return
+
+        reporter = self.config.pluginmanager.get_plugin("terminalreporter")
+
+        report = [x for i in item.hypothesis_report_information for x in i.split("\n")]
+        report = [self._reduce_path_strings(i) for i in report]
+        report = [
+            reporter._tw._highlight(i).rstrip("\n") if not i.lstrip().startswith("\x1b") else i
+            for i in report
+        ]
+
+        item.hypothesis_report_information = report
+
     def pytest_terminal_summary(self, terminalreporter):
         """
         Add a section to terminal summary reporting.

brownie/test/plugin.py

@@ -60,14 +60,15 @@ def pytest_configure(config):
             print(f"{color.format_tb(e)}\n")
             raise pytest.UsageError("Unable to load project")
 
-        # apply __tracebackhide__ so brownie internals aren't included in tracebacks
+        # do not include brownie internals in tracebacks
         base_path = Path(sys.modules["brownie"].__file__).parent.as_posix()
         for module in [
             v
             for v in sys.modules.values()
             if getattr(v, "__file__", None) and v.__file__.startswith(base_path)
         ]:
             module.__tracebackhide__ = True
+            module.__hypothesistracebackhide__ = True
 
         # enable verbose output if stdout capture is disabled
         if config.getoption("capture") == "no":

docs/config.rst

@@ -218,6 +218,7 @@ Default settings for :ref:`property-based<hypothesis>` and :ref:`stateful<hypoth
     hypothesis:
         deadline: null
         max_examples: 50
+        report_multiple_bugs: False
         stateful_step_count: 10
 
 Other Settings
@@ -249,4 +250,4 @@ Other Settings
 
     This is useful if another application, such as a front end framework, needs access to deployment artifacts while you are on a development network.
 
-    default value: ``false``
+    default value: ``false``

docs/tests-hypothesis-property.rst

@@ -393,6 +393,12 @@ Available Settings
 
     default-value: ``50``
 
+.. py:attributes:: report_multiple_bugs
+
+    Because Hypothesis runs each test many times, it can sometimes find multiple bugs in a single run. Reporting all of them at once can be useful, but also produces significantly longer and less descriptive output when compared to reporting a single error.
+
+    default-value: ``False``
+
 .. py:attribute:: stateful_step_count
 
     The maximum number of rules to execute in a stateful program before ending the run and considering it to have passed.