diff --git a/tests/benchmark/test_profile_against_numpy.py b/tests/benchmark/test_profile_against_numpy.py
new file mode 100644
index 0000000000..c0712e48f4
--- /dev/null
+++ b/tests/benchmark/test_profile_against_numpy.py
@@ -0,0 +1,48 @@
+""" Profiles basic -jX functionality """
+# Copyright (c) 2020 Frank Harrison <doublethefish@gmail.com>
+
+# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+# For details: https://github.com/PyCQA/pylint/blob/master/COPYING
+
+# pylint: disable=protected-access,missing-function-docstring,no-self-use
+
+import os
+
+import pytest
+
+from pylint.lint import Run
+from pylint.testutils import TestReporter as Reporter
+
+
+def _get_py_files(scanpath):
+    assert os.path.exists(scanpath), "Dir not found %s" % scanpath
+
+    filepaths = []
+    for _dirpath, dirnames, filenames in os.walk(scanpath):
+        dirnames[:] = [dirname for dirname in dirnames if dirname != "__pycache__"]
+        filepaths.extend(
+            [filename for filename in filenames if filename.endswith(".py")]
+        )
+    return filepaths
+
+
+@pytest.mark.skipif(
+    os.environ.get("PYTEST_PROFILE_NUMPY", False),
+    reason="PYTEST_PROFILE_NUMPY, not set, assuming not a profile run",
+)
+class TestEstablishBaselineBenchmarks:
+    """ Runs against numpy """
+
+    def test_run(self):
+
+        numpy_checkout_path = os.path.abspath(
+            os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+        )
+        filepaths = _get_py_files(scanpath=numpy_checkout_path)
+
+        Run(filepaths, reporter=Reporter(), do_exit=False)
+
+        # assert runner.linter.msg_status == 0, (
+        #    "Expected no errors to be thrown: %s"
+        #    % pprint.pformat(runner.linter.reporter.messages)
+        # )
diff --git a/tox.ini b/tox.ini
index eb6c5694d1..9805d50938 100644
--- a/tox.ini
+++ b/tox.ini
@@ -124,3 +124,28 @@ commands =
     python -Wi -m pytest --exitfirst --failed-first --benchmark-enable {toxinidir}/tests {posargs:} --profile-svg --benchmark-group-by="group" --benchmark-save-data --benchmark-autosave
 
 changedir = {toxworkdir}
+
+[testenv:profile_against_numpy]
+deps =
+   https://github.com/PyCQA/astroid/tarball/master#egg=astroid-master-2.0
+   coverage<5.0
+   gprof2dot
+   isort
+   mccabe
+   pytest
+   pytest-xdist
+   pytest-profiling
+
+setenv =
+  PYTEST_PROFILE_NUMPY = 1
+
+whitelist_externals =
+    git
+    mkdir
+
+commands =
+    mkdir -p {toxinidir}/.downstream_checkouts
+    git clone git@github.com:numpy/numpy.git {toxinidir}/.downstream_checkouts/num_py
+    python -Wi -m pytest --exitfirst --benchmark-disable {toxinidir}/tests/benchmark/test_profile_against_numpy.py --profile-svg
+
+changedir = {toxworkdir}