From 7a81d8e4d3b21a788e270f54eb7cb67b7d1cb7b7 Mon Sep 17 00:00:00 2001
From: Will Shanks <willshanks@us.ibm.com>
Date: Fri, 13 Oct 2023 23:43:05 -0400
Subject: [PATCH 1/8] Remove usage of QiskitTestCase

This change removes the dependence on `QiskitTestCase`, replacing it
with a direct dependence on `unittest.TestCase` and
`testtools.TestCase`.

As with `QiskitTestCase`, the ability to run the tests based either on
`unittest.TestCase` or `testtools.TestCase` (a `unittest.TestCase`
subclass) is preserved. For qiskit-experiments, the ability is actually
restored because the timeout feature added in
[#1246](https://github.com/Qiskit-Extensions/qiskit-experiments/pull/1246)
had introduced a hard dependence on `testtools`.

Specific changes:

* Add `testtools` and `fixtures` to `requirements-dev.txt` as required
  test dependencies.
* Use `QE_USE_TESTTOOLS` environment variable to control whether tests
  are based on `testtools.TestCase` rather than checking if `testtools`
is installed.
* Remove some checks for test writing best practices. `QiskitTestCase`
  used extra code to ensure that `setUp` and other test class methods
always called their parents and that those methods are not called from
individual tests.  `testtools.TestCase` does these checks as well. Since
qiskit-experiments always uses `testtools` in CI, it can rely on
`testtools` for these checks and just not do them for the alternate
`unittest` execution.
* Generate `QiskitExperimentsTestCase` from a `create_base_test_case`
  function. This function allows the base test class to be generated
based on either `testtools.TestCase` or `unittest.TestCase` so that the
`unittest` variant can be tested for regressions even when the
`testtools` variant is enabled.
---
 requirements-dev.txt                   |   4 +-
 test/base.py                           | 422 ++++++++++++++-----------
 test/framework/test_store_init_args.py |   5 +-
 test/test_base.py                      |  27 ++
 4 files changed, 269 insertions(+), 189 deletions(-)
 create mode 100644 test/test_base.py

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 0d0200aa7e..6c4707a806 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,5 +1,7 @@
 black~=22.0
+fixtures
 stestr
+testtools
 pylint~=2.16.2
 astroid~=2.14.2  # Must be kept aligned to what pylint wants
 jinja2==3.0.3
@@ -20,4 +22,4 @@ coverage>=5.5
 ipykernel<=6.21.3
 jupyter-client<=8.0.3
 ipython<8.13.0 ; python_version<"3.9"  # for python 3.8 compatibility
-sphinx-remove-toctrees
\ No newline at end of file
+sphinx-remove-toctrees
diff --git a/test/base.py b/test/base.py
index dc7384afb9..af66822971 100644
--- a/test/base.py
+++ b/test/base.py
@@ -16,12 +16,13 @@
 import os
 import json
 import pickle
+import unittest
 import warnings
 from typing import Any, Callable, Optional
 
 import fixtures
+import testtools
 import uncertainties
-from qiskit.test import QiskitTestCase
 from qiskit.utils.deprecation import deprecate_func
 
 from qiskit_experiments.framework import (
@@ -34,194 +35,243 @@
 
 # Fail tests that take longer than this
 TEST_TIMEOUT = int(os.environ.get("TEST_TIMEOUT", 60))
+# Use testtools by default as a (mostly) drop in replacement for
+# unittest's TestCase. This will enable the fixtures used for capturing stdout
+# stderr, and pylogging to attach the output to stestr's result stream.
+USE_TESTTOOLS = os.environ.get("QE_USE_TESTTOOLS", "TRUE").lower() not in ("false", "0")
 
 
-class QiskitExperimentsTestCase(QiskitTestCase):
-    """Qiskit Experiments specific extra functionality for test cases."""
-
-    def setUp(self):
-        super().setUp()
-        self.useFixture(fixtures.Timeout(TEST_TIMEOUT, gentle=True))
-
-    @classmethod
-    def setUpClass(cls):
-        """Set-up test class."""
-        super().setUpClass()
-
-        # Some functionality may be deprecated in Qiskit Experiments. If the deprecation warnings aren't
-        # filtered, the tests will fail as ``QiskitTestCase`` sets all warnings to be treated as an error
-        # by default.
-        # pylint: disable=invalid-name
-        allow_deprecationwarning_message = [
-            # TODO: Remove in 0.6, when submodule `.curve_analysis.visualization` is removed.
-            r".*Plotting and drawing functionality has been moved",
-            r".*Legacy drawers from `.curve_analysis.visualization are deprecated",
-        ]
-        for msg in allow_deprecationwarning_message:
-            warnings.filterwarnings("default", category=DeprecationWarning, message=msg)
-
-    def assertExperimentDone(
-        self,
-        experiment_data: ExperimentData,
-        timeout: float = 120,
-    ):
-        """Blocking execution of next line until all threads are completed then
-        checks if status returns Done.
-
-        Args:
-            experiment_data: Experiment data to evaluate.
-            timeout: The maximum time in seconds to wait for executor to complete.
-        """
-        experiment_data.block_for_results(timeout=timeout)
-
-        self.assertEqual(
-            experiment_data.status(),
-            ExperimentStatus.DONE,
-            msg="All threads are executed but status is not DONE. " + experiment_data.errors(),
+def create_base_test_case(use_testtools: bool) -> unittest.TestCase:
+    """Create the base test case class for package tests
+
+    This function produces the base class for qiskit-experiments tests using
+    either ``unittest.TestCase`` or ``testtools.TestCase`` for the base class.
+    The creation of the class is done in this function rather than directly
+    executed in the module so that, even when ``USE_TESTTOOLS`` is true, a
+    ``unittest`` base class can be produced for ``test_base.py`` to check that
+    no hard-dependence on ``testtools`` has been introduced.
+    """
+    if use_testtools:
+
+        class BaseTestCase(testtools.TestCase):
+            """Base test class."""
+
+            # testtools maintains their own version of assert functions which mostly
+            # behave as value adds to the std unittest assertion methods. However,
+            # for assertEquals and assertRaises modern unittest has diverged from
+            # the forks in testtools and offer more (or different) options that are
+            # incompatible testtools versions. Just use the stdlib versions so that
+            # our tests work as expected.
+            assertRaises = unittest.TestCase.assertRaises
+            assertEqual = unittest.TestCase.assertEqual
+
+    else:
+
+        class BaseTestCase(unittest.TestCase):
+            """Base test class."""
+
+            def useFixture(self, fixture):  # pylint: disable=invalid-name
+                """Shim so that useFixture can be called in subclasses
+
+                useFixture is a testtools.TestCase method. The actual fixture is
+                not used when using unittest.
+                """
+
+
+    class QETestCase(BaseTestCase):
+        """Qiskit Experiments specific extra functionality for test cases."""
+
+        def setUp(self):
+            super().setUp()
+            self.useFixture(fixtures.Timeout(TEST_TIMEOUT, gentle=True))
+
+        @classmethod
+        def setUpClass(cls):
+            """Set-up test class."""
+            super().setUpClass()
+
+            warnings.filterwarnings("error", category=DeprecationWarning)
+
+            # Some functionality may be deprecated in Qiskit Experiments. If
+            # the deprecation warnings aren't filtered, the tests will fail as
+            # ``QiskitTestCase`` sets all warnings to be treated as an error by
+            # default.
+            # pylint: disable=invalid-name
+            allow_deprecationwarning_message = [
+                # TODO: Remove in 0.6, when submodule `.curve_analysis.visualization` is removed.
+                r".*Plotting and drawing functionality has been moved",
+                r".*Legacy drawers from `.curve_analysis.visualization are deprecated",
+            ]
+            for msg in allow_deprecationwarning_message:
+                warnings.filterwarnings("default", category=DeprecationWarning, message=msg)
+
+        def assertExperimentDone(
+            self,
+            experiment_data: ExperimentData,
+            timeout: float = 120,
+        ):
+            """Blocking execution of next line until all threads are completed then
+            checks if status returns Done.
+
+            Args:
+                experiment_data: Experiment data to evaluate.
+                timeout: The maximum time in seconds to wait for executor to complete.
+            """
+            experiment_data.block_for_results(timeout=timeout)
+
+            self.assertEqual(
+                experiment_data.status(),
+                ExperimentStatus.DONE,
+                msg="All threads are executed but status is not DONE. " + experiment_data.errors(),
+            )
+
+        def assertEqualExtended(
+            self,
+            first: Any,
+            second: Any,
+            *,
+            msg: Optional[str] = None,
+            strict_type: bool = False,
+        ):
+            """Extended equality assertion which covers Qiskit Experiments classes.
+
+            .. note::
+                Some Qiskit Experiments class may intentionally avoid implementing
+                the equality dunder method, or may be used in some unusual situations.
+                These are mainly caused by to JSON round trip situation, and some custom classes
+                doesn't guarantee object equality after round trip.
+                This assertion function forcibly compares input two objects with
+                the custom equality checker, which is implemented for unittest purpose.
+
+            Args:
+                first: First object to compare.
+                second: Second object to compare.
+                msg: Optional. Custom error message issued when first and second object are not equal.
+                strict_type: Set True to enforce type check before comparison.
+            """
+            default_msg = f"{first} != {second}"
+
+            self.assertTrue(
+                is_equivalent(first, second, strict_type=strict_type),
+                msg=msg or default_msg,
+            )
+
+        def assertRoundTripSerializable(
+            self,
+            obj: Any,
+            *,
+            check_func: Optional[Callable] = None,
+            strict_type: bool = False,
+        ):
+            """Assert that an object is round trip serializable.
+
+            Args:
+                obj: the object to be serialized.
+                check_func: Optional, a custom function ``check_func(a, b) -> bool``
+                    to check equality of the original object with the decoded
+                    object. If None :meth:`.assertEqualExtended` is called.
+                strict_type: Set True to enforce type check before comparison.
+            """
+            try:
+                encoded = json.dumps(obj, cls=ExperimentEncoder)
+            except TypeError:
+                self.fail("JSON serialization raised unexpectedly.")
+            try:
+                decoded = json.loads(encoded, cls=ExperimentDecoder)
+            except TypeError:
+                self.fail("JSON deserialization raised unexpectedly.")
+
+            if check_func is not None:
+                self.assertTrue(check_func(obj, decoded), msg=f"{obj} != {decoded}")
+            else:
+                self.assertEqualExtended(obj, decoded, strict_type=strict_type)
+
+        def assertRoundTripPickle(
+            self,
+            obj: Any,
+            *,
+            check_func: Optional[Callable] = None,
+            strict_type: bool = False,
+        ):
+            """Assert that an object is round trip serializable using pickle module.
+
+            Args:
+                obj: the object to be serialized.
+                check_func: Optional, a custom function ``check_func(a, b) -> bool``
+                    to check equality of the original object with the decoded
+                    object. If None :meth:`.assertEqualExtended` is called.
+                strict_type: Set True to enforce type check before comparison.
+            """
+            try:
+                encoded = pickle.dumps(obj)
+            except TypeError:
+                self.fail("pickle raised unexpectedly.")
+            try:
+                decoded = pickle.loads(encoded)
+            except TypeError:
+                self.fail("pickle deserialization raised unexpectedly.")
+
+            if check_func is not None:
+                self.assertTrue(check_func(obj, decoded), msg=f"{obj} != {decoded}")
+            else:
+                self.assertEqualExtended(obj, decoded, strict_type=strict_type)
+
+        @classmethod
+        @deprecate_func(
+            since="0.6",
+            additional_msg="Use test.extended_equality.is_equivalent instead.",
+            pending=True,
+            package_name="qiskit-experiments",
         )
+        def json_equiv(cls, data1, data2) -> bool:
+            """Check if two experiments are equivalent by comparing their configs"""
+            return is_equivalent(data1, data2)
 
-    def assertEqualExtended(
-        self,
-        first: Any,
-        second: Any,
-        *,
-        msg: Optional[str] = None,
-        strict_type: bool = False,
-    ):
-        """Extended equality assertion which covers Qiskit Experiments classes.
-
-        .. note::
-            Some Qiskit Experiments class may intentionally avoid implementing
-            the equality dunder method, or may be used in some unusual situations.
-            These are mainly caused by to JSON round trip situation, and some custom classes
-            doesn't guarantee object equality after round trip.
-            This assertion function forcibly compares input two objects with
-            the custom equality checker, which is implemented for unittest purpose.
-
-        Args:
-            first: First object to compare.
-            second: Second object to compare.
-            msg: Optional. Custom error message issued when first and second object are not equal.
-            strict_type: Set True to enforce type check before comparison.
-        """
-        default_msg = f"{first} != {second}"
-
-        self.assertTrue(
-            is_equivalent(first, second, strict_type=strict_type),
-            msg=msg or default_msg,
+        @staticmethod
+        @deprecate_func(
+            since="0.6",
+            additional_msg="Use test.extended_equality.is_equivalent instead.",
+            pending=True,
+            package_name="qiskit-experiments",
         )
+        def ufloat_equiv(data1: uncertainties.UFloat, data2: uncertainties.UFloat) -> bool:
+            """Check if two values with uncertainties are equal. No correlation is considered."""
+            return is_equivalent(data1, data2)
+
+        @classmethod
+        @deprecate_func(
+            since="0.6",
+            additional_msg="Use test.extended_equality.is_equivalent instead.",
+            pending=True,
+            package_name="qiskit-experiments",
+        )
+        def analysis_result_equiv(cls, result1, result2):
+            """Test two analysis results are equivalent"""
+            return is_equivalent(result1, result2)
+
+        @classmethod
+        @deprecate_func(
+            since="0.6",
+            additional_msg="Use test.extended_equality.is_equivalent instead.",
+            pending=True,
+            package_name="qiskit-experiments",
+        )
+        def curve_fit_data_equiv(cls, data1, data2):
+            """Test two curve fit result are equivalent."""
+            return is_equivalent(data1, data2)
+
+        @classmethod
+        @deprecate_func(
+            since="0.6",
+            additional_msg="Use test.extended_equality.is_equivalent instead.",
+            pending=True,
+            package_name="qiskit-experiments",
+        )
+        def experiment_data_equiv(cls, data1, data2):
+            """Check two experiment data containers are equivalent"""
+            return is_equivalent(data1, data2)
+
+    return QETestCase
+
 
-    def assertRoundTripSerializable(
-        self,
-        obj: Any,
-        *,
-        check_func: Optional[Callable] = None,
-        strict_type: bool = False,
-    ):
-        """Assert that an object is round trip serializable.
-
-        Args:
-            obj: the object to be serialized.
-            check_func: Optional, a custom function ``check_func(a, b) -> bool``
-                to check equality of the original object with the decoded
-                object. If None :meth:`.assertEqualExtended` is called.
-            strict_type: Set True to enforce type check before comparison.
-        """
-        try:
-            encoded = json.dumps(obj, cls=ExperimentEncoder)
-        except TypeError:
-            self.fail("JSON serialization raised unexpectedly.")
-        try:
-            decoded = json.loads(encoded, cls=ExperimentDecoder)
-        except TypeError:
-            self.fail("JSON deserialization raised unexpectedly.")
-
-        if check_func is not None:
-            self.assertTrue(check_func(obj, decoded), msg=f"{obj} != {decoded}")
-        else:
-            self.assertEqualExtended(obj, decoded, strict_type=strict_type)
-
-    def assertRoundTripPickle(
-        self,
-        obj: Any,
-        *,
-        check_func: Optional[Callable] = None,
-        strict_type: bool = False,
-    ):
-        """Assert that an object is round trip serializable using pickle module.
-
-        Args:
-            obj: the object to be serialized.
-            check_func: Optional, a custom function ``check_func(a, b) -> bool``
-                to check equality of the original object with the decoded
-                object. If None :meth:`.assertEqualExtended` is called.
-            strict_type: Set True to enforce type check before comparison.
-        """
-        try:
-            encoded = pickle.dumps(obj)
-        except TypeError:
-            self.fail("pickle raised unexpectedly.")
-        try:
-            decoded = pickle.loads(encoded)
-        except TypeError:
-            self.fail("pickle deserialization raised unexpectedly.")
-
-        if check_func is not None:
-            self.assertTrue(check_func(obj, decoded), msg=f"{obj} != {decoded}")
-        else:
-            self.assertEqualExtended(obj, decoded, strict_type=strict_type)
-
-    @classmethod
-    @deprecate_func(
-        since="0.6",
-        additional_msg="Use test.extended_equality.is_equivalent instead.",
-        pending=True,
-        package_name="qiskit-experiments",
-    )
-    def json_equiv(cls, data1, data2) -> bool:
-        """Check if two experiments are equivalent by comparing their configs"""
-        return is_equivalent(data1, data2)
-
-    @staticmethod
-    @deprecate_func(
-        since="0.6",
-        additional_msg="Use test.extended_equality.is_equivalent instead.",
-        pending=True,
-        package_name="qiskit-experiments",
-    )
-    def ufloat_equiv(data1: uncertainties.UFloat, data2: uncertainties.UFloat) -> bool:
-        """Check if two values with uncertainties are equal. No correlation is considered."""
-        return is_equivalent(data1, data2)
-
-    @classmethod
-    @deprecate_func(
-        since="0.6",
-        additional_msg="Use test.extended_equality.is_equivalent instead.",
-        pending=True,
-        package_name="qiskit-experiments",
-    )
-    def analysis_result_equiv(cls, result1, result2):
-        """Test two analysis results are equivalent"""
-        return is_equivalent(result1, result2)
-
-    @classmethod
-    @deprecate_func(
-        since="0.6",
-        additional_msg="Use test.extended_equality.is_equivalent instead.",
-        pending=True,
-        package_name="qiskit-experiments",
-    )
-    def curve_fit_data_equiv(cls, data1, data2):
-        """Test two curve fit result are equivalent."""
-        return is_equivalent(data1, data2)
-
-    @classmethod
-    @deprecate_func(
-        since="0.6",
-        additional_msg="Use test.extended_equality.is_equivalent instead.",
-        pending=True,
-        package_name="qiskit-experiments",
-    )
-    def experiment_data_equiv(cls, data1, data2):
-        """Check two experiment data containers are equivalent"""
-        return is_equivalent(data1, data2)
+QiskitExperimentsTestCase = create_base_test_case(USE_TESTTOOLS)
diff --git a/test/framework/test_store_init_args.py b/test/framework/test_store_init_args.py
index df46c68a8d..87ebf90052 100644
--- a/test/framework/test_store_init_args.py
+++ b/test/framework/test_store_init_args.py
@@ -12,7 +12,8 @@
 
 """Tests for base experiment framework."""
 
-from qiskit.test import QiskitTestCase
+from test.base import QiskitExperimentsTestCase
+
 from qiskit_experiments.framework.store_init_args import StoreInitArgs
 
 
@@ -58,7 +59,7 @@ def __init__(self, a, b, c="default_c", d="default_d"):
         pass
 
 
-class TestSettings(QiskitTestCase):
+class TestSettings(QiskitExperimentsTestCase):
     """Test Settings mixin"""
 
     # pylint: disable = missing-function-docstring
diff --git a/test/test_base.py b/test/test_base.py
new file mode 100644
index 0000000000..339eb0ab81
--- /dev/null
+++ b/test/test_base.py
@@ -0,0 +1,27 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+"""
+Tests for qiskit-experiments base test module
+"""
+
+from test.base import create_base_test_case
+
+
+UnittestBase = create_base_test_case(use_testtools=False)
+
+
+class TestQiskitExperimentsTestCaseWithUnittest(UnittestBase):
+    """Test QiskitExperimentsTestCase behavior when not based on testtools.TestCase
+    """
+    def test_test(self):
+        """Test that a test not based on ``testtools`` can run"""
+        pass

From 5c2b50becd64031849eb9514518b948535de577a Mon Sep 17 00:00:00 2001
From: Will Shanks <willshanks@us.ibm.com>
Date: Thu, 2 Nov 2023 13:38:19 -0400
Subject: [PATCH 2/8] black

---
 test/base.py      | 1 -
 test/test_base.py | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/test/base.py b/test/base.py
index af66822971..a817c48b9e 100644
--- a/test/base.py
+++ b/test/base.py
@@ -77,7 +77,6 @@ def useFixture(self, fixture):  # pylint: disable=invalid-name
                 not used when using unittest.
                 """
 
-
     class QETestCase(BaseTestCase):
         """Qiskit Experiments specific extra functionality for test cases."""
 
diff --git a/test/test_base.py b/test/test_base.py
index 339eb0ab81..cf5c010c96 100644
--- a/test/test_base.py
+++ b/test/test_base.py
@@ -20,8 +20,8 @@
 
 
 class TestQiskitExperimentsTestCaseWithUnittest(UnittestBase):
-    """Test QiskitExperimentsTestCase behavior when not based on testtools.TestCase
-    """
+    """Test QiskitExperimentsTestCase behavior when not based on testtools.TestCase"""
+
     def test_test(self):
         """Test that a test not based on ``testtools`` can run"""
         pass

From 263d4ebee8a0350cd07c2b2a7c474b997a3723ce Mon Sep 17 00:00:00 2001
From: Will Shanks <willshanks@us.ibm.com>
Date: Wed, 8 Nov 2023 16:51:58 -0500
Subject: [PATCH 3/8] Add QISKIT_CAPTURE_OUTPUT_STREAMS behavior

---
 test/base.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/base.py b/test/base.py
index a817c48b9e..62fe10f102 100644
--- a/test/base.py
+++ b/test/base.py
@@ -65,6 +65,15 @@ class BaseTestCase(testtools.TestCase):
             assertRaises = unittest.TestCase.assertRaises
             assertEqual = unittest.TestCase.assertEqual
 
+            def setUp(self):
+                super().setUp()
+                if os.environ.get("QISKIT_TEST_CAPTURE_STREAMS"):
+                    stdout = self.useFixture(fixtures.StringStream("stdout")).stream
+                    self.useFixture(fixtures.MonkeyPatch("sys.stdout", stdout))
+                    stderr = self.useFixture(fixtures.StringStream("stderr")).stream
+                    self.useFixture(fixtures.MonkeyPatch("sys.stderr", stderr))
+                    self.useFixture(fixtures.LoggerFixture(nuke_handlers=False, level=None))
+
     else:
 
         class BaseTestCase(unittest.TestCase):

From dd7e165e7718b0a868317f4b1bfb42a41f1020ba Mon Sep 17 00:00:00 2001
From: Will Shanks <willshanks@us.ibm.com>
Date: Wed, 8 Nov 2023 16:55:04 -0500
Subject: [PATCH 4/8] Use TEST_TIMEOUT as the default for assertExperimentDone

---
 test/base.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/test/base.py b/test/base.py
index 62fe10f102..664bc1fefc 100644
--- a/test/base.py
+++ b/test/base.py
@@ -116,15 +116,18 @@ def setUpClass(cls):
         def assertExperimentDone(
             self,
             experiment_data: ExperimentData,
-            timeout: float = 120,
+            timeout: Optional[float] = None,
         ):
             """Blocking execution of next line until all threads are completed then
             checks if status returns Done.
 
             Args:
                 experiment_data: Experiment data to evaluate.
-                timeout: The maximum time in seconds to wait for executor to complete.
+                timeout: The maximum time in seconds to wait for executor to
+                    complete. Defaults to the value of ``TEST_TIMEOUT``.
             """
+            if timeout is None:
+                timeout = TEST_TIMEOUT
             experiment_data.block_for_results(timeout=timeout)
 
             self.assertEqual(

From e1eb8df85d27f7e92a42481a29b927136c2265eb Mon Sep 17 00:00:00 2001
From: Will Shanks <willshanks@us.ibm.com>
Date: Wed, 8 Nov 2023 16:57:06 -0500
Subject: [PATCH 5/8] Add QE_USE_TESTTOOLS to tox.ini passenv

---
 tox.ini | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 658e81e57a..e3914e9940 100644
--- a/tox.ini
+++ b/tox.ini
@@ -18,6 +18,7 @@ passenv =
   RAYON_NUM_THREADS
   QISKIT_IBM_*
   TEST_TIMEOUT
+  QE_USE_TESTTOOLS
 commands = stestr run {posargs}
 
 [testenv:cover]
@@ -45,6 +46,8 @@ passenv =
   QISKIT_PARALLEL
   RAYON_NUM_THREADS
   QISKIT_IBM_*
+  TEST_TIMEOUT
+  QE_USE_TESTTOOLS
 commands = stestr run {posargs}
 
 
@@ -105,4 +108,4 @@ commands =
 skip_install = true
 deps =
 allowlist_externals = rm
-commands = rm -rf {toxinidir}/docs/stubs/ {toxinidir}/docs/_build
\ No newline at end of file
+commands = rm -rf {toxinidir}/docs/stubs/ {toxinidir}/docs/_build

From e6643cfbddd6ea5e61a7e0f932100fc4be4dc146 Mon Sep 17 00:00:00 2001
From: Will Shanks <willshanks@us.ibm.com>
Date: Wed, 8 Nov 2023 17:06:11 -0500
Subject: [PATCH 6/8] Add description of test variables to contributing guide

---
 CONTRIBUTING.md | 16 ++++++++++++++--
 test/base.py    |  2 +-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7f4b45191f..85c5323f75 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -125,8 +125,8 @@ Note that tests will fail automatically if they do not finish execution within 6
 
 #### STDOUT/STDERR and logging capture
 
-When running tests in parallel using `stestr` either via tox, the Makefile (`make
-test_ci`), or in CI, we set the env variable `QISKIT_TEST_CAPTURE_STREAMS`, which will
+When running tests in parallel using `stestr` either via tox
+or in CI, we set the env variable `QISKIT_TEST_CAPTURE_STREAMS`, which will
 capture any text written to stdout, stderr, and log messages and add them as attachments
 to the tests run so output can be associated with the test case it originated from.
 However, if you run tests with `stestr` outside of these mechanisms, by default the
@@ -138,6 +138,18 @@ stdlib unittest runner, a similar result can be accomplished by using the
 [`--buffer`](https://docs.python.org/3/library/unittest.html#command-line-options)
 option (e.g. `python -m unittest discover --buffer ./test/python`).
 
+#### Other testing related settings
+
+The test code defines some environment variables that may occasionally be useful to set:
+
++ `TEST_TIMEOUT`: An integer representing the maximum time a test can take
+  before it is considered a failure.
++ `QE_USE_TESTTOOLS`: Set this variable to `FALSE`, `0`, or `NO` to have the
+  tests use `unittest.TestCase` as the base class. Otherwise, the default is
+`testtools.TestCase` which is an extension of `unittest.TestCase`. In some
+situations, a developer may wish to use a workflow that is not compatible with
+the `testtools` extensions.
+
 ### Code style
 
 The qiskit-experiments repository uses `black` for code formatting and style and
diff --git a/test/base.py b/test/base.py
index 664bc1fefc..028b549b7b 100644
--- a/test/base.py
+++ b/test/base.py
@@ -38,7 +38,7 @@
 # Use testtools by default as a (mostly) drop in replacement for
 # unittest's TestCase. This will enable the fixtures used for capturing stdout
 # stderr, and pylogging to attach the output to stestr's result stream.
-USE_TESTTOOLS = os.environ.get("QE_USE_TESTTOOLS", "TRUE").lower() not in ("false", "0")
+USE_TESTTOOLS = os.environ.get("QE_USE_TESTTOOLS", "TRUE").lower() not in ("false", "0", "no")
 
 
 def create_base_test_case(use_testtools: bool) -> unittest.TestCase:

From 3e4121b1fa6295387229ac7031ebf7d5cd68a2c1 Mon Sep 17 00:00:00 2001
From: Will Shanks <willshanks@us.ibm.com>
Date: Thu, 9 Nov 2023 17:03:59 -0500
Subject: [PATCH 7/8] Add QISKIT_TEST_CAPTURE_STREAMS to passenv

---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index 0bba6aeb69..71ba1b13d9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,6 +19,7 @@ passenv =
   QISKIT_IBM_*
   TEST_TIMEOUT
   QE_USE_TESTTOOLS
+  QISKIT_TEST_CAPTURE_STREAMS
 commands = stestr run {posargs}
 
 [testenv:cover]
@@ -48,6 +49,7 @@ passenv =
   QISKIT_IBM_*
   TEST_TIMEOUT
   QE_USE_TESTTOOLS
+  QISKIT_TEST_CAPTURE_STREAMS
 commands = stestr run {posargs}
 
 

From 46eaf16fada57a7eebc66d8b10273dceba11f690 Mon Sep 17 00:00:00 2001
From: Will Shanks <willshanks@us.ibm.com>
Date: Thu, 9 Nov 2023 17:05:58 -0500
Subject: [PATCH 8/8] Move QISKIT_TEST_CAPTURE_STREAMS to setenv

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 71ba1b13d9..bc7596d0bc 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,6 +9,7 @@ install_command = pip install -c{toxinidir}/constraints.txt -U {opts} {packages}
 setenv =
   VIRTUAL_ENV={envdir}
   QISKIT_SUPPRESS_PACKAGING_WARNINGS=Y
+  QISKIT_TEST_CAPTURE_STREAMS=1
 deps =
   -r{toxinidir}/requirements-dev.txt
   -r{toxinidir}/requirements-extras.txt
@@ -19,7 +20,6 @@ passenv =
   QISKIT_IBM_*
   TEST_TIMEOUT
   QE_USE_TESTTOOLS
-  QISKIT_TEST_CAPTURE_STREAMS
 commands = stestr run {posargs}
 
 [testenv:cover]
@@ -38,6 +38,7 @@ install_command = pip install -U {opts} {packages}
 setenv =
   VIRTUAL_ENV={envdir}
   QISKIT_SUPPRESS_PACKAGING_WARNINGS=Y
+  QISKIT_TEST_CAPTURE_STREAMS=1
 deps =
   git+https://github.com/Qiskit/qiskit-terra
   -r{toxinidir}/requirements-dev.txt
@@ -49,7 +50,6 @@ passenv =
   QISKIT_IBM_*
   TEST_TIMEOUT
   QE_USE_TESTTOOLS
-  QISKIT_TEST_CAPTURE_STREAMS
 commands = stestr run {posargs}