feat: improve included tap and target tests in singer_sdk.testing (#…

…1171)

Co-authored-by: Amanda Folson <afolson@users.noreply.github.com>
Co-authored-by: Dan Norman <buzzcutnorman@gmail.com>
Co-authored-by: Edgar R. M. <edgar@meltano.com>

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = W503, C901, ANN101
+ignore = W503, C901, ANN101, ANN102
 max-line-length = 88
 exclude = cookiecutter
 per-file-ignores =
@@ -13,6 +13,8 @@ per-file-ignores =
     # singer_sdk/helpers/_classproperty.py:D105
     # Ignore unused imports in __init__.py files
     singer_sdk/_singerlib/__init__.py:F401
+    # Templates support a generic resource of type Any.
+    singer_sdk/testing/templates.py:ANN401
 max-complexity = 10
 docstring-convention = google
 allow-star-arg-any = true

.pre-commit-config.yaml

@@ -47,7 +47,7 @@ repos:
         )$
 
 - repo: https://github.com/pycqa/isort
-  rev: 5.11.4
+  rev: 5.12.0
   hooks:
   - id: isort
     exclude: (cookiecutter/.*|singer_sdk/helpers/_simpleeval/.*)

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tests/conftest.py

@@ -0,0 +1,3 @@
+"""Test Configuration."""
+
+pytest_plugins = ("singer_sdk.testing.pytest_plugin",)

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tests/{{ 'test' }}_core.py

@@ -2,25 +2,22 @@
 
 import datetime
 
-from singer_sdk.testing import get_standard_tap_tests
+from singer_sdk.testing import get_tap_test_class
 
 from {{ cookiecutter.library_name }}.tap import Tap{{ cookiecutter.source_name }}
 
+
 SAMPLE_CONFIG = {
     "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
     # TODO: Initialize minimal tap config
 }
 
 
 # Run standard built-in tap tests from the SDK:
-def test_standard_tap_tests():
-    """Run standard tap tests from the SDK."""
-    tests = get_standard_tap_tests(
-        Tap{{ cookiecutter.source_name }},
-        config=SAMPLE_CONFIG
-    )
-    for test in tests:
-        test()
+TestTap{{ cookiecutter.source_name }} = get_tap_test_class(
+    tap_class=Tap{{ cookiecutter.source_name }},
+    config=SAMPLE_CONFIG
+)
 
 
 # TODO: Create additional tests as appropriate for your tap.

cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/tests/conftest.py

@@ -0,0 +1,3 @@
+"""Test Configuration."""
+
+pytest_plugins = ("singer_sdk.testing.pytest_plugin",)

cookiecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/tests/{{ 'test' }}_core.py

@@ -1,27 +1,39 @@
 """Tests standard target features using the built-in SDK tests library."""
 
-import datetime
-
+import pytest
 from typing import Dict, Any
 
-from singer_sdk.testing import get_standard_target_tests
+from singer_sdk.testing import get_target_test_class
 
 from {{ cookiecutter.library_name }}.target import Target{{ cookiecutter.destination_name }}
 
+
 SAMPLE_CONFIG: Dict[str, Any] = {
     # TODO: Initialize minimal target config
 }
 
 
 # Run standard built-in target tests from the SDK:
-def test_standard_target_tests():
-    """Run standard target tests from the SDK."""
-    tests = get_standard_target_tests(
-        Target{{ cookiecutter.destination_name }},
-        config=SAMPLE_CONFIG,
-    )
-    for test in tests:
-        test()
+StandardTargetTests = get_target_test_class(
+    target_class=Target{{ cookiecutter.destination_name }},
+    config=SAMPLE_CONFIG
+)
+
+
+class TestTarget{{ cookiecutter.destination_name }}(StandardTargetTests):
+    """Standard Target Tests."""
+
+    @pytest.fixture(scope="class")
+    def resource(self):
+        """Generic external resource.
+
+        This fixture is useful for setup and teardown of external resources,
+        such output folders, tables, buckets etc. for use during testing.
+
+        Example usage can be found in the SDK samples test suite:
+        https://github.com/meltano/sdk/tree/main/tests/samples
+        """
+        yield "resource"
 
 
 # TODO: Create additional tests as appropriate for your target.

docs/index.rst

@@ -73,6 +73,7 @@ Advanced Topics
    CONTRIBUTING
    implementation/index
    typing
+   testing
 
 .. _Singer: https://singer.io
 .. _Singer Spec: https://hub.meltano.com/singer/spec

docs/testing.md

@@ -0,0 +1,139 @@
+# Testing Taps & Targets
+
+The Meltano SDK includes suites of standard tests for both Taps and Targets to help you get started.
+These suites cover most common cases out-of-the-box, and tests are added to the standard suites as new errors are encountered by users in their deployments.
+
+## Test Framework
+
+The Meltano SDK test framework consists of 4 main components:
+
+1. A runner class (`TapTestRunner` and `TargetTestRunner`), responsible for executing Taps/Targets and capturing their output.
+1. A suite dataclass, containing a list of tests.
+1. A test template classes (`TapTestTemplate`, `StreamTestTemplate`, `AttributeTestTemplate` and `TargetTestTemplate`), with methods to `.setup()`, `.test()`, `.validate()` and `.teardown()` (called in that order using `.run()`).
+1. `get_tap_test_class` and `get_target_test_class` factory methods. These wrap a `get_test_class` factory method, which takes a runner and a list of suits and return a `pytest` test class.
+
+## Example Usage
+
+If you created your Tap/Target using the provided cookiecutter templates, you will find the following snippets in `<library_name>/tests/test_core.py`.
+You will also find a `conftest.py` file containing configuration of the SDK as a `pytest` plugin.
+This is required for tests to collect correctly:
+
+```python
+# register the singer_sdk pytest plugin
+pytest_plugins = ("singer_sdk.testing.pytest_plugin",)
+```
+
+### Testing Taps
+
+```python
+import datetime
+
+from singer_sdk.testing import get_tap_test_class
+
+from example.tap import TapExample
+
+SAMPLE_CONFIG = {
+    "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
+}
+
+
+# Run standard built-in tap tests from the SDK:
+TestTapExample = get_tap_test_class(
+    tap_class=TapExample,
+    config=SAMPLE_CONFIG
+)
+```
+
+### Testing Targets
+
+```python
+import pytest
+from typing import Dict, Any
+
+from singer_sdk.testing import get_target_test_class
+
+from example.target import TargetExample
+
+SAMPLE_CONFIG: Dict[str, Any] = {
+    # TODO: Initialize minimal target config
+}
+
+# Run standard built-in target tests from the SDK:
+StandardTargetTests = get_target_test_class(
+    target_class=TargetExample,
+    config=SAMPLE_CONFIG
+)
+
+
+class TestTargetExample(StandardTargetTests):
+    """Standard Target Tests."""
+
+    @pytest.fixture(scope="class")
+    def resource(self):
+        """Generic external resource.
+
+        This fixture is useful for setup and teardown of external resources,
+        such output folders, tables, buckets etc. for use during testing.
+
+        Example usage can be found in the SDK samples test suite:
+        https://github.com/meltano/sdk/tree/main/tests/samples
+        """
+        yield "resource"
+```
+
+## Configuring Tests
+
+Test suite behaviors can be configured by passing a `SuiteConfig` instance to the `get_test_class` functions:
+
+```python
+from singer_sdk.testing import SuiteConfig, get_tap_test_class
+
+from tap_stackexchange.tap import TapStackExchange
+
+SAMPLE_CONFIG = {
+    "site": "stackoverflow",
+    "tags": [
+        "meltano",
+        "singer-io",
+    ],
+    "metrics_log_level": "debug",
+}
+
+TEST_SUITE_CONFIG = SuiteConfig(
+    ignore_no_records_for_streams=["tag_synonyms"]
+)
+
+TestTapStackExchange = get_tap_test_class(
+    tap_class=TapStackExchange, config=SAMPLE_CONFIG, suite_config=TEST_SUITE_CONFIG
+)
+```
+
+Check out [`singer_sdk/testing/config.py`](https://github.com/meltano/sdk/tree/main/singer_sdk/testing/config.py) for available config options.
+
+## Writing New Tests
+
+Writing new tests is as easy as subclassing the appropriate class.
+Check out [`singer_sdk/testing/tap_tests.py`](https://github.com/meltano/sdk/tree/main/singer_sdk/testing/tap_tests.py) and [`singer_sdk/testing/target_tests.py`](https://github.com/meltano/sdk/tree/main/singer_sdk/testing/target_tests.py) for inspiration.
+
+```python
+class TapCLIPrintsTest(TapTestTemplate):
+    "Test that the tap is able to print standard metadata."
+    name = "cli_prints"
+
+    def test(self):
+        self.tap.print_version()
+        self.tap.print_about()
+        self.tap.print_about(format="json")
+```
+
+Once you have created some tests, add them to a suite:
+
+```python
+my_custom_tap_tests = TestSuite(
+    type="tap", tests=[TapCLIPrintsTest]
+)
+```
+
+This suite can now be passed to `get_tap_test_class` or `get_target_test_class` in a list of `custom_suites` along with any other suites, to generate your custom test class.
+
+If your new test covers a common or general case, consider contributing to the standard test library via a pull request to [meltano/sdk](https://github.com/meltano/sdk).

noxfile.py

@@ -32,6 +32,7 @@
     "coverage[toml]",
     "pytest",
     "pytest-snapshot",
+    "pytest-durations",
     "freezegun",
     "pandas",
     "requests-mock",
@@ -54,6 +55,7 @@ def mypy(session: Session) -> None:
     session.install(".")
     session.install(
         "mypy",
+        "pytest",
         "sqlalchemy2-stubs",
         "types-python-dateutil",
         "types-requests",
@@ -86,8 +88,8 @@ def tests(session: Session) -> None:
             "--parallel",
             "-m",
             "pytest",
-            "-x",
             "-v",
+            "--durations=10",
             *session.posargs,
         )
     finally: