test_discovery.py

# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import json
import os
import sys
from typing import Any, Dict, List, Optional

import pytest

from tests.tree_comparison_helper import is_same_tree

from . import expected_discovery_test_output, helpers


def test_import_error():
    """Test pytest discovery on a file that has a pytest marker but does not import pytest.

    Copies the contents of a .txt file to a .py file in the temporary directory
    to then run pytest discovery on.

    The json should still be returned but the errors list should be present.

    Keyword arguments:
    tmp_path -- pytest fixture that creates a temporary directory.
    """
    file_path = helpers.TEST_DATA_PATH / "error_pytest_import.txt"
    with helpers.text_to_python_file(file_path) as p:
        actual: Optional[List[Dict[str, Any]]] = helpers.runner(["--collect-only", os.fspath(p)])

    assert actual
    actual_list: List[Dict[str, Any]] = actual
    if actual_list is not None:
        for actual_item in actual_list:
            assert all(item in actual_item for item in ("status", "cwd", "error"))
            assert actual_item.get("status") == "error"
            assert actual_item.get("cwd") == os.fspath(helpers.TEST_DATA_PATH)

            # Ensure that 'error' is a list and then check its length
            error_content = actual_item.get("error")
            if error_content is not None and isinstance(
                error_content, (list, tuple, str)
            ):  # You can add other types if needed
                assert len(error_content) == 2
            else:
                pytest.fail(f"{error_content} is None or not a list, str, or tuple")


def test_syntax_error(tmp_path):  # noqa: ARG001
    """Test pytest discovery on a file that has a syntax error.

    Copies the contents of a .txt file to a .py file in the temporary directory
    to then run pytest discovery on.

    The json should still be returned but the errors list should be present.

    Keyword arguments:
    tmp_path -- pytest fixture that creates a temporary directory.
    """
    # Saving some files as .txt to avoid that file displaying a syntax error for
    # the extension as a whole. Instead, rename it before running this test
    # in order to test the error handling.
    file_path = helpers.TEST_DATA_PATH / "error_syntax_discovery.txt"
    with helpers.text_to_python_file(file_path) as p:
        actual = helpers.runner(["--collect-only", os.fspath(p)])

    assert actual
    actual_list: List[Dict[str, Any]] = actual
    if actual_list is not None:
        for actual_item in actual_list:
            assert all(item in actual_item for item in ("status", "cwd", "error"))
            assert actual_item.get("status") == "error"
            assert actual_item.get("cwd") == os.fspath(helpers.TEST_DATA_PATH)

            # Ensure that 'error' is a list and then check its length
            error_content = actual_item.get("error")
            if error_content is not None and isinstance(
                error_content, (list, tuple, str)
            ):  # You can add other types if needed
                assert len(error_content) == 2
            else:
                pytest.fail(f"{error_content} is None or not a list, str, or tuple")


def test_parameterized_error_collect():
    """Tests pytest discovery on specific file that incorrectly uses parametrize.

    The json should still be returned but the errors list should be present.
    """
    file_path_str = "error_parametrize_discovery.py"
    actual = helpers.runner(["--collect-only", file_path_str])
    assert actual
    actual_list: List[Dict[str, Any]] = actual
    if actual_list is not None:
        for actual_item in actual_list:
            assert all(item in actual_item for item in ("status", "cwd", "error"))
            assert actual_item.get("status") == "error"
            assert actual_item.get("cwd") == os.fspath(helpers.TEST_DATA_PATH)

            # Ensure that 'error' is a list and then check its length
            error_content = actual_item.get("error")
            if error_content is not None and isinstance(
                error_content, (list, tuple, str)
            ):  # You can add other types if needed
                assert len(error_content) == 2
            else:
                pytest.fail(f"{error_content} is None or not a list, str, or tuple")


@pytest.mark.parametrize(
    ("file", "expected_const"),
    [
        (
            "test_param_span_class.py",
            expected_discovery_test_output.test_param_span_class_expected_output,
        ),
        (
            "test_multi_class_nest.py",
            expected_discovery_test_output.nested_classes_expected_test_output,
        ),
        (
            "same_function_new_class_param.py",
            expected_discovery_test_output.same_function_new_class_param_expected_output,
        ),
        (
            "unittest_skiptest_file_level.py",
            expected_discovery_test_output.unittest_skip_file_level_expected_output,
        ),
        (
            "param_same_name",
            expected_discovery_test_output.param_same_name_expected_output,
        ),
        (
            "parametrize_tests.py",
            expected_discovery_test_output.parametrize_tests_expected_output,
        ),
        (
            "empty_discovery.py",
            expected_discovery_test_output.empty_discovery_pytest_expected_output,
        ),
        (
            "simple_pytest.py",
            expected_discovery_test_output.simple_discovery_pytest_expected_output,
        ),
        (
            "unittest_pytest_same_file.py",
            expected_discovery_test_output.unit_pytest_same_file_discovery_expected_output,
        ),
        (
            "unittest_folder",
            expected_discovery_test_output.unittest_folder_discovery_expected_output,
        ),
        (
            "dual_level_nested_folder",
            expected_discovery_test_output.dual_level_nested_folder_expected_output,
        ),
        (
            "folder_a",
            expected_discovery_test_output.double_nested_folder_expected_output,
        ),
        (
            "text_docstring.txt",
            expected_discovery_test_output.doctest_pytest_expected_output,
        ),
        (
            "pytest_describe_plugin" + os.path.sep + "describe_only.py",
            expected_discovery_test_output.expected_describe_only_output,
        ),
        (
            "pytest_describe_plugin" + os.path.sep + "nested_describe.py",
            expected_discovery_test_output.expected_nested_describe_output,
        ),
    ],
)
def test_pytest_collect(file, expected_const):
    """Test to test pytest discovery on a variety of test files/ folder structures.

    Uses variables from expected_discovery_test_output.py to store the expected
    dictionary return. Only handles discovery and therefore already contains the arg
    --collect-only. All test discovery will succeed, be in the correct cwd, and match
    expected test output.

    Keyword arguments:
    file -- a string with the file or folder to run pytest discovery on.
    expected_const -- the expected output from running pytest discovery on the file.
    """
    actual = helpers.runner(
        [
            os.fspath(helpers.TEST_DATA_PATH / file),
            "--collect-only",
        ]
    )

    assert actual
    actual_list: List[Dict[str, Any]] = actual
    if actual_list is not None:
        actual_item = actual_list.pop(0)
        assert all(item in actual_item for item in ("status", "cwd", "error"))
        assert actual_item.get("status") == "success", (
            f"Status is not 'success', error is: {actual_item.get('error')}"
        )
        assert actual_item.get("cwd") == os.fspath(helpers.TEST_DATA_PATH)
        assert is_same_tree(
            actual_item.get("tests"),
            expected_const,
            ["id_", "lineno", "name", "runID"],
        ), (
            f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_const, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}"
        )


@pytest.mark.skipif(
    sys.platform == "win32",
    reason="See https://stackoverflow.com/questions/32877260/privlege-error-trying-to-create-symlink-using-python-on-windows-10",
)
def test_symlink_root_dir():
    """Test to test pytest discovery with the command line arg --rootdir specified as a symlink path.

    Discovery should succeed and testids should be relative to the symlinked root directory.
    """
    with helpers.create_symlink(helpers.TEST_DATA_PATH, "root", "symlink_folder") as (
        source,
        destination,
    ):
        assert destination.is_symlink()

        # Run pytest with the cwd being the resolved symlink path (as it will be when we run the subprocess from node).
        actual = helpers.runner_with_cwd(
            ["--collect-only", f"--rootdir={os.fspath(destination)}"], source
        )
        expected = expected_discovery_test_output.symlink_expected_discovery_output
        assert actual
        actual_list: List[Dict[str, Any]] = actual
        if actual_list is not None:
            actual_item = actual_list.pop(0)
            try:
                # Check if all requirements
                assert all(item in actual_item for item in ("status", "cwd", "error")), (
                    "Required keys are missing"
                )
                assert actual_item.get("status") == "success", "Status is not 'success'"
                assert actual_item.get("cwd") == os.fspath(destination), (
                    f"CWD does not match: {os.fspath(destination)}"
                )
                assert actual_item.get("tests") == expected, "Tests do not match expected value"
            except AssertionError as e:
                # Print the actual_item in JSON format if an assertion fails
                print(json.dumps(actual_item, indent=4))
                pytest.fail(str(e))


def test_pytest_root_dir():
    """Test to test pytest discovery with the command line arg --rootdir specified to be a subfolder of the workspace root.

    Discovery should succeed and testids should be relative to workspace root.
    """
    rd = f"--rootdir={helpers.TEST_DATA_PATH / 'root' / 'tests'}"
    actual = helpers.runner_with_cwd(
        [
            "--collect-only",
            rd,
        ],
        helpers.TEST_DATA_PATH / "root",
    )
    assert actual
    actual_list: List[Dict[str, Any]] = actual
    if actual_list is not None:
        actual_item = actual_list.pop(0)

        assert all(item in actual_item for item in ("status", "cwd", "error"))
        assert actual_item.get("status") == "success"
        assert actual_item.get("cwd") == os.fspath(helpers.TEST_DATA_PATH / "root")
        assert is_same_tree(
            actual_item.get("tests"),
            expected_discovery_test_output.root_with_config_expected_output,
            ["id_", "lineno", "name", "runID"],
        ), (
            f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_discovery_test_output.root_with_config_expected_output, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}"
        )


def test_pytest_config_file():
    """Test to test pytest discovery with the command line arg -c with a specified config file which changes the workspace root.

    Discovery should succeed and testids should be relative to workspace root.
    """
    actual = helpers.runner_with_cwd(
        [
            "--collect-only",
            "tests/",
        ],
        helpers.TEST_DATA_PATH / "root",
    )
    assert actual
    actual_list: List[Dict[str, Any]] = actual
    if actual_list is not None:
        actual_item = actual_list.pop(0)

        assert all(item in actual_item for item in ("status", "cwd", "error"))
        assert actual_item.get("status") == "success"
        assert actual_item.get("cwd") == os.fspath(helpers.TEST_DATA_PATH / "root")
        assert is_same_tree(
            actual_item.get("tests"),
            expected_discovery_test_output.root_with_config_expected_output,
            ["id_", "lineno", "name", "runID"],
        ), (
            f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_discovery_test_output.root_with_config_expected_output, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}"
        )


def test_config_sub_folder():
    """Here the session node will be a subfolder of the workspace root and the test are in another subfolder.

    This tests checks to see if test node path are under the session node and if so the
    session node is correctly updated to the common path.
    """
    folder_path = helpers.TEST_DATA_PATH / "config_sub_folder"
    actual = helpers.runner_with_cwd(
        [
            "--collect-only",
            "-c=config/pytest.ini",
            "--rootdir=config/",
            "-vv",
        ],
        folder_path,
    )

    assert actual
    actual_list: List[Dict[str, Any]] = actual
    if actual_list is not None:
        actual_item = actual_list.pop(0)
        assert all(item in actual_item for item in ("status", "cwd", "error"))
        assert actual_item.get("status") == "success"
        assert actual_item.get("cwd") == os.fspath(helpers.TEST_DATA_PATH / "config_sub_folder")
        assert actual_item.get("tests") is not None
        if actual_item.get("tests") is not None:
            tests: Any = actual_item.get("tests")
            assert tests.get("name") == "config_sub_folder"


def test_ruff_plugin():
    """Here the session node will be a subfolder of the workspace root and the test are in another subfolder.

    This tests checks to see if test node path are under the session node and if so the
    session node is correctly updated to the common path.
    """
    file_path = helpers.TEST_DATA_PATH / "folder_with_script"
    actual = helpers.runner(
        [os.fspath(file_path), "--collect-only", "--ruff"],
    )

    assert actual
    actual_list: List[Dict[str, Any]] = actual
    if actual_list is not None:
        actual_item = actual_list.pop(0)
        assert all(item in actual_item for item in ("status", "cwd", "error"))
        assert actual_item.get("status") == "success", (
            f"Status is not 'success', error is: {actual_item.get('error')}"
        )
        assert actual_item.get("cwd") == os.fspath(helpers.TEST_DATA_PATH)
        assert is_same_tree(
            actual_item.get("tests"),
            expected_discovery_test_output.ruff_test_expected_output,
            ["id_", "lineno", "name", "runID"],
        ), (
            f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_discovery_test_output.ruff_test_expected_output, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}"
        )