Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Expand Testing for Unittest Run in PythonFiles #21028

Merged
Merged
21 changes: 21 additions & 0 deletions pythonFiles/tests/unittestadapter/.data/test_fail_simple.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import unittest

# Test class for the test_fail_simple test.
# The test_failed_tests function should return a dictionary with a "success" status
# and the two tests with their outcome as "failed".

class RunFailSimple(unittest.TestCase):
"""Test class for the test_fail_simple test.

The test_failed_tests function should return a dictionary with a "success" status
and the two tests with their outcome as "failed".
"""

def test_one_fail(self) -> None:
self.assertGreater(2, 3)

def test_two_fail(self) -> None:
self.assertNotEqual(1, 1)
20 changes: 20 additions & 0 deletions pythonFiles/tests/unittestadapter/.data/test_two_classes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import unittest

# Test class which runs for the test_multiple_ids_run test with the two class parameters.
# Both test functions will be returned in a dictionary with a "success" status,
# and the two tests with their outcome as "success".


class ClassOne(unittest.TestCase):

def test_one(self) -> None:
self.assertGreater(2, 1)

class ClassTwo(unittest.TestCase):

def test_two(self) -> None:
self.assertGreater(2, 1)

Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import unittest

# Test class for the two file pattern test. It is pattern *test.py.
# The test_ids_multiple_runs function should return a dictionary with a "success" status,
# and the two tests with their outcome as "success".



class DiscoveryA(unittest.TestCase):
"""Test class for the two file pattern test. It is pattern *test.py

The test_ids_multiple_runs function should return a dictionary with a "success" status,
and the two tests with their outcome as "success".
"""

def test_one_a(self) -> None:
self.assertGreater(2, 1)

def test_two_a(self) -> None:
self.assertNotEqual(2, 1)
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import unittest

# Test class for the two file pattern test. This file is pattern test*.py.
# The test_ids_multiple_runs function should return a dictionary with a "success" status,
# and the two tests with their outcome as "success".

class DiscoveryB(unittest.TestCase):

def test_one_b(self) -> None:
self.assertGreater(2, 1)

def test_two_b(self) -> None:
self.assertNotEqual(2, 1)
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import unittest

# Test class which runs for the test_multiple_ids_run test with the two test
# files in the same folder. The cwd is set to the parent folder. This should return
# a dictionary with a "success" status and the two tests with their outcome as "success".

def add(a, b):
return a + b


class TestAddFunction(unittest.TestCase):

def test_add_positive_numbers(self):
result = add(2, 3)
self.assertEqual(result, 5)


def test_add_negative_numbers(self):
result = add(-2, -3)
self.assertEqual(result, -5)
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import unittest

# Test class which runs for the test_multiple_ids_run test with the two test
# files in the same folder. The cwd is set to the parent folder. This should return
# a dictionary with a "success" status and the two tests with their outcome as "success".

def subtract(a, b):
return a - b


class TestSubtractFunction(unittest.TestCase):
def test_subtract_positive_numbers(self):
result = subtract(5, 3)
self.assertEqual(result, 2)


def test_subtract_negative_numbers(self):
result = subtract(-2, -3)
self.assertEqual(result, 1)
160 changes: 160 additions & 0 deletions pythonFiles/tests/unittestadapter/test_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,3 +92,163 @@ def test_single_ids_run() -> None:
assert id_result is not None
assert "outcome" in id_result
assert id_result["outcome"] == "success"


@pytest.mark.parametrize(
"test_ids, pattern, cwd, expected_outcome",
[
(
[
"test_add.TestAddFunction.test_add_negative_numbers",
"test_add.TestAddFunction.test_add_positive_numbers",
],
"test_add.py",
os.fspath(TEST_DATA_PATH / "unittest_folder"),
"success",
),
(
[
"test_add.TestAddFunction.test_add_negative_numbers",
"test_add.TestAddFunction.test_add_positive_numbers",
"test_subtract.TestSubtractFunction.test_subtract_negative_numbers",
"test_subtract.TestSubtractFunction.test_subtract_positive_numbers",
],
"test*",
os.fspath(TEST_DATA_PATH / "unittest_folder"),
"success",
),
(
[
"pattern_a_test.DiscoveryA.test_one_a",
"pattern_a_test.DiscoveryA.test_two_a",
],
"*test",
os.fspath(TEST_DATA_PATH / "two_patterns"),
"success",
),
(
[
"test_pattern_b.DiscoveryB.test_one_b",
"test_pattern_b.DiscoveryB.test_two_b",
],
"test_*",
os.fspath(TEST_DATA_PATH / "two_patterns"),
"success",
),
(
[
"file_one.CaseTwoFileOne.test_one",
"file_one.CaseTwoFileOne.test_two",
"folder.file_two.CaseTwoFileTwo.test_one",
"folder.file_two.CaseTwoFileTwo.test_two",
],
"*",
os.fspath(TEST_DATA_PATH / "utils_nested_cases"),
"success",
),
(
[
"test_two_classes.ClassOne.test_one",
"test_two_classes.ClassTwo.test_two",
],
"test_two_classes.py",
os.fspath(TEST_DATA_PATH),
"success",
),
],
)
def test_multiple_ids_run(test_ids, pattern, cwd, expected_outcome) -> None:
"""
The following are all successful tests of different formats.

# 1. Two tests with the `pattern` specified as a file
# 2. Two test files in the same folder called `unittest_folder`
# 3. A folder with two different test file patterns, this test gathers pattern `*test`
# 4. A folder with two different test file patterns, this test gathers pattern `test_*`
# 5. A nested structure where a test file is on the same level as a folder containing a test file
# 6. Test file with two test classes

All tests should have the outcome of `success`.
"""
actual = run_tests(cwd, test_ids, pattern, None, "fake-uuid")
assert actual
assert all(item in actual for item in ("cwd", "status"))
assert actual["status"] == "success"
assert actual["cwd"] == cwd
assert "result" in actual
result = actual["result"]
assert len(result) == len(test_ids)
for test_id in test_ids:
assert test_id in result
id_result = result[test_id]
assert id_result is not None
assert "outcome" in id_result
assert id_result["outcome"] == expected_outcome
assert True


def test_failed_tests():
"""This test runs on a single file `test_fail` with two tests that fail."""
test_ids = [
"test_fail_simple.RunFailSimple.test_one_fail",
"test_fail_simple.RunFailSimple.test_two_fail",
]
actual = run_tests(
os.fspath(TEST_DATA_PATH), test_ids, "test_fail_simple*", None, "fake-uuid"
)
assert actual
assert all(item in actual for item in ("cwd", "status"))
assert actual["status"] == "success"
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
assert "result" in actual
result = actual["result"]
assert len(result) == len(test_ids)
for test_id in test_ids:
assert test_id in result
id_result = result[test_id]
assert id_result is not None
assert "outcome" in id_result
assert id_result["outcome"] == "failure"
assert "message" and "traceback" in id_result
assert True


def test_unknown_id():
"""This test runs on a unknown test_id, therefore it should return
an error as the outcome as it attempts to find the given test.
"""
test_ids = ["unknown_id"]
actual = run_tests(
os.fspath(TEST_DATA_PATH), test_ids, "test_fail_simple*", None, "fake-uuid"
)
assert actual
assert all(item in actual for item in ("cwd", "status"))
assert actual["status"] == "success"
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
assert "result" in actual
result = actual["result"]
assert len(result) == len(test_ids)
assert "unittest.loader._FailedTest.unknown_id" in result
id_result = result["unittest.loader._FailedTest.unknown_id"]
assert id_result is not None
assert "outcome" in id_result
assert id_result["outcome"] == "error"
assert "message" and "traceback" in id_result


def test_incorrect_path():
"""This test runs on a non existent path, therefore it should return
an error as the outcome as it attempts to find the given folder.
"""
test_ids = ["unknown_id"]
actual = run_tests(
os.fspath(TEST_DATA_PATH / "unknown_folder"),
test_ids,
"test_fail_simple*",
None,
"fake-uuid",
)
assert actual
assert all(item in actual for item in ("cwd", "status", "error"))
assert actual["status"] == "error"
assert actual["cwd"] == os.fspath(TEST_DATA_PATH / "unknown_folder")
4 changes: 3 additions & 1 deletion pythonFiles/unittestadapter/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,9 @@ class TestOutcomeEnum(str, enum.Enum):


class UnittestTestResult(unittest.TextTestResult):
formatted: Dict[str, Dict[str, Union[str, None]]] = dict()
def __init__(self, *args, **kwargs):
self.formatted: Dict[str, Dict[str, Union[str, None]]] = dict()
super(UnittestTestResult, self).__init__(*args, **kwargs)

def startTest(self, test: unittest.TestCase):
super(UnittestTestResult, self).startTest(test)
Expand Down