Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[ci] Assert some tests are not skipped in the CI #12915

Merged
merged 1 commit into from
Nov 13, 2022
Merged

[ci] Assert some tests are not skipped in the CI #12915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 67 additions & 10 deletions ci/scripts/github_skipped_tests_comment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import inspect
import json
import os
import logging
import subprocess
Expand Down Expand Up @@ -102,10 +104,30 @@ def to_node_name(dir_name: str):
return dir_name.replace("_", ": ", 1)


def build_diff_comment_with_main(
common_commit_sha,
skipped_list,
commit_sha,
):
if len(skipped_list) == 0:
return f"No diff in skipped tests with main found in this branch for commit {commit_sha}.\n"

text = (
f"The list below shows tests that ran in main {common_commit_sha} but were "
f"skipped in the CI build of {commit_sha}:\n"
f"```\n"
)
for skip in skipped_list:
text += skip + "\n"
text += f"```\n"
return text


def build_comment(
common_commit_sha,
common_main_build,
skipped_list,
additional_skipped_list,
pr_number,
build_number,
commit_sha,
Expand All @@ -114,18 +136,21 @@ def build_comment(
if common_main_build["state"] != "success":
return f"Unable to run tests bot because main failed to pass CI at {common_commit_sha}."

if len(skipped_list) == 0:
return f"No additional skipped tests found in this branch for commit {commit_sha}."
text = build_diff_comment_with_main(common_commit_sha, skipped_list, commit_sha)

if len(additional_skipped_list) != 0:
text += "\n"
text += (
f"Additional tests that were skipped in the CI build and present in the [`required_tests_to_run`]"
f"(https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:"
f"\n```\n"
)
for skip in additional_skipped_list:
text += skip + "\n"
text += f"```\n"

text = (
f"The list below shows some tests that ran in main {common_commit_sha} but were "
f"skipped in the CI build of {commit_sha}:\n"
f"```\n"
)
for skip in skipped_list:
text += skip + "\n"
text += (
f"```\nA detailed report of ran tests is [here](https://{jenkins_prefix}/job/tvm/job/PR-{str(pr_number)}"
f"A detailed report of ran tests is [here](https://{jenkins_prefix}/job/tvm/job/PR-{str(pr_number)}"
f"/{str(build_number)}/testReport/)."
)
return text
Expand All @@ -148,6 +173,7 @@ def get_skipped_tests_comment(
main_test_report_dir: str = "main-reports",
common_commit_sha: Optional[str] = None,
common_main_build: Optional[Dict[str, Any]] = None,
additional_tests_to_check_file: str = "required_tests_to_run.json",
) -> str:
pr_head = pr["commits"]["nodes"][0]["commit"]
target_url = find_target_url(pr_head)
Expand Down Expand Up @@ -195,10 +221,41 @@ def get_skipped_tests_comment(
if len(skipped_list) == 0:
logging.info("No skipped tests found.")

if not is_dry_run:
current_file = Path(__file__).resolve()
additional_tests_to_check_file = Path(current_file).parent / "required_tests_to_run.json"

logging.info(
f"Checking additional tests in file {additional_tests_to_check_file} are not skipped."
)
try:
with open(additional_tests_to_check_file, "r") as f:
additional_tests_to_check = json.load(f)
except IOError:
logging.info(
f"Failed to read additional tests from file: {additional_tests_to_check_file}."
)
additional_tests_to_check = {}

# Assert that tests present in "required_tests_to_run.json" are not skipped.
additional_skipped_tests = []
for subdir, test_set in additional_tests_to_check.items():
if subdir not in build_tests.keys():
logging.warning(f"Could not find directory {subdir} in the build test set.")
continue

for test in test_set:
if test in build_tests[subdir]:
additional_skipped_tests.append(f"{to_node_name(subdir)} -> {test}")

if len(additional_skipped_tests) == 0:
logging.info("No skipped tests found in the additional list.")

body = build_comment(
common_commit_sha,
common_main_build,
skipped_list,
additional_skipped_tests,
pr_and_build["pr_number"],
pr_and_build["build_number"],
commit_sha,
Expand Down
11 changes: 11 additions & 0 deletions ci/scripts/required_tests_to_run.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"unittest_GPU":
[
"ctypes.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_integration_extract_from_bert_base",
"cython.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_integration_extract_from_bert_base",
"ctypes.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_dynamic_loop_extent",
"cython.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_dynamic_loop_extent",
"ctypes.tests.python.unittest.test_meta_schedule_integration#test_extract_task_arm_conv2d_nchwc",
"cython.tests.python.unittest.test_meta_schedule_integration#test_extract_task_arm_conv2d_nchwc"
]
}
108 changes: 105 additions & 3 deletions tests/python/ci/test_ci.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def parameterize_named(**kwargs):

# pylint: disable=line-too-long
TEST_DATA_SKIPPED_BOT = {
"found-diff": {
"found-diff-no-additional": {
"main_xml_file": "unittest/file1.xml",
"main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
Expand Down Expand Up @@ -78,12 +78,61 @@ def parameterize_named(**kwargs):
</testsuite>
</testsuites>
""",
"additional_tests_to_check": """{
"unittest": ["dummy_class#dummy_test"],
"unittest_GPU": ["another_dummy_class#another_dummy_test"]
}
""",
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
"common_main_build": """{"build_number": "4115", "state": "success"}""",
"commit_sha": "sha1234",
"expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
},
"found-diff-skipped-additional": {
"main_xml_file": "unittest/file1.xml",
"main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
<testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
</testcase>
</testsuite>
</testsuites>
""",
"pr_xml_file": "unittest/file2.xml",
"pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
<testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
<skipped message="This test is skipped" type="pytest.skip">
Skipped
</skipped>
</testcase>
<testcase classname="ctypes.tests.python.unittest.test_roofline"
name="test_estimate_peak_bandwidth[cuda]" time="4.679">
<skipped message="This is another skippe test" type="pytest.skip">
Skipped
</skipped>
</testcase>
</testsuite>
</testsuites>
""",
"additional_tests_to_check": """{
"unittest": ["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner", "dummy_class#dummy_test"],
"unittest_GPU": ["another_dummy_class#another_dummy_test"]
}
""",
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
"common_main_build": """{"build_number": "4115", "state": "success"}""",
"commit_sha": "sha1234",
"expected_body": "The list below shows some tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
"expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
},
"no-diff": {
"main_xml_file": "unittest/file1.xml",
Expand Down Expand Up @@ -114,12 +163,56 @@ def parameterize_named(**kwargs):
</testsuite>
</testsuites>
""",
"additional_tests_to_check": """{
}
""",
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
"common_main_build": """{"build_number": "4115", "state": "success"}""",
"commit_sha": "sha1234",
"expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
},
"no-diff-skipped-additional": {
"main_xml_file": "unittest/file1.xml",
"main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
<testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
<skipped message="This test is skipped" type="pytest.skip">
Skipped
</skipped>
</testcase>
</testsuite>
</testsuites>
""",
"pr_xml_file": "unittest/file2.xml",
"pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
<testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
<skipped message="This test is skipped" type="pytest.skip">
Skipped
</skipped>
</testcase>
</testsuite>
</testsuites>
""",
"additional_tests_to_check": """{
"unittest": ["dummy_class#dummy_test", "ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner"],
"unittest_GPU": ["another_dummy_class#another_dummy_test"]
}
""",
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
"common_main_build": """{"build_number": "4115", "state": "success"}""",
"commit_sha": "sha1234",
"expected_body": "No additional skipped tests found in this branch for commit sha1234.",
"expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
},
"unable-to-run": {
"main_xml_file": "unittest/file1.xml",
Expand All @@ -132,6 +225,11 @@ def parameterize_named(**kwargs):
<testsuites>
</testsuites>
""",
"additional_tests_to_check": """{
"unittest": ["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner", "dummy_class#dummy_test"],
"unittest_GPU": ["another_dummy_class#another_dummy_test"]
}
""",
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
Expand All @@ -153,6 +251,7 @@ def test_skipped_tests_comment(
main_xml_content,
pr_xml_file,
pr_xml_content,
additional_tests_to_check,
target_url,
s3_prefix,
jenkins_prefix,
Expand All @@ -176,6 +275,8 @@ def write_xml_file(root_dir, xml_file, xml_content):
write_xml_file(pr_test_report_dir, pr_xml_file, pr_xml_content)
main_test_report_dir = Path(git.cwd) / "main-reports"
write_xml_file(main_test_report_dir, main_xml_file, main_xml_content)
with open(Path(git.cwd) / "required_tests_to_run.json", "w") as f:
f.write(additional_tests_to_check)

pr_data = {
"commits": {
Expand Down Expand Up @@ -208,6 +309,7 @@ def write_xml_file(root_dir, xml_file, xml_content):
pr_test_report_dir=pr_test_report_dir,
main_test_report_dir=main_test_report_dir,
common_main_build=json.loads(common_main_build),
additional_tests_to_check_file=Path(git.cwd) / "required_tests_to_run.json",
)
assert_in(expected_body, comment)
assert_in(f"with target {target_url}", caplog.text)
Expand Down