Skip to content

Commit

Permalink
Report results for individual tests (#2751)
Browse files Browse the repository at this point in the history 
* Report results for individual tests

* mark failure

---------

Co-authored-by: narrieta <narrieta>
  • Loading branch information
@narrieta
narrieta authored Feb 3, 2023
1 parent becea90 commit ce369ac
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 29 deletions.
3 changes: 3 additions & 0 deletions tests_e2e/orchestrator/lib/agent_junit.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ def type_schema(cls) -> Type[schema.TypedSchema]:
return AgentJUnitSchema

def _received_message(self, message: MessageBase) -> None:
# The Agent sends its own TestResultMessage and marks them as "AgentTestResultMessage"; for the
# test results sent by LISA itself, we change the suite name to "_Setup_" in order to separate them
# from actual test results.
if isinstance(message, TestResultMessage) and message.type != "AgentTestResultMessage":
message.suite_full_name = "_Setup_"
message.suite_name = message.suite_full_name
Expand Down
96 changes: 67 additions & 29 deletions tests_e2e/orchestrator/lib/agent_test_suite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,33 +335,22 @@ def _execute_test_suite(self, suite: TestSuiteDescription) -> bool:
"""
suite_name = suite.name
suite_full_name = f"{suite_name}-{self.context.image_name}"
suite_start_time: datetime.datetime = datetime.datetime.now()

with _set_thread_name(suite_full_name): # The thread name is added to self._log
with set_current_thread_log(Path.home()/'logs'/f"{suite_full_name}.log"):
start_time: datetime.datetime = datetime.datetime.now()

message: TestResultMessage = TestResultMessage()
message.type = "AgentTestResultMessage"
message.id_ = str(uuid.uuid4())
message.status = TestStatus.RUNNING
message.suite_full_name = suite_name
message.suite_name = message.suite_full_name
message.full_name = f"{suite_name}-{self.context.image_name}"
message.name = message.full_name
message.elapsed = 0
notifier.notify(message)

try:
agent_test_logger.info("")
agent_test_logger.info("**************************************** %s ****************************************", suite_name)
agent_test_logger.info("")

failed: List[str] = []
failed: bool = False # True if any test fails
summary: List[str] = []

for test in suite.tests:
test_name = test.__name__
test_full_name = f"{suite_name}-{test_name}"
test_start_time: datetime.datetime = datetime.datetime.now()

agent_test_logger.info("******** Executing %s", test_name)
self._log.info("******** Executing %s", test_full_name)
Expand All @@ -373,16 +362,34 @@ def _execute_test_suite(self, suite: TestSuiteDescription) -> bool:
summary.append(f"[Passed] {test_name}")
agent_test_logger.info("******** [Passed] %s", test_name)
self._log.info("******** [Passed] %s", test_full_name)
self._report_test_result(
suite_full_name,
test_name,
TestStatus.PASSED,
test_start_time)
except AssertionError as e:
failed = True
summary.append(f"[Failed] {test_name}")
failed.append(test_name)
agent_test_logger.error("******** [Failed] %s: %s", test_name, e)
self._log.error("******** [Failed] %s", test_full_name)
self._report_test_result(
suite_full_name,
test_name,
TestStatus.FAILED,
test_start_time,
message=str(e))
except: # pylint: disable=bare-except
failed = True
summary.append(f"[Error] {test_name}")
failed.append(test_name)
agent_test_logger.exception("UNHANDLED EXCEPTION IN %s", test_name)
self._log.exception("UNHANDLED EXCEPTION IN %s", test_full_name)
self._report_test_result(
suite_full_name,
test_name,
TestStatus.FAILED,
test_start_time,
message="Unhandled exception.",
add_exception_stack_trace=True)

agent_test_logger.info("")

Expand All @@ -392,21 +399,52 @@ def _execute_test_suite(self, suite: TestSuiteDescription) -> bool:
agent_test_logger.info("\t%s", r)
agent_test_logger.info("")

if len(failed) == 0:
message.status = TestStatus.PASSED
else:
message.status = TestStatus.FAILED
message.message = f"Tests failed: {failed}"

except: # pylint: disable=bare-except
message.status = TestStatus.FAILED
message.message = "Unhandled exception while executing test suite."
message.stacktrace = traceback.format_exc()
finally:
message.elapsed = (datetime.datetime.now() - start_time).total_seconds()
notifier.notify(message)
failed = True
self._report_test_result(
suite_full_name,
suite_name,
TestStatus.FAILED,
suite_start_time,
message=f"Unhandled exception while executing test suite {suite_name}.",
add_exception_stack_trace=True)

return failed

return len(failed) == 0
@staticmethod
def _report_test_result(
suite_name: str,
test_name: str,
status: TestStatus,
start_time: datetime.datetime,
message: str = "",
add_exception_stack_trace: bool = False
) -> None:
"""
Reports a test result to the junit notifier
"""
# The junit notifier requires an initial RUNNING message in order to register the test in its internal cache.
msg: TestResultMessage = TestResultMessage()
msg.type = "AgentTestResultMessage"
msg.id_ = str(uuid.uuid4())
msg.status = TestStatus.RUNNING
msg.suite_full_name = suite_name
msg.suite_name = msg.suite_full_name
msg.full_name = test_name
msg.name = msg.full_name
msg.elapsed = 0

notifier.notify(msg)

# Now send the actual result. The notifier pipeline makes a deep copy of the message so it is OK to re-use the
# same object and just update a few fields. If using a different object, be sure that the "id_" is the same.
msg.status = status
msg.message = message
if add_exception_stack_trace:
msg.stacktrace = traceback.format_exc()
msg.elapsed = (datetime.datetime.now() - start_time).total_seconds()

notifier.notify(msg)

def execute_script_on_node(self, script_path: Path, parameters: str = "", sudo: bool = False) -> int:
"""
Expand Down

0 comments on commit ce369ac

Please sign in to comment.