diff --git a/tests/common/reboot.py b/tests/common/reboot.py index cf2f4bdc94..8c1d97ff04 100644 --- a/tests/common/reboot.py +++ b/tests/common/reboot.py @@ -1,8 +1,10 @@ import threading import time +import re import logging from multiprocessing.pool import ThreadPool, TimeoutError from errors import RunAnsibleModuleFail +from collections import deque logger = logging.getLogger(__name__) @@ -42,7 +44,7 @@ "wait": 120, # We are searching two types of reboot cause. # This change relates to changes of PR #6130 in sonic-buildimage repository - "cause": r"'reboot'|Non-Hardware \(reboot", + "cause": r"'reboot'|Non-Hardware \(reboot|^reboot", "test_reboot_cause_only": False }, REBOOT_TYPE_SOFT: { @@ -75,6 +77,11 @@ } } +MAX_NUM_REBOOT_CAUSE_HISTORY = 10 +REBOOT_TYPE_HISTOYR_QUEUE = deque([], MAX_NUM_REBOOT_CAUSE_HISTORY) +REBOOT_CAUSE_HISTORY_TITLE = ["name", "cause", "time", "user", "comment"] + + def get_warmboot_finalizer_state(duthost): try: res = duthost.command('systemctl is-active warmboot-finalizer.service',module_ignore_errors=True) @@ -223,3 +230,46 @@ def check_reboot_cause(dut, reboot_cause_expected): reboot_cause_got = get_reboot_cause(dut) logging.debug("dut {} last reboot-cause {}".format(dut.hostname, reboot_cause_got)) return reboot_cause_got == reboot_cause_expected + + +def check_reboot_cause_history(dut, reboot_type_history_queue): + """ + @summary: Check the reboot cause history on DUT. Can be used with wailt_until + @param dut: The AnsibleHost object of DUT. + @param reboot_type_history_queue: reboot type queue. + e.g. + show reboot-cause history + Name Cause Time User Comment + ------------------- ------------- ------------------------------- ------ --------- + 2021_09_09_14_15_13 Power Loss () N/A N/A N/A + 2021_09_09_14_06_17 reboot Thu 09 Sep 2021 02:05:17 PM UTC admin N/A + 2021_09_09_13_59_11 Watchdog () N/A N/A N/A + 2021_09_09_13_52_13 Power Loss () N/A N/A N/A + 2021_09_09_13_45_18 warm-reboot Thu 09 Sep 2021 01:44:14 PM UTC admin N/A + 2021_09_09_13_37_58 fast-reboot Thu 09 Sep 2021 01:37:09 PM UTC admin N/A + 2021_09_09_13_30_52 soft-reboot Thu 09 Sep 2021 01:30:24 PM UTC admin N/A + 2021_09_09_13_24_17 reboot Thu 09 Sep 2021 01:23:17 PM UTC admin N/A + """ + reboot_cause_history_got = dut.show_and_parse("show reboot-cause history") + logging.debug("dut {} reboot-cause history {}. reboot type history queue is {}".format( + dut.hostname, reboot_cause_history_got, reboot_type_history_queue)) + + logging.info("Verify reboot-cause history title") + if reboot_cause_history_got: + if not set(REBOOT_CAUSE_HISTORY_TITLE) == set(reboot_cause_history_got[0].keys()): + logging.error("Expected reboot-cause history title:{} not match actual reboot-cause history title:{}".format( + REBOOT_CAUSE_HISTORY_TITLE, reboot_cause_history_got[0].keys())) + return False + + logging.info("Verify reboot-cause output are sorted in reverse chronological order" ) + reboot_type_history_len = len(reboot_type_history_queue) + if reboot_type_history_len <= len(reboot_cause_history_got): + for index, reboot_type in enumerate(reboot_type_history_queue): + if not re.search(reboot_ctrl_dict[reboot_type]["cause"], reboot_cause_history_got[reboot_type_history_len-index-1]["cause"]): + logging.error("The {} reboot-cause not match. expected_reboot type={}, actual_reboot_cause={}".format( + index, reboot_ctrl_dict[reboot_type]["cause"], reboot_cause_history_got[reboot_type_history_len-index]["cause"])) + return False + return True + logging.error("The number of expected reboot-cause:{} is more than that of actual reboot-cuase:{}".format( + reboot_type_history_len, len(reboot_type_history_queue))) + return False diff --git a/tests/platform_tests/test_reboot.py b/tests/platform_tests/test_reboot.py index 73f0eb39d6..05a6198a4f 100644 --- a/tests/platform_tests/test_reboot.py +++ b/tests/platform_tests/test_reboot.py @@ -58,6 +58,7 @@ def reboot_and_check(localhost, dut, interfaces, xcvr_skip_list, reboot_type=REB logging.info("Run %s reboot on DUT" % reboot_type) reboot(dut, localhost, reboot_type=reboot_type, reboot_helper=reboot_helper, reboot_kwargs=reboot_kwargs) + REBOOT_TYPE_HISTOYR_QUEUE.append(reboot_type) check_interfaces_and_services(dut, interfaces, xcvr_skip_list, reboot_type) @@ -77,6 +78,12 @@ def check_interfaces_and_services(dut, interfaces, xcvr_skip_list, reboot_type = assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause, dut, reboot_type), \ "got reboot-cause failed after rebooted by %s" % reboot_type + if "201811" in dut.os_version or "201911" in dut.os_version: + logging.info("Skip check reboot-cause history for version before 202012") + else: + logger.info("Check reboot-cause history") + assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause_history, dut, + REBOOT_TYPE_HISTOYR_QUEUE), "Check reboot-cause history failed after rebooted by %s" % reboot_type if reboot_ctrl_dict[reboot_type]["test_reboot_cause_only"]: logging.info("Further checking skipped for %s test which intends to verify reboot-cause only" % reboot_type) return