diff --git a/tests_e2e/orchestrator/lib/agent_test_suite.py b/tests_e2e/orchestrator/lib/agent_test_suite.py index eb90e4edcc..4a14b1f665 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite.py @@ -467,8 +467,6 @@ def _execute(self, environment: Environment, variables: Dict[str, Any]): self.context.lisa_log.info("Executing Test Suite %s", suite.name) test_suite_success = self._execute_test_suite(suite) and test_suite_success - test_suite_success = self._check_agent_log() and test_suite_success - finally: collect = self.context.collect_logs if collect == CollectLogs.Always or collect == CollectLogs.Failed and not test_suite_success: @@ -512,6 +510,7 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool: log.info("") summary: List[str] = [] + ignore_error_rules: List[Dict[str, Any]] = [] for test in suite.tests: test_full_name = f"{suite_name}-{test.name}" @@ -522,9 +521,9 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool: test_success: bool = True + test_instance = test.test_class(self.context) try: - test.test_class(self.context).run() - + test_instance.run() summary.append(f"[Passed] {test.name}") log.info("******** [Passed] %s", test.name) self.context.lisa_log.info("[Passed] %s", test_full_name) @@ -583,6 +582,8 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool: suite_success = suite_success and test_success + ignore_error_rules.extend(test_instance.get_ignore_error_rules()) + if not test_success and test.blocks_suite: log.warning("%s failed and blocks the suite. Stopping suite execution.", test.name) break @@ -607,9 +608,11 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool: if not suite_success: self._mark_log_as_failed() + suite_success = suite_success and self._check_agent_log(ignore_error_rules) + return suite_success - def _check_agent_log(self) -> bool: + def _check_agent_log(self, ignore_error_rules: List[Dict[str, Any]]) -> bool: """ Checks the agent log for errors; returns true on success (no errors int the log) """ @@ -623,13 +626,6 @@ def _check_agent_log(self) -> bool: errors = json.loads(output, object_hook=AgentLogRecord.from_dictionary) # Individual tests may have rules to ignore known errors; filter those out - ignore_error_rules = [] - # pylint seems to think self.context.test_suites is not iterable. Suppressing warning, since its type is List[AgentTestSuite] - # E1133: Non-iterable value self.context.test_suites is used in an iterating context (not-an-iterable) - for suite in self.context.test_suites: # pylint: disable=E1133 - for test in suite.tests: - ignore_error_rules.extend(test.test_class(self.context).get_ignore_error_rules()) - if len(ignore_error_rules) > 0: new = [] for e in errors: diff --git a/tests_e2e/tests/agent_update/rsm_update.py b/tests_e2e/tests/agent_update/rsm_update.py index 22f295b744..46875a2a07 100644 --- a/tests_e2e/tests/agent_update/rsm_update.py +++ b/tests_e2e/tests/agent_update/rsm_update.py @@ -24,6 +24,7 @@ # For each scenario, we initiate the rsm request with target version and then verify agent updated to that target version. # import json +import re from typing import List, Dict, Any import requests @@ -49,6 +50,8 @@ def __init__(self, context: AgentTestContext): ip_address=self._context.vm_ip_address, username=self._context.username, private_key_file=self._context.private_key_file) + self._installed_agent_version = "9.9.9.9" + self._downgrade_version = "9.9.9.9" def get_ignore_error_rules(self) -> List[Dict[str, Any]]: ignore_rules = [ @@ -56,9 +59,10 @@ def get_ignore_error_rules(self) -> List[Dict[str, Any]]: # This is expected as we validate the downgrade scenario # # WARNING ExtHandler ExtHandler Agent WALinuxAgent-9.9.9.9 is permanently blacklisted - # + # Note: Version varies depending on the pipeline branch the test is running on { - 'message': r"Agent WALinuxAgent-9.9.9.9 is permanently blacklisted" + 'message': rf"Agent WALinuxAgent-{self._installed_agent_version} is permanently blacklisted", + 'if': lambda r: r.prefix == 'ExtHandler' and self._installed_agent_version > self._downgrade_version }, # We don't allow downgrades below then daemon version # 2023-07-11T02:28:21.249836Z WARNING ExtHandler ExtHandler [AgentUpdateError] The Agent received a request to downgrade to version 1.4.0.0, but downgrading to a version less than the Agent installed on the image (1.4.0.1) is not supported. Skipping downgrade. @@ -71,20 +75,22 @@ def get_ignore_error_rules(self) -> List[Dict[str, Any]]: return ignore_rules def run(self) -> None: + # retrieve the installed agent version in the vm before run the scenario + self._retrieve_installed_agent_version() # Allow agent to send supported feature flag self._verify_agent_reported_supported_feature_flag() log.info("*******Verifying the Agent Downgrade scenario*******") stdout: str = self._ssh_client.run_command("waagent-version", use_sudo=True) log.info("Current agent version running on the vm before update is \n%s", stdout) - downgrade_version: str = "1.5.0.0" - log.info("Attempting downgrade version %s", downgrade_version) - self._request_rsm_update(downgrade_version) - self._check_rsm_gs(downgrade_version) + self._downgrade_version: str = "1.5.0.0" + log.info("Attempting downgrade version %s", self._downgrade_version) + self._request_rsm_update(self._downgrade_version) + self._check_rsm_gs(self._downgrade_version) self._prepare_agent() # Verify downgrade scenario - self._verify_guest_agent_update(downgrade_version) - self._verify_agent_reported_update_status(downgrade_version) + self._verify_guest_agent_update(self._downgrade_version) + self._verify_agent_reported_update_status(self._downgrade_version) # Verify upgrade scenario @@ -241,6 +247,19 @@ def _verify_agent_reported_update_status(self, version: str): self._ssh_client.run_command(f"agent_update-verify_agent_reported_update_status.py --version {version}", use_sudo=True) log.info("Successfully Agent reported update status for version {0}".format(version)) + def _retrieve_installed_agent_version(self): + """ + Retrieve the installed agent version + """ + log.info("Retrieving installed agent version") + stdout: str = self._ssh_client.run_command("waagent-version", use_sudo=True) + log.info("Retrieved installed agent version \n {0}".format(stdout)) + match = re.search(r'.*Goal state agent: (\S*)', stdout) + if match: + self._installed_agent_version = match.groups()[0] + else: + log.warning("Unable to retrieve installed agent version and set to default value {0}".format(self._installed_agent_version)) + if __name__ == "__main__": RsmUpdateBvt.run_from_command_line()