fix the ignore rule in agent update test (#2915)

* ignore the agent installed version

* address comments

* address comments

* fixes

tests_e2e/orchestrator/lib/agent_test_suite.py

@@ -467,8 +467,6 @@ def _execute(self, environment: Environment, variables: Dict[str, Any]):
                             self.context.lisa_log.info("Executing Test Suite %s", suite.name)
                             test_suite_success = self._execute_test_suite(suite) and test_suite_success
 
-                        test_suite_success = self._check_agent_log() and test_suite_success
-
                     finally:
                         collect = self.context.collect_logs
                         if collect == CollectLogs.Always or collect == CollectLogs.Failed and not test_suite_success:
@@ -512,6 +510,7 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool:
                     log.info("")
 
                     summary: List[str] = []
+                    ignore_error_rules: List[Dict[str, Any]] = []
 
                     for test in suite.tests:
                         test_full_name = f"{suite_name}-{test.name}"
@@ -522,9 +521,9 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool:
 
                         test_success: bool = True
 
+                        test_instance = test.test_class(self.context)
                         try:
-                            test.test_class(self.context).run()
-
+                            test_instance.run()
                             summary.append(f"[Passed]  {test.name}")
                             log.info("******** [Passed] %s", test.name)
                             self.context.lisa_log.info("[Passed] %s", test_full_name)
@@ -583,6 +582,8 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool:
 
                         suite_success = suite_success and test_success
 
+                        ignore_error_rules.extend(test_instance.get_ignore_error_rules())
+
                         if not test_success and test.blocks_suite:
                             log.warning("%s failed and blocks the suite. Stopping suite execution.", test.name)
                             break
@@ -607,9 +608,11 @@ def _execute_test_suite(self, suite: TestSuiteInfo) -> bool:
                     if not suite_success:
                         self._mark_log_as_failed()
 
+                suite_success = suite_success and self._check_agent_log(ignore_error_rules)
+
                 return suite_success
 
-    def _check_agent_log(self) -> bool:
+    def _check_agent_log(self, ignore_error_rules: List[Dict[str, Any]]) -> bool:
         """
         Checks the agent log for errors; returns true on success (no errors int the log)
         """
@@ -623,13 +626,6 @@ def _check_agent_log(self) -> bool:
             errors = json.loads(output, object_hook=AgentLogRecord.from_dictionary)
 
             # Individual tests may have rules to ignore known errors; filter those out
-            ignore_error_rules = []
-            # pylint seems to think self.context.test_suites is not iterable. Suppressing warning, since its type is List[AgentTestSuite]
-            #  E1133: Non-iterable value self.context.test_suites is used in an iterating context (not-an-iterable)
-            for suite in self.context.test_suites:  # pylint: disable=E1133
-                for test in suite.tests:
-                    ignore_error_rules.extend(test.test_class(self.context).get_ignore_error_rules())
-
             if len(ignore_error_rules) > 0:
                 new = []
                 for e in errors:

tests_e2e/tests/agent_update/rsm_update.py

@@ -24,6 +24,7 @@
 # For each scenario, we initiate the rsm request with target version and then verify agent updated to that target version.
 #
 import json
+import re
 from typing import List, Dict, Any
 
 import requests
@@ -49,16 +50,19 @@ def __init__(self, context: AgentTestContext):
             ip_address=self._context.vm_ip_address,
             username=self._context.username,
             private_key_file=self._context.private_key_file)
+        self._installed_agent_version = "9.9.9.9"
+        self._downgrade_version = "9.9.9.9"
 
     def get_ignore_error_rules(self) -> List[Dict[str, Any]]:
         ignore_rules = [
             #
             # This is expected as we validate the downgrade scenario
             #
             # WARNING ExtHandler ExtHandler Agent WALinuxAgent-9.9.9.9 is permanently blacklisted
-            #
+            # Note: Version varies depending on the pipeline branch the test is running on
             {
-                'message': r"Agent WALinuxAgent-9.9.9.9 is permanently blacklisted"
+                'message': rf"Agent WALinuxAgent-{self._installed_agent_version} is permanently blacklisted",
+                'if': lambda r: r.prefix == 'ExtHandler' and self._installed_agent_version > self._downgrade_version
             },
             # We don't allow downgrades below then daemon version
             # 2023-07-11T02:28:21.249836Z WARNING ExtHandler ExtHandler [AgentUpdateError] The Agent received a request to downgrade to version 1.4.0.0, but downgrading to a version less than the Agent installed on the image (1.4.0.1) is not supported. Skipping downgrade.
@@ -71,20 +75,22 @@ def get_ignore_error_rules(self) -> List[Dict[str, Any]]:
         return ignore_rules
 
     def run(self) -> None:
+        # retrieve the installed agent version in the vm before run the scenario
+        self._retrieve_installed_agent_version()
         # Allow agent to send supported feature flag
         self._verify_agent_reported_supported_feature_flag()
 
         log.info("*******Verifying the Agent Downgrade scenario*******")
         stdout: str = self._ssh_client.run_command("waagent-version", use_sudo=True)
         log.info("Current agent version running on the vm before update is \n%s", stdout)
-        downgrade_version: str = "1.5.0.0"
-        log.info("Attempting downgrade version %s", downgrade_version)
-        self._request_rsm_update(downgrade_version)
-        self._check_rsm_gs(downgrade_version)
+        self._downgrade_version: str = "1.5.0.0"
+        log.info("Attempting downgrade version %s", self._downgrade_version)
+        self._request_rsm_update(self._downgrade_version)
+        self._check_rsm_gs(self._downgrade_version)
         self._prepare_agent()
         # Verify downgrade scenario
-        self._verify_guest_agent_update(downgrade_version)
-        self._verify_agent_reported_update_status(downgrade_version)
+        self._verify_guest_agent_update(self._downgrade_version)
+        self._verify_agent_reported_update_status(self._downgrade_version)
 
 
         # Verify upgrade scenario
@@ -241,6 +247,19 @@ def _verify_agent_reported_update_status(self, version: str):
         self._ssh_client.run_command(f"agent_update-verify_agent_reported_update_status.py --version {version}", use_sudo=True)
         log.info("Successfully Agent reported update status for version {0}".format(version))
 
+    def _retrieve_installed_agent_version(self):
+        """
+        Retrieve the installed agent version
+        """
+        log.info("Retrieving installed agent version")
+        stdout: str = self._ssh_client.run_command("waagent-version", use_sudo=True)
+        log.info("Retrieved installed agent version \n {0}".format(stdout))
+        match = re.search(r'.*Goal state agent: (\S*)', stdout)
+        if match:
+            self._installed_agent_version = match.groups()[0]
+        else:
+            log.warning("Unable to retrieve installed agent version and set to default value {0}".format(self._installed_agent_version))
+
 
 if __name__ == "__main__":
     RsmUpdateBvt.run_from_command_line()