Self update e2e test (#3000)

* self-update test

* addressed comments

* fix tests

* log

* added comment

* merge conflicts

azurelinuxagent/ga/rsm_version_updater.py

@@ -142,6 +142,5 @@ def proceed_with_update(self):
             # start the next available highest version which would be the target version
             prefix = "upgrade"
         raise AgentUpgradeExitException(
-            "Agent completed all update checks, exiting current process to {0} to the new Agent version {1}".format(
-                prefix,
-                self._version))
+            "Current Agent {0} completed all update checks, exiting current process to {1} to the new Agent version {2}".format(CURRENT_VERSION,
+                prefix, self._version))

azurelinuxagent/ga/self_update_version_updater.py

@@ -188,5 +188,5 @@ def proceed_with_update(self):
             # In case of an upgrade, we don't need to exclude anything as the daemon will automatically
             # start the next available highest version which would be the target version
             raise AgentUpgradeExitException(
-                "Agent completed all update checks, exiting current process to upgrade to the new Agent version {0}".format(
+                "Current Agent {0} completed all update checks, exiting current process to upgrade to the new Agent version {1}".format(CURRENT_VERSION,
                     self._version))

tests/ga/test_agent_update_handler.py

@@ -98,7 +98,7 @@ def _assert_no_agent_package_telemetry_emitted(self, mock_telemetry, version="9.
                              mock_telemetry.call_args_list))
 
     def _assert_agent_exit_process_telemetry_emitted(self, message):
-        self.assertIn("Agent completed all update checks, exiting current process", message)
+        self.assertIn("Current Agent {0} completed all update checks, exiting current process".format(CURRENT_VERSION), message)
 
     def test_it_should_not_update_when_autoupdate_disabled(self):
         self.prepare_agents(count=1)

tests/ga/test_update.py

@@ -1478,7 +1478,7 @@ def __assert_exit_code_successful(self, update_handler):
 
     def __assert_upgrade_telemetry_emitted(self, mock_telemetry, upgrade=True, version="9.9.9.10"):
         upgrade_event_msgs = [kwarg['message'] for _, kwarg in mock_telemetry.call_args_list if
-                              'Agent completed all update checks, exiting current process to {0} to the new Agent version {1}'.format(
+                              'Current Agent {0} completed all update checks, exiting current process to {1} to the new Agent version {2}'.format(CURRENT_VERSION,
                                   "upgrade" if upgrade else "downgrade", version) in kwarg['message'] and kwarg[
                                   'op'] == WALAEventOperation.AgentUpgrade]
         self.assertEqual(1, len(upgrade_event_msgs),

tests_e2e/orchestrator/runbook.yml

@@ -29,7 +29,7 @@ variable:
   # Test suites to execute
   #
   - name: test_suites
-    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline, ext_sequencing, agent_persist_firewall, publish_hostname"
+    value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline, ext_sequencing, agent_persist_firewall, publish_hostname, agent_update"
 
   #
   # Parameters used to create test VMs

tests_e2e/test_suites/agent_update.yml

@@ -1,7 +1,13 @@
+# Scenario validates RSM and Self-updates paths
+# RSM update: If vm enrolled into RSM, it will validate agent uses RSM to update to target version
+# Self-update: If vm not enrolled into RSM, it will validate agent uses self-update to update to latest version published
 name: "AgentUpdate"
 tests:
-  - "agent_update/rsm_update.py"
-images: "random(endorsed, 10)"
+#  - "agent_update/rsm_update.py"  will enable this test once we have a new test version published
+   - "agent_update/self_update.py"
+images:
+    - "random(endorsed, 10)"
+    - "random(endorsed-arm64, 2)"
 locations: "AzureCloud:eastus2euap"
 owns_vm: true
 skip_on_clouds:

tests_e2e/tests/agent_update/self_update.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+from pathlib import Path
+from threading import RLock
+
+from assertpy import fail
+
+import azurelinuxagent
+from tests_e2e.tests.lib.agent_test import AgentVmTest
+from tests_e2e.tests.lib.agent_test_context import AgentVmTestContext
+from tests_e2e.tests.lib.logging import log
+from tests_e2e.tests.lib.retry import retry_if_false
+from tests_e2e.tests.lib.shell import run_command
+
+
+class SelfUpdateBvt(AgentVmTest):
+    """
+    This test case is to verify that the agent can update itself to the latest version using self-update path when vm not enrolled to RSM updates
+    """
+
+    def __init__(self, context: AgentVmTestContext):
+        super().__init__(context)
+        self._ssh_client = self._context.create_ssh_client()
+        self._test_version = "2.8.9.9"
+        self._test_pkg_name = f"WALinuxAgent-{self._test_version}.zip"
+
+    _setup_lock = RLock()
+
+    def run(self):
+        self._test_setup()
+        self._verify_agent_updated_to_latest_version()
+
+    def _test_setup(self) -> None:
+        """
+        Builds the custom test agent pkg as some lower version and installs it on the vm
+        """
+        self._build_custom_test_agent()
+        self._ssh_client.run_command(f"agent_update-self_update_test_setup --package ~/tmp/{self._test_pkg_name} --version {self._test_version}", use_sudo=True)
+
+    def _build_custom_test_agent(self) -> None:
+        """
+        Builds the custom test pkg
+        """
+        with self._setup_lock:
+            agent_source_path: Path = self._context.working_directory / "source"
+            source_pkg_path: Path = agent_source_path / "eggs" / f"{self._test_pkg_name}"
+            if source_pkg_path.exists():
+                log.info("The test pkg already exists at %s, skipping build", source_pkg_path)
+            else:
+                if agent_source_path.exists():
+                    os.rmdir(agent_source_path)  # Remove if partial build exists
+                source_directory: Path = Path(azurelinuxagent.__path__[0]).parent
+                copy_cmd: str = f"cp -r {source_directory} {agent_source_path}"
+                log.info("Copying agent source %s to %s", source_directory, agent_source_path)
+                run_command(copy_cmd, shell=True)
+                if not agent_source_path.exists():
+                    raise Exception(
+                        f"The agent source was not copied to the expected path {agent_source_path}")
+                version_file: Path = agent_source_path / "azurelinuxagent" / "common" / "version.py"
+                version_cmd = rf"""sed -E -i "s/^AGENT_VERSION\s+=\s+'[0-9.]+'/AGENT_VERSION = '{self._test_version}'/g" {version_file}"""
+                log.info("Setting agent version to %s to build new pkg", self._test_version)
+                run_command(version_cmd, shell=True)
+                makepkg_file: Path = agent_source_path / "makepkg.py"
+                build_cmd: str = f"env PYTHONPATH={agent_source_path} python3 {makepkg_file} -o {agent_source_path}"
+                log.info("Building custom test agent pkg version %s", self._test_version)
+                run_command(build_cmd, shell=True)
+                if not source_pkg_path.exists():
+                    raise Exception(
+                        f"The test pkg was not created at the expected path {source_pkg_path}")
+            target_path: Path = Path("~") / "tmp"
+            log.info("Copying %s to %s:%s", source_pkg_path, self._context.vm, target_path)
+            self._ssh_client.copy_to_node(source_pkg_path, target_path)
+
+    def _verify_agent_updated_to_latest_version(self) -> None:
+        """
+        Verifies the agent updated to latest version from custom test version.
+        We retrieve latest version from goal state and compare with current agent version running as that latest version
+        """
+        latest_version: str = self._ssh_client.run_command("agent_update-self_update_latest_version.py", use_sudo=True).rstrip()
+        self._verify_guest_agent_update(latest_version)
+        # Verify agent updated to latest version by custom test agent
+        self._ssh_client.run_command("agent_update-self_update_check.py --latest-version {0} --current-version {1}".format(latest_version, self._test_version))
+
+    def _verify_guest_agent_update(self, latest_version: str) -> None:
+        """
+        Verify current agent version running on latest version
+        """
+
+        def _check_agent_version(latest_version: str) -> bool:
+            waagent_version: str = self._ssh_client.run_command("waagent-version", use_sudo=True)
+            expected_version = f"Goal state agent: {latest_version}"
+            if expected_version in waagent_version:
+                return True
+            else:
+                return False
+
+        waagent_version: str = ""
+        log.info("Verifying agent updated to latest version: {0}".format(latest_version))
+        success: bool = retry_if_false(lambda: _check_agent_version(latest_version), delay=60)
+        if not success:
+            fail("Guest agent didn't update to latest version {0} but found \n {1}".format(
+                latest_version, waagent_version))
+        waagent_version: str = self._ssh_client.run_command("waagent-version", use_sudo=True)
+        log.info(
+            f"Successfully verified agent updated to latest version. Current agent version running:\n {waagent_version}")

tests_e2e/tests/scripts/agent_update-self_update_check.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env pypy3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Script verifies agent update was done by test agent
+#
+import argparse
+import re
+
+from assertpy import fail
+
+from tests_e2e.tests.lib.agent_log import AgentLog
+from tests_e2e.tests.lib.logging import log
+from tests_e2e.tests.lib.retry import retry_if_false
+
+
+#2023-12-28T04:34:23.535652Z INFO ExtHandler ExtHandler Current Agent 2.8.9.9 completed all update checks, exiting current process to upgrade to the new Agent version 2.10.0.7
+_UPDATE_PATTERN = re.compile(r'Current Agent (\S*) completed all update checks, exiting current process to upgrade to the new Agent version (\S*)')
+
+
+def verify_agent_update_from_log(latest_version, current_version) -> bool:
+    """
+    Checks if the agent updated to the latest version from current version
+    """
+    agentlog = AgentLog()
+
+    for record in agentlog.read():
+        update_match = re.match(_UPDATE_PATTERN, record.message)
+        if update_match:
+            log.info('found the agent update log: %s', record.text)
+            if update_match.groups()[0] == current_version and update_match.groups()[1] == latest_version:
+                return True
+    return False
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-l', '--latest-version', required=True)
+    parser.add_argument('-c', '--current-version', required=True)
+    args = parser.parse_args()
+
+    found: bool = retry_if_false(lambda: verify_agent_update_from_log(args.latest_version, args.current_version))
+    if not found:
+        fail('agent update was not found in the logs for latest version {0} from current version {1}'.format(args.latest_version, args.current_version))
+
+
+if __name__ == "__main__":
+    main()

tests_e2e/tests/scripts/agent_update-self_update_latest_version.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env pypy3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# returns the agent latest version published
+#
+
+from azurelinuxagent.common.protocol.goal_state import GoalStateProperties
+from azurelinuxagent.common.protocol.util import get_protocol_util
+from azurelinuxagent.common.utils.flexible_version import FlexibleVersion
+from tests_e2e.tests.lib.retry import retry
+
+
+def get_agent_family_manifest(goal_state):
+    """
+    Get the agent_family from last GS for Test Family
+    """
+    agent_families = goal_state.extensions_goal_state.agent_families
+    agent_family_manifests = []
+    for m in agent_families:
+        if m.name == 'Test':
+            if len(m.uris) > 0:
+                agent_family_manifests.append(m)
+    return agent_family_manifests[0]
+
+
+def get_largest_version(agent_manifest):
+    """
+    Get the largest version from the agent manifest
+    """
+    largest_version = FlexibleVersion("0.0.0.0")
+    for pkg in agent_manifest.pkg_list.versions:
+        pkg_version = FlexibleVersion(pkg.version)
+        if pkg_version > largest_version:
+            largest_version = pkg_version
+    return largest_version
+
+
+def main():
+
+    try:
+        protocol = get_protocol_util().get_protocol(init_goal_state=False)
+        retry(lambda: protocol.client.reset_goal_state(
+            goal_state_properties=GoalStateProperties.ExtensionsGoalState))
+        goal_state = protocol.client.get_goal_state()
+        agent_family = get_agent_family_manifest(goal_state)
+        agent_manifest = goal_state.fetch_agent_manifest(agent_family.name, agent_family.uris)
+        largest_version = get_largest_version(agent_manifest)
+        print(str(largest_version))
+    except Exception as e:
+        raise Exception("Unable to verify agent updated to latest version since test failed to get the which is the latest version from the agent manifest: {0}".format(e))
+
+
+if __name__ == "__main__":
+    main()

tests_e2e/tests/scripts/agent_update-self_update_test_setup

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script prepares the new agent and install it on the vm
+#
+
+set -euo pipefail
+
+usage() (
+    echo "Usage: agent_update-self_update_test_setup -p|--package <path> -v|--version <version>"
+    exit 1
+)
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -p|--package)
+            shift
+            if [ "$#" -lt 1 ]; then
+                usage
+            fi
+            package=$1
+            shift
+            ;;
+        -v|--version)
+            shift
+            if [ "$#" -lt 1 ]; then
+                usage
+            fi
+            version=$1
+            shift
+            ;;
+        *)
+            usage
+    esac
+done
+if [ "$#" -ne 0 ] || [ -z ${package+x} ] || [ -z ${version+x} ]; then
+    usage
+fi
+
+echo "updating the related to self-update flags"
+update-waagent-conf Debug.EnableGAVersioning=n Debug.SelfUpdateHotfixFrequency=120 Debug.SelfUpdateRegularFrequency=120 Autoupdate.Frequency=120
+agent-service stop
+mv /var/log/waagent.log /var/log/waagent.$(date --iso-8601=seconds).log
+
+echo "Cleaning up the existing agents"
+rm -rf /var/lib/waagent/WALinuxAgent-*
+
+echo "Installing $package as version $version..."
+unzip.py $package /var/lib/waagent/WALinuxAgent-$version
+agent-service restart
+