Add test for extensions disabled; refactor VirtualMachine and VmExten…

…sion (#2824)

* Add test for extensions disabled; refactor VirtualMachine and VmExtension
---------

Co-authored-by: narrieta <narrieta>

test-requirements.txt

@@ -19,3 +19,4 @@ azure-identity
 azure-mgmt-compute>=22.1.0
 azure-mgmt-resource>=15.0.0
 msrestazure
+pytz

tests_e2e/orchestrator/docker/Dockerfile

@@ -80,7 +80,7 @@ RUN \
     #                                                                                                            \
     # Install additional test dependencies                                                                       \
     #                                                                                                            \
-    python3 -m pip install distro msrestazure                                                                 && \
+    python3 -m pip install distro msrestazure pytz                                                            && \
     python3 -m pip install azure-mgmt-compute --upgrade                                                       && \
                                                                                                                  \
     #                                                                                                            \

tests_e2e/orchestrator/lib/agent_test_suite.py

@@ -203,12 +203,26 @@ def context(self):
 
     #
     # Test suites within the same runbook may be executed concurrently, and setup needs to be done only once.
-    # We use this lock to allow only 1 thread to do the setup. Setup completion is marked using the 'completed'
+    # We use these locks to allow only 1 thread to do the setup. Setup completion is marked using the 'completed'
     # file: the thread doing the setup creates the file and threads that find that the file already exists
     # simply skip setup.
     #
+    _working_directory_lock = RLock()
     _setup_lock = RLock()
 
+    def _create_working_directory(self) -> None:
+        """
+        Creates the working directory for the test suite.
+        """
+        self._working_directory_lock.acquire()
+
+        try:
+            if not self.context.working_directory.exists():
+                log.info("Creating working directory: %s", self.context.working_directory)
+                self.context.working_directory.mkdir(parents=True)
+        finally:
+            self._working_directory_lock.release()
+
     def _setup(self) -> None:
         """
         Prepares the test suite for execution (currently, it just builds the agent package)
@@ -228,9 +242,6 @@ def _setup(self) -> None:
                 return
 
             self.context.lisa_log.info("Building test agent")
-            log.info("Creating working directory: %s", self.context.working_directory)
-            self.context.working_directory.mkdir(parents=True)
-
             self._build_agent_package()
 
             log.info("Completed setup, creating %s", completed)
@@ -407,6 +418,8 @@ def _execute(self, environment: Environment, variables: Dict[str, Any]):
                     test_suite_success = True
 
                     try:
+                        self._create_working_directory()
+
                         if not self.context.skip_setup:
                             self._setup()
 

tests_e2e/orchestrator/runbook.yml

@@ -49,7 +49,7 @@ variable:
   #
   # The test suites to execute
   - name: test_suites
-    value: "agent_bvt, no_outbound_connections"
+    value: "agent_bvt, no_outbound_connections, extensions_disabled"
   - name: cloud
     value: "AzureCloud"
     is_case_visible: true

tests_e2e/orchestrator/scripts/agent-service

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euo pipefail
+
+#
+# The service name is walinuxagent in Ubuntu/debian and waagent elsewhere
+#
+
+usage() (
+    echo "Usage: agent-service command"
+    exit 1
+)
+
+if [ "$#" -lt 1 ]; then
+    usage
+fi
+cmd=$1
+shift
+
+if [ "$#" -ne 0 ] || [ -z ${cmd+x} ] ; then
+    usage
+fi
+
+if command -v systemctl &> /dev/null; then
+  service-status() { systemctl --no-pager -l status $1; }
+  service-stop()   { systemctl stop $1; }
+  service-restart()  { systemctl restart $1; }
+  service-start()  { systemctl start $1; }
+else
+  service-status() { service $1 status; }
+  service-stop()   { service $1 stop; }
+  service-restart()   { service $1 restart; }
+  service-start()  { service $1 start; }
+fi
+
+python=$(get-agent-python)
+distro=$($python -c 'from azurelinuxagent.common.version import get_distro; print(get_distro()[0])')
+distro=$(echo $distro | tr '[:upper:]' '[:lower:]')
+
+if [[ $distro == *"ubuntu"* || $distro == *"debian"* ]]; then
+    service_name="walinuxagent"
+else
+    service_name="waagent"
+fi
+
+echo "Service name: $service_name"
+
+if [[ "$cmd" == "restart" ]]; then
+    echo "Restarting service..."
+    service-restart $service_name
+    echo "Service status..."
+    service-status $service_name
+fi
+
+if [[ "$cmd" == "start" ]]; then
+    echo "Starting service..."
+    service-start $service_name
+fi
+
+if [[ "$cmd" == "stop" ]]; then
+    echo "Stopping service..."
+    service-stop $service_name
+fi
+
+if [[ "$cmd" == "status" ]]; then
+    echo "Service status..."
+    service-status $service_name
+fi

tests_e2e/orchestrator/scripts/update-waagent-conf

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Updates waagent.conf with the specified setting and value and restarts the Agent.
+#
+
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+    echo "Usage: update-waagent-conf <setting> <value>"
+    exit 1
+fi
+
+name=$1
+value=$2
+
+PYTHON=$(get-agent-python)
+waagent_conf=$($PYTHON -c 'from azurelinuxagent.common.osutil import get_osutil; print(get_osutil().agent_conf_file_path)')
+echo "Setting $name=$value in $waagent_conf"
+sed -i -E "/^$name=/d" "$waagent_conf"
+sed -i -E "\$a $name=$value" "$waagent_conf"
+updated=$(grep "$name" "$waagent_conf")
+echo "Updated value: $updated"
+agent-service restart

tests_e2e/pipeline/pipeline.yml

@@ -9,7 +9,7 @@ parameters:
   - name: test_suites
     displayName: Test Suites
     type: string
-    default: agent_bvt, no_outbound_connections
+    default: agent_bvt, no_outbound_connections, extensions_disabled
 
     # NOTES:
     #         * 'image', 'location' and 'vm_size' override any values in the test suites/images definition

tests_e2e/test_suites/extensions_disabled.yml

@@ -0,0 +1,9 @@
+#
+# The test suite disables extension processing and verifies that extensions
+# are not processed, but the agent continues reporting status.
+#
+name: "ExtensionsDisabled"
+tests:
+  - "extensions_disabled.py"
+images: "random(endorsed)"
+owns_vm: true

tests_e2e/tests/bvts/extension_operations.py

@@ -35,7 +35,7 @@
 from tests_e2e.tests.lib.identifiers import VmExtensionIds, VmExtensionIdentifier
 from tests_e2e.tests.lib.logging import log
 from tests_e2e.tests.lib.ssh_client import SshClient
-from tests_e2e.tests.lib.vm_extension import VmExtension
+from tests_e2e.tests.lib.virtual_machine_extension_client import VirtualMachineExtensionClient
 
 
 class ExtensionOperationsBvt(AgentTest):
@@ -47,7 +47,7 @@ def run(self):
 
         is_arm64: bool = ssh_client.get_architecture() == "aarch64"
 
-        custom_script_2_0 = VmExtension(
+        custom_script_2_0 = VirtualMachineExtensionClient(
             self._context.vm,
             VmExtensionIds.CustomScript,
             resource_name="CustomScript")
@@ -65,15 +65,15 @@ def run(self):
             )
             custom_script_2_0.assert_instance_view(expected_version="2.0", expected_message=message)
 
-        custom_script_2_1 = VmExtension(
+        custom_script_2_1 = VirtualMachineExtensionClient(
             self._context.vm,
             VmExtensionIdentifier(VmExtensionIds.CustomScript.publisher, VmExtensionIds.CustomScript.type, "2.1"),
             resource_name="CustomScript")
 
         if is_arm64:
             log.info("Installing %s", custom_script_2_1)
         else:
-            log.info("Updating %s to %s", custom_script_2_0, custom_script_2_1)
+            log.info("Updating %s", custom_script_2_0)
 
         message = f"Hello {uuid.uuid4()}!"
         custom_script_2_1.enable(

tests_e2e/tests/bvts/run_command.py

@@ -35,12 +35,12 @@
 from tests_e2e.tests.lib.identifiers import VmExtensionIds
 from tests_e2e.tests.lib.logging import log
 from tests_e2e.tests.lib.ssh_client import SshClient
-from tests_e2e.tests.lib.vm_extension import VmExtension
+from tests_e2e.tests.lib.virtual_machine_extension_client import VirtualMachineExtensionClient
 
 
 class RunCommandBvt(AgentTest):
     class TestCase:
-        def __init__(self, extension: VmExtension, get_settings: Callable[[str], Dict[str, str]]):
+        def __init__(self, extension: VirtualMachineExtensionClient, get_settings: Callable[[str], Dict[str, str]]):
             self.extension = extension
             self.get_settings = get_settings
 
@@ -49,7 +49,7 @@ def run(self):
 
         test_cases = [
             RunCommandBvt.TestCase(
-                VmExtension(self._context.vm, VmExtensionIds.RunCommand, resource_name="RunCommand"),
+                VirtualMachineExtensionClient(self._context.vm, VmExtensionIds.RunCommand, resource_name="RunCommand"),
                 lambda s: {
                     "script": base64.standard_b64encode(bytearray(s, 'utf-8')).decode('utf-8')
                 })
@@ -60,7 +60,7 @@ def run(self):
         else:
             test_cases.append(
                 RunCommandBvt.TestCase(
-                    VmExtension(self._context.vm, VmExtensionIds.RunCommandHandler, resource_name="RunCommandHandler"),
+                    VirtualMachineExtensionClient(self._context.vm, VmExtensionIds.RunCommandHandler, resource_name="RunCommandHandler"),
                     lambda s: {
                         "source": {
                             "script": s

tests_e2e/tests/bvts/vm_access.py

@@ -33,7 +33,7 @@
 from tests_e2e.tests.lib.logging import log
 from tests_e2e.tests.lib.ssh_client import SshClient
 
-from tests_e2e.tests.lib.vm_extension import VmExtension
+from tests_e2e.tests.lib.virtual_machine_extension_client import VirtualMachineExtensionClient
 
 
 class VmAccessBvt(AgentTest):
@@ -58,7 +58,7 @@ def run(self):
             public_key = f.read()
 
         # Invoke the extension
-        vm_access = VmExtension(self._context.vm, VmExtensionIds.VmAccess, resource_name="VmAccess")
+        vm_access = VirtualMachineExtensionClient(self._context.vm, VmExtensionIds.VmAccess, resource_name="VmAccess")
         vm_access.enable(
             protected_settings={
                 'username': username,

tests_e2e/tests/extensions_disabled.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# This test disables extension processing on waagent.conf and verifies that extensions are not processed, but the
+# agent continues reporting status.
+#
+
+import datetime
+import pytz
+
+from assertpy import assert_that, fail
+
+from azure.mgmt.compute.models import VirtualMachineInstanceView
+
+from tests_e2e.tests.lib.agent_test import AgentTest
+from tests_e2e.tests.lib.identifiers import VmExtensionIds
+from tests_e2e.tests.lib.logging import log
+from tests_e2e.tests.lib.ssh_client import SshClient
+from tests_e2e.tests.lib.virtual_machine_client import VirtualMachineClient
+from tests_e2e.tests.lib.virtual_machine_extension_client import VirtualMachineExtensionClient
+
+
+class ExtensionsDisabled(AgentTest):
+    def run(self):
+        ssh_client: SshClient = self._context.create_ssh_client()
+
+        # Disable extension processing on the test VM
+        log.info("Disabling extension processing on the test VM [%s]", self._context.vm.name)
+        output = ssh_client.run_command("update-waagent-conf Extensions.Enabled n", use_sudo=True)
+        log.info("Disable completed:\n%s", output)
+
+        # From now on, extensions will time out; set the timeout to the minimum allowed(15 minutes)
+        log.info("Setting the extension timeout to 15 minutes")
+        vm: VirtualMachineClient = VirtualMachineClient(self._context.vm)
+
+        vm.update({"extensionsTimeBudget": "PT15M"})
+
+        disabled_timestamp: datetime.datetime = datetime.datetime.utcnow() - datetime.timedelta(minutes=60)
+
+        #
+        # Validate that the agent is not processing extensions by attempting to run CustomScript
+        #
+        log.info("Executing CustomScript; it should time out after 15 min or so.")
+        custom_script = VirtualMachineExtensionClient(self._context.vm, VmExtensionIds.CustomScript, resource_name="CustomScript")
+        try:
+            custom_script.enable(settings={'commandToExecute': "date"}, force_update=True, timeout=20 * 60)
+            fail("CustomScript should have timed out")
+        except Exception as error:
+            assert_that("VMExtensionProvisioningTimeout" in str(error)) \
+                .described_as(f"Expected a VMExtensionProvisioningTimeout: {error}") \
+                .is_true()
+            log.info("CustomScript timed out as expected")
+
+        #
+        # Validate that the agent continued reporting status even if it is not processing extensions
+        #
+        instance_view: VirtualMachineInstanceView = vm.get_instance_view()
+        log.info("Instance view of VM Agent:\n%s", instance_view.vm_agent.serialize())
+        assert_that(instance_view.vm_agent.statuses).described_as("The VM agent should have exactly 1 status").is_length(1)
+        assert_that(instance_view.vm_agent.statuses[0].display_status).described_as("The VM Agent should be ready").is_equal_to('Ready')
+        # The time in the status is time zone aware and 'disabled_timestamp' is not; we need to make the latter time zone aware before comparing them
+        assert_that(instance_view.vm_agent.statuses[0].time)\
+            .described_as("The VM Agent should be have reported status even after extensions were disabled")\
+            .is_greater_than(pytz.utc.localize(disabled_timestamp))
+        log.info("The VM Agent reported status after extensions were disabled, as expected.")
+
+
+if __name__ == "__main__":
+    ExtensionsDisabled.run_from_command_line()