Run tests using a container (#2704)

* Run tests within a container * cleanup Co-authored-by: narrieta <narrieta>
Azure · Nov 23, 2022 · a07e48a · a07e48a
1 parent 97e4d39
commit a07e48a
Show file tree

Hide file tree

Showing 13 changed files with 163 additions and 79 deletions.
diff --git a/tests_e2e/azure-pipelines.yml b/tests_e2e/azure-pipelines.yml
@@ -1,10 +1,6 @@
 variables:
   - name: azureConnection
-    value: 'AzLinux DCR Public (8e037ad4-618f-4466-8bc8-5099d41ac15b)'
-  - name: subId
-    value: '8e037ad4-618f-4466-8bc8-5099d41ac15b'
-  - name: testsSourcesDirectory
-    value: "$(Build.SourcesDirectory)/tests_e2e"
+    value: 'azuremanagement'
 
 trigger:
   - develop
@@ -15,7 +11,8 @@ pool:
   vmImage: ubuntu-latest
 
 stages:
-  - stage: "Execute"
+  - stage: "ExecuteTests"
+
     jobs:
       - template: 'templates/execute-tests.yml'
 
diff --git a/tests_e2e/docker/Dockerfile b/tests_e2e/docker/Dockerfile
@@ -0,0 +1,73 @@
+#
+# * Sample command to build the image:
+#
+#   docker build -t waagenttests .
+#
+# * Sample command to execute a container interactively:
+#
+#   docker run --rm -it -v /home/nam/src/WALinuxAgent:/home/waagent/WALinuxAgent waagenttests bash --login
+#
+FROM ubuntu:latest
+LABEL description="Test environment for WALinuxAgent"
+
+SHELL ["/bin/bash", "-c"]
+
+#
+# Install the required packages as root
+#
+USER root
+
+RUN \
+    apt-get update                                                                                            && \
+                                                                                                                 \
+    #                                                                                                            \
+    # Install basic dependencies                                                                                 \
+    #                                                                                                            \
+    apt-get install -y git python3.10 python3.10-dev curl                                                     && \
+    ln /usr/bin/python3.10 /usr/bin/python3                                                                   && \
+                                                                                                                 \
+    #                                                                                                            \
+    # Install LISA dependencies                                                                                  \
+    #                                                                                                            \
+    apt-get install -y git gcc libgirepository1.0-dev libcairo2-dev qemu-utils libvirt-dev python3-venv       && \
+                                                                                                                 \
+    #                                                                                                            \
+    # Create user waagent, which is used to execute the tests                                                    \
+    #                                                                                                            \
+    groupadd waagent                                                                                          && \
+    useradd --shell /bin/bash --create-home -g waagent waagent                                                && \
+    :
+
+#
+# Do the Poetry and LISA setup as waagent
+#
+USER waagent
+
+RUN \
+    #                                                                                                            \
+    # Install Poetry                                                                                             \
+    #                                                                                                            \
+    curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/install-poetry.py | python3 -     && \
+    export PATH="$HOME/.local/bin:$PATH"                                                                      && \
+                                                                                                                 \
+    #                                                                                                            \
+    # Install LISA                                                                                               \
+    #                                                                                                            \
+    cd $HOME                                                                                                  && \
+    git clone https://github.com/microsoft/lisa.git                                                           && \
+    cd lisa                                                                                                   && \
+    poetry config virtualenvs.in-project true                                                                 && \
+    make setup                                                                                                && \
+                                                                                                                 \
+    #                                                                                                            \
+    # Install additional test dependencies                                                                       \
+    #                                                                                                            \
+    poetry add msrestazure &&                                                                                    \
+                                                                                                                 \
+    #                                                                                                            \
+    # The setup for the tests depends on a couple of paths; add those to the profile                             \
+    #                                                                                                            \
+    echo 'export PYTHONPATH="$HOME/WALinuxAgent"' >> $HOME/.bash_profile                                      && \
+    echo 'export PATH="$HOME/.local/bin:$PATH"' >> $HOME/.bash_profile                                        && \
+    :
+
diff --git a/tests_e2e/requirements.txt b/tests_e2e/requirements.txt
@@ -10,4 +10,3 @@ cryptography
 distro
 junitparser
 msrestazure
-python-dotenv
diff --git a/tests_e2e/scenario_utils/azure_models.py b/tests_e2e/scenario_utils/azure_models.py
@@ -12,8 +12,8 @@
 from azure.mgmt.resource import ResourceManagementClient
 from msrestazure.azure_exceptions import CloudError
 
-from dcr.scenario_utils.logging_utils import LoggingHandler
-from dcr.scenario_utils.models import get_vm_data_from_env, VMModelType, VMMetaData
+from tests_e2e.scenario_utils.logging_utils import LoggingHandler
+from tests_e2e.scenario_utils.models import get_vm_data_from_env, VMModelType, VMMetaData
 
 
 class AzureComputeBaseClass(ABC, LoggingHandler):

diff --git a/tests_e2e/scenario_utils/extensions/BaseExtensionTestClass.py b/tests_e2e/scenario_utils/extensions/BaseExtensionTestClass.py
@@ -3,9 +3,9 @@
 
 from azure.core.polling import LROPoller
 
-from dcr.scenario_utils.azure_models import ComputeManager
-from dcr.scenario_utils.logging_utils import LoggingHandler
-from dcr.scenario_utils.models import ExtensionMetaData, get_vm_data_from_env
+from tests_e2e.scenario_utils.azure_models import ComputeManager
+from tests_e2e.scenario_utils.logging_utils import LoggingHandler
+from tests_e2e.scenario_utils.models import ExtensionMetaData, get_vm_data_from_env
 
 
 class BaseExtensionTestClass(LoggingHandler):

diff --git a/tests_e2e/scenario_utils/extensions/CustomScriptExtension.py b/tests_e2e/scenario_utils/extensions/CustomScriptExtension.py
@@ -1,7 +1,7 @@
 import uuid
 
-from dcr.scenario_utils.extensions.BaseExtensionTestClass import BaseExtensionTestClass
-from dcr.scenario_utils.models import ExtensionMetaData
+from tests_e2e.scenario_utils.extensions.BaseExtensionTestClass import BaseExtensionTestClass
+from tests_e2e.scenario_utils.models import ExtensionMetaData
 
 
 class CustomScriptExtension(BaseExtensionTestClass):

diff --git a/tests_e2e/scenario_utils/logging_utils.py b/tests_e2e/scenario_utils/logging_utils.py
@@ -0,0 +1,33 @@
+# Create a base class
+import logging
+
+
+def get_logger(name):
+    return LoggingHandler(name).log
+
+
+class LoggingHandler:
+    """
+    Base class for Logging
+    """
+    def __init__(self, name=None):
+        self.log = self.__setup_and_get_logger(name)
+
+    def __setup_and_get_logger(self, name):
+        logger = logging.getLogger(name if name is not None else self.__class__.__name__)
+        if logger.hasHandlers():
+            # Logging module inherits from base loggers if already setup, if a base logger found, reuse that
+            return logger
+
+        # No handlers found for logger, set it up
+        # This logging format is easier to read on the DevOps UI -
+        # https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=bash#formatting-commands
+        log_formatter = logging.Formatter("##[%(levelname)s] [%(asctime)s] [%(module)s] {%(pathname)s:%(lineno)d} %(message)s",
+                                          datefmt="%Y-%m-%dT%H:%M:%S%z")
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(log_formatter)
+        logger.addHandler(console_handler)
+        logger.setLevel(logging.INFO)
+
+        return logger
+
diff --git a/tests_e2e/scenario_utils/models.py b/tests_e2e/scenario_utils/models.py
@@ -2,8 +2,6 @@
 from enum import Enum, auto
 from typing import List
 
-from dotenv import load_dotenv
-
 
 class VMModelType(Enum):
     VM = auto()
@@ -123,13 +121,13 @@ def _get_ips(username) -> (list, list):
 
 def get_vm_data_from_env() -> VMMetaData:
     if get_vm_data_from_env.__instance is None:
-        load_dotenv()
         get_vm_data_from_env.__instance = VMMetaData(vm_name=os.environ["VMNAME"],
                                                      rg_name=os.environ['RGNAME'],
                                                      sub_id=os.environ["SUBID"],
                                                      location=os.environ['LOCATION'],
                                                      admin_username=os.environ['ADMINUSERNAME'])
 
+
     return get_vm_data_from_env.__instance
 
 

diff --git a/tests_e2e/scripts/__init__.py b/tests_e2e/scripts/__init__.py
diff --git a/tests_e2e/scripts/execute_tests.sh b/tests_e2e/scripts/execute_tests.sh
@@ -2,17 +2,19 @@
 
 set -euxo pipefail
 
-export PYTHONPATH=$BUILD_SOURCESDIRECTORY
+# The private ssh key is shared from the container host as $HOME/id_rsa; copy it to
+# HOME/.ssh, set the correct mode and generate the public key.
+mkdir "$HOME/.ssh"
+cp "$HOME/id_rsa" "$HOME/.ssh"
+chmod 700 "$HOME/.ssh/id_rsa"
+ssh-keygen -y -f "$HOME/.ssh/id_rsa" > "$HOME/.ssh/id_rsa.pub"
 
-cd $BUILD_SOURCESDIRECTORY/lisa
-
-# LISA needs both the public and private keys; generate the former
-chmod 700 $SSHKEY_SECUREFILEPATH
-ssh-keygen -y -f $SSHKEY_SECUREFILEPATH > "$SSHKEY_SECUREFILEPATH".pub
-
-./lisa.sh --runbook $BUILD_SOURCESDIRECTORY/tests_e2e/lisa/runbook/azure.yml \
-  --log_path $HOME/tmp \
-  --working_path $HOME/tmp \
-  -v subscription_id:$SUBID \
-  -v identity_file:$SSHKEY_SECUREFILEPATH
+# Execute the tests, this needs to be done from the LISA root directory
+cd "$HOME/lisa"
 
+./lisa.sh \
+  --runbook "$HOME/WALinuxAgent/tests_e2e/lisa/runbook/azure.yml" \
+  --log_path "$HOME/logs" \
+  --working_path "$HOME/logs" \
+  -v subscription_id:"$SUBSCRIPTION_ID" \
+  -v identity_file:"$HOME/.ssh/id_rsa.pub"
diff --git a/tests_e2e/scripts/execute_tests_container.sh b/tests_e2e/scripts/execute_tests_container.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+az login --service-principal --username "$AZURE_CLIENT_ID" --password "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID" > /dev/null
+
+az acr login --name waagenttests
+
+docker pull waagenttests.azurecr.io/waagenttests:latest
+
+# Logs will be placed in this location. Make waagent (UID 1000 in the container) the owner.
+mkdir "$HOME/logs"
+sudo chown 1000 "$HOME/logs"
+
+docker run --rm \
+      --volume "$BUILD_SOURCESDIRECTORY:/home/waagent/WALinuxAgent" \
+      --volume "$DOWNLOADSSHKEY_SECUREFILEPATH:/home/waagent/id_rsa" \
+      --volume "$HOME/logs:/home/waagent/logs" \
+      --env SUBSCRIPTION_ID \
+      --env AZURE_CLIENT_ID \
+      --env AZURE_CLIENT_SECRET \
+      --env AZURE_TENANT_ID \
+      waagenttests.azurecr.io/waagenttests \
+      bash --login -c '~/WALinuxAgent/tests_e2e/scripts/execute_tests.sh'
+
+ls -lR "$HOME/logs"
+
diff --git a/tests_e2e/scripts/install_dependencies.sh b/tests_e2e/scripts/install_dependencies.sh
diff --git a/tests_e2e/templates/execute-tests.yml b/tests_e2e/templates/execute-tests.yml
@@ -1,10 +1,10 @@
 jobs:
-  - job: ""
+  - job: "ExecuteTests"
 
     steps:
 
       - task: DownloadSecureFile@1
-        name: sshKey
+        name: downloadSshKey
         displayName: "Download SSH key"
         inputs:
           secureFile: 'id_rsa'
@@ -24,13 +24,11 @@ jobs:
           addToPath: true
           architecture: 'x64'
 
-      - bash: $(testsSourcesDirectory)/scripts/install_dependencies.sh
-        displayName: "Install Dependencies"
-
-      - bash: $(testsSourcesDirectory)/scripts/execute_tests.sh
+      - bash: $(Build.SourcesDirectory)/tests_e2e/scripts/execute_tests_container.sh
         displayName: "Execute tests"
         env:
           # Add all KeyVault secrets explicitly as they're not added by default to the environment vars
           AZURE_CLIENT_ID: $(AZURE-CLIENT-ID)
           AZURE_CLIENT_SECRET: $(AZURE-CLIENT-SECRET)
-          AZURE_TENANT_ID: $(AZURE-TENANT-ID)
+          AZURE_TENANT_ID: $(AZURE-TENANT-ID)
+          SUBSCRIPTION_ID: $(SUBSCRIPTION-ID)