From cc91dc67c3a70deec52b603a1491ea46c8e13d88 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Thu, 15 Dec 2022 15:18:43 -0800 Subject: [PATCH 1/5] Install agent on test VMs (#2714) --- .github/workflows/ci_pr.yml | 14 +- .gitignore | 2 - ci/nosetests_only.sh | 18 -- ci/pylint_and_nosetests.sh | 32 --- makepkg.py | 118 ++++++----- setup.py | 5 +- tests_e2e/docker/Dockerfile | 2 +- .../orchestrator/lib/agent_test_suite.py | 190 ++++++++++++++++++ tests_e2e/orchestrator/scripts/collect-logs | 19 ++ tests_e2e/orchestrator/scripts/install-agent | 105 ++++++++++ .../scripts/run-scenarios} | 9 +- tests_e2e/pipeline/scripts/execute_tests.sh | 10 +- tests_e2e/requirements.txt | 12 -- .../BaseExtensionTestClass.py | 6 +- .../{modules => lib}/CustomScriptExtension.py | 4 +- .../scenarios/{modules => lib}/__init__.py | 0 .../{modules => lib}/azure_models.py | 4 +- .../{modules => lib}/logging_utils.py | 0 .../scenarios/{modules => lib}/models.py | 0 tests_e2e/scenarios/runbooks/daily.yml | 1 - tests_e2e/scenarios/scripts/collect_logs.sh | 17 -- .../scenarios/tests/bvts/custom_script.py | 2 +- .../scenarios/tests/check_agent_version.py | 23 --- tests_e2e/scenarios/testsuites/__init__.py | 0 tests_e2e/scenarios/testsuites/agent_bvt.py | 14 +- .../scenarios/testsuites/agent_test_suite.py | 53 ----- 26 files changed, 419 insertions(+), 241 deletions(-) delete mode 100755 ci/nosetests_only.sh delete mode 100755 ci/pylint_and_nosetests.sh create mode 100644 tests_e2e/orchestrator/lib/agent_test_suite.py create mode 100755 tests_e2e/orchestrator/scripts/collect-logs create mode 100755 tests_e2e/orchestrator/scripts/install-agent rename tests_e2e/{scenarios/scripts/run_scenarios.sh => orchestrator/scripts/run-scenarios} (63%) delete mode 100644 tests_e2e/requirements.txt rename tests_e2e/scenarios/{modules => lib}/BaseExtensionTestClass.py (95%) rename tests_e2e/scenarios/{modules => lib}/CustomScriptExtension.py (82%) rename tests_e2e/scenarios/{modules => lib}/__init__.py (100%) rename tests_e2e/scenarios/{modules => lib}/azure_models.py (98%) rename tests_e2e/scenarios/{modules => lib}/logging_utils.py (100%) rename tests_e2e/scenarios/{modules => lib}/models.py (100%) delete mode 100755 tests_e2e/scenarios/scripts/collect_logs.sh delete mode 100755 tests_e2e/scenarios/tests/check_agent_version.py create mode 100644 tests_e2e/scenarios/testsuites/__init__.py delete mode 100644 tests_e2e/scenarios/testsuites/agent_test_suite.py diff --git a/.github/workflows/ci_pr.yml b/.github/workflows/ci_pr.yml index a4565198d8..279b7e76c4 100644 --- a/.github/workflows/ci_pr.yml +++ b/.github/workflows/ci_pr.yml @@ -41,22 +41,22 @@ jobs: include: - python-version: 2.7 - PYLINTOPTS: "--rcfile=ci/2.7.pylintrc" + PYLINTOPTS: "--rcfile=ci/2.7.pylintrc --ignore=tests_e2e,makepkg.py" - python-version: 3.4 - PYLINTOPTS: "--rcfile=ci/2.7.pylintrc" + PYLINTOPTS: "--rcfile=ci/2.7.pylintrc --ignore=tests_e2e,makepkg.py" - python-version: 3.6 - PYLINTOPTS: "--rcfile=ci/3.6.pylintrc" + PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e" - python-version: 3.7 - PYLINTOPTS: "--rcfile=ci/3.6.pylintrc" + PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e" - python-version: 3.8 - PYLINTOPTS: "--rcfile=ci/3.6.pylintrc" + PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e" - python-version: 3.9 - PYLINTOPTS: "--rcfile=ci/3.6.pylintrc" + PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=azure_models.py,BaseExtensionTestClass.py" additional-nose-opts: "--with-coverage --cover-erase --cover-inclusive --cover-branches --cover-package=azurelinuxagent" name: "Python ${{ matrix.python-version }} Unit Tests" @@ -64,7 +64,7 @@ jobs: env: PYLINTOPTS: ${{ matrix.PYLINTOPTS }} - PYLINTFILES: "azurelinuxagent setup.py makepkg.py tests" + PYLINTFILES: "azurelinuxagent setup.py makepkg.py tests tests_e2e" NOSEOPTS: "--with-timer ${{ matrix.additional-nose-opts }}" PYTHON_VERSION: ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index d4c7873f2b..fd64d3314e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,6 @@ develop-eggs/ dist/ downloads/ eggs/ -lib/ -lib64/ parts/ sdist/ var/ diff --git a/ci/nosetests_only.sh b/ci/nosetests_only.sh deleted file mode 100755 index 8f87ea2488..0000000000 --- a/ci/nosetests_only.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -u - -EXIT_CODE=0 - -echo -echo "=========================================" -echo "nosetests -a '!requires_sudo' output" -echo "=========================================" -nosetests -a '!requires_sudo' tests || EXIT_CODE=$(($EXIT_CODE || $?)) - -echo "=========================================" -echo "nosetests -a 'requires_sudo' output" -echo "=========================================" -sudo env "PATH=$PATH" nosetests -a 'requires_sudo' tests || EXIT_CODE=$(($EXIT_CODE || $?)) - -exit "$EXIT_CODE" diff --git a/ci/pylint_and_nosetests.sh b/ci/pylint_and_nosetests.sh deleted file mode 100755 index e3e6b93556..0000000000 --- a/ci/pylint_and_nosetests.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -u - -pylint $PYLINTOPTS --jobs=0 $PYLINTFILES &> pylint.output & PYLINT_PID=$! -nosetests -a '!requires_sudo' tests &> nosetests_no_sudo.output & NOSETESTS_PID=$! -sudo env "PATH=$PATH" nosetests -a 'requires_sudo' tests &> nosetests_sudo.output & NOSETESTS_SUDO_PID=$! - -EXIT_CODE=0 -wait $PYLINT_PID || EXIT_CODE=$(($EXIT_CODE || $?)) -wait $NOSETESTS_PID || EXIT_CODE=$(($EXIT_CODE || $?)) -wait $NOSETESTS_SUDO_PID || EXIT_CODE=$(($EXIT_CODE || $?)) - -echo "=========================================" -echo "pylint output:" -echo "=========================================" - -cat pylint.output - -echo -echo "=========================================" -echo "nosetests -a '!requires_sudo' output:" -echo "=========================================" -cat nosetests_no_sudo.output - -echo -echo "=========================================" -echo "nosetests -a 'requires_sudo' output:" -echo "=========================================" -cat nosetests_sudo.output - -exit "$EXIT_CODE" \ No newline at end of file diff --git a/makepkg.py b/makepkg.py index 11e90b95a7..e35b16e488 100755 --- a/makepkg.py +++ b/makepkg.py @@ -1,14 +1,15 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 +import argparse import glob -import os +import logging import os.path +import pathlib import shutil import subprocess import sys -from azurelinuxagent.common.version import AGENT_NAME, AGENT_VERSION, \ - AGENT_LONG_VERSION +from azurelinuxagent.common.version import AGENT_NAME, AGENT_VERSION, AGENT_LONG_VERSION from azurelinuxagent.ga.update import AGENT_MANIFEST_FILE MANIFEST = '''[{{ @@ -48,62 +49,77 @@ PUBLISH_MANIFEST_FILE = 'manifest.xml' -output_path = os.path.join(os.getcwd(), "eggs") # pylint: disable=invalid-name -target_path = os.path.join(output_path, AGENT_LONG_VERSION) # pylint: disable=invalid-name -bin_path = os.path.join(target_path, "bin") # pylint: disable=invalid-name -egg_path = os.path.join(bin_path, AGENT_LONG_VERSION + ".egg") # pylint: disable=invalid-name -manifest_path = os.path.join(target_path, AGENT_MANIFEST_FILE) # pylint: disable=invalid-name -publish_manifest_path = os.path.join(target_path, PUBLISH_MANIFEST_FILE) # pylint: disable=invalid-name -pkg_name = os.path.join(output_path, AGENT_LONG_VERSION + ".zip") # pylint: disable=invalid-name -family = 'Test' # pylint: disable=C0103 -if len(sys.argv) > 1: - family = sys.argv[1] # pylint: disable=invalid-name - -def do(*args): # pylint: disable=C0103,W0621 +def do(*args): try: - subprocess.check_output(args, stderr=subprocess.STDOUT) + return subprocess.check_output(args, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: # pylint: disable=C0103 - print("ERROR: {0}".format(str(e))) - print("\t{0}".format(" ".join(args))) - print(e.output) - sys.exit(1) - + raise Exception("[{0}] failed:\n{1}\n{2}".format(" ".join(args), str(e), e.stdout)) + + +def run(agent_family, output_directory, log): + output_path = os.path.join(output_directory, "eggs") + target_path = os.path.join(output_path, AGENT_LONG_VERSION) + bin_path = os.path.join(target_path, "bin") + egg_path = os.path.join(bin_path, AGENT_LONG_VERSION + ".egg") + manifest_path = os.path.join(target_path, AGENT_MANIFEST_FILE) + publish_manifest_path = os.path.join(target_path, PUBLISH_MANIFEST_FILE) + pkg_name = os.path.join(output_path, AGENT_LONG_VERSION + ".zip") + + if os.path.isdir(target_path): + shutil.rmtree(target_path) + elif os.path.isfile(target_path): + os.remove(target_path) + if os.path.isfile(pkg_name): + os.remove(pkg_name) + os.makedirs(bin_path) + log.info("Created {0} directory".format(target_path)) + + setup_script = str(pathlib.Path(__file__).parent.joinpath("setup.py")) + args = ["python3", setup_script, "bdist_egg", "--dist-dir={0}".format(bin_path)] + + log.info("Creating egg {0}".format(egg_path)) + do(*args) + + egg_name = os.path.join("bin", os.path.basename( + glob.glob(os.path.join(bin_path, "*"))[0])) + + log.info("Writing {0}".format(manifest_path)) + with open(manifest_path, mode='w') as manifest: + manifest.write(MANIFEST.format(AGENT_NAME, egg_name)) + + log.info("Writing {0}".format(publish_manifest_path)) + with open(publish_manifest_path, mode='w') as publish_manifest: + publish_manifest.write(PUBLISH_MANIFEST.format(AGENT_VERSION, agent_family)) + + cwd = os.getcwd() + os.chdir(target_path) + try: + log.info("Creating package {0}".format(pkg_name)) + do("zip", "-r", pkg_name, egg_name) + do("zip", "-j", pkg_name, AGENT_MANIFEST_FILE) + do("zip", "-j", pkg_name, PUBLISH_MANIFEST_FILE) + finally: + os.chdir(cwd) -if os.path.isdir(target_path): - shutil.rmtree(target_path) -elif os.path.isfile(target_path): - os.remove(target_path) -if os.path.isfile(pkg_name): - os.remove(pkg_name) -os.makedirs(bin_path) -print("Created {0} directory".format(target_path)) + log.info("Package {0} successfully created".format(pkg_name)) -args = ["python", "setup.py", "bdist_egg", "--dist-dir={0}".format(bin_path)] # pylint: disable=invalid-name -print("Creating egg {0}".format(egg_path)) -do(*args) +if __name__ == "__main__": + logging.basicConfig(format='%(message)s', level=logging.INFO) -egg_name = os.path.join("bin", os.path.basename( # pylint: disable=invalid-name - glob.glob(os.path.join(bin_path, "*"))[0])) + parser = argparse.ArgumentParser() + parser.add_argument('family', metavar='family', nargs='?', default='Test', help='Agent family') + parser.add_argument('-o', '--output', default=os.getcwd(), help='Output directory') -print("Writing {0}".format(manifest_path)) -with open(manifest_path, mode='w') as manifest: - manifest.write(MANIFEST.format(AGENT_NAME, egg_name)) + arguments = parser.parse_args() -print("Writing {0}".format(publish_manifest_path)) -with open(publish_manifest_path, mode='w') as publish_manifest: - publish_manifest.write(PUBLISH_MANIFEST.format(AGENT_VERSION, - family)) + try: + run(arguments.family, arguments.output, logging) -cwd = os.getcwd() # pylint: disable=invalid-name -os.chdir(target_path) -print("Creating package {0}".format(pkg_name)) -do("zip", "-r", pkg_name, egg_name) -do("zip", "-j", pkg_name, AGENT_MANIFEST_FILE) -do("zip", "-j", pkg_name, PUBLISH_MANIFEST_FILE) -os.chdir(cwd) + except Exception as exception: + logging.error(str(exception)) + sys.exit(1) -print("Package {0} successfully created".format(pkg_name)) -sys.exit(0) + sys.exit(0) diff --git a/setup.py b/setup.py index a4ce296c65..8f5d92b42e 100755 --- a/setup.py +++ b/setup.py @@ -288,7 +288,10 @@ def initialize_options(self): self.lnx_distro_version = DISTRO_VERSION self.lnx_distro_fullname = DISTRO_FULL_NAME self.register_service = False - self.skip_data_files = False + # All our data files are system-wide files that are not included in the egg; skip them when + # creating an egg. + self.skip_data_files = "bdist_egg" in sys.argv + # pylint: enable=attribute-defined-outside-init def finalize_options(self): diff --git a/tests_e2e/docker/Dockerfile b/tests_e2e/docker/Dockerfile index ee98f0f700..0489d3907f 100644 --- a/tests_e2e/docker/Dockerfile +++ b/tests_e2e/docker/Dockerfile @@ -59,7 +59,7 @@ RUN \ # \ # Install additional test dependencies \ # \ - python3 -m pip install msrestazure && \ + python3 -m pip install distro msrestazure && \ \ # \ # The setup for the tests depends on a couple of paths; add those to the profile \ diff --git a/tests_e2e/orchestrator/lib/agent_test_suite.py b/tests_e2e/orchestrator/lib/agent_test_suite.py new file mode 100644 index 0000000000..4ed68507cb --- /dev/null +++ b/tests_e2e/orchestrator/lib/agent_test_suite.py @@ -0,0 +1,190 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from pathlib import Path +import shutil + +import makepkg + +# E0401: Unable to import 'lisa' (import-error) +from lisa import ( # pylint: disable=E0401 + CustomScriptBuilder, + Logger, + Node, + TestSuite, + TestSuiteMetadata, +) +# E0401: Unable to import 'lisa.sut_orchestrator.azure.common' (import-error) +from lisa.sut_orchestrator.azure.common import get_node_context # pylint: disable=E0401 + +from azurelinuxagent.common.version import AGENT_VERSION + + +class AgentTestSuite(TestSuite): + """ + Base class for VM Agent tests. It provides initialization, cleanup, and utilities common to all VM Agent test suites. + """ + def __init__(self, metadata: TestSuiteMetadata): + super().__init__(metadata) + # The actual initialization happens in _initialize() + self._log = None + self._node = None + self._subscription_id = None + self._resource_group_name = None + self._vm_name = None + self._test_source_directory = None + self._working_directory = None + self._node_home_directory = None + + def before_case(self, *_, **kwargs) -> None: + self._initialize(kwargs['node'], kwargs['log']) + self._setup_node() + + def after_case(self, *_, **__) -> None: + try: + self._collect_node_logs() + finally: + self._clean_up() + + def _initialize(self, node: Node, log: Logger) -> None: + self._node = node + self._log = log + + node_context = get_node_context(node) + self._subscription_id = node.features._platform.subscription_id + self._resource_group_name = node_context.resource_group_name + self._vm_name = node_context.vm_name + + self._test_source_directory = AgentTestSuite._get_test_source_directory() + self._working_directory = Path().home()/"waagent-tmp" + self._node_home_directory = Path('/home')/self._node.connection_info['username'] + + self._log.info(f"Test Node: {self._vm_name}") + self._log.info(f"Resource Group: {self._resource_group_name}") + self._log.info(f"Working directory: {self._working_directory}...") + + if self._working_directory.exists(): + self._log.info(f"Removing existing working directory: {self._working_directory}...") + try: + shutil.rmtree(self._working_directory.as_posix()) + except Exception as exception: + self._log.warning(f"Failed to remove the working directory: {exception}") + self._working_directory.mkdir() + + def _clean_up(self) -> None: + self._log.info(f"Removing {self._working_directory}...") + shutil.rmtree(self._working_directory.as_posix(), ignore_errors=True) + + @staticmethod + def _get_test_source_directory() -> Path: + """ + Returns the root directory of the source code for the end-to-end tests (".../WALinuxAgent/tests_e2e") + """ + path = Path(__file__) + while path.name != '': + if path.name == "tests_e2e": + return path + path = path.parent + raise Exception("Can't find the test root directory (tests_e2e)") + + def _setup_node(self) -> None: + """ + Prepares the remote node for executing the test suite. + """ + agent_package_path = self._build_agent_package() + self._install_agent_on_node(agent_package_path) + + def _build_agent_package(self) -> Path: + """ + Builds the agent package and returns the path to the package. + """ + build_path = self._working_directory/"build" + + # The same orchestrator machine may be executing multiple suites on the same test VM, or + # the same suite on one or more test VMs; we use this file to mark the build is already done + build_done_path = self._working_directory/"build.done" + if build_done_path.exists(): + self._log.info("The agent build is already completed, will use existing package.") + else: + self._log.info(f"Building agent package to {build_path}") + makepkg.run(agent_family="Test", output_directory=str(build_path), log=self._log) + build_done_path.touch() + + package_path = build_path/"eggs"/f"WALinuxAgent-{AGENT_VERSION}.zip" + if not package_path.exists(): + raise Exception(f"Can't find the agent package at {package_path}") + + self._log.info(f"Agent package: {package_path}") + + return package_path + + def _install_agent_on_node(self, agent_package: Path) -> None: + """ + Installs the given agent package on the test node. + """ + # The same orchestrator machine may be executing multiple suites on the same test VM, + # we use this file to mark the agent is already installed on the test VM. + install_done_path = self._working_directory/f"agent-install.{self._vm_name}.done" + if install_done_path.exists(): + self._log.info(f"Package {agent_package} is already installed on {self._vm_name}...") + return + + # The install script needs to unzip the agent package; ensure unzip is installed on the test node + self._log.info(f"Installing unzip on {self._vm_name}...") + self._node.os.install_packages("unzip") + + self._log.info(f"Installing {agent_package} on {self._vm_name}...") + agent_package_remote_path = self._node_home_directory/agent_package.name + self._log.info(f"Copying {agent_package} to {self._vm_name}:{agent_package_remote_path}") + self._node.shell.copy(agent_package, agent_package_remote_path) + self._execute_script_on_node( + self._test_source_directory/"orchestrator"/"scripts"/"install-agent", + parameters=f"--package {agent_package_remote_path} --version {AGENT_VERSION}", + sudo=True) + + self._log.info("The agent was installed successfully.") + install_done_path.touch() + + def _collect_node_logs(self) -> None: + """ + Collects the test logs from the remote machine and copied them to the local machine + """ + # Collect the logs on the test machine into a compressed tarball + self._log.info("Collecting logs on test machine [%s]...", self._node.name) + self._execute_script_on_node(self._test_source_directory/"orchestrator"/"scripts"/"collect-logs", sudo=True) + + # Copy the tarball to the local logs directory + remote_path = self._node_home_directory/"logs.tgz" + local_path = Path.home()/'logs'/'vm-logs-{0}.tgz'.format(self._node.name) + self._log.info(f"Copying {self._node.name}:{remote_path} to {local_path}") + self._node.shell.copy_back(remote_path, local_path) + + def _execute_script_on_node(self, script_path: Path, parameters: str = "", sudo: bool = False) -> int: + custom_script_builder = CustomScriptBuilder(script_path.parent, [script_path.name]) + custom_script = self._node.tools[custom_script_builder] + + command_line = f"{script_path} {parameters}" + self._log.info(f"Executing {command_line}") + result = custom_script.run(parameters=parameters, sudo=sudo) + + # LISA appends stderr to stdout so no need to check for stderr + if result.exit_code != 0: + raise Exception(f"[{command_line}] failed.\n{result.stdout}") + + return result.exit_code + + diff --git a/tests_e2e/orchestrator/scripts/collect-logs b/tests_e2e/orchestrator/scripts/collect-logs new file mode 100755 index 0000000000..9872878a7f --- /dev/null +++ b/tests_e2e/orchestrator/scripts/collect-logs @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Collects the logs needed to debug agent issues into a compressed tarball. +# +set -euxo pipefail + +logs_file_name="$HOME/logs.tgz" + +echo "Collecting logs to $logs_file_name ..." + +tar --exclude='journal/*' --exclude='omsbundle' --exclude='omsagent' --exclude='mdsd' --exclude='scx*' \ + --exclude='*.so' --exclude='*__LinuxDiagnostic__*' --exclude='*.zip' --exclude='*.deb' --exclude='*.rpm' \ + -czf "$logs_file_name" \ + /var/log \ + /var/lib/waagent/ \ + /etc/waagent.conf + +chmod +r "$logs_file_name" + diff --git a/tests_e2e/orchestrator/scripts/install-agent b/tests_e2e/orchestrator/scripts/install-agent new file mode 100755 index 0000000000..08122a2780 --- /dev/null +++ b/tests_e2e/orchestrator/scripts/install-agent @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# +# +# +set -euo pipefail + +usage() ( + echo "Usage: install-agent -p|--package -v|--version " + exit 1 +) + +while [[ $# -gt 0 ]]; do + case $1 in + -p|--package) + shift + if [ "$#" -lt 1 ]; then + usage + fi + package=$1 + shift + ;; + -v|--version) + shift + if [ "$#" -lt 1 ]; then + usage + fi + version=$1 + shift + ;; + *) + usage + esac +done +if [ "$#" -ne 0 ] || [ -z ${package+x} ] || [ -z ${version+x} ]; then + usage +fi + +# +# The service name is walinuxagent in Ubuntu and waagent elsewhere +# +if service walinuxagent status > /dev/null;then + service_name="walinuxagent" +else + service_name="waagent" +fi +echo "Service name: $service_name" + +# +# Install the package +# +echo "Installing $package..." +unzip -d "/var/lib/waagent/WALinuxAgent-$version" -o "$package" +sed -i 's/AutoUpdate.Enabled=n/AutoUpdate.Enabled=y/g' /etc/waagent.conf + +# +# Restart the service +# +echo "Restarting service..." +service $service_name stop + +# Rename the previous log to ensure the new log starts with the agent we just installed +mv /var/log/waagent.log /var/log/waagent.pre-install.log + +if command -v systemctl &> /dev/null; then + systemctl daemon-reload +fi + +service $service_name start + +# +# Verify that the new agent is running and output its status. Note that the extension handler +# may take some time to start so give 1 minute. +# +echo "Verifying agent installation..." +check-version() { + for i in {0..5} + do + if waagent --version | grep -E "Goal state agent:\s+$1" > /dev/null; then + return 0 + fi + sleep 10 + done + + return 1 +} + +if check-version "$version"; then + printf "\nThe agent was installed successfully\n" + exit_code=0 +else + printf "\nThe agent was not installed correctly; expected version %s\n" "$version" + exit_code=1 +fi + +waagent --version + +printf "\n" + +if command -v systemctl &> /dev/null; then + systemctl --no-pager -l status $service_name +else + service $service_name status +fi + +exit $exit_code diff --git a/tests_e2e/scenarios/scripts/run_scenarios.sh b/tests_e2e/orchestrator/scripts/run-scenarios similarity index 63% rename from tests_e2e/scenarios/scripts/run_scenarios.sh rename to tests_e2e/orchestrator/scripts/run-scenarios index 75fd96ba3f..e5216e5b92 100755 --- a/tests_e2e/scenarios/scripts/run_scenarios.sh +++ b/tests_e2e/orchestrator/scripts/run-scenarios @@ -1,5 +1,11 @@ #!/usr/bin/env bash - +# +# This script runs on the container executing the tests. It creates the SSH keys (private and public) used +# to manage the test VMs taking the initial key value from the file shared by the container host, then it +# executes the daily test runbook. +# +# TODO: The runbook should be parameterized. +# set -euxo pipefail cd "$HOME" @@ -11,6 +17,7 @@ cp "$HOME/id_rsa" "$HOME/.ssh" chmod 700 "$HOME/.ssh/id_rsa" ssh-keygen -y -f "$HOME/.ssh/id_rsa" > "$HOME/.ssh/id_rsa.pub" +# Now start the runbook lisa \ --runbook "$HOME/WALinuxAgent/tests_e2e/scenarios/runbooks/daily.yml" \ --log_path "$HOME/logs" \ diff --git a/tests_e2e/pipeline/scripts/execute_tests.sh b/tests_e2e/pipeline/scripts/execute_tests.sh index 2f177faa73..1da857c3cb 100755 --- a/tests_e2e/pipeline/scripts/execute_tests.sh +++ b/tests_e2e/pipeline/scripts/execute_tests.sh @@ -2,13 +2,16 @@ set -euxo pipefail +# Pull the container image used to execute the tests az login --service-principal --username "$AZURE_CLIENT_ID" --password "$AZURE_CLIENT_SECRET" --tenant "$AZURE_TENANT_ID" > /dev/null az acr login --name waagenttests docker pull waagenttests.azurecr.io/waagenttests:latest -# Logs will be placed in the staging directory. Make waagent (UID 1000 in the container) the owner so that it can write to that location +# Building the agent package writes the egg info to the source code directory, and test write their logs to the staging directory. +# Make waagent (UID 1000 in the container) the owner of both locations, so that it can write to them. +sudo chown 1000 "$BUILD_SOURCESDIRECTORY" sudo chown 1000 "$BUILD_ARTIFACTSTAGINGDIRECTORY" docker run --rm \ @@ -20,9 +23,10 @@ docker run --rm \ --env AZURE_CLIENT_SECRET \ --env AZURE_TENANT_ID \ waagenttests.azurecr.io/waagenttests \ - bash --login -c '$HOME/WALinuxAgent/tests_e2e/scenarios/scripts/run_scenarios.sh' + bash --login -c '$HOME/WALinuxAgent/tests_e2e/orchestrator/scripts/run-scenarios' -# Retake ownership of the staging directory +# Retake ownership of the source and staging directory (note that the former does not need to be done recursively) +sudo chown "$USER" "$BUILD_SOURCESDIRECTORY" sudo find "$BUILD_ARTIFACTSTAGINGDIRECTORY" -exec chown "$USER" {} \; # LISA organizes its logs in a tree similar to diff --git a/tests_e2e/requirements.txt b/tests_e2e/requirements.txt deleted file mode 100644 index fa74f1bfeb..0000000000 --- a/tests_e2e/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -# This is a list of pip packages that will be installed on both the orchestrator and the test VM -# Only add the common packages here, for more specific modules, add them to the scenario itself -azure-identity -azure-keyvault-keys -azure-mgmt-compute>=22.1.0 -azure-mgmt-keyvault>=7.0.0 -azure-mgmt-network>=16.0.0 -azure-mgmt-resource>=15.0.0 -cryptography -distro -junitparser -msrestazure diff --git a/tests_e2e/scenarios/modules/BaseExtensionTestClass.py b/tests_e2e/scenarios/lib/BaseExtensionTestClass.py similarity index 95% rename from tests_e2e/scenarios/modules/BaseExtensionTestClass.py rename to tests_e2e/scenarios/lib/BaseExtensionTestClass.py index bc00f15c1d..0a7cf71b0c 100644 --- a/tests_e2e/scenarios/modules/BaseExtensionTestClass.py +++ b/tests_e2e/scenarios/lib/BaseExtensionTestClass.py @@ -3,9 +3,9 @@ from azure.core.polling import LROPoller -from tests_e2e.scenarios.modules.azure_models import ComputeManager -from tests_e2e.scenarios.modules.logging_utils import LoggingHandler -from tests_e2e.scenarios.modules.models import ExtensionMetaData, get_vm_data_from_env +from tests_e2e.scenarios.lib.azure_models import ComputeManager +from tests_e2e.scenarios.lib.logging_utils import LoggingHandler +from tests_e2e.scenarios.lib.models import ExtensionMetaData, get_vm_data_from_env class BaseExtensionTestClass(LoggingHandler): diff --git a/tests_e2e/scenarios/modules/CustomScriptExtension.py b/tests_e2e/scenarios/lib/CustomScriptExtension.py similarity index 82% rename from tests_e2e/scenarios/modules/CustomScriptExtension.py rename to tests_e2e/scenarios/lib/CustomScriptExtension.py index 7c67052ef6..369c710124 100644 --- a/tests_e2e/scenarios/modules/CustomScriptExtension.py +++ b/tests_e2e/scenarios/lib/CustomScriptExtension.py @@ -1,7 +1,7 @@ import uuid -from tests_e2e.scenarios.modules.BaseExtensionTestClass import BaseExtensionTestClass -from tests_e2e.scenarios.modules.models import ExtensionMetaData +from tests_e2e.scenarios.lib.BaseExtensionTestClass import BaseExtensionTestClass +from tests_e2e.scenarios.lib.models import ExtensionMetaData class CustomScriptExtension(BaseExtensionTestClass): diff --git a/tests_e2e/scenarios/modules/__init__.py b/tests_e2e/scenarios/lib/__init__.py similarity index 100% rename from tests_e2e/scenarios/modules/__init__.py rename to tests_e2e/scenarios/lib/__init__.py diff --git a/tests_e2e/scenarios/modules/azure_models.py b/tests_e2e/scenarios/lib/azure_models.py similarity index 98% rename from tests_e2e/scenarios/modules/azure_models.py rename to tests_e2e/scenarios/lib/azure_models.py index 99875d39c4..9a9c7a15bf 100644 --- a/tests_e2e/scenarios/modules/azure_models.py +++ b/tests_e2e/scenarios/lib/azure_models.py @@ -12,8 +12,8 @@ from azure.mgmt.resource import ResourceManagementClient from msrestazure.azure_exceptions import CloudError -from tests_e2e.scenarios.modules.logging_utils import LoggingHandler -from tests_e2e.scenarios.modules.models import get_vm_data_from_env, VMModelType, VMMetaData +from tests_e2e.scenarios.lib.logging_utils import LoggingHandler +from tests_e2e.scenarios.lib.models import get_vm_data_from_env, VMModelType, VMMetaData class AzureComputeBaseClass(ABC, LoggingHandler): diff --git a/tests_e2e/scenarios/modules/logging_utils.py b/tests_e2e/scenarios/lib/logging_utils.py similarity index 100% rename from tests_e2e/scenarios/modules/logging_utils.py rename to tests_e2e/scenarios/lib/logging_utils.py diff --git a/tests_e2e/scenarios/modules/models.py b/tests_e2e/scenarios/lib/models.py similarity index 100% rename from tests_e2e/scenarios/modules/models.py rename to tests_e2e/scenarios/lib/models.py diff --git a/tests_e2e/scenarios/runbooks/daily.yml b/tests_e2e/scenarios/runbooks/daily.yml index 27d9d5bb1c..fe723b0f9f 100644 --- a/tests_e2e/scenarios/runbooks/daily.yml +++ b/tests_e2e/scenarios/runbooks/daily.yml @@ -44,7 +44,6 @@ variable: value: "" is_secret: true notifier: - - type: html - type: env_stats - type: junit platform: diff --git a/tests_e2e/scenarios/scripts/collect_logs.sh b/tests_e2e/scenarios/scripts/collect_logs.sh deleted file mode 100755 index b557215d1c..0000000000 --- a/tests_e2e/scenarios/scripts/collect_logs.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -logs_file_name="$HOME/logs.tgz" - -echo "Collecting logs to $logs_file_name ..." - -sudo tar --exclude='journal/*' --exclude='omsbundle' --exclude='omsagent' --exclude='mdsd' --exclude='scx*' \ - --exclude='*.so' --exclude='*__LinuxDiagnostic__*' --exclude='*.zip' --exclude='*.deb' --exclude='*.rpm' \ - -czf "$logs_file_name" \ - /var/log \ - /var/lib/waagent/ \ - /etc/waagent.conf - -sudo chmod +r "$logs_file_name" - diff --git a/tests_e2e/scenarios/tests/bvts/custom_script.py b/tests_e2e/scenarios/tests/bvts/custom_script.py index a65c4fd9a8..2d716223bb 100644 --- a/tests_e2e/scenarios/tests/bvts/custom_script.py +++ b/tests_e2e/scenarios/tests/bvts/custom_script.py @@ -3,7 +3,7 @@ import uuid import sys -from tests_e2e.scenarios.modules.CustomScriptExtension import CustomScriptExtension +from tests_e2e.scenarios.lib.CustomScriptExtension import CustomScriptExtension def main(subscription_id, resource_group_name, vm_name): diff --git a/tests_e2e/scenarios/tests/check_agent_version.py b/tests_e2e/scenarios/tests/check_agent_version.py deleted file mode 100755 index c63402dfae..0000000000 --- a/tests_e2e/scenarios/tests/check_agent_version.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function - -import subprocess -import sys - - -def main(): - print("Executing waagent --version") - - pipe = subprocess.Popen(['waagent', '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout_lines = list(map(lambda s: s.decode('utf-8'), pipe.stdout.readlines())) - exit_code = pipe.wait() - - for line in stdout_lines: - print(line) - - return exit_code - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests_e2e/scenarios/testsuites/__init__.py b/tests_e2e/scenarios/testsuites/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_e2e/scenarios/testsuites/agent_bvt.py b/tests_e2e/scenarios/testsuites/agent_bvt.py index 34a228d68a..da39539a9c 100644 --- a/tests_e2e/scenarios/testsuites/agent_bvt.py +++ b/tests_e2e/scenarios/testsuites/agent_bvt.py @@ -1,10 +1,8 @@ -from assertpy import assert_that - -from tests_e2e.scenarios.testsuites.agent_test_suite import AgentTestSuite +from tests_e2e.orchestrator.lib.agent_test_suite import AgentTestSuite from tests_e2e.scenarios.tests.bvts import custom_script -from lisa import ( - simple_requirement, +# E0401: Unable to import 'lisa' (import-error) +from lisa import ( # pylint: disable=E0401 TestCaseMetadata, TestSuiteMetadata, ) @@ -16,18 +14,12 @@ description=""" A POC test suite for the waagent BVTs. """, - requirement=simple_requirement(unsupported_os=[]), ) class AgentBvt(AgentTestSuite): @TestCaseMetadata(description="", priority=0) def main(self, *_, **__) -> None: - self.check_agent_version() self.custom_script() - def check_agent_version(self) -> None: - exit_code = self._execute_remote_script(self._test_root.joinpath("scenarios", "tests"), "check_agent_version.py") - assert_that(exit_code).is_equal_to(0) - def custom_script(self) -> None: custom_script.main(self._subscription_id, self._resource_group_name, self._vm_name) diff --git a/tests_e2e/scenarios/testsuites/agent_test_suite.py b/tests_e2e/scenarios/testsuites/agent_test_suite.py deleted file mode 100644 index e5f995f3d4..0000000000 --- a/tests_e2e/scenarios/testsuites/agent_test_suite.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path, PurePath - -from lisa import ( - CustomScriptBuilder, - TestSuite, - TestSuiteMetadata, -) -from lisa.sut_orchestrator.azure.common import get_node_context - - -class AgentTestSuite(TestSuite): - def __init__(self, metadata: TestSuiteMetadata): - super().__init__(metadata) - self._log = None - self._node = None - self._test_root = None - self._subscription_id = None - self._resource_group_name = None - self._vm_name = None - - def before_case(self, *_, **kwargs) -> None: - node = kwargs['node'] - log = kwargs['log'] - node_context = get_node_context(node) - - self._log = log - self._node = node - self._test_root = Path(__file__).parent.parent.parent - self._subscription_id = node.features._platform.subscription_id - self._resource_group_name = node_context.resource_group_name - self._vm_name = node_context.vm_name - - def after_case(self, *_, **__) -> None: - # Collect the logs on the test machine into a compressed tarball - self._log.info("Collecting logs on test machine [%s]...", self._node.name) - self._execute_remote_script(self._test_root.joinpath("scenarios", "scripts"), "collect_logs.sh") - - # Copy the tarball to the local logs directory - remote_path = PurePath('/home') / self._node.connection_info['username'] / 'logs.tgz' - local_path = Path.home() / 'logs' / 'vm-logs-{0}.tgz'.format(self._node.name) - self._log.info("Copying %s:%s to %s...", self._node.name, remote_path, local_path) - self._node.shell.copy_back(remote_path, local_path) - - def _execute_remote_script(self, path: Path, script: str) -> int: - custom_script_builder = CustomScriptBuilder(path, [script]) - custom_script = self._node.tools[custom_script_builder] - self._log.info('Executing %s/%s...', path, script) - result = custom_script.run() - if result.stdout: - self._log.info('%s', result.stdout) - if result.stderr: - self._log.error('%s', result.stderr) - return result.exit_code From d56a3c782d6976f6c8086ee7d809a9b111fb81c5 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 16 Dec 2022 12:26:04 -0800 Subject: [PATCH 2/5] Remove dependency on pathlib from makepkg.py (#2717) Co-authored-by: narrieta --- makepkg.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/makepkg.py b/makepkg.py index e35b16e488..cef24e3513 100755 --- a/makepkg.py +++ b/makepkg.py @@ -4,7 +4,6 @@ import glob import logging import os.path -import pathlib import shutil import subprocess import sys @@ -75,8 +74,7 @@ def run(agent_family, output_directory, log): os.makedirs(bin_path) log.info("Created {0} directory".format(target_path)) - setup_script = str(pathlib.Path(__file__).parent.joinpath("setup.py")) - args = ["python3", setup_script, "bdist_egg", "--dist-dir={0}".format(bin_path)] + args = ["python3", "setup.py", "bdist_egg", "--dist-dir={0}".format(bin_path)] log.info("Creating egg {0}".format(egg_path)) do(*args) From a3a41bd1e565b9dcf298584181e36f101ac97d78 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Tue, 20 Dec 2022 13:49:16 -0800 Subject: [PATCH 3/5] Improvements in error handling on end-to-end tests (#2716) * Improvements in error handling on end-to-end tests * Undo changes to daily.yml * pylint Co-authored-by: narrieta --- .../orchestrator/lib/agent_test_suite.py | 186 +++++++++--------- tests_e2e/orchestrator/scripts/collect-logs | 6 + tests_e2e/orchestrator/scripts/install-agent | 17 +- tests_e2e/orchestrator/scripts/run-scenarios | 18 ++ .../scenarios/runbooks/samples/hello_world.py | 32 +++ .../scenarios/runbooks/samples/local.yml | 28 +++ tests_e2e/scenarios/testsuites/agent_bvt.py | 44 +++-- 7 files changed, 226 insertions(+), 105 deletions(-) create mode 100644 tests_e2e/scenarios/runbooks/samples/hello_world.py create mode 100644 tests_e2e/scenarios/runbooks/samples/local.yml diff --git a/tests_e2e/orchestrator/lib/agent_test_suite.py b/tests_e2e/orchestrator/lib/agent_test_suite.py index 4ed68507cb..e5ebf3dba1 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite.py @@ -14,9 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # - +from collections.abc import Callable from pathlib import Path -import shutil +from shutil import rmtree import makepkg @@ -25,8 +25,6 @@ CustomScriptBuilder, Logger, Node, - TestSuite, - TestSuiteMetadata, ) # E0401: Unable to import 'lisa.sut_orchestrator.azure.common' (import-error) from lisa.sut_orchestrator.azure.common import get_node_context # pylint: disable=E0401 @@ -34,60 +32,35 @@ from azurelinuxagent.common.version import AGENT_VERSION -class AgentTestSuite(TestSuite): +class AgentTestScenario(object): """ - Base class for VM Agent tests. It provides initialization, cleanup, and utilities common to all VM Agent test suites. + Instances of this class are used to execute Agent test scenarios. It also provides facilities to execute commands over SSH. """ - def __init__(self, metadata: TestSuiteMetadata): - super().__init__(metadata) - # The actual initialization happens in _initialize() - self._log = None - self._node = None - self._subscription_id = None - self._resource_group_name = None - self._vm_name = None - self._test_source_directory = None - self._working_directory = None - self._node_home_directory = None - - def before_case(self, *_, **kwargs) -> None: - self._initialize(kwargs['node'], kwargs['log']) - self._setup_node() - - def after_case(self, *_, **__) -> None: - try: - self._collect_node_logs() - finally: - self._clean_up() - - def _initialize(self, node: Node, log: Logger) -> None: - self._node = node - self._log = log - node_context = get_node_context(node) - self._subscription_id = node.features._platform.subscription_id - self._resource_group_name = node_context.resource_group_name - self._vm_name = node_context.vm_name - - self._test_source_directory = AgentTestSuite._get_test_source_directory() - self._working_directory = Path().home()/"waagent-tmp" - self._node_home_directory = Path('/home')/self._node.connection_info['username'] + class Context: + """ + Execution context for test scenarios, this information is passed to test scenarios by AgentTestScenario.execute() + """ + subscription_id: str + resource_group_name: str + vm_name: str + test_source_directory: Path + working_directory: Path + node_home_directory: Path - self._log.info(f"Test Node: {self._vm_name}") - self._log.info(f"Resource Group: {self._resource_group_name}") - self._log.info(f"Working directory: {self._working_directory}...") + def __init__(self, node: Node, log: Logger) -> None: + self._node: Node = node + self._log: Logger = log - if self._working_directory.exists(): - self._log.info(f"Removing existing working directory: {self._working_directory}...") - try: - shutil.rmtree(self._working_directory.as_posix()) - except Exception as exception: - self._log.warning(f"Failed to remove the working directory: {exception}") - self._working_directory.mkdir() + node_context = get_node_context(node) + self._context: AgentTestScenario.Context = AgentTestScenario.Context() + self._context.subscription_id = node.features._platform.subscription_id + self._context.resource_group_name = node_context.resource_group_name + self._context.vm_name = node_context.vm_name - def _clean_up(self) -> None: - self._log.info(f"Removing {self._working_directory}...") - shutil.rmtree(self._working_directory.as_posix(), ignore_errors=True) + self._context.test_source_directory = AgentTestScenario._get_test_source_directory() + self._context.working_directory = Path().home()/"waagent-tmp" + self._context.node_home_directory = Path('/home')/node.connection_info['username'] @staticmethod def _get_test_source_directory() -> Path: @@ -101,6 +74,29 @@ def _get_test_source_directory() -> Path: path = path.parent raise Exception("Can't find the test root directory (tests_e2e)") + def _setup(self) -> None: + """ + Prepares the test scenario for execution + """ + self._log.info(f"Test Node: {self._context.vm_name}") + self._log.info(f"Resource Group: {self._context.resource_group_name}") + self._log.info(f"Working directory: {self._context.working_directory}...") + + if self._context.working_directory.exists(): + self._log.info(f"Removing existing working directory: {self._context.working_directory}...") + try: + rmtree(self._context.working_directory.as_posix()) + except Exception as exception: + self._log.warning(f"Failed to remove the working directory: {exception}") + self._context.working_directory.mkdir() + + def _clean_up(self) -> None: + """ + Cleans up any leftovers from the test scenario run. + """ + self._log.info(f"Removing {self._context.working_directory}...") + rmtree(self._context.working_directory.as_posix(), ignore_errors=True) + def _setup_node(self) -> None: """ Prepares the remote node for executing the test suite. @@ -112,18 +108,10 @@ def _build_agent_package(self) -> Path: """ Builds the agent package and returns the path to the package. """ - build_path = self._working_directory/"build" - - # The same orchestrator machine may be executing multiple suites on the same test VM, or - # the same suite on one or more test VMs; we use this file to mark the build is already done - build_done_path = self._working_directory/"build.done" - if build_done_path.exists(): - self._log.info("The agent build is already completed, will use existing package.") - else: - self._log.info(f"Building agent package to {build_path}") - makepkg.run(agent_family="Test", output_directory=str(build_path), log=self._log) - build_done_path.touch() + build_path = self._context.working_directory/"build" + self._log.info(f"Building agent package to {build_path}") + makepkg.run(agent_family="Test", output_directory=str(build_path), log=self._log) package_path = build_path/"eggs"/f"WALinuxAgent-{AGENT_VERSION}.zip" if not package_path.exists(): raise Exception(f"Can't find the agent package at {package_path}") @@ -136,54 +124,70 @@ def _install_agent_on_node(self, agent_package: Path) -> None: """ Installs the given agent package on the test node. """ - # The same orchestrator machine may be executing multiple suites on the same test VM, - # we use this file to mark the agent is already installed on the test VM. - install_done_path = self._working_directory/f"agent-install.{self._vm_name}.done" - if install_done_path.exists(): - self._log.info(f"Package {agent_package} is already installed on {self._vm_name}...") - return - # The install script needs to unzip the agent package; ensure unzip is installed on the test node - self._log.info(f"Installing unzip on {self._vm_name}...") + self._log.info(f"Installing unzip on {self._context.vm_name}...") self._node.os.install_packages("unzip") - self._log.info(f"Installing {agent_package} on {self._vm_name}...") - agent_package_remote_path = self._node_home_directory/agent_package.name - self._log.info(f"Copying {agent_package} to {self._vm_name}:{agent_package_remote_path}") + self._log.info(f"Installing {agent_package} on {self._context.vm_name}...") + agent_package_remote_path = self._context.node_home_directory/agent_package.name + self._log.info(f"Copying {agent_package} to {self._context.vm_name}:{agent_package_remote_path}") self._node.shell.copy(agent_package, agent_package_remote_path) - self._execute_script_on_node( - self._test_source_directory/"orchestrator"/"scripts"/"install-agent", + self.execute_script_on_node( + self._context.test_source_directory/"orchestrator"/"scripts"/"install-agent", parameters=f"--package {agent_package_remote_path} --version {AGENT_VERSION}", sudo=True) self._log.info("The agent was installed successfully.") - install_done_path.touch() def _collect_node_logs(self) -> None: """ - Collects the test logs from the remote machine and copied them to the local machine + Collects the test logs from the remote machine and copies them to the local machine """ - # Collect the logs on the test machine into a compressed tarball - self._log.info("Collecting logs on test machine [%s]...", self._node.name) - self._execute_script_on_node(self._test_source_directory/"orchestrator"/"scripts"/"collect-logs", sudo=True) - - # Copy the tarball to the local logs directory - remote_path = self._node_home_directory/"logs.tgz" - local_path = Path.home()/'logs'/'vm-logs-{0}.tgz'.format(self._node.name) - self._log.info(f"Copying {self._node.name}:{remote_path} to {local_path}") - self._node.shell.copy_back(remote_path, local_path) + try: + # Collect the logs on the test machine into a compressed tarball + self._log.info("Collecting logs on test machine [%s]...", self._node.name) + self.execute_script_on_node(self._context.test_source_directory/"orchestrator"/"scripts"/"collect-logs", sudo=True) + + # Copy the tarball to the local logs directory + remote_path = self._context.node_home_directory/"logs.tgz" + local_path = Path.home()/'logs'/'vm-logs-{0}.tgz'.format(self._node.name) + self._log.info(f"Copying {self._node.name}:{remote_path} to {local_path}") + self._node.shell.copy_back(remote_path, local_path) + except Exception as e: + self._log.warning(f"Failed to collect logs from the test machine: {e}") + + def execute(self, scenario: Callable[[Context], None]) -> None: + """ + Executes the given scenario + """ + try: + self._setup() + try: + self._setup_node() + scenario(self._context) + finally: + self._collect_node_logs() + finally: + self._clean_up() - def _execute_script_on_node(self, script_path: Path, parameters: str = "", sudo: bool = False) -> int: + def execute_script_on_node(self, script_path: Path, parameters: str = "", sudo: bool = False) -> int: + """ + Executes the given script on the test node; if 'sudo' is True, the script is executed using the sudo command. + """ custom_script_builder = CustomScriptBuilder(script_path.parent, [script_path.name]) custom_script = self._node.tools[custom_script_builder] - command_line = f"{script_path} {parameters}" - self._log.info(f"Executing {command_line}") + if parameters == '': + command_line = f"{script_path}" + else: + command_line = f"{script_path} {parameters}" + + self._log.info(f"Executing [{command_line}]") result = custom_script.run(parameters=parameters, sudo=sudo) # LISA appends stderr to stdout so no need to check for stderr if result.exit_code != 0: - raise Exception(f"[{command_line}] failed.\n{result.stdout}") + raise Exception(f"Command [{command_line}] failed.\n{result.stdout}") return result.exit_code diff --git a/tests_e2e/orchestrator/scripts/collect-logs b/tests_e2e/orchestrator/scripts/collect-logs index 9872878a7f..46a23aff18 100755 --- a/tests_e2e/orchestrator/scripts/collect-logs +++ b/tests_e2e/orchestrator/scripts/collect-logs @@ -15,5 +15,11 @@ tar --exclude='journal/*' --exclude='omsbundle' --exclude='omsagent' --exclude=' /var/lib/waagent/ \ /etc/waagent.conf +# tar exits with 1 on warnings; ignore those +exit_code=$? +if [ "$exit_code" != "1" ] && [ "$exit_code" != "0" ]; then + exit $exit_code +fi + chmod +r "$logs_file_name" diff --git a/tests_e2e/orchestrator/scripts/install-agent b/tests_e2e/orchestrator/scripts/install-agent index 08122a2780..439bdcec65 100755 --- a/tests_e2e/orchestrator/scripts/install-agent +++ b/tests_e2e/orchestrator/scripts/install-agent @@ -1,7 +1,22 @@ #!/usr/bin/env bash + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation # +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + set -euo pipefail usage() ( @@ -59,7 +74,7 @@ echo "Restarting service..." service $service_name stop # Rename the previous log to ensure the new log starts with the agent we just installed -mv /var/log/waagent.log /var/log/waagent.pre-install.log +mv /var/log/waagent.log /var/log/waagent."$(date --iso-8601=seconds)".log if command -v systemctl &> /dev/null; then systemctl daemon-reload diff --git a/tests_e2e/orchestrator/scripts/run-scenarios b/tests_e2e/orchestrator/scripts/run-scenarios index e5216e5b92..43bc43a856 100755 --- a/tests_e2e/orchestrator/scripts/run-scenarios +++ b/tests_e2e/orchestrator/scripts/run-scenarios @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + # # This script runs on the container executing the tests. It creates the SSH keys (private and public) used # to manage the test VMs taking the initial key value from the file shared by the container host, then it diff --git a/tests_e2e/scenarios/runbooks/samples/hello_world.py b/tests_e2e/scenarios/runbooks/samples/hello_world.py new file mode 100644 index 0000000000..bf1a44a5c5 --- /dev/null +++ b/tests_e2e/scenarios/runbooks/samples/hello_world.py @@ -0,0 +1,32 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# E0401: Unable to import 'lisa' (import-error) +from lisa import ( # pylint: disable=E0401 + Logger, + Node, + TestCaseMetadata, + TestSuite, + TestSuiteMetadata, +) + + +@TestSuiteMetadata(area="sample", category="", description="") +class HelloWorld(TestSuite): + @TestCaseMetadata(description="") + def main(self, node: Node, log: Logger) -> None: + log.info(f"Hello world from {node.os.name}!") diff --git a/tests_e2e/scenarios/runbooks/samples/local.yml b/tests_e2e/scenarios/runbooks/samples/local.yml new file mode 100644 index 0000000000..f5edec65b2 --- /dev/null +++ b/tests_e2e/scenarios/runbooks/samples/local.yml @@ -0,0 +1,28 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +extension: + - "." +environment: + environments: + - nodes: + - type: local +notifier: + - type: console +testcase: + - criteria: + area: sample diff --git a/tests_e2e/scenarios/testsuites/agent_bvt.py b/tests_e2e/scenarios/testsuites/agent_bvt.py index da39539a9c..a63c3e34df 100644 --- a/tests_e2e/scenarios/testsuites/agent_bvt.py +++ b/tests_e2e/scenarios/testsuites/agent_bvt.py @@ -1,26 +1,44 @@ -from tests_e2e.orchestrator.lib.agent_test_suite import AgentTestSuite +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests_e2e.orchestrator.lib.agent_test_suite import AgentTestScenario from tests_e2e.scenarios.tests.bvts import custom_script # E0401: Unable to import 'lisa' (import-error) from lisa import ( # pylint: disable=E0401 + Logger, + Node, TestCaseMetadata, + TestSuite, TestSuiteMetadata, ) -@TestSuiteMetadata( - area="bvt", - category="functional", - description=""" - A POC test suite for the waagent BVTs. - """, -) -class AgentBvt(AgentTestSuite): +@TestSuiteMetadata(area="bvt", category="", description="Test suite for Agent BVTs") +class AgentBvt(TestSuite): + """ + Test suite for Agent BVTs + """ @TestCaseMetadata(description="", priority=0) - def main(self, *_, **__) -> None: - self.custom_script() + def main(self, log: Logger, node: Node) -> None: + def tests(ctx: AgentTestScenario.Context) -> None: + custom_script.main(ctx.subscription_id, ctx.resource_group_name, ctx.vm_name) + + AgentTestScenario(node, log).execute(tests) - def custom_script(self) -> None: - custom_script.main(self._subscription_id, self._resource_group_name, self._vm_name) From 2bd03c9abd6b8ddd94dfefad04ca957fff456537 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Thu, 5 Jan 2023 17:00:47 -0800 Subject: [PATCH 4/5] Add BVT for the agent (#2719) * Add BVT for extension workflow * logging * update scenario * test context * AgentTest * vmaccess bvt * arguments; protected settings * fix username * RunCommand * Test dependencies * ssh key mode * ssh key file * key mode * log type * PR review * Unused import Co-authored-by: narrieta --- .github/workflows/ci_pr.yml | 2 +- makepkg.py | 3 +- test-requirements.txt | 6 + .../orchestrator/lib/agent_test_suite.py | 239 ++++++++++++------ tests_e2e/orchestrator/scripts/run-scenarios | 2 +- .../scenarios/lib/BaseExtensionTestClass.py | 113 --------- .../scenarios/lib/CustomScriptExtension.py | 29 --- tests_e2e/scenarios/lib/agent_test.py | 54 ++++ tests_e2e/scenarios/lib/agent_test_context.py | 162 ++++++++++++ tests_e2e/scenarios/lib/azure_models.py | 239 ------------------ tests_e2e/scenarios/lib/identifiers.py | 63 +++++ tests_e2e/scenarios/lib/logging.py | 37 +++ tests_e2e/scenarios/lib/logging_utils.py | 33 --- tests_e2e/scenarios/lib/models.py | 135 ---------- tests_e2e/scenarios/lib/retry.py | 41 +++ tests_e2e/scenarios/lib/shell.py | 53 ++++ tests_e2e/scenarios/lib/ssh_client.py | 46 ++++ tests_e2e/scenarios/lib/virtual_machine.py | 143 +++++++++++ tests_e2e/scenarios/lib/vm_extension.py | 239 ++++++++++++++++++ .../scenarios/tests/bvts/custom_script.py | 45 ---- .../tests/bvts/extension_operations.py | 79 ++++++ tests_e2e/scenarios/tests/bvts/run_command.py | 89 +++++++ tests_e2e/scenarios/tests/bvts/vm_access.py | 75 ++++++ tests_e2e/scenarios/testsuites/agent_bvt.py | 24 +- 24 files changed, 1265 insertions(+), 686 deletions(-) delete mode 100644 tests_e2e/scenarios/lib/BaseExtensionTestClass.py delete mode 100644 tests_e2e/scenarios/lib/CustomScriptExtension.py create mode 100644 tests_e2e/scenarios/lib/agent_test.py create mode 100644 tests_e2e/scenarios/lib/agent_test_context.py delete mode 100644 tests_e2e/scenarios/lib/azure_models.py create mode 100644 tests_e2e/scenarios/lib/identifiers.py create mode 100644 tests_e2e/scenarios/lib/logging.py delete mode 100644 tests_e2e/scenarios/lib/logging_utils.py delete mode 100644 tests_e2e/scenarios/lib/models.py create mode 100644 tests_e2e/scenarios/lib/retry.py create mode 100644 tests_e2e/scenarios/lib/shell.py create mode 100644 tests_e2e/scenarios/lib/ssh_client.py create mode 100644 tests_e2e/scenarios/lib/virtual_machine.py create mode 100644 tests_e2e/scenarios/lib/vm_extension.py delete mode 100644 tests_e2e/scenarios/tests/bvts/custom_script.py create mode 100755 tests_e2e/scenarios/tests/bvts/extension_operations.py create mode 100755 tests_e2e/scenarios/tests/bvts/run_command.py create mode 100755 tests_e2e/scenarios/tests/bvts/vm_access.py diff --git a/.github/workflows/ci_pr.yml b/.github/workflows/ci_pr.yml index 279b7e76c4..589f0f5e7b 100644 --- a/.github/workflows/ci_pr.yml +++ b/.github/workflows/ci_pr.yml @@ -56,7 +56,7 @@ jobs: PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e" - python-version: 3.9 - PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=azure_models.py,BaseExtensionTestClass.py" + PYLINTOPTS: "--rcfile=ci/3.6.pylintrc" additional-nose-opts: "--with-coverage --cover-erase --cover-inclusive --cover-branches --cover-package=azurelinuxagent" name: "Python ${{ matrix.python-version }} Unit Tests" diff --git a/makepkg.py b/makepkg.py index cef24e3513..25b209229b 100755 --- a/makepkg.py +++ b/makepkg.py @@ -74,7 +74,8 @@ def run(agent_family, output_directory, log): os.makedirs(bin_path) log.info("Created {0} directory".format(target_path)) - args = ["python3", "setup.py", "bdist_egg", "--dist-dir={0}".format(bin_path)] + setup_path = os.path.join(os.path.dirname(__file__), "setup.py") + args = ["python3", setup_path, "bdist_egg", "--dist-dir={0}".format(bin_path)] log.info("Creating egg {0}".format(egg_path)) do(*args) diff --git a/test-requirements.txt b/test-requirements.txt index f335db2826..3c54ab9974 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,3 +13,9 @@ wrapt==1.12.0; python_version > '2.6' and python_version < '3.6' pylint; python_version > '2.6' and python_version < '3.6' pylint==2.8.3; python_version >= '3.6' +# Requirements to run pylint on the end-to-end tests source code +assertpy +azure-core +azure-identity +azure-mgmt-compute>=22.1.0 +azure-mgmt-resource>=15.0.0 diff --git a/tests_e2e/orchestrator/lib/agent_test_suite.py b/tests_e2e/orchestrator/lib/agent_test_suite.py index e5ebf3dba1..c4d94a2421 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite.py @@ -14,88 +14,109 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from collections.abc import Callable +from assertpy import assert_that from pathlib import Path from shutil import rmtree +from typing import List, Type -import makepkg - -# E0401: Unable to import 'lisa' (import-error) +# Disable those warnings, since 'lisa' is an external, non-standard, dependency +# E0401: Unable to import 'lisa' (import-error) +# E0401: Unable to import 'lisa.sut_orchestrator' (import-error) +# E0401: Unable to import 'lisa.sut_orchestrator.azure.common' (import-error) from lisa import ( # pylint: disable=E0401 CustomScriptBuilder, - Logger, Node, + TestSuite, + TestSuiteMetadata ) -# E0401: Unable to import 'lisa.sut_orchestrator.azure.common' (import-error) -from lisa.sut_orchestrator.azure.common import get_node_context # pylint: disable=E0401 +from lisa.sut_orchestrator import AZURE # pylint: disable=E0401 +from lisa.sut_orchestrator.azure.common import get_node_context, AzureNodeSchema # pylint: disable=E0401 +import makepkg from azurelinuxagent.common.version import AGENT_VERSION +from tests_e2e.scenarios.lib.agent_test import AgentTest +from tests_e2e.scenarios.lib.agent_test_context import AgentTestContext +from tests_e2e.scenarios.lib.identifiers import VmIdentifier +from tests_e2e.scenarios.lib.logging import log -class AgentTestScenario(object): +class AgentLisaTestContext(AgentTestContext): """ - Instances of this class are used to execute Agent test scenarios. It also provides facilities to execute commands over SSH. + Execution context for LISA tests. """ + def __init__(self, vm: VmIdentifier, node: Node): + super().__init__( + vm=vm, + paths=AgentTestContext.Paths(remote_working_directory=Path('/home')/node.connection_info['username']), + connection=AgentTestContext.Connection( + ip_address=node.connection_info['address'], + username=node.connection_info['username'], + private_key_file=node.connection_info['private_key_file'], + ssh_port=node.connection_info['port']) + ) + self._node = node - class Context: - """ - Execution context for test scenarios, this information is passed to test scenarios by AgentTestScenario.execute() - """ - subscription_id: str - resource_group_name: str - vm_name: str - test_source_directory: Path - working_directory: Path - node_home_directory: Path + @property + def node(self) -> Node: + return self._node + + +class AgentTestSuite(TestSuite): + """ + Base class for Agent test suites. It provides facilities for setup, execution of tests and reporting results. Derived + classes use the execute() method to run the tests in their corresponding suites. + """ + def __init__(self, metadata: TestSuiteMetadata) -> None: + super().__init__(metadata) + # The context is initialized by execute() + self.__context: AgentLisaTestContext = None - def __init__(self, node: Node, log: Logger) -> None: - self._node: Node = node - self._log: Logger = log + @property + def context(self) -> AgentLisaTestContext: + if self.__context is None: + raise Exception("The context for the AgentTestSuite has not been initialized") + return self.__context + def _set_context(self, node: Node): node_context = get_node_context(node) - self._context: AgentTestScenario.Context = AgentTestScenario.Context() - self._context.subscription_id = node.features._platform.subscription_id - self._context.resource_group_name = node_context.resource_group_name - self._context.vm_name = node_context.vm_name - self._context.test_source_directory = AgentTestScenario._get_test_source_directory() - self._context.working_directory = Path().home()/"waagent-tmp" - self._context.node_home_directory = Path('/home')/node.connection_info['username'] + runbook = node.capability.get_extended_runbook(AzureNodeSchema, AZURE) - @staticmethod - def _get_test_source_directory() -> Path: - """ - Returns the root directory of the source code for the end-to-end tests (".../WALinuxAgent/tests_e2e") - """ - path = Path(__file__) - while path.name != '': - if path.name == "tests_e2e": - return path - path = path.parent - raise Exception("Can't find the test root directory (tests_e2e)") + self.__context = AgentLisaTestContext( + VmIdentifier( + location=runbook.location, + subscription=node.features._platform.subscription_id, + resource_group=node_context.resource_group_name, + name=node_context.vm_name + ), + node + ) def _setup(self) -> None: """ - Prepares the test scenario for execution + Prepares the test suite for execution """ - self._log.info(f"Test Node: {self._context.vm_name}") - self._log.info(f"Resource Group: {self._context.resource_group_name}") - self._log.info(f"Working directory: {self._context.working_directory}...") + log.info("Test Node: %s", self.context.vm.name) + log.info("Resource Group: %s", self.context.vm.resource_group) + log.info("Working directory: %s", self.context.working_directory) - if self._context.working_directory.exists(): - self._log.info(f"Removing existing working directory: {self._context.working_directory}...") + if self.context.working_directory.exists(): + log.info("Removing existing working directory: %s", self.context.working_directory) try: - rmtree(self._context.working_directory.as_posix()) + rmtree(self.context.working_directory.as_posix()) except Exception as exception: - self._log.warning(f"Failed to remove the working directory: {exception}") - self._context.working_directory.mkdir() + log.warning("Failed to remove the working directory: %s", exception) + self.context.working_directory.mkdir() def _clean_up(self) -> None: """ - Cleans up any leftovers from the test scenario run. + Cleans up any leftovers from the test suite run. """ - self._log.info(f"Removing {self._context.working_directory}...") - rmtree(self._context.working_directory.as_posix(), ignore_errors=True) + try: + log.info("Removing %s", self.context.working_directory) + rmtree(self.context.working_directory.as_posix(), ignore_errors=True) + except: # pylint: disable=bare-except + log.exception("Failed to cleanup the test run") def _setup_node(self) -> None: """ @@ -108,15 +129,17 @@ def _build_agent_package(self) -> Path: """ Builds the agent package and returns the path to the package. """ - build_path = self._context.working_directory/"build" + build_path = self.context.working_directory/"build" + + log.info("Building agent package to %s", build_path) + + makepkg.run(agent_family="Test", output_directory=str(build_path), log=log) - self._log.info(f"Building agent package to {build_path}") - makepkg.run(agent_family="Test", output_directory=str(build_path), log=self._log) package_path = build_path/"eggs"/f"WALinuxAgent-{AGENT_VERSION}.zip" if not package_path.exists(): raise Exception(f"Can't find the agent package at {package_path}") - self._log.info(f"Agent package: {package_path}") + log.info("Agent package: %s", package_path) return package_path @@ -125,19 +148,19 @@ def _install_agent_on_node(self, agent_package: Path) -> None: Installs the given agent package on the test node. """ # The install script needs to unzip the agent package; ensure unzip is installed on the test node - self._log.info(f"Installing unzip on {self._context.vm_name}...") - self._node.os.install_packages("unzip") + log.info("Installing unzip tool on %s", self.context.node.name) + self.context.node.os.install_packages("unzip") - self._log.info(f"Installing {agent_package} on {self._context.vm_name}...") - agent_package_remote_path = self._context.node_home_directory/agent_package.name - self._log.info(f"Copying {agent_package} to {self._context.vm_name}:{agent_package_remote_path}") - self._node.shell.copy(agent_package, agent_package_remote_path) + log.info("Installing %s on %s", agent_package, self.context.node.name) + agent_package_remote_path = self.context.remote_working_directory / agent_package.name + log.info("Copying %s to %s:%s", agent_package, self.context.node.name, agent_package_remote_path) + self.context.node.shell.copy(agent_package, agent_package_remote_path) self.execute_script_on_node( - self._context.test_source_directory/"orchestrator"/"scripts"/"install-agent", + self.context.test_source_directory/"orchestrator"/"scripts"/"install-agent", parameters=f"--package {agent_package_remote_path} --version {AGENT_VERSION}", sudo=True) - self._log.info("The agent was installed successfully.") + log.info("The agent was installed successfully.") def _collect_node_logs(self) -> None: """ @@ -145,49 +168,107 @@ def _collect_node_logs(self) -> None: """ try: # Collect the logs on the test machine into a compressed tarball - self._log.info("Collecting logs on test machine [%s]...", self._node.name) - self.execute_script_on_node(self._context.test_source_directory/"orchestrator"/"scripts"/"collect-logs", sudo=True) + log.info("Collecting logs on test machine [%s]...", self.context.node.name) + self.execute_script_on_node(self.context.test_source_directory/"orchestrator"/"scripts"/"collect-logs", sudo=True) # Copy the tarball to the local logs directory - remote_path = self._context.node_home_directory/"logs.tgz" - local_path = Path.home()/'logs'/'vm-logs-{0}.tgz'.format(self._node.name) - self._log.info(f"Copying {self._node.name}:{remote_path} to {local_path}") - self._node.shell.copy_back(remote_path, local_path) - except Exception as e: - self._log.warning(f"Failed to collect logs from the test machine: {e}") + remote_path = self.context.remote_working_directory / "logs.tgz" + local_path = Path.home()/'logs'/'vm-logs-{0}.tgz'.format(self.context.node.name) + log.info("Copying %s:%s to %s", self.context.node.name, remote_path, local_path) + self.context.node.shell.copy_back(remote_path, local_path) + except: # pylint: disable=bare-except + log.exception("Failed to collect logs from the test machine") - def execute(self, scenario: Callable[[Context], None]) -> None: + def execute(self, node: Node, test_suite: List[Type[AgentTest]]) -> None: """ - Executes the given scenario + Executes each of the AgentTests in the given List. Note that 'test_suite' is a list of test classes, rather than + instances of the test class (this method will instantiate each of these test classes). """ + self._set_context(node) + + log.info("") + log.info("**************************************** [Setup] ****************************************") + log.info("") + + failed: List[str] = [] + try: self._setup() + try: self._setup_node() - scenario(self._context) + + log.info("") + log.info("**************************************** [%s] ****************************************", self._metadata.full_name) + log.info("") + + results: List[str] = [] + + for test in test_suite: + try: + log.info("******************** [%s]", test.__name__) + log.info("") + + test(self.context).run() + + result = f"[Passed] {test.__name__}" + + log.info("") + log.info("******************** %s", result) + log.info("") + + results.append(result) + except: # pylint: disable=bare-except + result = f"[Failed] {test.__name__}" + + log.info("") + log.exception("******************** %s\n", result) + log.info("") + + results.append(result) + failed.append(test.__name__) + + log.info("**************************************** [Test Results] ****************************************") + log.info("") + for r in results: + log.info("\t%s", r) + log.info("") + finally: self._collect_node_logs() + finally: self._clean_up() + # Fail the entire test suite if any test failed + assert_that(failed).described_as("One or more tests failed").is_length(0) + def execute_script_on_node(self, script_path: Path, parameters: str = "", sudo: bool = False) -> int: """ Executes the given script on the test node; if 'sudo' is True, the script is executed using the sudo command. """ custom_script_builder = CustomScriptBuilder(script_path.parent, [script_path.name]) - custom_script = self._node.tools[custom_script_builder] + custom_script = self.context.node.tools[custom_script_builder] if parameters == '': command_line = f"{script_path}" else: command_line = f"{script_path} {parameters}" - self._log.info(f"Executing [{command_line}]") + log.info("Executing [%s]", command_line) + result = custom_script.run(parameters=parameters, sudo=sudo) - # LISA appends stderr to stdout so no need to check for stderr if result.exit_code != 0: - raise Exception(f"Command [{command_line}] failed.\n{result.stdout}") + output = result.stdout if result.stderr == "" else f"{result.stdout}\n{result.stderr}" + raise Exception(f"[{command_line}] failed:\n{output}") + + if result.stdout != "": + separator = "\n" if "\n" in result.stdout else " " + log.info("stdout:%s%s", separator, result.stdout) + if result.stderr != "": + separator = "\n" if "\n" in result.stderr else " " + log.error("stderr:%s%s", separator, result.stderr) return result.exit_code diff --git a/tests_e2e/orchestrator/scripts/run-scenarios b/tests_e2e/orchestrator/scripts/run-scenarios index 43bc43a856..8eecec40d9 100755 --- a/tests_e2e/orchestrator/scripts/run-scenarios +++ b/tests_e2e/orchestrator/scripts/run-scenarios @@ -41,4 +41,4 @@ lisa \ --log_path "$HOME/logs" \ --working_path "$HOME/logs" \ -v subscription_id:"$SUBSCRIPTION_ID" \ - -v identity_file:"$HOME/.ssh/id_rsa.pub" + -v identity_file:"$HOME/.ssh/id_rsa" diff --git a/tests_e2e/scenarios/lib/BaseExtensionTestClass.py b/tests_e2e/scenarios/lib/BaseExtensionTestClass.py deleted file mode 100644 index 0a7cf71b0c..0000000000 --- a/tests_e2e/scenarios/lib/BaseExtensionTestClass.py +++ /dev/null @@ -1,113 +0,0 @@ -import time -from typing import List - -from azure.core.polling import LROPoller - -from tests_e2e.scenarios.lib.azure_models import ComputeManager -from tests_e2e.scenarios.lib.logging_utils import LoggingHandler -from tests_e2e.scenarios.lib.models import ExtensionMetaData, get_vm_data_from_env - - -class BaseExtensionTestClass(LoggingHandler): - - def __init__(self, extension_data: ExtensionMetaData): - super().__init__() - self.__extension_data = extension_data - self.__vm_data = get_vm_data_from_env() - self.__compute_manager = ComputeManager().compute_manager - - def get_ext_props(self, settings=None, protected_settings=None, auto_upgrade_minor_version=True, - force_update_tag=None): - - return self.__compute_manager.get_ext_props( - extension_data=self.__extension_data, - settings=settings, - protected_settings=protected_settings, - auto_upgrade_minor_version=auto_upgrade_minor_version, - force_update_tag=force_update_tag - ) - - def run(self, ext_props: List, remove: bool = True, continue_on_error: bool = False): - - def __add_extension(): - extension: LROPoller = self.__compute_manager.extension_func.begin_create_or_update( - self.__vm_data.rg_name, - self.__vm_data.name, - self.__extension_data.name, - ext_prop - ) - self.log.info("Add extension: {0}".format(extension.result(timeout=5 * 60))) - - def __remove_extension(): - self.__compute_manager.extension_func.begin_delete( - self.__vm_data.rg_name, - self.__vm_data.name, - self.__extension_data.name - ).result() - self.log.info(f"Delete vm extension {self.__extension_data.name} successful") - - def _retry_on_retryable_error(func): - retry = 1 - while retry < 5: - try: - func() - break - except Exception as err_: - if "RetryableError" in str(err_) and retry < 5: - self.log.warning(f"({retry}/5) Ran into RetryableError, retrying in 30 secs: {err_}") - time.sleep(30) - retry += 1 - continue - raise - - try: - for ext_prop in ext_props: - try: - _retry_on_retryable_error(__add_extension) - # Validate success from instance view - _retry_on_retryable_error(self.validate_ext) - except Exception as err: - if continue_on_error: - self.log.exception("Ran into error but ignoring it as asked: {0}".format(err)) - continue - else: - raise - finally: - # Always try to delete extensions if asked to remove even on errors - if remove: - _retry_on_retryable_error(__remove_extension) - - def validate_ext(self): - """ - Validate if the extension operation was successful from the Instance View - :raises: Exception if either unable to fetch instance view or if extension not successful - """ - retry = 0 - max_retry = 3 - ext_instance_view = None - status = None - - while retry < max_retry: - try: - ext_instance_view = self.__compute_manager.get_extension_instance_view(self.__extension_data.name) - if ext_instance_view is None: - raise Exception("Extension not found") - elif not ext_instance_view.instance_view: - raise Exception("Instance view not present") - elif not ext_instance_view.instance_view.statuses or len(ext_instance_view.instance_view.statuses) < 1: - raise Exception("Instance view status not present") - else: - status = ext_instance_view.instance_view.statuses[0].code - status_message = ext_instance_view.instance_view.statuses[0].message - self.log.info('Extension Status: \n\tCode: [{0}]\n\tMessage: {1}'.format(status, status_message)) - break - except Exception as err: - self.log.exception(f"Ran into error: {err}") - retry += 1 - if retry < max_retry: - self.log.info("Retrying in 30 secs") - time.sleep(30) - raise - - if 'succeeded' not in status: - raise Exception(f"Extension did not succeed. Last Instance view: {ext_instance_view}") diff --git a/tests_e2e/scenarios/lib/CustomScriptExtension.py b/tests_e2e/scenarios/lib/CustomScriptExtension.py deleted file mode 100644 index 369c710124..0000000000 --- a/tests_e2e/scenarios/lib/CustomScriptExtension.py +++ /dev/null @@ -1,29 +0,0 @@ -import uuid - -from tests_e2e.scenarios.lib.BaseExtensionTestClass import BaseExtensionTestClass -from tests_e2e.scenarios.lib.models import ExtensionMetaData - - -class CustomScriptExtension(BaseExtensionTestClass): - META_DATA = ExtensionMetaData( - publisher='Microsoft.Azure.Extensions', - ext_type='CustomScript', - version="2.1" - ) - - def __init__(self, extension_name: str): - extension_data = self.META_DATA - extension_data.name = extension_name - super().__init__(extension_data) - - -def add_cse(): - # Install and remove CSE - cse = CustomScriptExtension(extension_name="testCSE") - - ext_props = [ - cse.get_ext_props(settings={'commandToExecute': f"echo \'Hello World! {uuid.uuid4()} \'"}), - cse.get_ext_props(settings={'commandToExecute': "echo \'Hello again\'"}) - ] - - cse.run(ext_props=ext_props) \ No newline at end of file diff --git a/tests_e2e/scenarios/lib/agent_test.py b/tests_e2e/scenarios/lib/agent_test.py new file mode 100644 index 0000000000..6bbb8eaede --- /dev/null +++ b/tests_e2e/scenarios/lib/agent_test.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +from abc import ABC, abstractmethod + +from tests_e2e.scenarios.lib.agent_test_context import AgentTestContext +from tests_e2e.scenarios.lib.logging import log + + +class AgentTest(ABC): + """ + Defines the interface for agent tests, which are simply constructed from an AgentTestContext and expose a single method, + run(), to execute the test. + """ + def __init__(self, context: AgentTestContext): + self._context = context + + @abstractmethod + def run(self): + pass + + @classmethod + def run_from_command_line(cls): + """ + Convenience method to execute the test when it is being invoked directly from the command line (as opposed as + being invoked from a test framework or library. + """ + try: + cls(AgentTestContext.from_args()).run() + except SystemExit: # Bad arguments + pass + except: # pylint: disable=bare-except + log.exception("Test failed") + sys.exit(1) + + sys.exit(0) diff --git a/tests_e2e/scenarios/lib/agent_test_context.py b/tests_e2e/scenarios/lib/agent_test_context.py new file mode 100644 index 0000000000..b35e93a80d --- /dev/null +++ b/tests_e2e/scenarios/lib/agent_test_context.py @@ -0,0 +1,162 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import os + +from pathlib import Path + +import tests_e2e +from tests_e2e.scenarios.lib.identifiers import VmIdentifier + + +class AgentTestContext: + """ + Execution context for agent tests. Defines the test VM, working directories and connection info for the tests. + """ + + class Paths: + # E1101: Instance of 'list' has no '_path' member (no-member) + DEFAULT_TEST_SOURCE_DIRECTORY = Path(tests_e2e.__path__._path[0]) # pylint: disable=E1101 + DEFAULT_WORKING_DIRECTORY = Path().home() / "waagent-tmp" + + def __init__( + self, + remote_working_directory: Path, + test_source_directory: Path = DEFAULT_TEST_SOURCE_DIRECTORY, + working_directory: Path = DEFAULT_WORKING_DIRECTORY + ): + self._test_source_directory: Path = test_source_directory + self._working_directory: Path = working_directory + self._remote_working_directory: Path = remote_working_directory + + class Connection: + DEFAULT_SSH_PORT = 22 + + def __init__( + self, + ip_address: str, + username: str, + private_key_file: Path, + ssh_port: int = DEFAULT_SSH_PORT + ): + self._ip_address: str = ip_address + self._username: str = username + self._private_key_file: Path = private_key_file + self._ssh_port: int = ssh_port + + def __init__(self, vm: VmIdentifier, paths: Paths, connection: Connection): + self._vm: VmIdentifier = vm + self._paths = paths + self._connection = connection + + @property + def vm(self) -> VmIdentifier: + """ + The test VM (the VM on which the tested Agent is running) + """ + return self._vm + + @property + def vm_ip_address(self) -> str: + """ + The IP address of the test VM + """ + return self._connection._ip_address + + @property + def test_source_directory(self) -> Path: + """ + Root directory for the source code of the tests. Used to build paths to specific scripts. + """ + return self._paths._test_source_directory + + @property + def working_directory(self) -> Path: + """ + Tests create temporary files under this directory + """ + return self._paths._working_directory + + @property + def remote_working_directory(self) -> Path: + """ + Tests create temporary files under this directory on the test VM + """ + return self._paths._remote_working_directory + + @property + def username(self) -> str: + """ + The username to use for SSH connections + """ + return self._connection._username + + @property + def private_key_file(self) -> Path: + """ + The file containing the private SSH key for the username + """ + return self._connection._private_key_file + + @property + def ssh_port(self) -> int: + """ + Port for SSH connections + """ + return self._connection._ssh_port + + @staticmethod + def from_args(): + """ + Creates an AgentTestContext from the command line arguments. + """ + parser = argparse.ArgumentParser() + parser.add_argument('-g', '--group', required=True) + parser.add_argument('-l', '--location', required=True) + parser.add_argument('-s', '--subscription', required=True) + parser.add_argument('-vm', '--vm', required=True) + + parser.add_argument('-rw', '--remote-working-directory', dest="remote_working_directory", required=False, default=str(Path('/home')/os.getenv("USER"))) + parser.add_argument('-t', '--test-source-directory', dest="test_source_directory", required=False, default=str(AgentTestContext.Paths.DEFAULT_TEST_SOURCE_DIRECTORY)) + parser.add_argument('-w', '--working-directory', dest="working_directory", required=False, default=str(AgentTestContext.Paths.DEFAULT_WORKING_DIRECTORY)) + + parser.add_argument('-a', '--ip-address', dest="ip_address", required=False) # Use the vm name as default + parser.add_argument('-u', '--username', required=False, default=os.getenv("USER")) + parser.add_argument('-k', '--private-key-file', dest="private_key_file", required=False, default=Path.home()/".ssh"/"id_rsa") + parser.add_argument('-p', '--ssh-port', dest="ssh_port", required=False, default=AgentTestContext.Connection.DEFAULT_SSH_PORT) + + args = parser.parse_args() + + working_directory = Path(args.working_directory) + if not working_directory.exists(): + working_directory.mkdir(exist_ok=True) + + return AgentTestContext( + vm=VmIdentifier( + location=args.location, + subscription=args.subscription, + resource_group=args.group, + name=args.vm), + paths=AgentTestContext.Paths( + remote_working_directory=Path(args.remote_working_directory), + test_source_directory=Path(args.test_source_directory), + working_directory=working_directory), + connection=AgentTestContext.Connection( + ip_address=args.ip_address if args.ip_address is not None else args.vm, + username=args.username, + private_key_file=Path(args.private_key_file), + ssh_port=args.ssh_port)) diff --git a/tests_e2e/scenarios/lib/azure_models.py b/tests_e2e/scenarios/lib/azure_models.py deleted file mode 100644 index 9a9c7a15bf..0000000000 --- a/tests_e2e/scenarios/lib/azure_models.py +++ /dev/null @@ -1,239 +0,0 @@ -import time -from abc import ABC, abstractmethod -from builtins import TimeoutError -from typing import List - -from azure.core.exceptions import HttpResponseError -from azure.core.polling import LROPoller -from azure.identity import DefaultAzureCredential -from azure.mgmt.compute import ComputeManagementClient -from azure.mgmt.compute.models import VirtualMachineExtension, VirtualMachineScaleSetExtension, \ - VirtualMachineInstanceView, VirtualMachineScaleSetInstanceView, VirtualMachineExtensionInstanceView -from azure.mgmt.resource import ResourceManagementClient -from msrestazure.azure_exceptions import CloudError - -from tests_e2e.scenarios.lib.logging_utils import LoggingHandler -from tests_e2e.scenarios.lib.models import get_vm_data_from_env, VMModelType, VMMetaData - - -class AzureComputeBaseClass(ABC, LoggingHandler): - - def __init__(self): - super().__init__() - self.__vm_data = get_vm_data_from_env() - self.__compute_client = None - self.__resource_client = None - - @property - def vm_data(self) -> VMMetaData: - return self.__vm_data - - @property - def compute_client(self) -> ComputeManagementClient: - if self.__compute_client is None: - self.__compute_client = ComputeManagementClient( - credential=DefaultAzureCredential(), - subscription_id=self.vm_data.sub_id - ) - return self.__compute_client - - @property - def resource_client(self) -> ResourceManagementClient: - if self.__resource_client is None: - self.__resource_client = ResourceManagementClient( - credential=DefaultAzureCredential(), - subscription_id=self.vm_data.sub_id - ) - return self.__resource_client - - @property - @abstractmethod - def vm_func(self): - pass - - @property - @abstractmethod - def extension_func(self): - pass - - @abstractmethod - def get_vm_instance_view(self): - pass - - @abstractmethod - def get_extensions(self): - pass - - @abstractmethod - def get_extension_instance_view(self, extension_name): - pass - - @abstractmethod - def get_ext_props(self, extension_data, settings=None, protected_settings=None, auto_upgrade_minor_version=True, - force_update_tag=None): - pass - - @abstractmethod - def restart(self, timeout=5): - pass - - def _run_azure_op_with_retry(self, get_func): - max_retries = 3 - retries = max_retries - while retries > 0: - try: - ext = get_func() - return ext - except (CloudError, HttpResponseError) as ce: - if retries > 0: - self.log.exception(f"Got Azure error: {ce}") - self.log.warning("...retrying [{0} attempts remaining]".format(retries)) - retries -= 1 - time.sleep(30 * (max_retries - retries)) - else: - raise - - -class VirtualMachineHelper(AzureComputeBaseClass): - - def __init__(self): - super().__init__() - - @property - def vm_func(self): - return self.compute_client.virtual_machines - - @property - def extension_func(self): - return self.compute_client.virtual_machine_extensions - - def get_vm_instance_view(self) -> VirtualMachineInstanceView: - return self._run_azure_op_with_retry(lambda: self.vm_func.get( - resource_group_name=self.vm_data.rg_name, - vm_name=self.vm_data.name, - expand="instanceView" - )) - - def get_extensions(self) -> List[VirtualMachineExtension]: - return self._run_azure_op_with_retry(lambda: self.extension_func.list( - resource_group_name=self.vm_data.rg_name, - vm_name=self.vm_data.name - )) - - def get_extension_instance_view(self, extension_name) -> VirtualMachineExtensionInstanceView: - return self._run_azure_op_with_retry(lambda: self.extension_func.get( - resource_group_name=self.vm_data.rg_name, - vm_name=self.vm_data.name, - vm_extension_name=extension_name, - expand="instanceView" - )) - - def get_ext_props(self, extension_data, settings=None, protected_settings=None, auto_upgrade_minor_version=True, - force_update_tag=None) -> VirtualMachineExtension: - return VirtualMachineExtension( - location=self.vm_data.location, - publisher=extension_data.publisher, - type_properties_type=extension_data.ext_type, - type_handler_version=extension_data.version, - auto_upgrade_minor_version=auto_upgrade_minor_version, - settings=settings, - protected_settings=protected_settings, - force_update_tag=force_update_tag - ) - - def restart(self, timeout=5): - self.log.info(f"Initiating restart of machine: {self.vm_data.name}") - poller : LROPoller = self._run_azure_op_with_retry(lambda: self.vm_func.begin_restart( - resource_group_name=self.vm_data.rg_name, - vm_name=self.vm_data.name - )) - poller.wait(timeout=timeout * 60) - if not poller.done(): - raise TimeoutError(f"Machine {self.vm_data.name} failed to restart after {timeout} mins") - self.log.info(f"Restarted machine: {self.vm_data.name}") - - -class VirtualMachineScaleSetHelper(AzureComputeBaseClass): - - def restart(self, timeout=5): - poller: LROPoller = self._run_azure_op_with_retry(lambda: self.vm_func.begin_restart( - resource_group_name=self.vm_data.rg_name, - vm_scale_set_name=self.vm_data.name - )) - poller.wait(timeout=timeout * 60) - if not poller.done(): - raise TimeoutError(f"ScaleSet {self.vm_data.name} failed to restart after {timeout} mins") - - def __init__(self): - super().__init__() - - @property - def vm_func(self): - return self.compute_client.virtual_machine_scale_set_vms - - @property - def extension_func(self): - return self.compute_client.virtual_machine_scale_set_extensions - - def get_vm_instance_view(self) -> VirtualMachineScaleSetInstanceView: - # Since this is a VMSS, return the instance view of the first VMSS VM. For the instance view of the complete VMSS, - # use the compute_client.virtual_machine_scale_sets function - - # https://docs.microsoft.com/en-us/python/api/azure-mgmt-compute/azure.mgmt.compute.v2019_12_01.operations.virtualmachinescalesetsoperations?view=azure-python - - for vm in self._run_azure_op_with_retry(lambda: self.vm_func.list(self.vm_data.rg_name, self.vm_data.name)): - try: - return self._run_azure_op_with_retry(lambda: self.vm_func.get_instance_view( - resource_group_name=self.vm_data.rg_name, - vm_scale_set_name=self.vm_data.name, - instance_id=vm.instance_id - )) - except Exception as err: - self.log.warning( - f"Unable to fetch instance view of VMSS VM: {vm}. Trying out other instances.\nError: {err}") - continue - - raise Exception(f"Unable to fetch instance view of any VMSS instances for {self.vm_data.name}") - - def get_extensions(self) -> List[VirtualMachineScaleSetExtension]: - return self._run_azure_op_with_retry(lambda: self.extension_func.list( - resource_group_name=self.vm_data.rg_name, - vm_scale_set_name=self.vm_data.name - )) - - def get_extension_instance_view(self, extension_name) -> VirtualMachineExtensionInstanceView: - return self._run_azure_op_with_retry(lambda: self.extension_func.get( - resource_group_name=self.vm_data.rg_name, - vm_scale_set_name=self.vm_data.name, - vmss_extension_name=extension_name, - expand="instanceView" - )) - - def get_ext_props(self, extension_data, settings=None, protected_settings=None, auto_upgrade_minor_version=True, - force_update_tag=None) -> VirtualMachineScaleSetExtension: - return VirtualMachineScaleSetExtension( - publisher=extension_data.publisher, - type_properties_type=extension_data.ext_type, - type_handler_version=extension_data.version, - auto_upgrade_minor_version=auto_upgrade_minor_version, - settings=settings, - protected_settings=protected_settings - ) - - -class ComputeManager: - """ - The factory class for setting the Helper class based on the setting. - """ - def __init__(self): - self.__vm_data = get_vm_data_from_env() - self.__compute_manager = None - - @property - def is_vm(self) -> bool: - return self.__vm_data.model_type == VMModelType.VM - - @property - def compute_manager(self): - if self.__compute_manager is None: - self.__compute_manager = VirtualMachineHelper() if self.is_vm else VirtualMachineScaleSetHelper() - return self.__compute_manager diff --git a/tests_e2e/scenarios/lib/identifiers.py b/tests_e2e/scenarios/lib/identifiers.py new file mode 100644 index 0000000000..48794140b3 --- /dev/null +++ b/tests_e2e/scenarios/lib/identifiers.py @@ -0,0 +1,63 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class VmIdentifier(object): + def __init__(self, location, subscription, resource_group, name): + """ + Represents the information that identifies a VM to the ARM APIs + """ + self.location = location + self.subscription: str = subscription + self.resource_group: str = resource_group + self.name: str = name + + def __str__(self): + return f"{self.resource_group}:{self.name}" + + +class VmExtensionIdentifier(object): + def __init__(self, publisher, ext_type, version): + """ + Represents the information that identifies an extension to the ARM APIs + + publisher - e.g. Microsoft.Azure.Extensions + type - e.g. CustomScript + version - e.g. 2.1, 2.* + name - arbitrary name for the extension ARM resource + """ + self.publisher: str = publisher + self.type: str = ext_type + self.version: str = version + + def __str__(self): + return f"{self.publisher}.{self.type}" + + +class VmExtensionIds(object): + """ + A set of extensions used by the tests, listed here for convenience (easy to reference them by name). + + Only the major version is specified, and the minor version is set to 0 (set autoUpgradeMinorVersion to True in the call to enable + to use the latest version) + """ + CustomScript: VmExtensionIdentifier = VmExtensionIdentifier(publisher='Microsoft.Azure.Extensions', ext_type='CustomScript', version="2.0") + # Older run command extension, still used by the Portal as of Dec 2022 + RunCommand: VmExtensionIdentifier = VmExtensionIdentifier(publisher='Microsoft.CPlat.Core', ext_type='RunCommandLinux', version="1.0") + # New run command extension, with support for multi-config + RunCommandHandler: VmExtensionIdentifier = VmExtensionIdentifier(publisher='Microsoft.CPlat.Core', ext_type='RunCommandHandlerLinux', version="1.0") + VmAccess: VmExtensionIdentifier = VmExtensionIdentifier(publisher='Microsoft.OSTCExtensions', ext_type='VMAccessForLinux', version="1.0") diff --git a/tests_e2e/scenarios/lib/logging.py b/tests_e2e/scenarios/lib/logging.py new file mode 100644 index 0000000000..2cb523d6bc --- /dev/null +++ b/tests_e2e/scenarios/lib/logging.py @@ -0,0 +1,37 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging + +# +# This module defines a single object, 'log', which test use for logging. +# +# When the test is invoked as part of a LISA test suite, 'log' references the LISA root logger. +# Otherwise, it references a new Logger named 'waagent'. +# + +log: logging.Logger = logging.getLogger("lisa") + +if not log.hasHandlers(): + log = logging.getLogger("waagent") + console_handler = logging.StreamHandler() + log.addHandler(console_handler) + +log.setLevel(logging.INFO) + +formatter = logging.Formatter('%(asctime)s.%(msecs)03d [%(levelname)s] %(message)s', datefmt="%Y-%m-%dT%H:%M:%SZ") +for handler in log.handlers: + handler.setFormatter(formatter) diff --git a/tests_e2e/scenarios/lib/logging_utils.py b/tests_e2e/scenarios/lib/logging_utils.py deleted file mode 100644 index 462f6a957a..0000000000 --- a/tests_e2e/scenarios/lib/logging_utils.py +++ /dev/null @@ -1,33 +0,0 @@ -# Create a base class -import logging - - -def get_logger(name): - return LoggingHandler(name).log - - -class LoggingHandler: - """ - Base class for Logging - """ - def __init__(self, name=None): - self.log = self.__setup_and_get_logger(name) - - def __setup_and_get_logger(self, name): - logger = logging.getLogger(name if name is not None else self.__class__.__name__) - if logger.hasHandlers(): - # Logging module inherits from base loggers if already setup, if a base logger found, reuse that - return logger - - # No handlers found for logger, set it up - # This logging format is easier to read on the DevOps UI - - # https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=bash#formatting-commands - log_formatter = logging.Formatter("##[%(levelname)s] [%(asctime)s] [%(module)s] {%(pathname)s:%(lineno)d} %(message)s", - datefmt="%Y-%m-%dT%H:%M:%S%z") - console_handler = logging.StreamHandler() - console_handler.setFormatter(log_formatter) - logger.addHandler(console_handler) - logger.setLevel(logging.INFO) - - return logger - diff --git a/tests_e2e/scenarios/lib/models.py b/tests_e2e/scenarios/lib/models.py deleted file mode 100644 index a9e3e8cf01..0000000000 --- a/tests_e2e/scenarios/lib/models.py +++ /dev/null @@ -1,135 +0,0 @@ -import os -from enum import Enum, auto -from typing import List - - -class VMModelType(Enum): - VM = auto() - VMSS = auto() - - -class ExtensionMetaData: - def __init__(self, publisher: str, ext_type: str, version: str, ext_name: str = ""): - self.__publisher = publisher - self.__ext_type = ext_type - self.__version = version - self.__ext_name = ext_name - - @property - def publisher(self) -> str: - return self.__publisher - - @property - def ext_type(self) -> str: - return self.__ext_type - - @property - def version(self) -> str: - return self.__version - - @property - def name(self): - return self.__ext_name - - @name.setter - def name(self, ext_name): - self.__ext_name = ext_name - - @property - def handler_name(self): - return f"{self.publisher}.{self.ext_type}" - - -class VMMetaData: - - def __init__(self, vm_name: str, rg_name: str, sub_id: str, location: str, admin_username: str, - ips: List[str] = None): - self.__vm_name = vm_name - self.__rg_name = rg_name - self.__sub_id = sub_id - self.__location = location - self.__admin_username = admin_username - - vm_ips, vmss_ips = _get_ips(admin_username) - # By default assume the test is running on a VM - self.__type = VMModelType.VM - self.__ips = vm_ips - if any(vmss_ips): - self.__type = VMModelType.VMSS - self.__ips = vmss_ips - - if ips is not None: - self.__ips = ips - - print(f"IPs: {self.__ips}") - - @property - def name(self) -> str: - return self.__vm_name - - @property - def rg_name(self) -> str: - return self.__rg_name - - @property - def location(self) -> str: - return self.__location - - @property - def sub_id(self) -> str: - return self.__sub_id - - @property - def admin_username(self): - return self.__admin_username - - @property - def ips(self) -> List[str]: - return self.__ips - - @property - def model_type(self): - return self.__type - - -def _get_ips(username) -> (list, list): - """ - Try fetching Ips from the files that we create via az-cli. - We do a best effort to fetch this from both orchestrator or the test VM. Its located in different locations on both - scenarios. - Returns: Tuple of (VmIps, VMSSIps). - """ - - vms, vmss = [], [] - orchestrator_path = os.path.join(os.environ['BUILD_SOURCESDIRECTORY'], "dcr") - test_vm_path = os.path.join("/home", username, "dcr") - - for ip_path in [orchestrator_path, test_vm_path]: - - vm_ip_path = os.path.join(ip_path, ".vm_ips") - if os.path.exists(vm_ip_path): - with open(vm_ip_path, 'r') as vm_ips: - vms.extend(ip.strip() for ip in vm_ips.readlines()) - - vmss_ip_path = os.path.join(ip_path, ".vmss_ips") - if os.path.exists(vmss_ip_path): - with open(vmss_ip_path, 'r') as vmss_ips: - vmss.extend(ip.strip() for ip in vmss_ips.readlines()) - - return vms, vmss - - -def get_vm_data_from_env() -> VMMetaData: - if get_vm_data_from_env.__instance is None: - get_vm_data_from_env.__instance = VMMetaData(vm_name=os.environ["VMNAME"], - rg_name=os.environ['RGNAME'], - sub_id=os.environ["SUBID"], - location=os.environ['LOCATION'], - admin_username=os.environ['ADMINUSERNAME']) - - - return get_vm_data_from_env.__instance - - -get_vm_data_from_env.__instance = None - diff --git a/tests_e2e/scenarios/lib/retry.py b/tests_e2e/scenarios/lib/retry.py new file mode 100644 index 0000000000..1b78f0a13b --- /dev/null +++ b/tests_e2e/scenarios/lib/retry.py @@ -0,0 +1,41 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import time + +from typing import Callable, Any + +from tests_e2e.scenarios.lib.logging import log + + +def execute_with_retry(operation: Callable[[], Any]) -> Any: + """ + Some Azure errors (e.g. throttling) are retryable; this method attempts the given operation retrying a few times + (after a short delay) if the error includes the string "RetryableError" + """ + attempts = 3 + while attempts > 0: + attempts -= 1 + try: + return operation() + except Exception as e: + # TODO: Do we need to retry on msrestazure.azure_exceptions.CloudError? + if "RetryableError" not in str(e) or attempts == 0: + raise + log.warning("The operation failed with a RetryableError, retrying in 30 secs. Error: %s", e) + time.sleep(30) + + diff --git a/tests_e2e/scenarios/lib/shell.py b/tests_e2e/scenarios/lib/shell.py new file mode 100644 index 0000000000..894ba90ca2 --- /dev/null +++ b/tests_e2e/scenarios/lib/shell.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from subprocess import Popen, PIPE +from typing import Any + + +class CommandError(Exception): + """ + Exception raised by run_command when the command returns an error + """ + def __init__(self, command: Any, exit_code: int, stdout: str, stderr: str): + super().__init__(f"'{command}' failed (exit code: {exit_code}): {stderr}") + self.command: Any = command + self.exit_code: int = exit_code + self.stdout: str = stdout + self.stderr: str = stderr + + +def run_command(command: Any, shell=False) -> str: + """ + This function is a thin wrapper around Popen/communicate in the subprocess module. It executes the given command + and returns its stdout. If the command returns a non-zero exit code, the function raises a RunCommandException. + + Similarly to Popen, the 'command' can be a string or a list of strings, and 'shell' indicates whether to execute + the command through the shell. + + NOTE: The command's stdout and stderr are read as text streams. + """ + process = Popen(command, stdout=PIPE, stderr=PIPE, shell=shell, text=True) + + stdout, stderr = process.communicate() + + if process.returncode != 0: + raise CommandError(command, process.returncode, stdout, stderr) + + return stdout + diff --git a/tests_e2e/scenarios/lib/ssh_client.py b/tests_e2e/scenarios/lib/ssh_client.py new file mode 100644 index 0000000000..5e0afbd41a --- /dev/null +++ b/tests_e2e/scenarios/lib/ssh_client.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pathlib import Path + +from tests_e2e.scenarios.lib import shell + + +class SshClient(object): + def __init__(self, ip_address: str, username: str, private_key_file: Path, port: int = 22): + self._ip_address: str = ip_address + self._username:str = username + self._private_key_file: Path = private_key_file + self._port: int = port + + def run_command(self, command: str) -> str: + """ + Executes the given command over SSH and returns its stdout. If the command returns a non-zero exit code, + the function raises a RunCommandException. + """ + destination = f"ssh://{self._username}@{self._ip_address}:{self._port}" + + return shell.run_command(["ssh", "-o", "StrictHostKeyChecking=no", "-i", self._private_key_file, destination, command]) + + @staticmethod + def generate_ssh_key(private_key_file: Path): + """ + Generates an SSH key on the given Path + """ + shell.run_command(["ssh-keygen", "-m", "PEM", "-t", "rsa", "-b", "4096", "-q", "-N", "", "-f", str(private_key_file)]) + diff --git a/tests_e2e/scenarios/lib/virtual_machine.py b/tests_e2e/scenarios/lib/virtual_machine.py new file mode 100644 index 0000000000..6a1f76f961 --- /dev/null +++ b/tests_e2e/scenarios/lib/virtual_machine.py @@ -0,0 +1,143 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This module includes facilities to execute some operations on virtual machines and scale sets (list extensions, restart, etc). +# + +from abc import ABC, abstractmethod +from builtins import TimeoutError +from typing import Any, List + +from azure.core.polling import LROPoller +from azure.identity import DefaultAzureCredential +from azure.mgmt.compute import ComputeManagementClient +from azure.mgmt.compute.models import VirtualMachineExtension, VirtualMachineScaleSetExtension, VirtualMachineInstanceView, VirtualMachineScaleSetInstanceView +from azure.mgmt.resource import ResourceManagementClient + +from tests_e2e.scenarios.lib.identifiers import VmIdentifier +from tests_e2e.scenarios.lib.logging import log +from tests_e2e.scenarios.lib.retry import execute_with_retry + + +class VirtualMachineBaseClass(ABC): + """ + Abstract base class for VirtualMachine and VmScaleSet. + + Defines the interface common to both classes and provides the implementation of some methods in that interface. + """ + def __init__(self, vm: VmIdentifier): + super().__init__() + self._identifier: VmIdentifier = vm + self._compute_client = ComputeManagementClient(credential=DefaultAzureCredential(), subscription_id=vm.subscription) + self._resource_client = ResourceManagementClient(credential=DefaultAzureCredential(), subscription_id=vm.subscription) + + @abstractmethod + def get_instance_view(self) -> Any: # Returns VirtualMachineInstanceView or VirtualMachineScaleSetInstanceView + """ + Retrieves the instance view of the virtual machine or scale set + """ + + @abstractmethod + def get_extensions(self) -> Any: # Returns List[VirtualMachineExtension] or List[VirtualMachineScaleSetExtension] + """ + Retrieves the extensions installed on the virtual machine or scale set + """ + + def restart(self, timeout=5 * 60) -> None: + """ + Restarts the virtual machine or scale set + """ + log.info("Initiating restart of %s", self._identifier) + + poller: LROPoller = execute_with_retry(self._begin_restart) + + poller.wait(timeout=timeout) + + if not poller.done(): + raise TimeoutError(f"Failed to restart {self._identifier.name} after {timeout} seconds") + + log.info("Restarted %s", self._identifier.name) + + @abstractmethod + def _begin_restart(self) -> LROPoller: + """ + Derived classes must provide the implementation for this method using their corresponding begin_restart() implementation + """ + + def __str__(self): + return f"{self._identifier}" + + +class VirtualMachine(VirtualMachineBaseClass): + def get_instance_view(self) -> VirtualMachineInstanceView: + log.info("Retrieving instance view for %s", self._identifier) + return execute_with_retry(self._compute_client.virtual_machines.get( + resource_group_name=self._identifier.resource_group, + vm_name=self._identifier.name, + expand="instanceView" + ).instance_view) + + def get_extensions(self) -> List[VirtualMachineExtension]: + log.info("Retrieving extensions for %s", self._identifier) + return execute_with_retry(self._compute_client.virtual_machine_extensions.list( + resource_group_name=self._identifier.resource_group, + vm_name=self._identifier.name)) + + def _begin_restart(self) -> LROPoller: + return self._compute_client.virtual_machines.begin_restart( + resource_group_name=self._identifier.resource_group, + vm_name=self._identifier.name) + + +class VmScaleSet(VirtualMachineBaseClass): + def get_instance_view(self) -> VirtualMachineScaleSetInstanceView: + log.info("Retrieving instance view for %s", self._identifier) + + # TODO: Revisit this implementation. Currently this method returns the instance view of the first VM instance available. + # For the instance view of the complete VMSS, use the compute_client.virtual_machine_scale_sets function + # https://docs.microsoft.com/en-us/python/api/azure-mgmt-compute/azure.mgmt.compute.v2019_12_01.operations.virtualmachinescalesetsoperations?view=azure-python + for vm in execute_with_retry(lambda: self._compute_client.virtual_machine_scale_set_vms.list(self._identifier.resource_group, self._identifier.name)): + try: + return execute_with_retry(lambda: self._compute_client.virtual_machine_scale_set_vms.get_instance_view( + resource_group_name=self._identifier.resource_group, + vm_scale_set_name=self._identifier.name, + instance_id=vm.instance_id)) + except Exception as e: + log.warning("Unable to retrieve instance view for scale set instance %s. Trying out other instances.\nError: %s", vm, e) + + raise Exception(f"Unable to retrieve instance view of any instances for scale set {self._identifier}") + + + @property + def vm_func(self): + return self._compute_client.virtual_machine_scale_set_vms + + @property + def extension_func(self): + return self._compute_client.virtual_machine_scale_set_extensions + + def get_extensions(self) -> List[VirtualMachineScaleSetExtension]: + log.info("Retrieving extensions for %s", self._identifier) + return execute_with_retry(self._compute_client.virtual_machine_scale_set_extensions.list( + resource_group_name=self._identifier.resource_group, + vm_scale_set_name=self._identifier.name)) + + def _begin_restart(self) -> LROPoller: + return self._compute_client.virtual_machine_scale_sets.begin_restart( + resource_group_name=self._identifier.resource_group, + vm_scale_set_name=self._identifier.name) diff --git a/tests_e2e/scenarios/lib/vm_extension.py b/tests_e2e/scenarios/lib/vm_extension.py new file mode 100644 index 0000000000..1a30ce8b5b --- /dev/null +++ b/tests_e2e/scenarios/lib/vm_extension.py @@ -0,0 +1,239 @@ +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This module includes facilities to execute VM extension operations (enable, remove, etc) on single virtual machines (using +# class VmExtension) or virtual machine scale sets (using class VmssExtension). +# + +import uuid + +from abc import ABC, abstractmethod +from assertpy import assert_that, soft_assertions +from typing import Any, Callable, Dict, Type + +from azure.core.polling import LROPoller +from azure.mgmt.compute import ComputeManagementClient +from azure.mgmt.compute.models import VirtualMachineExtension, VirtualMachineScaleSetExtension, VirtualMachineExtensionInstanceView +from azure.identity import DefaultAzureCredential + +from tests_e2e.scenarios.lib.identifiers import VmIdentifier, VmExtensionIdentifier +from tests_e2e.scenarios.lib.logging import log +from tests_e2e.scenarios.lib.retry import execute_with_retry + + +_TIMEOUT = 5 * 60 # Timeout for extension operations (in seconds) + + +class _VmExtensionBaseClass(ABC): + """ + Abstract base class for VmExtension and VmssExtension. + + Implements the operations that are common to virtual machines and scale sets. Derived classes must provide the specific types and methods for the + virtual machine or scale set. + """ + def __init__(self, vm: VmIdentifier, extension: VmExtensionIdentifier, resource_name: str): + super().__init__() + self._vm: VmIdentifier = vm + self._identifier = extension + self._resource_name = resource_name + self._compute_client: ComputeManagementClient = ComputeManagementClient(credential=DefaultAzureCredential(), subscription_id=vm.subscription) + + def enable( + self, + settings: Dict[str, Any] = None, + protected_settings: Dict[str, Any] = None, + auto_upgrade_minor_version: bool = True, + force_update: bool = False, + force_update_tag: str = None + ) -> None: + """ + Performs an enable operation on the extension. + + NOTE: 'force_update' is not a parameter of the actual ARM API. It is provided for convenience: If set to True, + the 'force_update_tag' can be left unspecified and this method will generate a random tag. + """ + if force_update_tag is not None and not force_update: + raise ValueError("If force_update_tag is provided then force_update must be set to true") + + if force_update and force_update_tag is None: + force_update_tag = str(uuid.uuid4()) + + extension_parameters = self._ExtensionType( + publisher=self._identifier.publisher, + location=self._vm.location, + type_properties_type=self._identifier.type, + type_handler_version=self._identifier.version, + auto_upgrade_minor_version=auto_upgrade_minor_version, + settings=settings, + protected_settings=protected_settings, + force_update_tag=force_update_tag) + + # Hide the protected settings from logging + if protected_settings is not None: + extension_parameters.protected_settings = "*****[REDACTED]*****" + log.info("Enabling %s", self._identifier) + log.info("%s", extension_parameters) + # Now set the actual protected settings before invoking the extension + extension_parameters.protected_settings = protected_settings + + result: VirtualMachineExtension = execute_with_retry( + lambda: self._begin_create_or_update( + self._vm.resource_group, + self._vm.name, + self._resource_name, + extension_parameters + ).result(timeout=_TIMEOUT)) + + if result.provisioning_state != 'Succeeded': + raise Exception(f"Enable {self._identifier} failed. Provisioning state: {result.provisioning_state}") + log.info("Enable succeeded.") + + def get_instance_view(self) -> VirtualMachineExtensionInstanceView: # TODO: Check type for scale sets + """ + Retrieves the instance view of the extension + """ + log.info("Retrieving instance view for %s...", self._identifier) + + return execute_with_retry(lambda: self._get( + resource_group_name=self._vm.resource_group, + vm_name=self._vm.name, + vm_extension_name=self._resource_name, + expand="instanceView" + ).instance_view) + + def assert_instance_view( + self, + expected_status_code: str = "ProvisioningState/succeeded", + expected_version: str = None, + expected_message: str = None, + assert_function: Callable[[VirtualMachineExtensionInstanceView], None] = None + ) -> None: + """ + Asserts that the extension's instance view matches the given expected values. If 'expected_version' and/or 'expected_message' + are omitted, they are not validated. + + If 'assert_function' is provided, it is invoked passing as parameter the instance view. This function can be used to perform + additional validations. + """ + instance_view = self.get_instance_view() + + with soft_assertions(): + if expected_version is not None: + # Compare only the major and minor versions (i.e. the first 2 items in the result of split()) + installed_version = instance_view.type_handler_version + assert_that(expected_version.split(".")[0:2]).described_as("Unexpected extension version").is_equal_to(installed_version.split(".")[0:2]) + + assert_that(instance_view.statuses).described_as(f"Expected 1 status, got: {instance_view.statuses}").is_length(1) + status = instance_view.statuses[0] + + if expected_message is not None: + assert_that(expected_message in status.message).described_as(f"{expected_message} should be in the InstanceView message ({status.message})").is_true() + + assert_that(status.code).described_as("InstanceView status code").is_equal_to(expected_status_code) + + if assert_function is not None: + assert_function(instance_view) + + log.info("The instance view matches the expected values") + + @abstractmethod + def delete(self) -> None: + """ + Performs a delete operation on the extension + """ + + @property + @abstractmethod + def _ExtensionType(self) -> Type: + """ + Type of the extension object for the virtual machine or scale set (i.e. VirtualMachineExtension or VirtualMachineScaleSetExtension) + """ + + @property + @abstractmethod + def _begin_create_or_update(self) -> Callable[[str, str, str, Any], LROPoller[Any]]: # "Any" can be VirtualMachineExtension or VirtualMachineScaleSetExtension + """ + The begin_create_or_update method for the virtual machine or scale set extension + """ + + @property + @abstractmethod + def _get(self) -> Any: # VirtualMachineExtension or VirtualMachineScaleSetExtension + """ + The get method for the virtual machine or scale set extension + """ + + def __str__(self): + return f"{self._identifier}" + + +class VmExtension(_VmExtensionBaseClass): + """ + Extension operations on a single virtual machine. + """ + @property + def _ExtensionType(self) -> Type: + return VirtualMachineExtension + + @property + def _begin_create_or_update(self) -> Callable[[str, str, str, VirtualMachineExtension], LROPoller[VirtualMachineExtension]]: + return self._compute_client.virtual_machine_extensions.begin_create_or_update + + @property + def _get(self) -> VirtualMachineExtension: + return self._compute_client.virtual_machine_extensions.get + + def delete(self) -> None: + log.info("Deleting %s", self._identifier) + + execute_with_retry(lambda: self._compute_client.virtual_machine_extensions.begin_delete( + self._vm.resource_group, + self._vm.name, + self._resource_name + ).wait(timeout=_TIMEOUT)) + + +class VmssExtension(_VmExtensionBaseClass): + """ + Extension operations on virtual machine scale sets. + """ + @property + def _ExtensionType(self) -> Type: + return VirtualMachineScaleSetExtension + + @property + def _begin_create_or_update(self) -> Callable[[str, str, str, VirtualMachineScaleSetExtension], LROPoller[VirtualMachineScaleSetExtension]]: + return self._compute_client.virtual_machine_scale_set_extensions.begin_create_or_update + + @property + def _get(self) -> VirtualMachineScaleSetExtension: + return self._compute_client.virtual_machine_scale_set_extensions.get + + def delete(self) -> None: # TODO: Implement this method + raise NotImplementedError() + + def delete_from_instance(self, instance_id: str) -> None: + log.info("Deleting %s from scale set instance %s", self._identifier, instance_id) + + execute_with_retry(lambda: self._compute_client.virtual_machine_scale_set_vm_extensions.begin_delete( + resource_group_name=self._vm.resource_group, + vm_scale_set_name=self._vm.name, + vm_extension_name=self._resource_name, + instance_id=instance_id + ).wait(timeout=_TIMEOUT)) + diff --git a/tests_e2e/scenarios/tests/bvts/custom_script.py b/tests_e2e/scenarios/tests/bvts/custom_script.py deleted file mode 100644 index 2d716223bb..0000000000 --- a/tests_e2e/scenarios/tests/bvts/custom_script.py +++ /dev/null @@ -1,45 +0,0 @@ -import argparse -import os -import uuid -import sys - -from tests_e2e.scenarios.lib.CustomScriptExtension import CustomScriptExtension - - -def main(subscription_id, resource_group_name, vm_name): - os.environ["VMNAME"] = vm_name - os.environ['RGNAME'] = resource_group_name - os.environ["SUBID"] = subscription_id - os.environ["SCENARIONAME"] = "BVT" - os.environ["LOCATION"] = "westus2" - os.environ["ADMINUSERNAME"] = "somebody" - os.environ["BUILD_SOURCESDIRECTORY"] = "/somewhere" - - cse = CustomScriptExtension(extension_name="testCSE") - - ext_props = [ - cse.get_ext_props(settings={'commandToExecute': f"echo \'Hello World! {uuid.uuid4()} \'"}), - cse.get_ext_props(settings={'commandToExecute': "echo \'Hello again\'"}) - ] - - cse.run(ext_props=ext_props) - - -if __name__ == "__main__": - try: - parser = argparse.ArgumentParser() - parser.add_argument('--subscription') - parser.add_argument('--group') - parser.add_argument('--vm') - - args = parser.parse_args() - - main(args.subscription, args.group, args.vm) - - except Exception as exception: - print(str(exception)) - sys.exit(1) - - sys.exit(0) - - diff --git a/tests_e2e/scenarios/tests/bvts/extension_operations.py b/tests_e2e/scenarios/tests/bvts/extension_operations.py new file mode 100755 index 0000000000..ae5e0c13b1 --- /dev/null +++ b/tests_e2e/scenarios/tests/bvts/extension_operations.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# BVT for extension operations (Install/Enable/Update/Uninstall). +# +# The test executes an older version of an extension, then updates it to a newer version, and lastly +# it removes it. The actual extension is irrelevant, but the test uses CustomScript for simplicity, +# since it's invocation is trivial and the entire extension workflow can be tested end-to-end by +# checking the message in the status produced by the extension. +# +import uuid + +from assertpy import assert_that + +from azure.core.exceptions import ResourceNotFoundError + +from tests_e2e.scenarios.lib.agent_test import AgentTest +from tests_e2e.scenarios.lib.identifiers import VmExtensionIds, VmExtensionIdentifier +from tests_e2e.scenarios.lib.logging import log +from tests_e2e.scenarios.lib.vm_extension import VmExtension + + +class ExtensionOperationsBvt(AgentTest): + def run(self): + custom_script_2_0 = VmExtension( + self._context.vm, + VmExtensionIds.CustomScript, + resource_name="CustomScript") + + custom_script_2_1 = VmExtension( + self._context.vm, + VmExtensionIdentifier(VmExtensionIds.CustomScript.publisher, VmExtensionIds.CustomScript.type, "2.1"), + resource_name="CustomScript") + + log.info("Installing %s", custom_script_2_0) + message = f"Hello {uuid.uuid4()}!" + custom_script_2_0.enable( + settings={ + 'commandToExecute': f"echo \'{message}\'" + }, + auto_upgrade_minor_version=False + ) + custom_script_2_0.assert_instance_view(expected_version="2.0", expected_message=message) + + log.info("Updating %s to %s", custom_script_2_0, custom_script_2_1) + message = f"Hello {uuid.uuid4()}!" + custom_script_2_1.enable( + settings={ + 'commandToExecute': f"echo \'{message}\'" + } + ) + custom_script_2_1.assert_instance_view(expected_version="2.1", expected_message=message) + + custom_script_2_1.delete() + + assert_that(custom_script_2_1.get_instance_view).\ + described_as("Fetching the instance view should fail after removing the extension").\ + raises(ResourceNotFoundError) + + +if __name__ == "__main__": + ExtensionOperationsBvt.run_from_command_line() diff --git a/tests_e2e/scenarios/tests/bvts/run_command.py b/tests_e2e/scenarios/tests/bvts/run_command.py new file mode 100755 index 0000000000..25258cef39 --- /dev/null +++ b/tests_e2e/scenarios/tests/bvts/run_command.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# BVT for RunCommand. +# +# Note that there are two incarnations of RunCommand (which are actually two different extensions): +# Microsoft.CPlat.Core.RunCommandHandlerLinux and Microsoft.CPlat.Core.RunCommandLinux. This test +# exercises both using the same strategy: execute the extension to create a file on the test VM, +# then fetch the contents of the file over SSH and compare against the known value. +# +import base64 +import uuid + +from assertpy import assert_that, soft_assertions +from typing import Callable, Dict + +from tests_e2e.scenarios.lib.agent_test import AgentTest +from tests_e2e.scenarios.lib.identifiers import VmExtensionIds +from tests_e2e.scenarios.lib.logging import log +from tests_e2e.scenarios.lib.ssh_client import SshClient +from tests_e2e.scenarios.lib.vm_extension import VmExtension + + +class RunCommandBvt(AgentTest): + class TestCase: + def __init__(self, extension: VmExtension, get_settings: Callable[[str], Dict[str, str]]): + self.extension = extension + self.get_settings = get_settings + + def run(self): + test_cases = [ + RunCommandBvt.TestCase( + VmExtension(self._context.vm, VmExtensionIds.RunCommand, resource_name="RunCommand"), + lambda s: { + "script": base64.standard_b64encode(bytearray(s, 'utf-8')).decode('utf-8') + }), + RunCommandBvt.TestCase( + VmExtension(self._context.vm, VmExtensionIds.RunCommandHandler, resource_name="RunCommandHandler"), + lambda s: { + "source": { + "script": s + } + }) + ] + + ssh_client = SshClient( + ip_address=self._context.vm_ip_address, + username=self._context.username, + private_key_file=self._context.private_key_file) + + with soft_assertions(): + for t in test_cases: + log.info("Test case: %s", t.extension) + + unique = str(uuid.uuid4()) + test_file = f"/tmp/waagent-test.{unique}" + script = f"echo '{unique}' > {test_file}" + log.info("Script to execute: %s", script) + + t.extension.enable(settings=t.get_settings(script)) + t.extension.assert_instance_view() + + log.info("Verifying contents of the file created by the extension") + contents = ssh_client.run_command(f"cat {test_file}").rstrip() # remove the \n + assert_that(contents).\ + described_as("Contents of the file created by the extension").\ + is_equal_to(unique) + log.info("The contents match") + + +if __name__ == "__main__": + RunCommandBvt.run_from_command_line() diff --git a/tests_e2e/scenarios/tests/bvts/vm_access.py b/tests_e2e/scenarios/tests/bvts/vm_access.py new file mode 100755 index 0000000000..36919e3f30 --- /dev/null +++ b/tests_e2e/scenarios/tests/bvts/vm_access.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +# Microsoft Azure Linux Agent +# +# Copyright 2018 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# BVT for the VmAccess extension +# +# The test executes VmAccess to add a user and then verifies that an SSH connection to the VM can +# be established with that user's identity. +# +import uuid + +from assertpy import assert_that +from pathlib import Path + +from tests_e2e.scenarios.lib.agent_test import AgentTest +from tests_e2e.scenarios.lib.identifiers import VmExtensionIds +from tests_e2e.scenarios.lib.logging import log +from tests_e2e.scenarios.lib.ssh_client import SshClient + +from tests_e2e.scenarios.lib.vm_extension import VmExtension + + +class VmAccessBvt(AgentTest): + def run(self): + # Try to use a unique username for each test run (note that we truncate to 32 chars to + # comply with the rules for usernames) + log.info("Generating a new username and SSH key") + username: str = f"test-{uuid.uuid4()}"[0:32] + log.info("Username: %s", username) + + # Create an SSH key for the user and fetch the public key + private_key_file: Path = self._context.working_directory/f"{username}_rsa" + public_key_file: Path = self._context.working_directory/f"{username}_rsa.pub" + log.info("Generating SSH key as %s", private_key_file) + ssh: SshClient = SshClient(ip_address=self._context.vm_ip_address, username=username, private_key_file=private_key_file) + ssh.generate_ssh_key(private_key_file) + with public_key_file.open() as f: + public_key = f.read() + + # Invoke the extension + vm_access = VmExtension(self._context.vm, VmExtensionIds.VmAccess, resource_name="VmAccess") + vm_access.enable( + protected_settings={ + 'username': username, + 'ssh_key': public_key, + 'reset_ssh': 'false' + } + ) + vm_access.assert_instance_view() + + # Verify the user was added correctly by starting an SSH session to the VM + log.info("Verifying SSH connection to the test VM") + stdout = ssh.run_command("echo -n $USER") + assert_that(stdout).described_as("Output from SSH command").is_equal_to(username) + log.info("SSH command output ($USER): %s", stdout) + + +if __name__ == "__main__": + VmAccessBvt.run_from_command_line() diff --git a/tests_e2e/scenarios/testsuites/agent_bvt.py b/tests_e2e/scenarios/testsuites/agent_bvt.py index a63c3e34df..0b0383e99a 100644 --- a/tests_e2e/scenarios/testsuites/agent_bvt.py +++ b/tests_e2e/scenarios/testsuites/agent_bvt.py @@ -15,30 +15,34 @@ # limitations under the License. # -from tests_e2e.orchestrator.lib.agent_test_suite import AgentTestScenario -from tests_e2e.scenarios.tests.bvts import custom_script +from tests_e2e.orchestrator.lib.agent_test_suite import AgentTestSuite +from tests_e2e.scenarios.tests.bvts.extension_operations import ExtensionOperationsBvt +from tests_e2e.scenarios.tests.bvts.vm_access import VmAccessBvt +from tests_e2e.scenarios.tests.bvts.run_command import RunCommandBvt # E0401: Unable to import 'lisa' (import-error) from lisa import ( # pylint: disable=E0401 - Logger, Node, TestCaseMetadata, - TestSuite, TestSuiteMetadata, ) @TestSuiteMetadata(area="bvt", category="", description="Test suite for Agent BVTs") -class AgentBvt(TestSuite): +class AgentBvt(AgentTestSuite): """ Test suite for Agent BVTs """ @TestCaseMetadata(description="", priority=0) - def main(self, log: Logger, node: Node) -> None: - def tests(ctx: AgentTestScenario.Context) -> None: - custom_script.main(ctx.subscription_id, ctx.resource_group_name, ctx.vm_name) - - AgentTestScenario(node, log).execute(tests) + def main(self, node: Node) -> None: + self.execute( + node, + [ + ExtensionOperationsBvt, # Tests the basic operations (install, enable, update, uninstall) using CustomScript + RunCommandBvt, + VmAccessBvt + ] + ) From 1fe3de80ba4a1c5477400d0212bf61d3f3f32004 Mon Sep 17 00:00:00 2001 From: Norberto Arrieta Date: Fri, 6 Jan 2023 12:49:04 -0800 Subject: [PATCH 5/5] Disable DCR v2 Pipeline (#2722) Co-authored-by: narrieta --- dcr/azure-pipelines.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dcr/azure-pipelines.yml b/dcr/azure-pipelines.yml index 3fcffc6aef..c2bbe9d191 100644 --- a/dcr/azure-pipelines.yml +++ b/dcr/azure-pipelines.yml @@ -59,14 +59,6 @@ trigger: # no PR triggers pr: none -schedules: - - cron: "0 */8 * * *" # Run every 8 hours - displayName: Daily validation builds - branches: - include: - - develop - always: true - variables: - template: templates/vars.yml