From bbfc703729af4c03911f00ec40108271f2ad7969 Mon Sep 17 00:00:00 2001 From: narrieta Date: Mon, 13 Feb 2023 15:59:33 -0800 Subject: [PATCH 1/3] Add support for VHDs; add image, location and vm_size as parameters to the pipeline --- .../orchestrator/lib/agent_test_suite.py | 13 +- .../lib/agent_test_suite_combinator.py | 126 +++++++++++++----- tests_e2e/orchestrator/runbook.yml | 45 +++++-- tests_e2e/pipeline/pipeline.yml | 30 ++++- tests_e2e/pipeline/scripts/execute_tests.sh | 17 ++- 5 files changed, 180 insertions(+), 51 deletions(-) diff --git a/tests_e2e/orchestrator/lib/agent_test_suite.py b/tests_e2e/orchestrator/lib/agent_test_suite.py index 54551541b..6c79786d6 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite.py @@ -112,6 +112,7 @@ def __init__(self, vm: VmIdentifier, paths: AgentTestContext.Paths, connection: self.node: Node = None self.runbook_name: str = None self.image_name: str = None + self.is_vhd: bool = None self.test_suites: List[AgentTestSuite] = None self.collect_logs: str = None self.skip_setup: bool = None @@ -146,8 +147,9 @@ def _set_context(self, node: Node, variables: Dict[str, Any], log: Logger): self.__context.log = log self.__context.node = node - self.__context.image_name = f"{runbook.marketplace.offer}-{runbook.marketplace.sku}" - self.__context.test_suites = self._get_required_parameter(variables, "test_suites_info") + self.__context.is_vhd = 'c_vhd' in variables + self.__context.image_name = f"{node.os.name}-vhd" if self.__context.is_vhd else f"{runbook.marketplace.offer}-{runbook.marketplace.sku}" + self.__context.test_suites = self._get_required_parameter(variables, "c_test_suites") self.__context.collect_logs = self._get_required_parameter(variables, "collect_logs") self.__context.skip_setup = self._get_required_parameter(variables, "skip_setup") @@ -249,7 +251,10 @@ def _setup_node(self) -> None: self._log.info("Resource Group: %s", self.context.vm.resource_group) self._log.info("") - self._install_agent_on_node() + if self.context.is_vhd: + self._log.info("Using a VHD; will not install the test Agent.") + else: + self._install_agent_on_node() def _install_agent_on_node(self) -> None: """ @@ -292,7 +297,7 @@ def _collect_node_logs(self) -> None: @TestCaseMetadata(description="", priority=0) def agent_test_suite(self, node: Node, variables: Dict[str, Any], log: Logger) -> None: """ - Executes each of the AgentTests included in "test_suites_info" variable (which is generated by the AgentTestSuitesCombinator). + Executes each of the AgentTests included in the "c_test_suites" variable (which is generated by the AgentTestSuitesCombinator). """ self._set_context(node, variables, log) diff --git a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py index 8cf91dc5a..a83ce9d58 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. import logging +import re +import urllib.parse from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Type @@ -15,7 +17,7 @@ from lisa.combinator import Combinator # pylint: disable=E0401 from lisa.util import field_metadata # pylint: disable=E0401 -from tests_e2e.orchestrator.lib.agent_test_loader import AgentTestLoader +from tests_e2e.orchestrator.lib.agent_test_loader import AgentTestLoader, VmImageInfo @dataclass_json() @@ -24,6 +26,15 @@ class AgentTestSuitesCombinatorSchema(schema.Combinator): test_suites: str = field( default_factory=str, metadata=field_metadata(required=True) ) + image: str = field( + default_factory=str, metadata=field_metadata(required=True) + ) + location: str = field( + default_factory=str, metadata=field_metadata(required=True) + ) + vm_size: str = field( + default_factory=str, metadata=field_metadata(required=True) + ) class AgentTestSuitesCombinator(Combinator): @@ -31,19 +42,19 @@ class AgentTestSuitesCombinator(Combinator): The "agent_test_suites" combinator returns a list of items containing five variables that specify the environments that the agent test suites must be executed on: - * marketplace_image: e.g. "Canonical UbuntuServer 18.04-LTS latest", - * location: e.g. "westus2", - * vm_size: e.g. "Standard_D2pls_v5" - * vhd: e.g "https://rhel.blob.core.windows.net/images/RHEL_8_Standard-8.3.202006170423.vhd?se=..." - * test_suites_info: e.g. [AgentBvt, FastTrack] + * c_marketplace_image: e.g. "Canonical UbuntuServer 18.04-LTS latest", + * c_location: e.g. "westus2", + * c_vm_size: e.g. "Standard_D2pls_v5" + * c_vhd: e.g "https://rhel.blob.core.windows.net/images/RHEL_8_Standard-8.3.202006170423.vhd?se=..." + * c_test_suites: e.g. [AgentBvt, FastTrack] - (marketplace_image, location, vm_size) and vhd are mutually exclusive and define the environment (i.e. the test VM) - in which the test will be executed. test_suites_info defines the test suites that should be executed in that + (c_marketplace_image, c_location, c_vm_size) and vhd are mutually exclusive and define the environment (i.e. the test VM) + in which the test will be executed. c_test_suites defines the test suites that should be executed in that environment. """ def __init__(self, runbook: AgentTestSuitesCombinatorSchema) -> None: super().__init__(runbook) - self._environments = self.create_environment_list(self.runbook.test_suites) + self._environments = self.create_environment_list() self._index = 0 @classmethod @@ -63,47 +74,87 @@ def _next(self) -> Optional[Dict[str, Any]]: _DEFAULT_LOCATION = "westus2" - @staticmethod - def create_environment_list(test_suites: str) -> List[Dict[str, Any]]: + def create_environment_list(self) -> List[Dict[str, Any]]: + loader = AgentTestLoader(self.runbook.test_suites) + + # + # If the runbook provides any of 'image', 'location', or 'vm_size', those values + # override any configuration values on the test suite. + # + # Check 'images' first and add them to 'runbook_images', if any + # + if self.runbook.image == "": + runbook_images = [] + else: + runbook_images = loader.images.get(self.runbook.image) + if runbook_images is None: + if not self._is_urn(self.runbook.image) and not self._is_vhd(self.runbook.image): + raise Exception(f"The 'image' parameter must be an image or image set name, a urn, or a vhd: {self.runbook.image}") + i = VmImageInfo() + i.urn = self.runbook.image # Note that this could be a URN or the URI for a VHD + i.locations = [] + i.vm_sizes = [] + runbook_images = [i] + + # + # Now walk through all the test_suites and create a list of the environments (test VMs) that need to be created. + # environment_list: List[Dict[str, Any]] = [] shared_environments: Dict[str, Dict[str, Any]] = {} - loader = AgentTestLoader(test_suites) - for suite_info in loader.test_suites: - images_info = loader.images[suite_info.images] + images_info = runbook_images if len(runbook_images) > 0 else loader.images[suite_info.images] + for image in images_info: - # If the suite specifies a location, use it. Else, if the image specifies a list of locations, use - # any of them. Otherwise, use the default location. - if suite_info.location != '': + # The URN can be a VHD if the runbook provided a VHD in the 'images' parameter + if self._is_vhd(image.urn): + marketplace_image = "" + vhd = image.urn + else: + marketplace_image = image.urn + vhd = "" + + # If the runbook specified a location, use it. Then try the suite location, if any. Otherwise, check if the image specifies + # a list of locations and use any of them. If no location is specified so far, use the default. + if self.runbook.location != "": + location = self.runbook.location + elif suite_info.location != '': location = suite_info.location elif len(image.locations) > 0: location = image.locations[0] else: location = AgentTestSuitesCombinator._DEFAULT_LOCATION - # If the image specifies a list of VM sizes, use any of them. Otherwise, set the size to empty and let LISA choose it. - vm_size = image.vm_sizes[0] if len(image.vm_sizes) > 0 else "" + # If the runbook specified a VM size, use it. Else if the image specifies a list of VM sizes, use any of them. Otherwise, + # set the size to empty and let LISA choose it. + if self.runbook.vm_size != '': + vm_size = self.runbook.vm_size + elif len(image.vm_sizes) > 0: + vm_size = image.vm_sizes[0] + else: + vm_size = "" if suite_info.owns_vm: + # create an environment for exclusive use by this suite environment_list.append({ - "marketplace_image": image.urn, - "location": location, - "vm_size": vm_size, - "vhd": "", - "test_suites_info": [suite_info] + "c_marketplace_image": marketplace_image, + "c_location": location, + "c_vm_size": vm_size, + "c_vhd": vhd, + "c_test_suites": [suite_info] }) else: + # add this suite to the shared environments key: str = f"{image.urn}:{location}" if key in shared_environments: - shared_environments[key]["test_suites_info"].append(suite_info) + shared_environments[key]["c_test_suites"].append(suite_info) else: shared_environments[key] = { - "marketplace_image": image.urn, - "location": location, - "vm_size": vm_size, - "vhd": "", - "test_suites_info": [suite_info] + "c_marketplace_image": marketplace_image, + "c_location": location, + "c_vm_size": vm_size, + "c_vhd": vhd, + "c_test_suites": [suite_info] } environment_list.extend(shared_environments.values()) @@ -112,8 +163,19 @@ def create_environment_list(test_suites: str) -> List[Dict[str, Any]]: log.info("******** Environments *****") for e in environment_list: log.info( - "{ marketplace_image: '%s', location: '%s', vm_size: '%s', vhd: '%s', test_suites_info: '%s' }", - e['marketplace_image'], e['location'], e['vm_size'], e['vhd'], [s.name for s in e['test_suites_info']]) + "{ marketplace_image: '%s', location: '%s', vm_size: '%s', vhd: '%s', test_suites: '%s' }", + e['c_marketplace_image'], e['c_location'], e['c_vm_size'], e['c_vhd'], [s.name for s in e['c_test_suites']]) log.info("***************************") return environment_list + + @staticmethod + def _is_urn(urn: str) -> bool: + # URNs can be given as ' ' or ':::' + return re.match(r"(\S+\s\S+\s\S+\s\S+)|([^:]+:[^:]+:[^:]+:[^:]+)", urn) is not None + + @staticmethod + def _is_vhd(vhd: str) -> bool: + # VHDs are given as URIs to storage; do some basic validation, not intending to be exhaustive. + parsed = urllib.parse.urlparse(vhd) + return parsed.scheme == 'https' and parsed.netloc != "" and parsed.path != "" diff --git a/tests_e2e/orchestrator/runbook.yml b/tests_e2e/orchestrator/runbook.yml index 2f67cf7ff..542f4acfe 100644 --- a/tests_e2e/orchestrator/runbook.yml +++ b/tests_e2e/orchestrator/runbook.yml @@ -9,7 +9,7 @@ extension: variable: # - # These variables define runbook parameters; they are handled by LISA. + # These variables define parameters handled by LISA. # - name: subscription_id value: "" @@ -26,6 +26,9 @@ variable: # # These variables define parameters for the AgentTestSuite; see the test wiki for details. # + # NOTE: c_test_suites, generated by the AgentTestSuitesCombinator, is also a parameter + # for the AgentTestSuite + # # Whether to collect logs from the test VM - name: collect_logs value: "failed" @@ -37,25 +40,38 @@ variable: is_case_visible: true # - # These variables parameters for the AgentTestSuitesCombinator combinator + # These variables are parameters for the AgentTestSuitesCombinator # # The test suites to execute - name: test_suites value: "agent_bvt" - is_case_visible: true + - name: image + value: "" + - name: location + value: "" + - name: vm_size + value: "" # - # These variables are set by the AgentTestSuitesCombinator combinator + # The values for these variables are generated by the AgentTestSuitesCombinator combinator. They are + # prefixed with "c_" to distinguish them from the rest of the variables, whose value can be set from + # the command line. + # + # c_marketplace_image, c_vm_size, c_location, and c_vhd are handled by LISA and define + # the set of test VMs that need to be created, while c_test_suites is a parameter + # for the AgentTestSuite and defines the test suites that must be executed on each + # of those test VMs (the AgentTestSuite also uses c_vhd) # - - name: marketplace_image + - name: c_marketplace_image value: "" - - name: vm_size + - name: c_vm_size value: "" - - name: location + - name: c_location value: "" - - name: vhd + - name: c_vhd value: "" - - name: test_suites_info + is_case_visible: true + - name: c_test_suites value: [] is_case_visible: true @@ -86,14 +102,17 @@ platform: core_count: min: 2 azure: - marketplace: $(marketplace_image) - vhd: $(vhd) - location: $(location) - vm_size: $(vm_size) + marketplace: $(c_marketplace_image) + vhd: $(c_vhd) + location: $(c_location) + vm_size: $(c_vm_size) combinator: type: agent_test_suites test_suites: $(test_suites) + image: $(image) + location: $(location) + vm_size: $(vm_size) concurrency: 10 diff --git a/tests_e2e/pipeline/pipeline.yml b/tests_e2e/pipeline/pipeline.yml index 69154d08d..3251ff24e 100644 --- a/tests_e2e/pipeline/pipeline.yml +++ b/tests_e2e/pipeline/pipeline.yml @@ -1,10 +1,31 @@ parameters: - # see the test wiki for a description of the parameters + # See the test wiki for a description of the parameters - name: test_suites displayName: Test Suites type: string default: agent_bvt + # NOTES: + # * 'image', 'location' and 'vm_size' override any values in the test suites/images definition + # files. Those parameters are useful for 1-off tests, like testing a VHD or checking if + # an image is supported in a particular location. + # * Azure Pipelines do not allow empty string for the parameter value, using "-" instead. + # + - name: image + displayName: Image (image/image set name, URN, or VHD) + type: string + default: "-" + + - name: location + displayName: Location (region) + type: string + default: "-" + + - name: vm_size + displayName: VM size + type: string + default: "-" + - name: collect_logs displayName: Collect logs from test VMs type: string @@ -26,8 +47,15 @@ parameters: variables: - name: azureConnection value: 'azuremanagement' + # These variables exposed the above parameters as environment variables - name: test_suites value: ${{ parameters.test_suites }} + - name: image + value: ${{ parameters.image }} + - name: location + value: ${{ parameters.location }} + - name: vm_size + value: ${{ parameters.vm_size }} - name: collect_logs value: ${{ parameters.collect_logs }} - name: keep_environment diff --git a/tests_e2e/pipeline/scripts/execute_tests.sh b/tests_e2e/pipeline/scripts/execute_tests.sh index b02df16bc..cece7cd84 100755 --- a/tests_e2e/pipeline/scripts/execute_tests.sh +++ b/tests_e2e/pipeline/scripts/execute_tests.sh @@ -30,6 +30,18 @@ az acr login --name waagenttests docker pull waagenttests.azurecr.io/waagenttests:latest +# Azure Pipelines does not allow an empty string as the value for a pipeline parameter; instead we use "-" to indicate +# an empty value. Change "-" to "" for the variables that capture the parameter values. +if [[ $IMAGE == "-" ]]; then + IMAGE="" +fi +if [[ $LOCATION == "-" ]]; then + LOCATION="" +fi +if [[ $VM_SIZE == "-" ]]; then + VM_SIZE="" +fi + # A test failure will cause automation to exit with an error code and we don't want this script to stop so we force the command # to succeed and capture the exit code to return it at the end of the script. echo "exit 0" > /tmp/exit.sh @@ -51,7 +63,10 @@ docker run --rm \ -v identity_file:\$HOME/.ssh/id_rsa \ -v test_suites:\"$TEST_SUITES\" \ -v collect_logs:\"$COLLECT_LOGS\" \ - -v keep_environment:\"$KEEP_ENVIRONMENT\"" \ + -v keep_environment:\"$KEEP_ENVIRONMENT\" \ + -v image:\"$IMAGE\" \ + -v location:\"$LOCATION\" \ + -v vm_size:\"$VM_SIZE\"" \ || echo "exit $?" > /tmp/exit.sh # From e179eb0a00ff964e4287dd07720dc7c6386013d0 Mon Sep 17 00:00:00 2001 From: narrieta Date: Mon, 13 Feb 2023 16:29:32 -0800 Subject: [PATCH 2/3] check for empty --- tests_e2e/orchestrator/lib/agent_test_suite.py | 2 +- tests_e2e/orchestrator/lib/agent_test_suite_combinator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests_e2e/orchestrator/lib/agent_test_suite.py b/tests_e2e/orchestrator/lib/agent_test_suite.py index 6c79786d6..d0b430221 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite.py @@ -147,7 +147,7 @@ def _set_context(self, node: Node, variables: Dict[str, Any], log: Logger): self.__context.log = log self.__context.node = node - self.__context.is_vhd = 'c_vhd' in variables + self.__context.is_vhd = self._get_required_parameter(variables, "c_test_suites") != "" self.__context.image_name = f"{node.os.name}-vhd" if self.__context.is_vhd else f"{runbook.marketplace.offer}-{runbook.marketplace.sku}" self.__context.test_suites = self._get_required_parameter(variables, "c_test_suites") self.__context.collect_logs = self._get_required_parameter(variables, "collect_logs") diff --git a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py index a83ce9d58..39fb10458 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py @@ -163,7 +163,7 @@ def create_environment_list(self) -> List[Dict[str, Any]]: log.info("******** Environments *****") for e in environment_list: log.info( - "{ marketplace_image: '%s', location: '%s', vm_size: '%s', vhd: '%s', test_suites: '%s' }", + "{ c_marketplace_image: '%s', c_location: '%s', c_vm_size: '%s', c_vhd: '%s', c_test_suites: '%s' }", e['c_marketplace_image'], e['c_location'], e['c_vm_size'], e['c_vhd'], [s.name for s in e['c_test_suites']]) log.info("***************************") From 2e02991ad70c7e9a2dae9ab246d5e11b0edd1698 Mon Sep 17 00:00:00 2001 From: narrieta Date: Tue, 14 Feb 2023 10:19:07 -0800 Subject: [PATCH 3/3] typo --- tests_e2e/orchestrator/lib/agent_test_suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_e2e/orchestrator/lib/agent_test_suite.py b/tests_e2e/orchestrator/lib/agent_test_suite.py index d0b430221..0f663a8ba 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite.py @@ -147,7 +147,7 @@ def _set_context(self, node: Node, variables: Dict[str, Any], log: Logger): self.__context.log = log self.__context.node = node - self.__context.is_vhd = self._get_required_parameter(variables, "c_test_suites") != "" + self.__context.is_vhd = self._get_required_parameter(variables, "c_vhd") != "" self.__context.image_name = f"{node.os.name}-vhd" if self.__context.is_vhd else f"{runbook.marketplace.offer}-{runbook.marketplace.sku}" self.__context.test_suites = self._get_required_parameter(variables, "c_test_suites") self.__context.collect_logs = self._get_required_parameter(variables, "collect_logs")