diff --git a/tests_e2e/orchestrator/lib/agent_test_loader.py b/tests_e2e/orchestrator/lib/agent_test_loader.py index 11e665c13f..f952f1160b 100644 --- a/tests_e2e/orchestrator/lib/agent_test_loader.py +++ b/tests_e2e/orchestrator/lib/agent_test_loader.py @@ -88,7 +88,7 @@ class AgentTestLoader(object): """ Loads a given set of test suites from the YAML configuration files. """ - def __init__(self, test_suites: str, cloud: str): + def __init__(self, test_suites: List[str], cloud: str): """ Loads the specified 'test_suites', which are given as a string of comma-separated suite names or a YAML description of a single test_suite. @@ -175,25 +175,9 @@ def _parse_image(image: str) -> str: if suite_skip_image not in self.images: raise Exception(f"Invalid image reference in test suite {suite.name}: Can't find {suite_skip_image} in images.yml") - @staticmethod - def _load_test_suites(test_suites: str) -> List[TestSuiteInfo]: - # - # Attempt to parse 'test_suites' as the YML description of a single suite - # - parsed = yaml.safe_load(test_suites) - - # - # A comma-separated list (e.g. "foo", "foo, bar", etc.) is valid YAML, but it is parsed as a string. An actual test suite would - # be parsed as a dictionary. If it is a dict, take is as the YML description of a single test suite - # - if isinstance(parsed, dict): - return [AgentTestLoader._load_test_suite(parsed)] - - # - # If test_suites is not YML, then it should be a comma-separated list of description files - # - description_files: List[Path] = [AgentTestLoader._SOURCE_CODE_ROOT/"test_suites"/f"{t.strip()}.yml" for t in test_suites.split(',')] + def _load_test_suites(test_suites: List[str]) -> List[TestSuiteInfo]: + description_files: List[Path] = [AgentTestLoader._SOURCE_CODE_ROOT/"test_suites"/f"{t}.yml" for t in test_suites] return [AgentTestLoader._load_test_suite(f) for f in description_files] @staticmethod diff --git a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py index ffecaf3630..1450398c8d 100644 --- a/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py +++ b/tests_e2e/orchestrator/lib/agent_test_suite_combinator.py @@ -46,6 +46,7 @@ class AgentTestSuitesCombinatorSchema(schema.Combinator): resource_group_name: str = field(default_factory=str, metadata=field_metadata(required=True)) subscription_id: str = field(default_factory=str, metadata=field_metadata(required=True)) test_suites: str = field(default_factory=str, metadata=field_metadata(required=True)) + default_test_suites: List[str] = field(default_factory=list, metadata=field_metadata(required=True)) user: str = field(default_factory=str, metadata=field_metadata(required=True)) vm_name: str = field(default_factory=str, metadata=field_metadata(required=True)) vm_size: str = field(default_factory=str, metadata=field_metadata(required=True)) @@ -81,20 +82,25 @@ def __init__(self, runbook: AgentTestSuitesCombinatorSchema) -> None: if self.runbook.resource_group_name == '': raise Exception("Invalid runbook parameters: The 'vmss_name' parameter indicates an existing VMSS, a 'resource_group_name' must be specified.") + if self.runbook.test_suites != "": + test_suites = [t.strip() for t in self.runbook.test_suites.split(',')] + else: + test_suites = self.runbook.default_test_suites + self._log: logging.Logger = logging.getLogger("lisa") with set_thread_name("AgentTestSuitesCombinator"): if self.runbook.vm_name != '': - self._environments = [self.create_existing_vm_environment()] + self._environments = [self.create_existing_vm_environment(test_suites)] elif self.runbook.vmss_name != '': - self._environments = [self.create_existing_vmss_environment()] + self._environments = [self.create_existing_vmss_environment(test_suites)] else: - self._environments = self.create_environment_list() + self._environments = self.create_environment_list(test_suites) self._index = 0 @classmethod def type_name(cls) -> str: - return "agent_test_suites" + return "agent_test_suite_combinator" @classmethod def type_schema(cls) -> Type[schema.TypedSchema]: @@ -125,7 +131,7 @@ def _next(self) -> Optional[Dict[str, Any]]: "AzureUSGovernment": "usgovarizona", } - def create_environment_list(self) -> List[Dict[str, Any]]: + def create_environment_list(self, test_suites: List[str]) -> List[Dict[str, Any]]: """ Examines the test_suites specified in the runbook and returns a list of the environments (i.e. test VMs or scale sets) that need to be created in order to execute these suites. @@ -136,7 +142,7 @@ def create_environment_list(self) -> List[Dict[str, Any]]: environments: List[Dict[str, Any]] = [] shared_environments: Dict[str, Dict[str, Any]] = {} # environments shared by multiple test suites - loader = AgentTestLoader(self.runbook.test_suites, self.runbook.cloud) + loader = AgentTestLoader(test_suites, self.runbook.cloud) runbook_images = self._get_runbook_images(loader) @@ -260,8 +266,8 @@ def create_environment_list(self) -> List[Dict[str, Any]]: return environments - def create_existing_vm_environment(self) -> Dict[str, Any]: - loader = AgentTestLoader(self.runbook.test_suites, self.runbook.cloud) + def create_existing_vm_environment(self, test_suites: List[str]) -> Dict[str, Any]: + loader = AgentTestLoader(test_suites, self.runbook.cloud) vm: VirtualMachineClient = VirtualMachineClient( cloud=self.runbook.cloud, @@ -300,8 +306,8 @@ def create_existing_vm_environment(self) -> Dict[str, Any]: "c_test_suites": loader.test_suites, } - def create_existing_vmss_environment(self) -> Dict[str, Any]: - loader = AgentTestLoader(self.runbook.test_suites, self.runbook.cloud) + def create_existing_vmss_environment(self, test_suites: List[str]) -> Dict[str, Any]: + loader = AgentTestLoader(test_suites, self.runbook.cloud) vmss = VirtualMachineScaleSetClient( cloud=self.runbook.cloud, diff --git a/tests_e2e/orchestrator/runbook.yml b/tests_e2e/orchestrator/runbook.yml index 8b0ef37ec7..722ceba61d 100644 --- a/tests_e2e/orchestrator/runbook.yml +++ b/tests_e2e/orchestrator/runbook.yml @@ -26,10 +26,33 @@ variable: is_case_visible: true # - # Test suites to execute + # Test suites to execute. + # + # Use "test_suites" to specify from the command-line the test suites to execute. If not specifies, the "default_test_suites" are executed. # - name: test_suites - value: "agent_bvt, no_outbound_connections, extensions_disabled, agent_not_provisioned, fips, agent_ext_workflow, agent_status, multi_config_ext, agent_cgroups, ext_cgroups, agent_firewall, ext_telemetry_pipeline, ext_sequencing, agent_persist_firewall, publish_hostname, agent_update, recover_network_interface" + value: "" + + - name: default_test_suites + value: + - agent_bvt + - agent_cgroups + - agent_ext_workflow + - agent_firewall + - agent_not_provisioned + - agent_persist_firewall + - agent_status + - agent_update + - ext_cgroups + - extensions_disabled + - ext_sequencing + - ext_telemetry_pipeline + - fips + - keyvault_certificates + - multi_config_ext + - no_outbound_connections + - publish_hostname + - recover_network_interface # # Parameters used to create test VMs @@ -183,7 +206,7 @@ environment: $(c_environment) platform: $(c_platform) combinator: - type: agent_test_suites + type: agent_test_suite_combinator allow_ssh: $(allow_ssh) cloud: $(cloud) identity_file: $(identity_file) @@ -193,6 +216,7 @@ combinator: resource_group_name: $(resource_group_name) subscription_id: $(subscription_id) test_suites: $(test_suites) + default_test_suites: $(default_test_suites) user: $(user) vm_name: $(vm_name) vm_size: $(vm_size)