Skip to content

Commit

Permalink
Add support for Azure Clouds (#2795)
Browse files Browse the repository at this point in the history
* Add support for Azure Clouds
---------

Co-authored-by: narrieta <narrieta>
  • Loading branch information
narrieta authored Apr 11, 2023
1 parent c89647c commit 9390ce5
Show file tree
Hide file tree
Showing 14 changed files with 261 additions and 106 deletions.
1 change: 1 addition & 0 deletions test-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@ azure-core
azure-identity
azure-mgmt-compute>=22.1.0
azure-mgmt-resource>=15.0.0
msrestazure
15 changes: 10 additions & 5 deletions tests_e2e/orchestrator/lib/agent_test_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ class VmImageInfo(object):
# The URN of the image (publisher, offer, version separated by spaces)
urn: str
# Indicates that the image is available only on those locations. If empty, the image should be available in all locations
locations: List[str]
locations: Dict[str, List[str]]
# Indicates that the image is available only for those VM sizes. If empty, the image should be available for all VM sizes
vm_sizes: List[str]

Expand Down Expand Up @@ -109,8 +109,9 @@ def _validate(self):
if suite.location != '':
for suite_image in suite.images:
for image in self.images[suite_image]:
if len(image.locations) > 0:
if suite.location not in image.locations:
# If the image has a location restriction, validate that it is available on the location the suite must run on
if image.locations:
if not any(suite.location in l for l in image.locations.values()):
raise Exception(f"Test suite {suite.name} must be executed in {suite.location}, but <{image.urn}> is not available in that location")

@staticmethod
Expand Down Expand Up @@ -223,14 +224,18 @@ def _load_images() -> Dict[str, List[VmImageInfo]]:
i = VmImageInfo()
if isinstance(description, str):
i.urn = description
i.locations = []
i.locations = {}
i.vm_sizes = []
else:
if "urn" not in description:
raise Exception(f"Image {name} is missing the 'urn' property: {description}")
i.urn = description["urn"]
i.locations = description["locations"] if "locations" in description else []
i.locations = description["locations"] if "locations" in description else {}
i.vm_sizes = description["vm_sizes"] if "vm_sizes" in description else []
for cloud in i.locations.keys():
if cloud not in ["AzureCloud", "AzureChinaCloud", "AzureUSGovernment"]:
raise Exception(f"Invalid cloud {cloud} for image {name} in images.yml")

images[name] = [i]

# now load the image-sets, mapping them to the images that we just computed
Expand Down
7 changes: 3 additions & 4 deletions tests_e2e/orchestrator/lib/agent_test_suite.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,7 @@
)
from lisa.environment import EnvironmentStatus # pylint: disable=E0401
from lisa.messages import TestStatus, TestResultMessage # pylint: disable=E0401
from lisa.sut_orchestrator import AZURE # pylint: disable=E0401
from lisa.sut_orchestrator.azure.common import get_node_context, AzureNodeSchema # pylint: disable=E0401
from lisa.sut_orchestrator.azure.common import get_node_context # pylint: disable=E0401

import makepkg
from azurelinuxagent.common.version import AGENT_VERSION
Expand Down Expand Up @@ -133,11 +132,11 @@ def __init__(self, metadata: TestSuiteMetadata) -> None:
def _initialize(self, node: Node, variables: Dict[str, Any], lisa_working_path: str, lisa_log_path: str, lisa_log: Logger):
connection_info = node.connection_info
node_context = get_node_context(node)
runbook = node.capability.get_extended_runbook(AzureNodeSchema, AZURE)

self.__context = self._Context(
vm=VmIdentifier(
location=runbook.location,
cloud=self._get_required_parameter(variables, "c_cloud"),
location=self._get_required_parameter(variables, "c_location"),
subscription=node.features._platform.subscription_id,
resource_group=node_context.resource_group_name,
name=node_context.vm_name),
Expand Down
61 changes: 46 additions & 15 deletions tests_e2e/orchestrator/lib/agent_test_suite_combinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,21 @@ def _next(self) -> Optional[Dict[str, Any]]:
return result

_DEFAULT_LOCATIONS = {
"china": "china north 2",
"government": "usgovarizona",
"public": "westus2"
"AzureCloud": "westus2",
"AzureChinaCloud": "chinanorth2",
"AzureUSGovernment": "usgovarizona",
}

_MARKETPLACE_IMAGE_INFORMATION_LOCATIONS = {
"AzureCloud": "", # empty indicates the default location used by LISA
"AzureChinaCloud": "chinanorth2",
"AzureUSGovernment": "usgovarizona",
}

_SHARED_RESOURCE_GROUP_LOCATIONS = {
"AzureCloud": "", # empty indicates the default location used by LISA
"AzureChinaCloud": "chinanorth2",
"AzureUSGovernment": "usgovarizona",
}

def create_environment_for_existing_vm(self) -> List[Dict[str, Any]]:
Expand Down Expand Up @@ -178,15 +190,23 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
raise Exception(f"Invalid URN: {image.urn}")
name = f"{match.group('offer')}-{match.group('sku')}"

# If the runbook specified a location, use it. Then try the suite location, if any. Otherwise, check if the image specifies
# a list of locations and use any of them. If no location is specified so far, use the default.
location: str = None
# If the runbook specified a location, use it.
if self.runbook.location != "":
location = self.runbook.location
# Then try the suite location, if any.
elif suite_info.location != '':
location = suite_info.location
elif len(image.locations) > 0:
location = image.locations[0]
else:
# If the image has a location restriction, use any location where it is available.
# However, if it is not available on any location, skip the image.
elif image.locations:
image_locations = image.locations.get(self.runbook.cloud)
if image_locations is not None:
if len(image_locations) == 0:
continue
location = image_locations[0]
# If no location has been selected, use the default.
if location is None:
location = AgentTestSuitesCombinator._DEFAULT_LOCATIONS[self.runbook.cloud]

# If the runbook specified a VM size, use it. Else if the image specifies a list of VM sizes, use any of them. Otherwise,
Expand All @@ -202,11 +222,14 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
# create an environment for exclusive use by this suite
environment_list.append({
"c_marketplace_image": marketplace_image,
"c_cloud": self.runbook.cloud,
"c_location": location,
"c_vm_size": vm_size,
"c_vhd": vhd,
"c_test_suites": [suite_info],
"c_env_name": f"{name}-{suite_info.name}"
"c_env_name": f"{name}-{suite_info.name}",
"c_marketplace_image_information_location": self._MARKETPLACE_IMAGE_INFORMATION_LOCATIONS[self.runbook.cloud],
"c_shared_resource_group_location": self._SHARED_RESOURCE_GROUP_LOCATIONS[self.runbook.cloud]
})
else:
# add this suite to the shared environments
Expand All @@ -216,27 +239,35 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
else:
shared_environments[key] = {
"c_marketplace_image": marketplace_image,
"c_cloud": self.runbook.cloud,
"c_location": location,
"c_vm_size": vm_size,
"c_vhd": vhd,
"c_test_suites": [suite_info],
"c_env_name": key
"c_env_name": key,
"c_marketplace_image_information_location": self._MARKETPLACE_IMAGE_INFORMATION_LOCATIONS[self.runbook.cloud],
"c_shared_resource_group_location": self._SHARED_RESOURCE_GROUP_LOCATIONS[self.runbook.cloud]
}

environment_list.extend(shared_environments.values())

if len(environment_list) == 0:
raise Exception("No VM images were found to execute the test suites.")

log: logging.Logger = logging.getLogger("lisa")
log.info("******** Environments *****")
for e in environment_list:
log.info(
"{ c_marketplace_image: '%s', c_location: '%s', c_vm_size: '%s', c_vhd: '%s', c_test_suites: '%s', c_env_name: '%s' }",
e['c_marketplace_image'], e['c_location'], e['c_vm_size'], e['c_vhd'], [s.name for s in e['c_test_suites']], e['c_env_name'])
log.info("")
log.info("******** Agent Test Environments *****")
for environment in environment_list:
test_suites = [s.name for s in environment['c_test_suites']]
log.info("Settings for %s:\n%s\n", environment['c_env_name'], '\n'.join([f"\t{name}: {value if name != 'c_test_suites' else test_suites}" for name, value in environment.items()]))
log.info("***************************")
log.info("")

return environment_list

_URN = re.compile(r"(?P<publisher>[^\s:]+)[\s:](?P<offer>[^\s:]+)[\s:](?P<sku>[^\s:]+)[\s:](?P<version>[^\s:]+)")


@staticmethod
def _is_urn(urn: str) -> bool:
# URNs can be given as '<Publisher> <Offer> <Sku> <Version>' or '<Publisher>:<Offer>:<Sku>:<Version>'
Expand Down
23 changes: 17 additions & 6 deletions tests_e2e/orchestrator/runbook.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ variable:
- name: test_suites
value: "agent_bvt"
- name: cloud
value: "public"
value: "AzureCloud"
- name: image
value: ""
- name: location
Expand All @@ -64,21 +64,26 @@ variable:
# prefixed with "c_" to distinguish them from the rest of the variables, whose value can be set from
# the command line.
#
# c_marketplace_image, c_vm_size, c_location, and c_vhd are handled by LISA and define
# the set of test VMs that need to be created, while c_test_suites and c_env_name are parameters
# for the AgentTestSuite; the former defines the test suites that must be executed on each
# of those test VMs and the latter is the name of the environment, which is used for logging
# purposes (NOTE: the AgentTestSuite also uses c_vhd).
# Most of these variables are handled by LISA and are used to define the set of test VMs that need to be
# created. The variables marked with 'is_case_visible' are also referenced by the AgentTestSuite.
#
- name: c_env_name
value: ""
is_case_visible: true
- name: c_marketplace_image
value: ""
- name: c_marketplace_image_information_location
value: ""
- name: c_shared_resource_group_location
value: ""
- name: c_vm_size
value: ""
- name: c_cloud
value: ""
is_case_visible: true
- name: c_location
value: ""
is_case_visible: true
- name: c_vhd
value: ""
is_case_visible: true
Expand Down Expand Up @@ -107,6 +112,12 @@ platform:
keep_environment: $(keep_environment)
azure:
deploy: True
#
# TODO: Enable these parameters once LISA supports all Azure clouds
#
# cloud: $(cloud)
# marketplace_image_information_location: $(c_marketplace_image_information_location)
# shared_resource_group_location: $(c_shared_resource_group_location)
subscription_id: $(subscription_id)
wait_delete: false
requirement:
Expand Down
22 changes: 17 additions & 5 deletions tests_e2e/orchestrator/sample_runbooks/existing_vm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ variable:
# These variables identify the existing VM, and the user for SSH connections
#
- name: cloud
value: "public"
value: "AzureCloud"
- name: subscription_id
value: ""
- name: resource_group_name
Expand Down Expand Up @@ -80,18 +80,24 @@ variable:
# prefixed with "c_" to distinguish them from the rest of the variables, whose value can be set from
# the command line.
#
# c_marketplace_image, c_vm_size, c_location, and c_vhd are handled by LISA and define
# the set of test VMs that need to be created, while c_test_suites is a parameter
# for the AgentTestSuite and defines the test suites that must be executed on each
# of those test VMs (the AgentTestSuite also uses c_vhd)
# Most of these variables are handled by LISA and are used to define the set of test VMs that need to be
# created. The variables marked with 'is_case_visible' are also referenced by the AgentTestSuite.
#
- name: c_env_name
value: ""
is_case_visible: true
- name: c_vm_name
value: ""
- name: c_marketplace_image_information_location
value: ""
- name: c_shared_resource_group_location
value: ""
- name: c_cloud
value: ""
is_case_visible: true
- name: c_location
value: ""
is_case_visible: true
- name: c_test_suites
value: []
is_case_visible: true
Expand All @@ -114,6 +120,12 @@ platform:
admin_username: $(user)
admin_private_key_file: $(identity_file)
azure:
#
# TODO: Enable these parameters once LISA supports all Azure clouds
#
# cloud: $(cloud)
# marketplace_image_information_location: $(c_marketplace_image_information_location)
# shared_resource_group_location: $(c_shared_resource_group_location)
resource_group_name: $(resource_group_name)
deploy: false
subscription_id: $(subscription_id)
Expand Down
91 changes: 46 additions & 45 deletions tests_e2e/pipeline/pipeline-cleanup.yml
Original file line number Diff line number Diff line change
@@ -1,58 +1,59 @@
#
# Pipeline for cleaning up any remaining Resource Groups generated by the Azure.WALinuxAgent pipeline.
#
# Deletes any resource groups that are more than a day old and contain string "lisa-WALinuxAgent-"
# Deletes any resource groups that are older than 'older_than' and match the 'name_pattern' regular expression
#
schedules:
- cron: "0 */12 * * *" # Run twice a day (every 12 hours)
displayName: cleanup build
branches:
include:
- develop
always: true

trigger:
- develop
parameters:
- name: name_pattern
displayName: Regular expression to match the name of the resource groups to delete
type: string
default: lisa-WALinuxAgent-.*

pr: none
- name: older_than
displayName: Delete resources older than (use the syntax of the "date -d" command)
type: string
default: 1 day ago

- name: service_connections
type: object
default:
- azuremanagement
#
# TODO: Enable these services connections once we create test pipelines for all Azure clouds
#
# - azuremanagement.china
# - azuremanagement.government

pool:
vmImage: ubuntu-latest

variables:
- name: azureConnection
value: 'azuremanagement'
- name: rgPrefix
value: 'lisa-WALinuxAgent-'

steps:
- ${{ each service_connection in parameters.service_connections }}:
- task: AzureCLI@2
inputs:
azureSubscription: ${{ service_connection }}
scriptType: 'bash'
scriptLocation: 'inlineScript'
inlineScript: |
set -euxo pipefail
- task: AzureKeyVault@2
displayName: "Fetch secrets from KV"
inputs:
azureSubscription: '$(azureConnection)'
KeyVaultName: 'dcrV2SPs'
SecretsFilter: '*'
RunAsPreJob: true
#
# We use the REST API to list the resource groups because we need the createdTime and that
# property is not available via the az-cli commands.
#
subscription_id=$(az account list --all --query "[?isDefault].id" -o tsv)
date=$(date --utc +%Y-%m-%d'T'%H:%M:%S.%N'Z' -d "${{ parameters.older_than }}")
- task: AzureCLI@2
inputs:
azureSubscription: '$(azureConnection)'
scriptType: 'bash'
scriptLocation: 'inlineScript'
inlineScript: |
set -euxo pipefail
date=`date --utc +%Y-%m-%d'T'%H:%M:%S.%N'Z' -d "1 day ago"`
# Using the Azure REST GET resourceGroups API call as we can add the createdTime to the results.
# This feature is not available via the az-cli commands directly so we have to use the Azure REST APIs

az rest --method GET \
--url "https://management.azure.com/subscriptions/$(SUBSCRIPTION-ID)/resourcegroups" \
--url-parameters api-version=2021-04-01 \$expand=createdTime \
--output json \
--query value \
| jq --arg date "$date" '.[] | select (.createdTime < $date).name' \
| grep "$(rgPrefix)" \
| xargs -l -t -r az group delete --no-wait -y -n \
|| echo "No resource groups found to delete"
rest_endpoint=$(az cloud show --query "endpoints.resourceManager" -o tsv)
az rest --method GET \
--url "${rest_endpoint}/subscriptions/${subscription_id}/resourcegroups" \
--url-parameters api-version=2021-04-01 \$expand=createdTime \
--output json \
--query value \
| jq --arg date "$date" '.[] | select (.createdTime < $date).name' \
| grep '${{ parameters.name_pattern }}' \
| xargs -l -t -r az group delete --no-wait -y -n \
|| echo "No resource groups found to delete"
Loading

0 comments on commit 9390ce5

Please sign in to comment.