Add support for Azure Clouds

test-requirements.txt

@@ -19,3 +19,4 @@ azure-core
 azure-identity
 azure-mgmt-compute>=22.1.0
 azure-mgmt-resource>=15.0.0
+msrestazure

tests_e2e/orchestrator/lib/agent_test_loader.py

@@ -48,7 +48,7 @@ class VmImageInfo(object):
     # The URN of the image (publisher, offer, version separated by spaces)
     urn: str
     # Indicates that the image is available only on those locations. If empty, the image should be available in all locations
-    locations: List[str]
+    locations: Dict[str, List[str]]
     # Indicates that the image is available only for those VM sizes. If empty, the image should be available for all VM sizes
     vm_sizes: List[str]
 
@@ -109,8 +109,9 @@ def _validate(self):
             if suite.location != '':
                 for suite_image in suite.images:
                     for image in self.images[suite_image]:
-                        if len(image.locations) > 0:
-                            if suite.location not in image.locations:
+                        # If the image has a location restriction, validate that it is available on the location the suite must run on
+                        if image.locations:
+                            if not any(suite.location in l for l in image.locations.values()):
                                 raise Exception(f"Test suite {suite.name} must be executed in {suite.location}, but <{image.urn}> is not available in that location")
 
     @staticmethod
@@ -223,14 +224,18 @@ def _load_images() -> Dict[str, List[VmImageInfo]]:
             i = VmImageInfo()
             if isinstance(description, str):
                 i.urn = description
-                i.locations = []
+                i.locations = {}
                 i.vm_sizes = []
             else:
                 if "urn" not in description:
                     raise Exception(f"Image {name} is missing the 'urn' property: {description}")
                 i.urn = description["urn"]
-                i.locations = description["locations"] if "locations" in description else []
+                i.locations = description["locations"] if "locations" in description else {}
                 i.vm_sizes = description["vm_sizes"] if "vm_sizes" in description else []
+                for cloud in i.locations.keys():
+                    if cloud not in ["AzureCloud", "AzureChinaCloud", "AzureUSGovernment"]:
+                        raise Exception(f"Invalid cloud {cloud} for image {name} in images.yml")
+
             images[name] = [i]
 
         # now load the image-sets, mapping them to the images that we just computed

tests_e2e/orchestrator/lib/agent_test_suite.py

@@ -40,8 +40,7 @@
 )
 from lisa.environment import EnvironmentStatus  # pylint: disable=E0401
 from lisa.messages import TestStatus, TestResultMessage  # pylint: disable=E0401
-from lisa.sut_orchestrator import AZURE  # pylint: disable=E0401
-from lisa.sut_orchestrator.azure.common import get_node_context, AzureNodeSchema  # pylint: disable=E0401
+from lisa.sut_orchestrator.azure.common import get_node_context  # pylint: disable=E0401
 
 import makepkg
 from azurelinuxagent.common.version import AGENT_VERSION
@@ -133,11 +132,11 @@ def __init__(self, metadata: TestSuiteMetadata) -> None:
     def _initialize(self, node: Node, variables: Dict[str, Any], lisa_working_path: str, lisa_log_path: str, lisa_log: Logger):
         connection_info = node.connection_info
         node_context = get_node_context(node)
-        runbook = node.capability.get_extended_runbook(AzureNodeSchema, AZURE)
 
         self.__context = self._Context(
             vm=VmIdentifier(
-                location=runbook.location,
+                cloud=self._get_required_parameter(variables, "c_cloud"),
+                location=self._get_required_parameter(variables, "c_location"),
                 subscription=node.features._platform.subscription_id,
                 resource_group=node_context.resource_group_name,
                 name=node_context.vm_name),

tests_e2e/orchestrator/lib/agent_test_suite_combinator.py

@@ -98,9 +98,21 @@ def _next(self) -> Optional[Dict[str, Any]]:
         return result
 
     _DEFAULT_LOCATIONS = {
-        "china": "china north 2",
-        "government": "usgovarizona",
-        "public": "westus2"
+        "AzureCloud": "westus2",
+        "AzureChinaCloud": "chinanorth2",
+        "AzureUSGovernment": "usgovarizona",
+    }
+
+    _MARKETPLACE_IMAGE_INFORMATION_LOCATIONS = {
+        "AzureCloud": "",  # empty indicates the default location used by LISA
+        "AzureChinaCloud": "chinanorth2",
+        "AzureUSGovernment": "usgovarizona",
+    }
+
+    _SHARED_RESOURCE_GROUP_LOCATIONS = {
+        "AzureCloud": "",   # empty indicates the default location used by LISA
+        "AzureChinaCloud": "chinanorth2",
+        "AzureUSGovernment": "usgovarizona",
     }
 
     def create_environment_for_existing_vm(self) -> List[Dict[str, Any]]:
@@ -178,15 +190,23 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
                         raise Exception(f"Invalid URN: {image.urn}")
                     name = f"{match.group('offer')}-{match.group('sku')}"
 
-                # If the runbook specified a location, use it. Then try the suite location, if any. Otherwise, check if the image specifies
-                # a list of locations and use any of them. If no location is specified so far, use the default.
+                location: str = None
+                # If the runbook specified a location, use it.
                 if self.runbook.location != "":
                     location = self.runbook.location
+                #  Then try the suite location, if any.
                 elif suite_info.location != '':
                     location = suite_info.location
-                elif len(image.locations) > 0:
-                    location = image.locations[0]
-                else:
+                # If the image has a location restriction, use any location where it is available.
+                # However, if it is not available on any location, skip the image.
+                elif image.locations:
+                    image_locations = image.locations.get(self.runbook.cloud)
+                    if image_locations is not None:
+                        if len(image_locations) == 0:
+                            continue
+                        location = image_locations[0]
+                # If no location has been selected, use the default.
+                if location is None:
                     location = AgentTestSuitesCombinator._DEFAULT_LOCATIONS[self.runbook.cloud]
 
                 # If the runbook specified a VM size, use it. Else if the image specifies a list of VM sizes, use any of them. Otherwise,
@@ -202,11 +222,14 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
                     # create an environment for exclusive use by this suite
                     environment_list.append({
                         "c_marketplace_image": marketplace_image,
+                        "c_cloud": self.runbook.cloud,
                         "c_location": location,
                         "c_vm_size": vm_size,
                         "c_vhd": vhd,
                         "c_test_suites": [suite_info],
-                        "c_env_name": f"{name}-{suite_info.name}"
+                        "c_env_name": f"{name}-{suite_info.name}",
+                        "c_marketplace_image_information_location": self._MARKETPLACE_IMAGE_INFORMATION_LOCATIONS[self.runbook.cloud],
+                        "c_shared_resource_group_location": self._SHARED_RESOURCE_GROUP_LOCATIONS[self.runbook.cloud]
                     })
                 else:
                     # add this suite to the shared environments
@@ -216,27 +239,35 @@ def create_environment_list(self) -> List[Dict[str, Any]]:
                     else:
                         shared_environments[key] = {
                             "c_marketplace_image": marketplace_image,
+                            "c_cloud": self.runbook.cloud,
                             "c_location": location,
                             "c_vm_size": vm_size,
                             "c_vhd": vhd,
                             "c_test_suites": [suite_info],
-                            "c_env_name": key
+                            "c_env_name": key,
+                            "c_marketplace_image_information_location": self._MARKETPLACE_IMAGE_INFORMATION_LOCATIONS[self.runbook.cloud],
+                            "c_shared_resource_group_location": self._SHARED_RESOURCE_GROUP_LOCATIONS[self.runbook.cloud]
                         }
 
         environment_list.extend(shared_environments.values())
 
+        if len(environment_list) == 0:
+            raise Exception("No VM images were found to execute the test suites.")
+
         log: logging.Logger = logging.getLogger("lisa")
-        log.info("******** Environments *****")
-        for e in environment_list:
-            log.info(
-                "{ c_marketplace_image: '%s', c_location: '%s', c_vm_size: '%s', c_vhd: '%s', c_test_suites: '%s', c_env_name: '%s' }",
-                e['c_marketplace_image'], e['c_location'], e['c_vm_size'], e['c_vhd'], [s.name for s in e['c_test_suites']], e['c_env_name'])
+        log.info("")
+        log.info("******** Agent Test Environments *****")
+        for environment in environment_list:
+            test_suites = [s.name for s in environment['c_test_suites']]
+            log.info("Settings for %s:\n%s\n", environment['c_env_name'], '\n'.join([f"\t{name}: {value if name != 'c_test_suites' else test_suites}" for name, value in environment.items()]))
         log.info("***************************")
+        log.info("")
 
         return environment_list
 
     _URN = re.compile(r"(?P<publisher>[^\s:]+)[\s:](?P<offer>[^\s:]+)[\s:](?P<sku>[^\s:]+)[\s:](?P<version>[^\s:]+)")
 
+
     @staticmethod
     def _is_urn(urn: str) -> bool:
         # URNs can be given as '<Publisher> <Offer> <Sku> <Version>' or '<Publisher>:<Offer>:<Sku>:<Version>'

tests_e2e/orchestrator/runbook.yml

@@ -51,7 +51,7 @@ variable:
   - name: test_suites
     value: "agent_bvt"
   - name: cloud
-    value: "public"
+    value: "AzureCloud"
   - name: image
     value: ""
   - name: location
@@ -64,21 +64,26 @@ variable:
   # prefixed with "c_" to distinguish them from the rest of the variables, whose value can be set from
   # the command line.
   #
-  # c_marketplace_image, c_vm_size, c_location, and c_vhd are handled by LISA and define
-  # the set of test VMs that need to be created, while c_test_suites and c_env_name are parameters
-  # for the AgentTestSuite; the former defines the test suites that must be executed on each
-  # of those test VMs and the latter is the name of the environment, which is used for logging
-  # purposes (NOTE: the AgentTestSuite also uses c_vhd).
+  # Most of these variables are handled by LISA and are used to define the set of test VMs that need to be
+  # created. The variables marked with 'is_case_visible' are also referenced by the AgentTestSuite.
   #
   - name: c_env_name
     value: ""
     is_case_visible: true
   - name: c_marketplace_image
     value: ""
+  - name: c_marketplace_image_information_location
+    value: ""
+  - name: c_shared_resource_group_location
+    value: ""
   - name: c_vm_size
     value: ""
+  - name: c_cloud
+    value: ""
+    is_case_visible: true
   - name: c_location
     value: ""
+    is_case_visible: true
   - name: c_vhd
     value: ""
     is_case_visible: true
@@ -107,6 +112,12 @@ platform:
     keep_environment: $(keep_environment)
     azure:
       deploy: True
+#
+# TODO: Enable these parameters once LISA supports all Azure clouds
+#
+#      cloud: $(cloud)
+#      marketplace_image_information_location: $(c_marketplace_image_information_location)
+#      shared_resource_group_location: $(c_shared_resource_group_location)
       subscription_id: $(subscription_id)
       wait_delete: false
     requirement:

tests_e2e/orchestrator/sample_runbooks/existing_vm.yml

@@ -32,7 +32,7 @@ variable:
   # These variables identify the existing VM, and the user for SSH connections
   #
   - name: cloud
-    value: "public"
+    value: "AzureCloud"
   - name: subscription_id
     value: ""
   - name: resource_group_name
@@ -80,18 +80,24 @@ variable:
   # prefixed with "c_" to distinguish them from the rest of the variables, whose value can be set from
   # the command line.
   #
-  # c_marketplace_image, c_vm_size, c_location, and c_vhd are handled by LISA and define
-  # the set of test VMs that need to be created, while c_test_suites is a parameter
-  # for the AgentTestSuite and defines the test suites that must be executed on each
-  # of those test VMs (the AgentTestSuite also uses c_vhd)
+  # Most of these variables are handled by LISA and are used to define the set of test VMs that need to be
+  # created. The variables marked with 'is_case_visible' are also referenced by the AgentTestSuite.
   #
   - name: c_env_name
     value: ""
     is_case_visible: true
   - name: c_vm_name
     value: ""
+  - name: c_marketplace_image_information_location
+    value: ""
+  - name: c_shared_resource_group_location
+    value: ""
+  - name: c_cloud
+    value: ""
+    is_case_visible: true
   - name: c_location
     value: ""
+    is_case_visible: true
   - name: c_test_suites
     value: []
     is_case_visible: true
@@ -114,6 +120,12 @@ platform:
     admin_username: $(user)
     admin_private_key_file: $(identity_file)
     azure:
+#
+# TODO: Enable these parameters once LISA supports all Azure clouds
+#
+#      cloud: $(cloud)
+#      marketplace_image_information_location: $(c_marketplace_image_information_location)
+#      shared_resource_group_location: $(c_shared_resource_group_location)
       resource_group_name: $(resource_group_name)
       deploy: false
       subscription_id: $(subscription_id)

tests_e2e/pipeline/pipeline-cleanup.yml

@@ -1,58 +1,59 @@
 #
 # Pipeline for cleaning up any remaining Resource Groups generated by the Azure.WALinuxAgent pipeline.
 #
-# Deletes any resource groups that are more than a day old and contain string "lisa-WALinuxAgent-"
+# Deletes any resource groups that are older than 'older_than' and match the 'name_pattern' regular expression
 #
-schedules:
-  - cron: "0 */12 * * *" # Run twice a day (every 12 hours)
-    displayName: cleanup build
-    branches:
-      include:
-      - develop
-    always: true
 
-trigger:
-  - develop
+parameters:
+  - name: name_pattern
+    displayName: Regular expression to match the name of the resource groups to delete
+    type: string
+    default: lisa-WALinuxAgent-.*
 
-pr: none
+  - name: older_than
+    displayName: Delete resources older than (use the syntax of the "date -d" command)
+    type: string
+    default: 1 day ago
+
+  - name: service_connections
+    type: object
+    default:
+      - azuremanagement
+#
+# TODO: Enable these services connections once we create test pipelines for all Azure clouds
+#
+#      - azuremanagement.china
+#      - azuremanagement.government
 
 pool:
   vmImage: ubuntu-latest
 
-variables:
-  - name: azureConnection
-    value: 'azuremanagement'
-  - name: rgPrefix
-    value: 'lisa-WALinuxAgent-'
-
 steps:
+  - ${{ each service_connection in parameters.service_connections }}:
+    - task: AzureCLI@2
+      inputs:
+        azureSubscription: ${{ service_connection }}
+        scriptType: 'bash'
+        scriptLocation: 'inlineScript'
+        inlineScript: |
+          set -euxo pipefail
 
-  - task: AzureKeyVault@2
-    displayName: "Fetch secrets from KV"
-    inputs:
-      azureSubscription: '$(azureConnection)'
-      KeyVaultName: 'dcrV2SPs'
-      SecretsFilter: '*'
-      RunAsPreJob: true
+          #
+          # We use the REST API to list the resource groups because we need the createdTime and that
+          # property is not available via the az-cli commands.
+          #
+          subscription_id=$(az account list --all --query "[?isDefault].id" -o tsv)
+          
+          date=$(date --utc +%Y-%m-%d'T'%H:%M:%S.%N'Z' -d "${{ parameters.older_than }}")
 
-  - task: AzureCLI@2
-    inputs:
-      azureSubscription: '$(azureConnection)'
-      scriptType: 'bash'
-      scriptLocation: 'inlineScript'
-      inlineScript: |
-        set -euxo pipefail
-        date=`date --utc +%Y-%m-%d'T'%H:%M:%S.%N'Z' -d "1 day ago"`
-
-        # Using the Azure REST GET resourceGroups API call as we can add the createdTime to the results.
-        # This feature is not available via the az-cli commands directly so we have to use the Azure REST APIs
-
-        az rest --method GET \
-          --url "https://management.azure.com/subscriptions/$(SUBSCRIPTION-ID)/resourcegroups" \
-          --url-parameters api-version=2021-04-01 \$expand=createdTime \
-          --output json \
-          --query value \
-        | jq --arg date "$date" '.[] | select (.createdTime < $date).name' \
-        | grep "$(rgPrefix)" \
-        | xargs -l -t -r az group delete --no-wait -y -n \
-        || echo "No resource groups found to delete"
+          rest_endpoint=$(az cloud show --query "endpoints.resourceManager" -o tsv)
+
+          az rest --method GET \
+            --url "${rest_endpoint}/subscriptions/${subscription_id}/resourcegroups" \
+            --url-parameters api-version=2021-04-01 \$expand=createdTime \
+            --output json \
+            --query value \
+          | jq --arg date "$date" '.[] | select (.createdTime < $date).name' \
+          | grep '${{ parameters.name_pattern }}' \
+          | xargs -l -t -r az group delete --no-wait -y -n \
+          || echo "No resource groups found to delete"