Add support for VHDs; add image, location and vm_size as parameters to the pipeline

tests_e2e/orchestrator/lib/agent_test_suite.py

@@ -112,6 +112,7 @@ def __init__(self, vm: VmIdentifier, paths: AgentTestContext.Paths, connection:
             self.node: Node = None
             self.runbook_name: str = None
             self.image_name: str = None
+            self.is_vhd: bool = None
             self.test_suites: List[AgentTestSuite] = None
             self.collect_logs: str = None
             self.skip_setup: bool = None
@@ -146,8 +147,9 @@ def _set_context(self, node: Node, variables: Dict[str, Any], log: Logger):
 
         self.__context.log = log
         self.__context.node = node
-        self.__context.image_name = f"{runbook.marketplace.offer}-{runbook.marketplace.sku}"
-        self.__context.test_suites = self._get_required_parameter(variables, "test_suites_info")
+        self.__context.is_vhd = self._get_required_parameter(variables, "c_vhd") != ""
+        self.__context.image_name = f"{node.os.name}-vhd" if self.__context.is_vhd else f"{runbook.marketplace.offer}-{runbook.marketplace.sku}"
+        self.__context.test_suites = self._get_required_parameter(variables, "c_test_suites")
         self.__context.collect_logs = self._get_required_parameter(variables, "collect_logs")
         self.__context.skip_setup = self._get_required_parameter(variables, "skip_setup")
 
@@ -249,7 +251,10 @@ def _setup_node(self) -> None:
         self._log.info("Resource Group: %s", self.context.vm.resource_group)
         self._log.info("")
 
-        self._install_agent_on_node()
+        if self.context.is_vhd:
+            self._log.info("Using a VHD; will not install the test Agent.")
+        else:
+            self._install_agent_on_node()
 
     def _install_agent_on_node(self) -> None:
         """
@@ -292,7 +297,7 @@ def _collect_node_logs(self) -> None:
     @TestCaseMetadata(description="", priority=0)
     def agent_test_suite(self, node: Node, variables: Dict[str, Any], log: Logger) -> None:
         """
-        Executes each of the AgentTests included in "test_suites_info" variable (which is generated by the AgentTestSuitesCombinator).
+        Executes each of the AgentTests included in the "c_test_suites" variable (which is generated by the AgentTestSuitesCombinator).
         """
         self._set_context(node, variables, log)
 

tests_e2e/orchestrator/lib/agent_test_suite_combinator.py

@@ -1,6 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 import logging
+import re
+import urllib.parse
 
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional, Type
@@ -15,7 +17,7 @@
 from lisa.combinator import Combinator  # pylint: disable=E0401
 from lisa.util import field_metadata  # pylint: disable=E0401
 
-from tests_e2e.orchestrator.lib.agent_test_loader import AgentTestLoader
+from tests_e2e.orchestrator.lib.agent_test_loader import AgentTestLoader, VmImageInfo
 
 
 @dataclass_json()
@@ -24,26 +26,35 @@ class AgentTestSuitesCombinatorSchema(schema.Combinator):
     test_suites: str = field(
         default_factory=str, metadata=field_metadata(required=True)
     )
+    image: str = field(
+        default_factory=str, metadata=field_metadata(required=True)
+    )
+    location: str = field(
+        default_factory=str, metadata=field_metadata(required=True)
+    )
+    vm_size: str = field(
+        default_factory=str, metadata=field_metadata(required=True)
+    )
 
 
 class AgentTestSuitesCombinator(Combinator):
     """
     The "agent_test_suites" combinator returns a list of items containing five variables that specify the environments
     that the agent test suites must be executed on:
 
-        * marketplace_image: e.g. "Canonical UbuntuServer 18.04-LTS latest",
-        * location: e.g. "westus2",
-        * vm_size: e.g. "Standard_D2pls_v5"
-        * vhd: e.g "https://rhel.blob.core.windows.net/images/RHEL_8_Standard-8.3.202006170423.vhd?se=..."
-        * test_suites_info: e.g. [AgentBvt, FastTrack]
+        * c_marketplace_image: e.g. "Canonical UbuntuServer 18.04-LTS latest",
+        * c_location: e.g. "westus2",
+        * c_vm_size: e.g. "Standard_D2pls_v5"
+        * c_vhd: e.g "https://rhel.blob.core.windows.net/images/RHEL_8_Standard-8.3.202006170423.vhd?se=..."
+        * c_test_suites: e.g. [AgentBvt, FastTrack]
 
-    (marketplace_image, location, vm_size) and vhd are mutually exclusive and define the environment (i.e. the test VM)
-    in which the test will be executed. test_suites_info defines the test suites that should be executed in that
+    (c_marketplace_image, c_location, c_vm_size) and vhd are mutually exclusive and define the environment (i.e. the test VM)
+    in which the test will be executed. c_test_suites defines the test suites that should be executed in that
     environment.
     """
     def __init__(self, runbook: AgentTestSuitesCombinatorSchema) -> None:
         super().__init__(runbook)
-        self._environments = self.create_environment_list(self.runbook.test_suites)
+        self._environments = self.create_environment_list()
         self._index = 0
 
     @classmethod
@@ -63,47 +74,87 @@ def _next(self) -> Optional[Dict[str, Any]]:
 
     _DEFAULT_LOCATION = "westus2"
 
-    @staticmethod
-    def create_environment_list(test_suites: str) -> List[Dict[str, Any]]:
+    def create_environment_list(self) -> List[Dict[str, Any]]:
+        loader = AgentTestLoader(self.runbook.test_suites)
+
+        #
+        # If the runbook provides any of 'image', 'location', or 'vm_size', those values
+        # override any configuration values on the test suite.
+        #
+        # Check 'images' first and add them to 'runbook_images', if any
+        #
+        if self.runbook.image == "":
+            runbook_images = []
+        else:
+            runbook_images = loader.images.get(self.runbook.image)
+            if runbook_images is None:
+                if not self._is_urn(self.runbook.image) and not self._is_vhd(self.runbook.image):
+                    raise Exception(f"The 'image' parameter must be an image or image set name, a urn, or a vhd: {self.runbook.image}")
+                i = VmImageInfo()
+                i.urn = self.runbook.image  # Note that this could be a URN or the URI for a VHD
+                i.locations = []
+                i.vm_sizes = []
+                runbook_images = [i]
+
+        #
+        # Now walk through all the test_suites and create a list of the environments (test VMs) that need to be created.
+        #
         environment_list: List[Dict[str, Any]] = []
         shared_environments: Dict[str, Dict[str, Any]] = {}
 
-        loader = AgentTestLoader(test_suites)
-
         for suite_info in loader.test_suites:
-            images_info = loader.images[suite_info.images]
+            images_info = runbook_images if len(runbook_images) > 0 else loader.images[suite_info.images]
+
             for image in images_info:
-                # If the suite specifies a location, use it. Else, if the image specifies a list of locations, use
-                # any of them. Otherwise, use the default location.
-                if suite_info.location != '':
+                # The URN can be a VHD if the runbook provided a VHD in the 'images' parameter
+                if self._is_vhd(image.urn):
+                    marketplace_image = ""
+                    vhd = image.urn
+                else:
+                    marketplace_image = image.urn
+                    vhd = ""
+
+                # If the runbook specified a location, use it. Then try the suite location, if any. Otherwise, check if the image specifies
+                # a list of locations and use any of them. If no location is specified so far, use the default.
+                if self.runbook.location != "":
+                    location = self.runbook.location
+                elif suite_info.location != '':
                     location = suite_info.location
                 elif len(image.locations) > 0:
                     location = image.locations[0]
                 else:
                     location = AgentTestSuitesCombinator._DEFAULT_LOCATION
 
-                # If the image specifies a list of VM sizes, use any of them. Otherwise, set the size to empty and let LISA choose it.
-                vm_size = image.vm_sizes[0] if len(image.vm_sizes) > 0 else ""
+                # If the runbook specified a VM size, use it. Else if the image specifies a list of VM sizes, use any of them. Otherwise,
+                # set the size to empty and let LISA choose it.
+                if self.runbook.vm_size != '':
+                    vm_size = self.runbook.vm_size
+                elif len(image.vm_sizes) > 0:
+                    vm_size = image.vm_sizes[0]
+                else:
+                    vm_size = ""
 
                 if suite_info.owns_vm:
+                    # create an environment for exclusive use by this suite
                     environment_list.append({
-                        "marketplace_image": image.urn,
-                        "location": location,
-                        "vm_size": vm_size,
-                        "vhd": "",
-                        "test_suites_info": [suite_info]
+                        "c_marketplace_image": marketplace_image,
+                        "c_location": location,
+                        "c_vm_size": vm_size,
+                        "c_vhd": vhd,
+                        "c_test_suites": [suite_info]
                     })
                 else:
+                    # add this suite to the shared environments
                     key: str = f"{image.urn}:{location}"
                     if key in shared_environments:
-                        shared_environments[key]["test_suites_info"].append(suite_info)
+                        shared_environments[key]["c_test_suites"].append(suite_info)
                     else:
                         shared_environments[key] = {
-                            "marketplace_image": image.urn,
-                            "location": location,
-                            "vm_size": vm_size,
-                            "vhd": "",
-                            "test_suites_info": [suite_info]
+                            "c_marketplace_image": marketplace_image,
+                            "c_location": location,
+                            "c_vm_size": vm_size,
+                            "c_vhd": vhd,
+                            "c_test_suites": [suite_info]
                         }
 
         environment_list.extend(shared_environments.values())
@@ -112,8 +163,19 @@ def create_environment_list(test_suites: str) -> List[Dict[str, Any]]:
         log.info("******** Environments *****")
         for e in environment_list:
             log.info(
-                "{ marketplace_image: '%s', location: '%s', vm_size: '%s', vhd: '%s', test_suites_info: '%s' }",
-                e['marketplace_image'], e['location'], e['vm_size'], e['vhd'], [s.name for s in e['test_suites_info']])
+                "{ c_marketplace_image: '%s', c_location: '%s', c_vm_size: '%s', c_vhd: '%s', c_test_suites: '%s' }",
+                e['c_marketplace_image'], e['c_location'], e['c_vm_size'], e['c_vhd'], [s.name for s in e['c_test_suites']])
         log.info("***************************")
 
         return environment_list
+
+    @staticmethod
+    def _is_urn(urn: str) -> bool:
+        # URNs can be given as '<Publisher> <Offer> <Sku> <Version>' or '<Publisher>:<Offer>:<Sku>:<Version>'
+        return re.match(r"(\S+\s\S+\s\S+\s\S+)|([^:]+:[^:]+:[^:]+:[^:]+)", urn) is not None
+
+    @staticmethod
+    def _is_vhd(vhd: str) -> bool:
+        # VHDs are given as URIs to storage; do some basic validation, not intending to be exhaustive.
+        parsed = urllib.parse.urlparse(vhd)
+        return parsed.scheme == 'https' and parsed.netloc != "" and parsed.path != ""

tests_e2e/orchestrator/runbook.yml

@@ -9,7 +9,7 @@ extension:
 
 variable:
   #
-  # These variables define runbook parameters; they are handled by LISA.
+  # These variables define parameters handled by LISA.
   #
   - name: subscription_id
     value: ""
@@ -26,6 +26,9 @@ variable:
   #
   # These variables define parameters for the AgentTestSuite; see the test wiki for details.
   #
+  # NOTE: c_test_suites, generated by the AgentTestSuitesCombinator, is also a parameter
+  #       for the AgentTestSuite
+  #
   # Whether to collect logs from the test VM
   - name: collect_logs
     value: "failed"
@@ -37,25 +40,38 @@ variable:
     is_case_visible: true
 
   #
-  # These variables parameters for the AgentTestSuitesCombinator combinator
+  # These variables are parameters for the AgentTestSuitesCombinator
   #
   # The test suites to execute
   - name: test_suites
     value: "agent_bvt"
-    is_case_visible: true
+  - name: image
+    value: ""
+  - name: location
+    value: ""
+  - name: vm_size
+    value: ""
 
   #
-  # These variables are set by the AgentTestSuitesCombinator combinator
+  # The values for these variables are generated by the AgentTestSuitesCombinator combinator. They are
+  # prefixed with "c_" to distinguish them from the rest of the variables, whose value can be set from
+  # the command line.
+  #
+  # c_marketplace_image, c_vm_size, c_location, and c_vhd are handled by LISA and define
+  # the set of test VMs that need to be created, while c_test_suites is a parameter
+  # for the AgentTestSuite and defines the test suites that must be executed on each
+  # of those test VMs (the AgentTestSuite also uses c_vhd)
   #
-  - name: marketplace_image
+  - name: c_marketplace_image
     value: ""
-  - name: vm_size
+  - name: c_vm_size
     value: ""
-  - name: location
+  - name: c_location
     value: ""
-  - name: vhd
+  - name: c_vhd
     value: ""
-  - name: test_suites_info
+    is_case_visible: true
+  - name: c_test_suites
     value: []
     is_case_visible: true
 
@@ -86,14 +102,17 @@ platform:
       core_count:
         min: 2
       azure:
-        marketplace: $(marketplace_image)
-        vhd: $(vhd)
-        location: $(location)
-        vm_size: $(vm_size)
+        marketplace: $(c_marketplace_image)
+        vhd: $(c_vhd)
+        location: $(c_location)
+        vm_size: $(c_vm_size)
 
 combinator:
   type: agent_test_suites
   test_suites: $(test_suites)
+  image: $(image)
+  location: $(location)
+  vm_size: $(vm_size)
 
 concurrency: 10
 

tests_e2e/pipeline/pipeline.yml

@@ -1,10 +1,31 @@
 parameters:
-    # see the test wiki for a description of the parameters
+    # See the test wiki for a description of the parameters
   - name: test_suites
     displayName: Test Suites
     type: string
     default: agent_bvt
 
+    # NOTES:
+    #         * 'image', 'location' and 'vm_size' override any values in the test suites/images definition
+    #            files. Those parameters are useful for 1-off tests, like testing a VHD or checking if
+    #            an image is supported in a particular location.
+    #         * Azure Pipelines do not allow empty string for the parameter value, using "-" instead.
+    #
+  - name: image
+    displayName: Image (image/image set name, URN, or VHD)
+    type: string
+    default: "-"
+
+  - name: location
+    displayName: Location (region)
+    type: string
+    default: "-"
+
+  - name: vm_size
+    displayName: VM size
+    type: string
+    default: "-"
+
   - name: collect_logs
     displayName: Collect logs from test VMs
     type: string
@@ -26,8 +47,15 @@ parameters:
 variables:
   - name: azureConnection
     value: 'azuremanagement'
+    # These variables exposed the above parameters as environment variables
   - name: test_suites
     value: ${{ parameters.test_suites }}
+  - name: image
+    value: ${{ parameters.image }}
+  - name: location
+    value: ${{ parameters.location }}
+  - name: vm_size
+    value: ${{ parameters.vm_size }}
   - name: collect_logs
     value: ${{ parameters.collect_logs }}
   - name: keep_environment