diff --git a/azurelinuxagent/common/protocol/restapi.py b/azurelinuxagent/common/protocol/restapi.py
index 5497e76465..08b25c6512 100644
--- a/azurelinuxagent/common/protocol/restapi.py
+++ b/azurelinuxagent/common/protocol/restapi.py
@@ -151,18 +151,19 @@ def __init__(self,
sequenceNumber=None,
publicSettings=None,
protectedSettings=None,
- certificateThumbprint=None):
+ certificateThumbprint=None,
+ dependencyLevel=0):
self.name = name
self.sequenceNumber = sequenceNumber
self.publicSettings = publicSettings
self.protectedSettings = protectedSettings
self.certificateThumbprint = certificateThumbprint
+ self.dependencyLevel = dependencyLevel
class ExtHandlerProperties(DataContract):
def __init__(self):
self.version = None
- self.dependencyLevel = None
self.state = None
self.extensions = DataContractList(Extension)
@@ -179,9 +180,11 @@ def __init__(self, name=None):
self.versionUris = DataContractList(ExtHandlerVersionUri)
def sort_key(self):
- level = self.properties.dependencyLevel
- if level is None:
+ levels = [e.dependencyLevel for e in self.properties.extensions]
+ if len(levels) == 0:
level = 0
+ else:
+ level = min(levels)
# Process uninstall or disabled before enabled, in reverse order
# remap 0 to -1, 1 to -2, 2 to -3, etc
if self.properties.state != u"enabled":
diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py
index 752f62c601..de39fa37d1 100644
--- a/azurelinuxagent/common/protocol/wire.py
+++ b/azurelinuxagent/common/protocol/wire.py
@@ -1591,11 +1591,6 @@ def parse_plugin(self, plugin):
ext_handler.properties.version = getattrib(plugin, "version")
ext_handler.properties.state = getattrib(plugin, "state")
- try:
- ext_handler.properties.dependencyLevel = int(getattrib(plugin, "dependencyLevel"))
- except ValueError:
- ext_handler.properties.dependencyLevel = 0
-
location = getattrib(plugin, "location")
failover_location = getattrib(plugin, "failoverlocation")
for uri in [location, failover_location]:
@@ -1627,6 +1622,15 @@ def parse_plugin_settings(self, ext_handler, plugin_settings):
logger.error("Invalid extension settings")
return
+ depends_on_level = 0
+ depends_on_node = find(settings[0], "DependsOn")
+ if depends_on_node != None:
+ try:
+ depends_on_level = int(getattrib(depends_on_node, "dependencyLevel"))
+ except (ValueError, TypeError):
+ logger.warn("Could not parse dependencyLevel for handler {0}. Setting it to 0".format(name))
+ depends_on_level = 0
+
for plugin_settings_list in runtime_settings["runtimeSettings"]:
handler_settings = plugin_settings_list["handlerSettings"]
ext = Extension()
@@ -1636,6 +1640,7 @@ def parse_plugin_settings(self, ext_handler, plugin_settings):
ext.sequenceNumber = seqNo
ext.publicSettings = handler_settings.get("publicSettings")
ext.protectedSettings = handler_settings.get("protectedSettings")
+ ext.dependencyLevel = depends_on_level
thumbprint = handler_settings.get(
"protectedSettingsCertThumbprint")
ext.certificateThumbprint = thumbprint
diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py
index 497f8d9776..74109ddeb9 100644
--- a/azurelinuxagent/ga/exthandlers.py
+++ b/azurelinuxagent/ga/exthandlers.py
@@ -57,7 +57,9 @@
HANDLER_ENVIRONMENT_VERSION = 1.0
EXTENSION_STATUS_ERROR = 'error'
+EXTENSION_STATUS_SUCCESS = 'success'
VALID_EXTENSION_STATUS = ['transitioning', 'error', 'success', 'warning']
+EXTENSION_TERMINAL_STATUSES = ['error', 'success']
VALID_HANDLER_STATUS = ['Ready', 'NotReady', "Installing", "Unresponsive"]
@@ -66,6 +68,7 @@
HANDLER_PKG_EXT = ".zip"
HANDLER_PKG_PATTERN = re.compile(HANDLER_PATTERN + r"\.zip$", re.IGNORECASE)
+DEFAULT_EXT_TIMEOUT_MINUTES = 90
def validate_has_key(obj, key, fullname):
if key not in obj:
@@ -120,9 +123,7 @@ def parse_ext_status(ext_status, data):
ext_status.code = status_data.get('code', 0)
formatted_message = status_data.get('formattedMessage')
ext_status.message = parse_formatted_message(formatted_message)
- substatus_list = status_data.get('substatus')
- if substatus_list is None:
- return
+ substatus_list = status_data.get('substatus', [])
for substatus in substatus_list:
if substatus is not None:
ext_status.substatusList.append(parse_ext_substatus(substatus))
@@ -310,11 +311,62 @@ def handle_ext_handlers(self, etag=None):
logger.info("Extension handling is on hold")
return
+ wait_until = datetime.datetime.utcnow() + datetime.timedelta(minutes=DEFAULT_EXT_TIMEOUT_MINUTES)
+ max_dep_level = max([handler.sort_key() for handler in self.ext_handlers.extHandlers])
+
self.ext_handlers.extHandlers.sort(key=operator.methodcaller('sort_key'))
for ext_handler in self.ext_handlers.extHandlers:
- # TODO: handle install in sequence, enable in parallel
self.handle_ext_handler(ext_handler, etag)
-
+
+ # Wait for the extension installation until it is handled.
+ # This is done for the install and enable. Not for the uninstallation.
+ # If handled successfully, proceed with the current handler.
+ # Otherwise, skip the rest of the extension installation.
+ dep_level = ext_handler.sort_key()
+ if dep_level >= 0 and dep_level < max_dep_level:
+ if not self.wait_for_handler_successful_completion(ext_handler, wait_until):
+ logger.warn("An extension failed or timed out, will skip processing the rest of the extensions")
+ break
+
+ def wait_for_handler_successful_completion(self, ext_handler, wait_until):
+ '''
+ Check the status of the extension being handled.
+ Wait until it has a terminal state or times out.
+ Return True if it is handled successfully. False if not.
+ '''
+ handler_i = ExtHandlerInstance(ext_handler, self.protocol)
+ for ext in ext_handler.properties.extensions:
+ ext_completed, status = handler_i.is_ext_handling_complete(ext)
+
+ # Keep polling for the extension status until it becomes success or times out
+ while not ext_completed and datetime.datetime.utcnow() <= wait_until:
+ time.sleep(5)
+ ext_completed, status = handler_i.is_ext_handling_complete(ext)
+
+ # In case of timeout or terminal error state, we log it and return false
+ # so that the extensions waiting on this one can be skipped processing
+ if datetime.datetime.utcnow() > wait_until:
+ msg = "Extension {0} did not reach a terminal state within the allowed timeout. Last status was {1}".format(ext.name, status)
+ logger.warn(msg)
+ add_event(AGENT_NAME,
+ version=CURRENT_VERSION,
+ op=WALAEventOperation.ExtensionProcessing,
+ is_success=False,
+ message=msg)
+ return False
+
+ if status != EXTENSION_STATUS_SUCCESS:
+ msg = "Extension {0} did not succeed. Status was {1}".format(ext.name, status)
+ logger.warn(msg)
+ add_event(AGENT_NAME,
+ version=CURRENT_VERSION,
+ op=WALAEventOperation.ExtensionProcessing,
+ is_success=False,
+ message=msg)
+ return False
+
+ return True
+
def handle_ext_handler(self, ext_handler, etag):
ext_handler_i = ExtHandlerInstance(ext_handler, self.protocol)
@@ -871,6 +923,35 @@ def collect_ext_status(self, ext):
return ext_status
+ def get_ext_handling_status(self, ext):
+ seq_no, ext_status_file = self.get_status_file_path(ext)
+ if seq_no < 0 or ext_status_file is None:
+ return None
+
+ # Missing status file is considered a non-terminal state here
+ # so that extension sequencing can wait until it becomes existing
+ if not os.path.exists(ext_status_file):
+ status = "warning"
+ else:
+ ext_status = self.collect_ext_status(ext)
+ status = ext_status.status if ext_status is not None else None
+
+ return status
+
+ def is_ext_handling_complete(self, ext):
+ status = self.get_ext_handling_status(ext)
+
+ # when seq < 0 (i.e. no new user settings), the handling is complete and return None status
+ if status is None:
+ return (True, None)
+
+ # If not in terminal state, it is incomplete
+ if status not in EXTENSION_TERMINAL_STATUSES:
+ return (False, status)
+
+ # Extension completed, return its status
+ return (True, status)
+
def report_ext_status(self):
active_exts = []
# TODO Refactor or remove this common code pattern (for each extension subordinate to an ext_handler, do X).
diff --git a/tests/data/wire/ext_conf_sequencing.xml b/tests/data/wire/ext_conf_sequencing.xml
index 7f4c9bc82d..3120a979b7 100644
--- a/tests/data/wire/ext_conf_sequencing.xml
+++ b/tests/data/wire/ext_conf_sequencing.xml
@@ -15,14 +15,19 @@
-
-
+
+
+
+
+
{"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3","protectedSettings":"MIICWgYJK","publicSettings":{"foo":"bar"}}}]}
+
+
{"runtimeSettings":[{"handlerSettings":{"protectedSettingsCertThumbprint":"4037FBF5F1F3014F99B5D6C7799E9B20E6871CB3","protectedSettings":"MIICWgYJK","publicSettings":{"foo":"bar"}}}]}
diff --git a/tests/ga/test_extension.py b/tests/ga/test_extension.py
index 8940419d3a..e77d68ec5f 100644
--- a/tests/ga/test_extension.py
+++ b/tests/ga/test_extension.py
@@ -18,6 +18,8 @@
import os.path
from tests.protocol.mockwiredata import *
+
+from azurelinuxagent.common.protocol.restapi import Extension
from azurelinuxagent.ga.exthandlers import *
from azurelinuxagent.common.protocol.wire import WireProtocol
@@ -409,8 +411,8 @@ def test_ext_handler_sequencing(self, *args):
self.assertTrue(exthandlers_handler.ext_handlers is not None)
self.assertTrue(exthandlers_handler.ext_handlers.extHandlers is not None)
self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2)
- self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.dependencyLevel, 1)
- self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.dependencyLevel, 2)
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 1)
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 2)
#Test goal state not changed
exthandlers_handler.run()
@@ -422,6 +424,7 @@ def test_ext_handler_sequencing(self, *args):
"2<")
test_data.ext_conf = test_data.ext_conf.replace("seqNo=\"0\"",
"seqNo=\"1\"")
+ # Swap the dependency ordering
test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"2\"",
"dependencyLevel=\"3\"")
test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"1\"",
@@ -431,10 +434,13 @@ def test_ext_handler_sequencing(self, *args):
self._assert_ext_status(protocol.report_ext_status, "success", 1)
self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2)
- self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.dependencyLevel, 3)
- self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.dependencyLevel, 4)
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 3)
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 4)
#Test disable
+ # In the case of disable, the last extension to be enabled should be
+ # the first extension disabled. The first extension enabled should be
+ # the last one disabled.
test_data.goal_state = test_data.goal_state.replace("2<",
"3<")
test_data.ext_conf = test_data.ext_conf.replace("enabled", "disabled")
@@ -443,13 +449,17 @@ def test_ext_handler_sequencing(self, *args):
1, "1.0.0",
expected_handler_name="OSTCExtensions.OtherExampleHandlerLinux")
self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2)
- self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.dependencyLevel, 4)
- self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.dependencyLevel, 3)
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 4)
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 3)
#Test uninstall
+ # In the case of uninstall, the last extension to be installed should be
+ # the first extension uninstalled. The first extension installed
+ # should be the last one uninstalled.
test_data.goal_state = test_data.goal_state.replace("3<",
"4<")
test_data.ext_conf = test_data.ext_conf.replace("disabled", "uninstall")
+ # Swap the dependency ordering AGAIN
test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"3\"",
"dependencyLevel=\"6\"")
test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"4\"",
@@ -457,8 +467,32 @@ def test_ext_handler_sequencing(self, *args):
exthandlers_handler.run()
self._assert_no_handler_status(protocol.report_vm_status)
self.assertEqual(len(exthandlers_handler.ext_handlers.extHandlers), 2)
- self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.dependencyLevel, 6)
- self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.dependencyLevel, 5)
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 6)
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[1].properties.extensions[0].dependencyLevel, 5)
+
+ def test_ext_handler_sequencing_default_dependency_level(self, *args):
+ test_data = WireProtocolData(DATA_FILE)
+ exthandlers_handler, protocol = self._create_mock(test_data, *args)
+ exthandlers_handler.run()
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0)
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0)
+
+ def test_ext_handler_sequencing_invalid_dependency_level(self, *args):
+ test_data = WireProtocolData(DATA_FILE_EXT_SEQUENCING)
+ exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+ test_data.goal_state = test_data.goal_state.replace("1<",
+ "2<")
+ test_data.ext_conf = test_data.ext_conf.replace("seqNo=\"0\"",
+ "seqNo=\"1\"")
+ test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"1\"",
+ "dependencyLevel=\"a6\"")
+ test_data.ext_conf = test_data.ext_conf.replace("dependencyLevel=\"2\"",
+ "dependencyLevel=\"5b\"")
+ exthandlers_handler.run()
+
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0)
+ self.assertEqual(exthandlers_handler.ext_handlers.extHandlers[0].properties.extensions[0].dependencyLevel, 0)
def test_ext_handler_rollingupgrade(self, *args):
test_data = WireProtocolData(DATA_FILE_EXT_ROLLINGUPGRADE)
@@ -666,6 +700,164 @@ def test_ext_handler_no_reporting_status(self, *args):
self._assert_handler_status(protocol.report_vm_status, "Ready", 1, "1.0.0")
self._assert_ext_status(protocol.report_ext_status, "error", 0)
+ def test_wait_for_handler_successful_completion_empty_exts(self, *args):
+ '''
+ Testing wait_for_handler_successful_completion() when there is no extension in a handler.
+ Expected to return True.
+ '''
+ test_data = WireProtocolData(DATA_FILE)
+ exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+ handler = ExtHandler(name="handler")
+
+ ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=None)
+ self.assertTrue(exthandlers_handler.wait_for_handler_successful_completion(handler, datetime.datetime.utcnow()))
+
+ def _helper_wait_for_handler_successful_completion(self, exthandlers_handler):
+ '''
+ Call wait_for_handler_successful_completion() passing a handler with an extension.
+ Override the wait time to be 5 seconds to minimize the timout duration.
+ Return the value returned by wait_for_handler_successful_completion().
+ '''
+ handler_name = "Handler"
+ exthandler = ExtHandler(name=handler_name)
+ extension = Extension(name=handler_name)
+ exthandler.properties.extensions.append(extension)
+
+ # Override the timeout value to minimize the test duration
+ wait_until = datetime.datetime.utcnow() + datetime.timedelta(seconds=5)
+ return exthandlers_handler.wait_for_handler_successful_completion(exthandler, wait_until)
+
+ def test_wait_for_handler_successful_completion_no_status(self, *args):
+ '''
+ Testing wait_for_handler_successful_completion() when there is no status file or seq_no is negative.
+ Expected to return False.
+ '''
+ test_data = WireProtocolData(DATA_FILE)
+ exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+ ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=None)
+ self.assertFalse(self._helper_wait_for_handler_successful_completion(exthandlers_handler))
+
+ def test_wait_for_handler_successful_completion_success_status(self, *args):
+ '''
+ Testing wait_for_handler_successful_completion() when there is successful status.
+ Expected to return True.
+ '''
+ test_data = WireProtocolData(DATA_FILE)
+ exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+ status = "success"
+
+ ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=status)
+ self.assertTrue(self._helper_wait_for_handler_successful_completion(exthandlers_handler))
+
+ def test_wait_for_handler_successful_completion_error_status(self, *args):
+ '''
+ Testing wait_for_handler_successful_completion() when there is error status.
+ Expected to return False.
+ '''
+ test_data = WireProtocolData(DATA_FILE)
+ exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+ status = "error"
+
+ ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=status)
+ self.assertFalse(self._helper_wait_for_handler_successful_completion(exthandlers_handler))
+
+ def test_wait_for_handler_successful_completion_timeout(self, *args):
+ '''
+ Testing wait_for_handler_successful_completion() when there is non terminal status.
+ Expected to return False.
+ '''
+ test_data = WireProtocolData(DATA_FILE)
+ exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+ # Choose a non-terminal status
+ status = "warning"
+
+ ExtHandlerInstance.get_ext_handling_status = MagicMock(return_value=status)
+ self.assertFalse(self._helper_wait_for_handler_successful_completion(exthandlers_handler))
+
+ def test_get_ext_handling_status(self, *args):
+ '''
+ Testing get_ext_handling_status() function with various cases and
+ verifying against the expected values
+ '''
+ test_data = WireProtocolData(DATA_FILE)
+ exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+ handler_name = "Handler"
+ exthandler = ExtHandler(name=handler_name)
+ extension = Extension(name=handler_name)
+ exthandler.properties.extensions.append(extension)
+
+ # In the following list of test cases, the first element corresponds to seq_no.
+ # the second element is the status file name, the third element indicates if the status file exits or not.
+ # The fourth element is the expected value from get_ext_handling_status()
+ test_cases = [
+ [-5, None, False, None],
+ [-1, None, False, None],
+ [0, None, False, None],
+ [0, "filename", False, "warning"],
+ [0, "filename", True, ExtensionStatus(status="success")],
+ [5, "filename", False, "warning"],
+ [5, "filename", True, ExtensionStatus(status="success")]
+ ]
+
+ orig_state = os.path.exists
+ for case in test_cases:
+ ext_handler_i = ExtHandlerInstance(exthandler, protocol)
+ ext_handler_i.get_status_file_path = MagicMock(return_value=(case[0], case[1]))
+ os.path.exists = MagicMock(return_value=case[2])
+ if case[2]:
+ # when the status file exists, it is expected return the value from collect_ext_status()
+ ext_handler_i.collect_ext_status= MagicMock(return_value=case[3])
+
+ status = ext_handler_i.get_ext_handling_status(extension)
+ if case[2]:
+ self.assertEqual(status, case[3].status)
+ else:
+ self.assertEqual(status, case[3])
+
+ os.path.exists = orig_state
+
+ def test_is_ext_handling_complete(self, *args):
+ '''
+ Testing is_ext_handling_complete() with various input and
+ verifying against the expected output values.
+ '''
+ test_data = WireProtocolData(DATA_FILE)
+ exthandlers_handler, protocol = self._create_mock(test_data, *args)
+
+ handler_name = "Handler"
+ exthandler = ExtHandler(name=handler_name)
+ extension = Extension(name=handler_name)
+ exthandler.properties.extensions.append(extension)
+
+ ext_handler_i = ExtHandlerInstance(exthandler, protocol)
+
+ # Testing no status case
+ ext_handler_i.get_ext_handling_status = MagicMock(return_value=None)
+ completed, status = ext_handler_i.is_ext_handling_complete(extension)
+ self.assertTrue(completed)
+ self.assertEqual(status, None)
+
+ # Here the key represents the possible input value to is_ext_handling_complete()
+ # the value represents the output tuple from is_ext_handling_complete()
+ expected_results = {
+ "error": (True, "error"),
+ "success": (True, "success"),
+ "warning": (False, "warning"),
+ "transitioning": (False, "transitioning")
+ }
+
+ for key in expected_results.keys():
+ ext_handler_i.get_ext_handling_status = MagicMock(return_value=key)
+ completed, status = ext_handler_i.is_ext_handling_complete(extension)
+ self.assertEqual(completed, expected_results[key][0])
+ self.assertEqual(status, expected_results[key][1])
+
def test_ext_handler_version_decide_autoupgrade_internalversion(self, *args):
for internal in [False, True]:
for autoupgrade in [False, True]:
@@ -917,6 +1109,180 @@ def test_upgrade(self, patch_get_update_command, *args):
self._assert_handler_status(protocol.report_vm_status, "NotReady", expected_ext_count=1, version="1.0.1")
+@patch("azurelinuxagent.common.protocol.wire.CryptUtil")
+@patch("azurelinuxagent.common.utils.restutil.http_get")
+class TestExtensionSequencing(AgentTestCase):
+
+ def _create_mock(self, mock_http_get, MockCryptUtil):
+ test_data = WireProtocolData(DATA_FILE)
+
+ #Mock protocol to return test data
+ mock_http_get.side_effect = test_data.mock_http_get
+ MockCryptUtil.side_effect = test_data.mock_crypt_util
+
+ protocol = WireProtocol("foo.bar")
+ protocol.detect()
+ protocol.report_ext_status = MagicMock()
+ protocol.report_vm_status = MagicMock()
+ protocol.get_artifacts_profile = MagicMock()
+
+ handler = get_exthandlers_handler()
+ handler.protocol_util.get_protocol = Mock(return_value=protocol)
+ handler.ext_handlers, handler.last_etag = protocol.get_ext_handlers()
+ conf.get_enable_overprovisioning = Mock(return_value=False)
+
+ def wait_for_handler_successful_completion(prev_handler, wait_until):
+ return orig_wait_for_handler_successful_completion(prev_handler, datetime.datetime.utcnow() + datetime.timedelta(seconds=5))
+
+ orig_wait_for_handler_successful_completion = handler.wait_for_handler_successful_completion
+ handler.wait_for_handler_successful_completion = wait_for_handler_successful_completion
+ return handler
+
+ def _set_dependency_levels(self, dependency_levels, exthandlers_handler):
+ '''
+ Creates extensions with the given dependencyLevel
+ '''
+ handler_map = dict()
+ all_handlers = []
+ for h, level in dependency_levels:
+ if handler_map.get(h) is None:
+ handler = ExtHandler(name=h)
+ extension = Extension(name=h)
+ handler.properties.state = "enabled"
+ handler.properties.extensions.append(extension)
+ handler_map[h] = handler
+ all_handlers.append(handler)
+
+ handler = handler_map[h]
+ for ext in handler.properties.extensions:
+ ext.dependencyLevel = level
+
+ exthandlers_handler.ext_handlers.extHandlers = []
+ for handler in all_handlers:
+ exthandlers_handler.ext_handlers.extHandlers.append(handler)
+
+ def _validate_extension_sequence(self, expected_sequence, exthandlers_handler):
+ installed_extensions = [a[0].name for a, k in exthandlers_handler.handle_ext_handler.call_args_list]
+ self.assertListEqual(expected_sequence, installed_extensions, "Expected and actual list of extensions are not equal")
+
+ def _run_test(self, extensions_to_be_failed, expected_sequence, exthandlers_handler):
+ '''
+ Mocks get_ext_handling_status() to mimic error status for a given extension.
+ Calls ExtHandlersHandler.run()
+ Verifies if the ExtHandlersHandler.handle_ext_handler() was called with appropriate extensions
+ in the expected order.
+ '''
+
+ def get_ext_handling_status(ext):
+ status = "error" if ext.name in extensions_to_be_failed else "success"
+ return status
+
+ ExtHandlerInstance.get_ext_handling_status = MagicMock(side_effect = get_ext_handling_status)
+ exthandlers_handler.handle_ext_handler = MagicMock()
+ exthandlers_handler.run()
+ self._validate_extension_sequence(expected_sequence, exthandlers_handler)
+
+ def test_handle_ext_handlers(self, *args):
+ '''
+ Tests extension sequencing among multiple extensions with dependencies.
+ This test introduces failure in all possible levels and extensions.
+ Verifies that the sequencing is in the expected order and a failure in one extension
+ skips the rest of the extensions in the sequence.
+ '''
+ exthandlers_handler = self._create_mock(*args)
+
+ self._set_dependency_levels([("A", 3), ("B", 2), ("C", 2), ("D", 1), ("E", 1), ("F", 1), ("G", 1)],
+ exthandlers_handler)
+
+ extensions_to_be_failed = []
+ expected_sequence = ["D", "E", "F", "G", "B", "C", "A"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["D"]
+ expected_sequence = ["D"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["E"]
+ expected_sequence = ["D", "E"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["F"]
+ expected_sequence = ["D", "E", "F"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["G"]
+ expected_sequence = ["D", "E", "F", "G"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["B"]
+ expected_sequence = ["D", "E", "F", "G", "B"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["C"]
+ expected_sequence = ["D", "E", "F", "G", "B", "C"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["A"]
+ expected_sequence = ["D", "E", "F", "G", "B", "C", "A"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ def test_handle_ext_handlers_with_uninstallation(self, *args):
+ '''
+ Tests extension sequencing among multiple extensions with dependencies when
+ some extension are to be uninstalled.
+ Verifies that the sequencing is in the expected order and the uninstallation takes place
+ prior to all the installation/enable.
+ '''
+ exthandlers_handler = self._create_mock(*args)
+
+ # "A", "D" and "F" are marked as to be uninstalled
+ self._set_dependency_levels([("A", 0), ("B", 2), ("C", 2), ("D", 0), ("E", 1), ("F", 0), ("G", 1)],
+ exthandlers_handler)
+
+ extensions_to_be_failed = []
+ expected_sequence = ["A", "D", "F", "E", "G", "B", "C"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ def test_handle_ext_handlers_fallback(self, *args):
+ '''
+ This test makes sure that the extension sequencing is applied only when the user specifies
+ dependency information in the extension.
+ When there is no dependency specified, the agent is expected to assign dependencyLevel=0 to all extension.
+ Also, it is expected to install all the extension no matter if there is any failure in any of the extensions.
+ '''
+ exthandlers_handler = self._create_mock(*args)
+
+ self._set_dependency_levels([("A", 1), ("B", 1), ("C", 1), ("D", 1), ("E", 1), ("F", 1), ("G", 1)],
+ exthandlers_handler)
+
+ # Expected sequence must contain all the extensions in the given order.
+ # The following test cases verfy against this same expected sequence no matter if any extension failed
+ expected_sequence = ["A", "B", "C", "D", "E", "F", "G"]
+
+ # Make sure that failure in any extension does not prevent other extensions to be installed
+ extensions_to_be_failed = []
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["A"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["B"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["C"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["D"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["E"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["F"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
+
+ extensions_to_be_failed = ["G"]
+ self._run_test(extensions_to_be_failed, expected_sequence, exthandlers_handler)
if __name__ == '__main__':
unittest.main()