Merge branch 'develop' into test-suites

.github/workflows/ci_pr.yml

@@ -10,31 +10,48 @@ on:
 jobs:
   test-legacy-python-versions:
 
-    name: "Python 2.6 Unit Tests"
-    runs-on: ubuntu-18.04
-
     strategy:
       fail-fast: false
+      matrix:
+        include:
+          - python-version: 2.6
+          - python-version: 3.4
+
+    name: "Python ${{ matrix.python-version }} Unit Tests"
+    runs-on: ubuntu-20.04
+    container:
+      image: ubuntu:16.04
+      volumes:
+        - /home/waagent:/home/waagent
+    defaults:
+      run:
+        shell: bash -l {0}
+
     env:
       NOSEOPTS: "--verbose"
 
     steps:
+    - uses: actions/checkout@v3
 
-    - name: Install Python 2.6
+    - name: Install Python ${{ matrix.python-version }}
       run: |
-        curl https://dcrdata.blob.core.windows.net/python/python-2.6.tar.bz2 -o python-2.6.tar.bz2
-        sudo tar xjvf python-2.6.tar.bz2 --directory /
-
-    - uses: actions/checkout@v2
+        apt-get update
+        apt-get install -y curl bzip2 sudo python3
+        curl https://dcrdata.blob.core.windows.net/python/python-${{ matrix.python-version }}.tar.bz2 -o python-${{ matrix.python-version }}.tar.bz2
+        sudo tar xjvf python-${{ matrix.python-version }}.tar.bz2 --directory /
 
     - name: Test with nosetests
       run: |
-        source /home/waagent/virtualenv/python2.6.9/bin/activate
+        if [[ ${{ matrix.python-version }} == 2.6 ]]; then
+          source /home/waagent/virtualenv/python2.6.9/bin/activate
+        else
+          source /home/waagent/virtualenv/python3.4.8/bin/activate
+        fi
         ./ci/nosetests.sh
         exit $?
 
   test-current-python-versions:
-    
+
     strategy:
       fail-fast: false
       matrix:
@@ -43,24 +60,24 @@ jobs:
           - python-version: 2.7
             PYLINTOPTS: "--rcfile=ci/2.7.pylintrc --ignore=tests_e2e,makepkg.py"
 
-          - python-version: 3.4
-            PYLINTOPTS: "--rcfile=ci/2.7.pylintrc --ignore=tests_e2e,makepkg.py"
+          - python-version: 3.5
+            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e,makepkg.py"
 
           - python-version: 3.6
             PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e"
 
           - python-version: 3.7
             PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e"
-            
+
           - python-version: 3.8
             PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e"
 
           - python-version: 3.9
             PYLINTOPTS: "--rcfile=ci/3.6.pylintrc"
             additional-nose-opts: "--with-coverage --cover-erase --cover-inclusive --cover-branches --cover-package=azurelinuxagent"
-    
+
     name: "Python ${{ matrix.python-version }} Unit Tests"
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     env:
       PYLINTOPTS: ${{ matrix.PYLINTOPTS }}
@@ -69,15 +86,15 @@ jobs:
       PYTHON_VERSION: ${{ matrix.python-version }}
 
     steps:
-    
+
     - name: Checkout WALinuxAgent repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
-    
+
     - name: Install dependencies
       id: install-dependencies
       run: |
@@ -106,6 +123,6 @@ jobs:
 
     - name: Upload Coverage
       if: matrix.python-version == 3.9
-      uses: codecov/codecov-action@v1
+      uses: codecov/codecov-action@v2
       with:
         file: ./coverage.xml

azurelinuxagent/common/cgroupconfigurator.py

@@ -368,10 +368,9 @@ def __setup_azure_slice():
             if not os.path.exists(vmextensions_slice):
                 files_to_create.append((vmextensions_slice, _VMEXTENSIONS_SLICE_CONTENTS))
 
-            if not os.path.exists(logcollector_slice):
-                slice_contents = _LOGCOLLECTOR_SLICE_CONTENTS_FMT.format(cpu_quota=_LOGCOLLECTOR_CPU_QUOTA)
-
-                files_to_create.append((logcollector_slice, slice_contents))
+            # Update log collector slice contents
+            slice_contents = _LOGCOLLECTOR_SLICE_CONTENTS_FMT.format(cpu_quota=_LOGCOLLECTOR_CPU_QUOTA)
+            files_to_create.append((logcollector_slice, slice_contents))
 
             if fileutil.findre_in_file(agent_unit_file, r"Slice=") is not None:
                 CGroupConfigurator._Impl.__cleanup_unit_file(agent_drop_in_file_slice)

azurelinuxagent/common/logcollector.py

@@ -119,7 +119,7 @@ def _set_resource_usage_cgroups(cpu_cgroup_path, memory_cgroup_path):
     @staticmethod
     def _initialize_telemetry():
         protocol = get_protocol_util().get_protocol(init_goal_state=False)
-        protocol.client.reset_goal_state(goalstate_properties=GoalStateProperties.RoleConfig | GoalStateProperties.HostingEnv)
+        protocol.client.reset_goal_state(goal_state_properties=GoalStateProperties.RoleConfig | GoalStateProperties.HostingEnv)
         # Initialize the common parameters for telemetry events
         initialize_event_logger_vminfo_common_parameters(protocol)
 

tests/common/mock_cgroup_environment.py

@@ -91,6 +91,7 @@
 
 class UnitFilePaths:
     walinuxagent = "/lib/systemd/system/walinuxagent.service"
+    logcollector = "/lib/systemd/system/azure-walinuxagent-logcollector.slice"
     azure = "/lib/systemd/system/azure.slice"
     vmextensions = "/lib/systemd/system/azure-vmextensions.slice"
     extensionslice = "/lib/systemd/system/azure-vmextensions-Microsoft.CPlat.Extension.slice"

tests/common/osutil/test_default.py

@@ -35,7 +35,7 @@
 from azurelinuxagent.common.utils.flexible_version import FlexibleVersion
 from azurelinuxagent.common.utils.networkutil import AddFirewallRules
 from tests.common.mock_environment import MockEnvironment
-from tests.tools import AgentTestCase, patch, open_patch, load_data, data_dir
+from tests.tools import AgentTestCase, patch, open_patch, load_data, data_dir, is_python_version_26_or_34, skip_if_predicate_true
 
 actual_get_proc_net_route = 'azurelinuxagent.common.osutil.default.DefaultOSUtil._get_proc_net_route'
 
@@ -950,6 +950,7 @@ def test_remove_firewall_should_not_retry_invalid_rule(self):
 
                 self.assertFalse(osutil._enable_firewall)
 
+    @skip_if_predicate_true(is_python_version_26_or_34, "Disabled on Python 2.6 and 3.4 for now. Need to revisit to fix it")
     def test_get_nic_state(self):
         state = osutil.DefaultOSUtil().get_nic_state()
         self.assertNotEqual(state, {})

tests/common/test_cgroupconfigurator.py

@@ -39,7 +39,7 @@
 from azurelinuxagent.common.utils import shellutil, fileutil
 from tests.common.mock_environment import MockCommand
 from tests.common.mock_cgroup_environment import mock_cgroup_environment, UnitFilePaths
-from tests.tools import AgentTestCase, patch, mock_sleep, i_am_root, data_dir
+from tests.tools import AgentTestCase, patch, mock_sleep, i_am_root, data_dir, is_python_version_26_or_34, skip_if_predicate_true
 from tests.utils.miscellaneous_tools import format_processes, wait_for
 
 
@@ -188,6 +188,27 @@ def test_initialize_should_create_unit_files_when_the_agent_service_file_is_not_
             self.assertTrue(os.path.exists(agent_drop_in_file_cpu_accounting), "{0} was not created".format(agent_drop_in_file_cpu_accounting))
             self.assertTrue(os.path.exists(agent_drop_in_file_memory_accounting), "{0} was not created".format(agent_drop_in_file_memory_accounting))
 
+    def test_initialize_should_update_logcollector_memorylimit(self):
+        with self._get_cgroup_configurator(initialize=False) as configurator:
+            log_collector_unit_file = configurator.mocks.get_mapped_path(UnitFilePaths.logcollector)
+            original_memory_limit = "MemoryLimit=30M"
+
+            # The mock creates the slice unit file with memory limit
+            configurator.mocks.add_data_file(os.path.join(data_dir, 'init', "azure-walinuxagent-logcollector.slice"),
+                                             UnitFilePaths.logcollector)
+            if not os.path.exists(log_collector_unit_file):
+                raise Exception("{0} should have been created during test setup".format(log_collector_unit_file))
+            if not fileutil.findre_in_file(log_collector_unit_file, original_memory_limit):
+                raise Exception("MemoryLimit was not set correctly. Expected: {0}. Got:\n{1}".format(
+                    original_memory_limit, fileutil.read_file(log_collector_unit_file)))
+
+            configurator.initialize()
+
+            # initialize() should update the unit file to remove the memory limit
+            self.assertFalse(fileutil.findre_in_file(log_collector_unit_file, original_memory_limit),
+                             "Log collector slice unit file was not updated correctly. Expected no memory limit. Got:\n{0}".format(
+                                 fileutil.read_file(log_collector_unit_file)))
+
     def test_setup_extension_slice_should_create_unit_files(self):
         with self._get_cgroup_configurator() as configurator:
             # get the paths to the mocked files
@@ -505,6 +526,7 @@ def test_start_extension_command_should_disable_cgroups_and_invoke_the_command_d
 
                         self.assertEqual(len(CGroupsTelemetry._tracked), 0, "No cgroups should have been created")
 
+    @skip_if_predicate_true(is_python_version_26_or_34, "Disabled on Python 2.6 and 3.4 for now. Need to revisit to fix it")
     @attr('requires_sudo')
     @patch('time.sleep', side_effect=lambda _: mock_sleep())
     def test_start_extension_command_should_not_use_fallback_option_if_extension_fails(self, *args):
@@ -542,6 +564,7 @@ def test_start_extension_command_should_not_use_fallback_option_if_extension_fai
                     # wasn't truncated.
                     self.assertIn("Running scope as unit", ustr(context_manager.exception))
 
+    @skip_if_predicate_true(is_python_version_26_or_34, "Disabled on Python 2.6 and 3.4 for now. Need to revisit to fix it")
     @attr('requires_sudo')
     @patch('time.sleep', side_effect=lambda _: mock_sleep())
     @patch("azurelinuxagent.common.utils.extensionprocessutil.TELEMETRY_MESSAGE_MAX_LEN", 5)

tests/common/test_event.py

@@ -44,7 +44,7 @@
 from tests.protocol import mockwiredata
 from tests.protocol.mocks import mock_wire_protocol, MockHttpResponse
 from tests.protocol.HttpRequestPredicates import HttpRequestPredicates
-from tests.tools import AgentTestCase, data_dir, load_data, patch, skip_if_predicate_true
+from tests.tools import AgentTestCase, data_dir, load_data, patch, skip_if_predicate_true, is_python_version_26_or_34
 from tests.utils.event_logger_tools import EventLoggerTools
 
 
@@ -414,6 +414,7 @@ def test_collect_events_should_be_able_to_process_events_with_non_ascii_characte
         self.assertEqual(len(event_list), 1)
         self.assertEqual(TestEvent._get_event_message(event_list[0]), u'World\u05e2\u05d9\u05d5\u05ea \u05d0\u05d7\u05e8\u05d5\u05ea\u0906\u091c')
 
+    @skip_if_predicate_true(is_python_version_26_or_34, "Disabled on Python 2.6 and 3.4 for now. Need to revisit to fix it")
     def test_collect_events_should_ignore_invalid_event_files(self):
         self._create_test_event_file("custom_script_1.tld")  # a valid event
         self._create_test_event_file("custom_script_utf-16.tld")

tests/data/init/azure-walinuxagent-logcollector.slice

@@ -0,0 +1,9 @@
+[Unit]
+Description=Slice for Azure VM Agent Periodic Log Collector
+DefaultDependencies=no
+Before=slices.target
+[Slice]
+CPUAccounting=yes
+CPUQuota=5%
+MemoryAccounting=yes
+MemoryLimit=30M

tests/ga/test_update.py

@@ -53,7 +53,7 @@
 from tests.protocol.mocks import mock_wire_protocol, MockHttpResponse
 from tests.protocol.mockwiredata import DATA_FILE, DATA_FILE_MULTIPLE_EXT, DATA_FILE_VM_SETTINGS
 from tests.tools import AgentTestCase, AgentTestCaseWithGetVmSizeMock, data_dir, DEFAULT, patch, load_bin_data, Mock, MagicMock, \
-    clear_singleton_instances
+    clear_singleton_instances, is_python_version_26_or_34, skip_if_predicate_true
 from tests.protocol import mockwiredata
 from tests.protocol.HttpRequestPredicates import HttpRequestPredicates
 
@@ -1410,6 +1410,7 @@ def _mock_popen(cmd, *args, **kwargs):
             "Not setting up persistent firewall rules as OS.EnableFirewall=False" == args[0] for (args, _) in
             patch_info.call_args_list), "Info not logged properly, got: {0}".format(patch_info.call_args_list))
 
+    @skip_if_predicate_true(is_python_version_26_or_34, "Disabled on Python 2.6 and 3.4 for now. Need to revisit to fix it")
     def test_it_should_setup_persistent_firewall_rules_on_startup(self):
         iterations = 1
         executed_commands = []

tests/test_agent.py

@@ -218,6 +218,7 @@ def test_rejects_invalid_log_collector_mode(self, mock_exit, mock_stderr):  # py
     @patch("azurelinuxagent.agent.LogCollector")
     def test_calls_collect_logs_with_proper_mode(self, mock_log_collector, *args):  # pylint: disable=unused-argument
         agent = Agent(False, conf_file_path=os.path.join(data_dir, "test_waagent.conf"))
+        mock_log_collector.run = Mock()
 
         agent.collect_logs(is_full_mode=True)
         full_mode = mock_log_collector.call_args_list[0][0][0]
@@ -231,6 +232,7 @@ def test_calls_collect_logs_with_proper_mode(self, mock_log_collector, *args):
     def test_calls_collect_logs_on_valid_cgroups(self, mock_log_collector):
         try:
             CollectLogsHandler.enable_cgroups_validation()
+            mock_log_collector.run = Mock()
 
             def mock_cgroup_paths(*args, **kwargs):
                 if args and args[0] == "self":
@@ -246,9 +248,11 @@ def mock_cgroup_paths(*args, **kwargs):
         finally:
             CollectLogsHandler.disable_cgroups_validation()
 
-    def test_doesnt_call_collect_logs_on_invalid_cgroups(self):
+    @patch("azurelinuxagent.agent.LogCollector")
+    def test_doesnt_call_collect_logs_on_invalid_cgroups(self, mock_log_collector):
         try:
             CollectLogsHandler.enable_cgroups_validation()
+            mock_log_collector.run = Mock()
 
             def mock_cgroup_paths(*args, **kwargs):
                 if args and args[0] == "self":

tests/tools.py

@@ -121,6 +121,14 @@ def is_python_version_26():
     return sys.version_info[0] == 2 and sys.version_info[1] == 6
 
 
+def is_python_version_34():
+    return sys.version_info[0] == 3 and sys.version_info[1] == 4
+
+
+def is_python_version_26_or_34():
+    return is_python_version_26() or is_python_version_34()
+
+
 class AgentTestCase(unittest.TestCase):
     @classmethod
     def setUpClass(cls):