Install agent on test VMs (#2714)

.github/workflows/ci_pr.yml

@@ -41,30 +41,30 @@ jobs:
         include:
 
           - python-version: 2.7
-            PYLINTOPTS: "--rcfile=ci/2.7.pylintrc"
+            PYLINTOPTS: "--rcfile=ci/2.7.pylintrc --ignore=tests_e2e,makepkg.py"
 
           - python-version: 3.4
-            PYLINTOPTS: "--rcfile=ci/2.7.pylintrc"
+            PYLINTOPTS: "--rcfile=ci/2.7.pylintrc --ignore=tests_e2e,makepkg.py"
 
           - python-version: 3.6
-            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc"
+            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e"
 
           - python-version: 3.7
-            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc"
+            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e"
 
           - python-version: 3.8
-            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc"
+            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=tests_e2e"
 
           - python-version: 3.9
-            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc"
+            PYLINTOPTS: "--rcfile=ci/3.6.pylintrc --ignore=azure_models.py,BaseExtensionTestClass.py"
             additional-nose-opts: "--with-coverage --cover-erase --cover-inclusive --cover-branches --cover-package=azurelinuxagent"
 
     name: "Python ${{ matrix.python-version }} Unit Tests"
     runs-on: ubuntu-18.04
 
     env:
       PYLINTOPTS: ${{ matrix.PYLINTOPTS }}
-      PYLINTFILES: "azurelinuxagent setup.py makepkg.py tests"
+      PYLINTFILES: "azurelinuxagent setup.py makepkg.py tests tests_e2e"
       NOSEOPTS: "--with-timer ${{ matrix.additional-nose-opts }}"
       PYTHON_VERSION: ${{ matrix.python-version }}
 

.gitignore

@@ -17,8 +17,6 @@ develop-eggs/
 dist/
 downloads/
 eggs/
-lib/
-lib64/
 parts/
 sdist/
 var/

makepkg.py

@@ -1,14 +1,15 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
+import argparse
 import glob
-import os
+import logging
 import os.path
+import pathlib
 import shutil
 import subprocess
 import sys
 
-from azurelinuxagent.common.version import AGENT_NAME, AGENT_VERSION, \
-    AGENT_LONG_VERSION
+from azurelinuxagent.common.version import AGENT_NAME, AGENT_VERSION, AGENT_LONG_VERSION
 from azurelinuxagent.ga.update import AGENT_MANIFEST_FILE
 
 MANIFEST = '''[{{
@@ -48,62 +49,77 @@
 
 PUBLISH_MANIFEST_FILE = 'manifest.xml'
 
-output_path = os.path.join(os.getcwd(), "eggs")  # pylint: disable=invalid-name
-target_path = os.path.join(output_path, AGENT_LONG_VERSION)  # pylint: disable=invalid-name
-bin_path = os.path.join(target_path, "bin")  # pylint: disable=invalid-name
-egg_path = os.path.join(bin_path, AGENT_LONG_VERSION + ".egg")  # pylint: disable=invalid-name
-manifest_path = os.path.join(target_path, AGENT_MANIFEST_FILE)  # pylint: disable=invalid-name
-publish_manifest_path = os.path.join(target_path, PUBLISH_MANIFEST_FILE)  # pylint: disable=invalid-name
-pkg_name = os.path.join(output_path, AGENT_LONG_VERSION + ".zip")  # pylint: disable=invalid-name
 
-family = 'Test'  # pylint: disable=C0103
-if len(sys.argv) > 1:
-    family = sys.argv[1]  # pylint: disable=invalid-name
-
-def do(*args):  # pylint: disable=C0103,W0621
+def do(*args):
     try:
-        subprocess.check_output(args, stderr=subprocess.STDOUT)
+        return subprocess.check_output(args, stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as e:  # pylint: disable=C0103
-        print("ERROR: {0}".format(str(e)))
-        print("\t{0}".format(" ".join(args)))
-        print(e.output)
-        sys.exit(1)
-
+        raise Exception("[{0}] failed:\n{1}\n{2}".format(" ".join(args), str(e), e.stdout))
+
+
+def run(agent_family, output_directory, log):
+    output_path = os.path.join(output_directory, "eggs")
+    target_path = os.path.join(output_path, AGENT_LONG_VERSION)
+    bin_path = os.path.join(target_path, "bin")
+    egg_path = os.path.join(bin_path, AGENT_LONG_VERSION + ".egg")
+    manifest_path = os.path.join(target_path, AGENT_MANIFEST_FILE)
+    publish_manifest_path = os.path.join(target_path, PUBLISH_MANIFEST_FILE)
+    pkg_name = os.path.join(output_path, AGENT_LONG_VERSION + ".zip")
+
+    if os.path.isdir(target_path):
+        shutil.rmtree(target_path)
+    elif os.path.isfile(target_path):
+        os.remove(target_path)
+    if os.path.isfile(pkg_name):
+        os.remove(pkg_name)
+    os.makedirs(bin_path)
+    log.info("Created {0} directory".format(target_path))
+
+    setup_script = str(pathlib.Path(__file__).parent.joinpath("setup.py"))
+    args = ["python3", setup_script, "bdist_egg", "--dist-dir={0}".format(bin_path)]
+
+    log.info("Creating egg {0}".format(egg_path))
+    do(*args)
+
+    egg_name = os.path.join("bin", os.path.basename(
+        glob.glob(os.path.join(bin_path, "*"))[0]))
+
+    log.info("Writing {0}".format(manifest_path))
+    with open(manifest_path, mode='w') as manifest:
+        manifest.write(MANIFEST.format(AGENT_NAME, egg_name))
+
+    log.info("Writing {0}".format(publish_manifest_path))
+    with open(publish_manifest_path, mode='w') as publish_manifest:
+        publish_manifest.write(PUBLISH_MANIFEST.format(AGENT_VERSION, agent_family))
+
+    cwd = os.getcwd()
+    os.chdir(target_path)
+    try:
+        log.info("Creating package {0}".format(pkg_name))
+        do("zip", "-r", pkg_name, egg_name)
+        do("zip", "-j", pkg_name, AGENT_MANIFEST_FILE)
+        do("zip", "-j", pkg_name, PUBLISH_MANIFEST_FILE)
+    finally:
+        os.chdir(cwd)
 
-if os.path.isdir(target_path):
-    shutil.rmtree(target_path)
-elif os.path.isfile(target_path):
-    os.remove(target_path)
-if os.path.isfile(pkg_name):
-    os.remove(pkg_name)
-os.makedirs(bin_path)
-print("Created {0} directory".format(target_path))
+    log.info("Package {0} successfully created".format(pkg_name))
 
-args = ["python", "setup.py", "bdist_egg", "--dist-dir={0}".format(bin_path)]  # pylint: disable=invalid-name
 
-print("Creating egg {0}".format(egg_path))
-do(*args)
+if __name__ == "__main__":
+    logging.basicConfig(format='%(message)s', level=logging.INFO)
 
-egg_name = os.path.join("bin", os.path.basename(  # pylint: disable=invalid-name
-    glob.glob(os.path.join(bin_path, "*"))[0]))
+    parser = argparse.ArgumentParser()
+    parser.add_argument('family', metavar='family', nargs='?', default='Test', help='Agent family')
+    parser.add_argument('-o', '--output', default=os.getcwd(), help='Output directory')
 
-print("Writing {0}".format(manifest_path))
-with open(manifest_path, mode='w') as manifest:
-    manifest.write(MANIFEST.format(AGENT_NAME, egg_name))
+    arguments = parser.parse_args()
 
-print("Writing {0}".format(publish_manifest_path))
-with open(publish_manifest_path, mode='w') as publish_manifest:
-    publish_manifest.write(PUBLISH_MANIFEST.format(AGENT_VERSION,
-                                                   family))
+    try:
 
+        run(arguments.family, arguments.output, logging)
 
-cwd = os.getcwd()  # pylint: disable=invalid-name
-os.chdir(target_path)
-print("Creating package {0}".format(pkg_name))
-do("zip", "-r", pkg_name, egg_name)
-do("zip", "-j", pkg_name, AGENT_MANIFEST_FILE)
-do("zip", "-j", pkg_name, PUBLISH_MANIFEST_FILE)
-os.chdir(cwd)
+    except Exception as exception:
+        logging.error(str(exception))
+        sys.exit(1)
 
-print("Package {0} successfully created".format(pkg_name))
-sys.exit(0)
+    sys.exit(0)

setup.py

@@ -288,7 +288,10 @@ def initialize_options(self):
         self.lnx_distro_version = DISTRO_VERSION
         self.lnx_distro_fullname = DISTRO_FULL_NAME
         self.register_service = False
-        self.skip_data_files = False
+        # All our data files are system-wide files that are not included in the egg; skip them when
+        # creating an egg.
+        self.skip_data_files = "bdist_egg" in sys.argv
+
         # pylint: enable=attribute-defined-outside-init
 
     def finalize_options(self):

tests_e2e/docker/Dockerfile

@@ -59,7 +59,7 @@ RUN \
     #                                                                                                            \
     # Install additional test dependencies                                                                       \
     #                                                                                                            \
-    python3 -m pip install msrestazure                                                                        && \
+    python3 -m pip install distro msrestazure                                                                 && \
                                                                                                                  \
     #                                                                                                            \
     # The setup for the tests depends on a couple of paths; add those to the profile                             \