Merge pull request #553 from cmatsuoka/charm-python-packages

parts,charm_builder: add charm-python-packages plugin property (CRAFT-552)

.github/workflows/tests.yaml

@@ -162,6 +162,19 @@ jobs:
           unzip -l build-packages-test_*.charm | grep "venv/ops/charm.py"
           popd
 
+          mkdir -p python-packages-test
+          pushd python-packages-test
+          charmcraft -v init --author testuser
+          sed -i "s|20.04|$VERSION_ID|g" charmcraft.yaml
+          cat <<- EOF >> charmcraft.yaml
+          parts:
+            charm:
+              charm-python-packages: [bump2version]
+          EOF
+          sg lxd -c "charmcraft -v pack"
+          unzip -l python-packages-test_*.charm | grep "venv/bumpversion/__init__.py"
+          popd
+
           sudo snap set charmcraft provider=lxd
           sudo snap set charmcraft provider=multipass
           if sudo snap set charmcraft provider=invalid; then

charmcraft/charm_builder.py

@@ -225,20 +225,27 @@ def handle_dependencies(self):
         """Handle from-directory and virtualenv dependencies."""
         logger.debug("Installing dependencies")
 
-        # virtualenv with other dependencies (if any)
-        if self.requirement_paths:
+        if self.requirement_paths or self.python_packages:
+            # create virtualenv using the host environment python
             staging_venv_dir = self.charmdir / STAGING_VENV_DIRNAME
-
-            # use the host environment python
             _process_run(["python3", "-m", "venv", str(staging_venv_dir)])
             pip_cmd = str(_find_venv_bin(staging_venv_dir, "pip3"))
 
             _process_run([pip_cmd, "--version"])
 
-            cmd = [pip_cmd, "install", "--upgrade", "--no-binary", ":all:"]  # base command
-            for reqspath in self.requirement_paths:
-                cmd.append("--requirement={}".format(reqspath))  # the dependencies file(s)
-            _process_run(cmd)
+            if self.python_packages:
+                # install python packages
+                cmd = [pip_cmd, "install", "--upgrade", "--no-binary", ":all:"]  # base command
+                for pkg in self.python_packages:
+                    cmd.append(pkg)  # the python package to install
+                _process_run(cmd)
+
+            if self.requirement_paths:
+                # install dependencies from requirement files
+                cmd = [pip_cmd, "install", "--upgrade", "--no-binary", ":all:"]  # base command
+                for reqspath in self.requirement_paths:
+                    cmd.append("--requirement={}".format(reqspath))  # the dependencies file(s)
+                _process_run(cmd)
 
             # copy the virtualvenv site-packages directory to /venv in charm
             basedir = pathlib.Path(STAGING_VENV_DIRNAME)
@@ -312,13 +319,21 @@ def _parse_arguments() -> argparse.Namespace:
         required=True,
         help="The build destination directory",
     )
+    parser.add_argument(
+        "-p",
+        "--package",
+        metavar="pkg",
+        action="append",
+        default=None,
+        help="Python package to install before requirements.",
+    )
     parser.add_argument(
         "-r",
         "--requirement",
         metavar="reqfile",
         action="append",
         default=None,
-        help="Comma-separated list of requirements files.",
+        help="Requirements file to install dependencies from.",
     )
 
     return parser.parse_args()
@@ -336,6 +351,7 @@ def main():
         charmdir=pathlib.Path(options.charmdir),
         builddir=pathlib.Path(options.builddir),
         entrypoint=pathlib.Path(options.entrypoint),
+        python_packages=options.package,
         requirements=options.requirement,
     )
     builder.build_charm()

charmcraft/parts.py

@@ -39,6 +39,7 @@ class CharmPluginProperties(plugins.PluginProperties, plugins.PluginModel):
 
     source: str = ""
     charm_entrypoint: str = ""  # TODO: add default after removing --entrypoint
+    charm_python_packages: List[str] = []
     charm_requirements: List[str] = []
 
     @classmethod
@@ -68,6 +69,10 @@ class CharmPlugin(plugins.Plugin):
         (string)
         The path to the main charm executable, relative to the charm root.
 
+      - ``charm-python-packages``
+        (list of strings)
+        A list of python packages to install from PyPI before installing requirements.
+
       - ``charm-requirements``
         (list of strings)
         List of paths to requirements files.
@@ -135,6 +140,9 @@ def get_build_commands(self) -> List[str]:
             entrypoint = self._part_info.part_build_dir / options.charm_entrypoint
             build_cmd.extend(["--entrypoint", str(entrypoint)])
 
+        for pkg in options.charm_python_packages:
+            build_cmd.extend(["-p", pkg])
+
         for req in options.charm_requirements:
             build_cmd.extend(["-r", req])
 

tests/test_charm_builder.py

@@ -602,6 +602,7 @@ def test_build_dependencies_virtualenv_simple(tmp_path):
         charmdir=tmp_path,
         builddir=build_dir,
         entrypoint=pathlib.Path("whatever"),
+        python_packages=[],
         requirements=["reqs.txt"],
     )
 
@@ -632,6 +633,7 @@ def test_build_dependencies_virtualenv_multiple(tmp_path):
         charmdir=tmp_path,
         builddir=build_dir,
         entrypoint=pathlib.Path("whatever"),
+        python_packages=[],
         requirements=["reqs1.txt", "reqs2.txt"],
     )
 
@@ -671,6 +673,7 @@ def test_build_dependencies_virtualenv_none(tmp_path):
         charmdir=tmp_path,
         builddir=build_dir,
         entrypoint=pathlib.Path("whatever"),
+        python_packages=[],
         requirements=[],
     )
 
@@ -680,6 +683,79 @@ def test_build_dependencies_virtualenv_none(tmp_path):
     mock_run.assert_not_called()
 
 
+def test_build_dependencies_virtualenv_packages(tmp_path):
+    """A virtualenv is created with the specified packages."""
+    metadata = tmp_path / CHARM_METADATA
+    metadata.write_text("name: crazycharm")
+    build_dir = tmp_path / BUILD_DIRNAME
+    build_dir.mkdir()
+
+    builder = CharmBuilder(
+        charmdir=tmp_path,
+        builddir=build_dir,
+        entrypoint=pathlib.Path("whatever"),
+        python_packages=["pkg1", "pkg2"],
+        requirements=[],
+    )
+
+    with patch("charmcraft.charm_builder._process_run") as mock:
+        with patch("shutil.copytree") as mock_copytree:
+            builder.handle_dependencies()
+
+    pip_cmd = str(charm_builder._find_venv_bin(tmp_path / STAGING_VENV_DIRNAME, "pip3"))
+
+    assert mock.mock_calls == [
+        call(["python3", "-m", "venv", str(tmp_path / STAGING_VENV_DIRNAME)]),
+        call([pip_cmd, "--version"]),
+        call([pip_cmd, "install", "--upgrade", "--no-binary", ":all:", "pkg1", "pkg2"]),
+    ]
+
+    site_packages_dir = charm_builder._find_venv_site_packages(pathlib.Path(STAGING_VENV_DIRNAME))
+    assert mock_copytree.mock_calls == [call(site_packages_dir, build_dir / VENV_DIRNAME)]
+
+
+def test_build_dependencies_virtualenv_both(tmp_path):
+    """A virtualenv is created with the specified packages."""
+    metadata = tmp_path / CHARM_METADATA
+    metadata.write_text("name: crazycharm")
+    build_dir = tmp_path / BUILD_DIRNAME
+    build_dir.mkdir()
+
+    builder = CharmBuilder(
+        charmdir=tmp_path,
+        builddir=build_dir,
+        entrypoint=pathlib.Path("whatever"),
+        python_packages=["pkg1", "pkg2"],
+        requirements=["reqs1.txt", "reqs2.txt"],
+    )
+
+    with patch("charmcraft.charm_builder._process_run") as mock:
+        with patch("shutil.copytree") as mock_copytree:
+            builder.handle_dependencies()
+
+    pip_cmd = str(charm_builder._find_venv_bin(tmp_path / STAGING_VENV_DIRNAME, "pip3"))
+
+    assert mock.mock_calls == [
+        call(["python3", "-m", "venv", str(tmp_path / STAGING_VENV_DIRNAME)]),
+        call([pip_cmd, "--version"]),
+        call([pip_cmd, "install", "--upgrade", "--no-binary", ":all:", "pkg1", "pkg2"]),
+        call(
+            [
+                pip_cmd,
+                "install",
+                "--upgrade",
+                "--no-binary",
+                ":all:",
+                "--requirement=reqs1.txt",
+                "--requirement=reqs2.txt",
+            ]
+        ),
+    ]
+
+    site_packages_dir = charm_builder._find_venv_site_packages(pathlib.Path(STAGING_VENV_DIRNAME))
+    assert mock_copytree.mock_calls == [call(site_packages_dir, build_dir / VENV_DIRNAME)]
+
+
 def test_builder_without_jujuignore(tmp_path):
     """Without a .jujuignore we still have a default set of ignores"""
     metadata = tmp_path / CHARM_METADATA

tests/test_parts.py

@@ -38,6 +38,7 @@ def setup_method_fixture(self, tmp_path):
         spec = {
             "plugin": "charm",
             "charm-entrypoint": "entrypoint",
+            "charm-python-packages": ["pkg1", "pkg2"],
             "charm-requirements": ["reqs1.txt", "reqs2.txt"],
         }
         plugin_properties = parts.CharmPluginProperties.unmarshal(spec)
@@ -93,6 +94,8 @@ def test_get_build_commands(self, tmp_path, monkeypatch):
             "--charmdir {work_dir}/parts/foo/build "
             "--builddir {work_dir}/parts/foo/install "
             "--entrypoint {work_dir}/parts/foo/build/entrypoint "
+            "-p pkg1 "
+            "-p pkg2 "
             "-r reqs1.txt "
             "-r reqs2.txt".format(
                 python=sys.executable,
@@ -133,6 +136,7 @@ def test_run_new_entrypoint(self, tmp_path, monkeypatch):
             "plugin": "charm",
             "source": ".",
             "charm-entrypoint": "my-entrypoint",
+            "charm-python-packages": ["pkg1", "pkg2"],
             "charm-requirements": ["reqs1.txt", "reqs2.txt"],
         }
 
@@ -164,6 +168,7 @@ def test_run_same_entrypoint(self, tmp_path, monkeypatch):
             "plugin": "charm",
             "source": ".",
             "charm-entrypoint": "src/charm.py",
+            "charm-python-packages": ["pkg1", "pkg2"],
             "charm-requirements": ["reqs1.txt", "reqs2.txt"],
         }
 
@@ -195,6 +200,7 @@ def test_run_no_previous_entrypoint(self, tmp_path, monkeypatch):
             "plugin": "charm",
             "source": ".",
             "charm-entrypoint": "my-entrypoint",
+            "charm-python-packages": ["pkg1", "pkg2"],
             "charm-requirements": ["reqs1.txt", "reqs2.txt"],
         }