feat: add Python and Poetry plugins (#1899)

This adds Charmcraft-specific versions of the craft-parts `python` and
`poetry` plugins. Notable differences:

- The venv is in a subdirectory
- Uses the system's `pip` so it doesn't have to install `pip` in the
charm venv.
- Neither plugin tries to install the current directory as a module
- Defaults to not using binaries
- Adds the `src` and `lib` directories to the charm if they exist.

.github/workflows/tests.yaml

@@ -80,6 +80,11 @@ jobs:
         run: |
           sudo apt update
           sudo apt install -y python3-pip python3-setuptools python3-wheel python3-venv libapt-pkg-dev
+          pipx install poetry
+          # Jammy runners have too old a version of pip.
+          if [[ $(lsb_release --codename --short) == 'jammy' ]]; then
+            python3 -m pip install -U pip
+          fi
       - name: Install skopeo (mac)
         # This is only necessary for Linux until skopeo >= 1.11 is in repos.
         # Once we're running on Noble, we can get skopeo from apt.

charmcraft/application/main.py

@@ -26,9 +26,8 @@
 from craft_parts.plugins.plugins import PluginType
 from overrides import override
 
-from charmcraft import extensions, models, preprocess, services
+from charmcraft import extensions, models, parts, preprocess, services
 from charmcraft.application import commands
-from charmcraft.parts import plugins
 from charmcraft.services import CharmcraftServiceFactory
 
 GENERAL_SUMMARY = """
@@ -132,11 +131,7 @@ def _get_dispatcher(self) -> craft_cli.Dispatcher:
 
     @override
     def _get_app_plugins(self) -> dict[str, PluginType]:
-        return {
-            "charm": plugins.CharmPlugin,
-            "bundle": plugins.BundlePlugin,
-            "reactive": plugins.ReactivePlugin,
-        }
+        return parts.get_app_plugins()
 
     @override
     def _pre_run(self, dispatcher: craft_cli.Dispatcher) -> None:

charmcraft/parts/__init__.py

@@ -26,21 +26,27 @@
 
 __all__ = [
     "plugins",
+    "get_app_plugins",
     "setup_parts",
     "process_part_config",
     "PartsLifecycle",
 ]
 
 
-def setup_parts():
+def get_app_plugins() -> dict[str, type[craft_parts.plugins.Plugin]]:
+    """Get the app-specific plugins for Charmcraft."""
+    return {
+        "bundle": plugins.BundlePlugin,
+        "charm": plugins.CharmPlugin,
+        "poetry": plugins.PoetryPlugin,
+        "python": plugins.PythonPlugin,
+        "reactive": plugins.ReactivePlugin,
+    }
+
+
+def setup_parts() -> None:
     """Initialize craft-parts plugins."""
-    craft_parts.plugins.register(
-        {
-            "charm": plugins.CharmPlugin,
-            "bundle": plugins.BundlePlugin,
-            "reactive": plugins.ReactivePlugin,
-        }
-    )
+    craft_parts.plugins.register(get_app_plugins())
 
 
 def process_part_config(data: dict[str, Any]) -> dict[str, Any]:

charmcraft/parts/plugins/__init__.py

@@ -18,13 +18,19 @@
 
 from ._bundle import BundlePlugin, BundlePluginProperties
 from ._charm import CharmPlugin, CharmPluginProperties
+from ._poetry import PoetryPlugin, PoetryPluginProperties
+from ._python import PythonPlugin, PythonPluginProperties
 from ._reactive import ReactivePlugin, ReactivePluginProperties
 
 __all__ = [
     "BundlePlugin",
     "BundlePluginProperties",
     "CharmPlugin",
     "CharmPluginProperties",
+    "PoetryPlugin",
+    "PoetryPluginProperties",
+    "PythonPlugin",
+    "PythonPluginProperties",
     "ReactivePlugin",
     "ReactivePluginProperties",
 ]

charmcraft/parts/plugins/_poetry.py

@@ -0,0 +1,98 @@
+# Copyright 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# For further info, check https://github.com/canonical/charmcraft
+"""Charmcraft-specific poetry plugin."""
+
+import pathlib
+from pathlib import Path
+
+from craft_parts.plugins import poetry_plugin
+from overrides import override
+
+from charmcraft import utils
+
+
+class PoetryPluginProperties(poetry_plugin.PoetryPluginProperties, frozen=True):
+
+    poetry_keep_bins: bool = False
+    """Keep the virtual environment's 'bin' directory."""
+
+
+class PoetryPlugin(poetry_plugin.PoetryPlugin):
+    """Charmcraft-specific version of the poetry plugin."""
+
+    properties_class = PoetryPluginProperties
+    _options: PoetryPluginProperties  # type: ignore[reportIncompatibleVariableOverride]
+
+    def get_build_environment(self) -> dict[str, str]:
+        return utils.extend_python_build_environment(super().get_build_environment())
+
+    def _get_venv_directory(self) -> Path:
+        return self._part_info.part_install_dir / "venv"
+
+    def _get_pip(self) -> str:
+        """Get the pip command to use."""
+        return f"{self._get_system_python_interpreter()} -m pip --python=${{PARTS_PYTHON_VENV_INTERP_PATH}}"
+
+    def _get_pip_install_commands(self, requirements_path: pathlib.Path) -> list[str]:
+        """Get the commands for installing with pip.
+
+        This only installs the dependencies from requirements, unlike the upstream
+        version, because charms are not installable Python packages.
+
+        :param requirements_path: The path of the requirements.txt file to write to.
+        :returns: A list of strings forming the install script.
+        """
+        pip = self._get_pip()
+        return [
+            # These steps need to be separate because poetry export defaults to including
+            # hashes, which don't work with installing from a directory.
+            f"{pip} install --no-deps '--requirement={requirements_path}'",
+            # Check that the virtualenv is consistent.
+            f"{pip} check",
+        ]
+
+    def _get_package_install_commands(self) -> list[str]:
+        """Get the package installation commands.
+
+        This overrides the generic class to also:
+
+        1. Copy the charm source into the charm.
+        2. Copy the charmlibs into the charm.
+        """
+        return [
+            *super()._get_package_install_commands(),
+            *utils.get_charm_copy_commands(
+                self._part_info.part_build_dir, self._part_info.part_install_dir
+            ),
+        ]
+
+    def _should_remove_symlinks(self) -> bool:
+        return True
+
+    def _get_rewrite_shebangs_commands(self) -> list[str]:
+        """Get the commands used to rewrite shebangs in the install dir.
+
+        Charms don't need the shebangs to be rewritten.
+        """
+        return []
+
+    @override
+    def get_build_commands(self) -> list[str]:
+        """Get the build commands for the Python plugin."""
+        if self._options.poetry_keep_bins:
+            return super().get_build_commands()
+        venv_bin = self._get_venv_directory() / "bin"
+        return [*super().get_build_commands(), f"rm -rf {venv_bin}"]

charmcraft/parts/plugins/_python.py

@@ -0,0 +1,100 @@
+# Copyright 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# For further info, check https://github.com/canonical/charmcraft
+"""Charmcraft-specific poetry plugin."""
+
+import shlex
+from pathlib import Path
+
+from craft_parts.plugins import python_plugin
+from overrides import override
+
+from charmcraft import utils
+
+
+class PythonPluginProperties(python_plugin.PythonPluginProperties, frozen=True):
+
+    python_packages: list[str] = []  # No default packages.
+    python_keep_bins: bool = False
+    """Keep the virtual environment's 'bin' directory."""
+
+
+class PythonPlugin(python_plugin.PythonPlugin):
+    """Charmcraft-specific version of the python plugin."""
+
+    properties_class = PythonPluginProperties
+    _options: PythonPluginProperties  # type: ignore[reportIncompatibleVariableOverride]
+
+    @override
+    def get_build_environment(self) -> dict[str, str]:
+        return utils.extend_python_build_environment(super().get_build_environment())
+
+    @override
+    def _get_venv_directory(self) -> Path:
+        return self._part_info.part_install_dir / "venv"
+
+    @override
+    def _get_pip(self) -> str:
+        """Get the pip command to use."""
+        return f"{self._get_system_python_interpreter()} -m pip --python=${{PARTS_PYTHON_VENV_INTERP_PATH}}"
+
+    @override
+    def _get_package_install_commands(self) -> list[str]:
+        """Get the package installation commands.
+
+        This overrides the generic class in the following ways:
+
+        1. Doesn't try to install '.' (charms are not installable packages)
+        2. Copy the charm source into the charm.
+        3. Copy the charmlibs into the charm.
+        """
+        pip = self._get_pip()
+        install_params = shlex.join(
+            (
+                *(f"--constraint={constraint}" for constraint in self._options.python_constraints),
+                *(
+                    f"--requirement={requirement}"
+                    for requirement in self._options.python_requirements
+                ),
+                *self._options.python_packages,
+            )
+        )
+        return [
+            f"{pip} install --no-deps {install_params}",
+            f"{pip} check",
+            *utils.get_charm_copy_commands(
+                self._part_info.part_build_dir, self._part_info.part_install_dir
+            ),
+        ]
+
+    @override
+    def _should_remove_symlinks(self) -> bool:
+        return True
+
+    @override
+    def _get_rewrite_shebangs_commands(self) -> list[str]:
+        """Get the commands used to rewrite shebangs in the install dir.
+
+        Charms don't need the shebangs to be rewritten.
+        """
+        return []
+
+    @override
+    def get_build_commands(self) -> list[str]:
+        """Get the build commands for the Python plugin."""
+        if self._options.python_keep_bins:
+            return super().get_build_commands()
+        venv_bin = self._get_venv_directory() / "bin"
+        return [*super().get_build_commands(), f"rm -rf {venv_bin}"]

charmcraft/utils/__init__.py

@@ -55,6 +55,7 @@
     get_requirements_file_package_names,
     validate_strict_dependencies,
 )
+from charmcraft.utils.parts import extend_python_build_environment, get_charm_copy_commands
 from charmcraft.utils.project import (
     find_charm_sources,
     get_charm_name_from_path,
@@ -100,6 +101,8 @@
     "confirm_with_user",
     "format_content",
     "humanize_list",
+    "extend_python_build_environment",
+    "get_charm_copy_commands",
     "find_charm_sources",
     "get_charm_name_from_path",
     "get_templates_environment",

charmcraft/utils/parts.py

@@ -0,0 +1,50 @@
+# Copyright 2024 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# For further info, check https://github.com/canonical/charmcraft
+"""Utility functions for craft-parts plugins."""
+
+import pathlib
+import shlex
+from collections.abc import Collection
+
+
+def extend_python_build_environment(environment: dict[str, str]) -> dict[str, str]:
+    """Extend the build environment for all Python plugins.
+
+    :param environment: the existing environment dictionary
+    :returns: the environment dictionary with charmcraft-specific additions.
+    """
+    return environment | {
+        "PIP_NO_BINARY": ":all:",  # Build from source
+        "PARTS_PYTHON_VENV_ARGS": "--without-pip",
+    }
+
+
+def get_charm_copy_commands(build_dir: pathlib.Path, install_dir: pathlib.Path) -> Collection[str]:
+    """Get the commands to copy charm source and charmlibs into the install directory.
+
+    The commands will only be included if the relevant directories exist.
+    """
+    copy_command_base = ["cp", "--archive", "--recursive", "--reflink=auto"]
+    src_dir = build_dir / "src"
+    libs_dir = build_dir / "lib"
+
+    commands = []
+    if src_dir.exists():
+        commands.append(shlex.join([*copy_command_base, str(src_dir), str(install_dir)]))
+    if libs_dir.exists():
+        commands.append(shlex.join([*copy_command_base, str(libs_dir), str(install_dir)]))
+
+    return commands

pyproject.toml

@@ -7,7 +7,7 @@ dependencies = [
     "craft-application~=4.1",
     "craft-cli>=2.3.0",
     "craft-grammar>=2.0.0",
-    "craft-parts>=2.0.0",
+    "craft-parts>=2.1.0",
     "craft-providers>=2.0.0",
     "craft-platforms~=0.3",
     "craft-providers>=2.0.0",

requirements-dev.txt

@@ -10,7 +10,7 @@ craft-application==4.1.0
 craft-archives==2.0.0
 craft-cli==2.7.0
 craft-grammar==2.0.0
-craft-parts==2.0.0
+craft-parts==2.1.0
 craft-platforms==0.1.1
 craft-providers==2.0.0
 craft-store==3.0.0

requirements.txt

@@ -8,7 +8,7 @@ craft-application==4.1.0
 craft-archives==2.0.0
 craft-cli==2.7.0
 craft-grammar==2.0.0
-craft-parts==2.0.0
+craft-parts==2.1.0
 craft-platforms==0.1.1
 craft-providers==2.0.0
 craft-store==3.0.0