Skip to content

Commit

Permalink
feat: use debian arch in platforms (#380)
Browse files Browse the repository at this point in the history 
This commit puts Rockcraft in line with other craft tools: the
architectures listed in the platforms are in Debian format, and we
convert to GOARCH when necessary (such as when fetching images
from a Docker registry or creating empty ones with umoci).

As a side-effect, we also drop the "build variant" notion from the
platforms - currently this variant is only used when fetching/creating
images, so keep the variant logic localized to rockcraft.oci.
  • Loading branch information
@tigarmo
tigarmo committed Oct 26, 2023
1 parent 2eb32fd commit 340064d
Show file tree
Hide file tree
Showing 8 changed files with 157 additions and 215 deletions.
4 changes: 2 additions & 2 deletions docs/reference/rockcraft.yaml.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,10 +159,10 @@ entry corresponding to a check. Each check can be one of three types:
**Required**: Yes

The set of architecture-specific ROCKs to be built. Supported architectures are:
``amd64``, ``arm64``, ``arm``, ``i386``, ``ppc64le``, ``riscv64`` and ``s390x``.
``amd64``, ``arm64``, ``armhf``, ``i386``, ``ppc64el``, ``riscv64`` and ``s390x``.

Entries in the ``platforms`` dict can be free-form strings, or the name of a
supported architecture.
supported architecture (in Debian format).

.. warning::
**All** target architectures must be compatible with the architecture of
Expand Down
64 changes: 64 additions & 0 deletions rockcraft/architectures.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
#
# Copyright 2023 Canonical Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Architecture definitions and conversions for Debian and Go/Docker."""

from __future__ import annotations

import dataclasses


@dataclasses.dataclass(frozen=True)
class ArchitectureMapping:
"""Mapping of a Debian arch to Go arch/variant (for use with registries).
The Go-related values must conform to:
https://github.com/opencontainers/image-spec/blob/67d2d5658fe0476ab9bf414cec164077ebff3920/config.md#properties
"""

description: str
go_arch: str
go_variant: str | None = None


# The keys are valid debian architectures.
SUPPORTED_ARCHS: dict[str, ArchitectureMapping] = {
"amd64": ArchitectureMapping(
description="Intel 64",
go_arch="amd64",
),
"armhf": ArchitectureMapping(
description="ARM 32-bit", go_arch="arm", go_variant="v7"
),
"arm64": ArchitectureMapping(
description="ARM 64-bit", go_arch="arm64", go_variant="v8"
),
"i386": ArchitectureMapping(
description="Intel 386",
go_arch="386",
),
"ppc64el": ArchitectureMapping(
description="PowerPC 64-bit",
go_arch="ppc64le",
),
"riscv64": ArchitectureMapping(
description="RISCV 64-bit",
go_arch="riscv64",
),
"s390x": ArchitectureMapping(
description="IBM Z 64-bit",
go_arch="s390x",
),
}
9 changes: 2 additions & 7 deletions rockcraft/models/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,6 @@
"""Rockcraft models."""


from rockcraft.models.project import (
Project,
RockcraftBuildInfo,
load_project,
transform_yaml,
)
from rockcraft.models.project import Project, load_project, transform_yaml

__all__ = ["Project", "RockcraftBuildInfo", "load_project", "transform_yaml"]
__all__ = ["Project", "load_project", "transform_yaml"]
129 changes: 7 additions & 122 deletions rockcraft/models/project.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.

"""Project definition and helpers."""
import dataclasses
import operator
import platform as host_platform
import re
from builtins import super
from functools import reduce
from pathlib import Path
from typing import (
TYPE_CHECKING,
Expand All @@ -45,6 +40,7 @@
from craft_providers import bases
from pydantic_yaml import YamlModelMixin

from rockcraft.architectures import SUPPORTED_ARCHS
from rockcraft.errors import ProjectLoadError, ProjectValidationError
from rockcraft.extensions import apply_extensions
from rockcraft.parts import part_has_overlay, validate_part
Expand All @@ -55,69 +51,6 @@
from pydantic.error_wrappers import ErrorDict


@dataclasses.dataclass
class RockcraftBuildInfo(BuildInfo):
"""BuildInfo with Rockcraft-specific entries."""

build_for_variant: Optional[str] = None
"""Used for arm archs"""


class ArchitectureMapping(pydantic.BaseModel):
"""Maps different denominations of the same architecture."""

description: str
deb_arch: str
compatible_uts_machine_archs: List[str]
go_arch: str


_SUPPORTED_ARCHS: Dict[str, ArchitectureMapping] = {
"amd64": ArchitectureMapping(
description="Intel 64",
deb_arch="amd64",
compatible_uts_machine_archs=["amd64", "x86_64"],
go_arch="amd64",
),
"arm": ArchitectureMapping(
description="ARM 32-bit",
deb_arch="armhf",
compatible_uts_machine_archs=["arm"],
go_arch="arm",
),
"arm64": ArchitectureMapping(
description="ARM 64-bit",
deb_arch="arm64",
compatible_uts_machine_archs=["aarch64"],
go_arch="arm64",
),
"i386": ArchitectureMapping(
description="Intel 386",
deb_arch="i386",
compatible_uts_machine_archs=["i386"], # TODO: also include "i686", "x86_64"?
go_arch="386",
),
"ppc64le": ArchitectureMapping(
description="PowerPC 64-bit",
deb_arch="ppc64el",
compatible_uts_machine_archs=["ppc64le"],
go_arch="ppc64le",
),
"riscv64": ArchitectureMapping(
description="RISCV 64-bit",
deb_arch="riscv64",
compatible_uts_machine_archs=["riscv64"],
go_arch="riscv64",
),
"s390x": ArchitectureMapping(
description="IBM Z 64-bit",
deb_arch="s390x",
compatible_uts_machine_archs=["s390x"],
go_arch="s390x",
),
}


class Platform(pydantic.BaseModel):
"""Rockcraft project platform definition."""

Expand Down Expand Up @@ -290,8 +223,6 @@ def _validate_build_base(cls, build_base: Optional[str], values: Any) -> str:
@classmethod
def _validate_all_platforms(cls, platforms: Dict[str, Any]) -> Dict[str, Any]:
"""Make sure all provided platforms are tangible and sane."""
_self_uts_machine = host_platform.machine().lower()

for platform_label in platforms:
platform = platforms[platform_label] if platforms[platform_label] else {}
error_prefix = f"Error for platform entry '{platform_label}'"
Expand All @@ -314,10 +245,7 @@ def _validate_all_platforms(cls, platforms: Dict[str, Any]) -> Dict[str, Any]:
# otherwise the project is invalid.
if platform["build_for"]:
build_target = platform["build_for"][0]
if (
platform_label in _SUPPORTED_ARCHS
and platform_label != build_target
):
if platform_label in SUPPORTED_ARCHS and platform_label != build_target:
raise ProjectValidationError(
str(
f"{error_prefix}: if 'build_for' is provided and the "
Expand All @@ -329,65 +257,24 @@ def _validate_all_platforms(cls, platforms: Dict[str, Any]) -> Dict[str, Any]:
build_target = platform_label

# Both build and target architectures must be supported
if not any(b_o in _SUPPORTED_ARCHS for b_o in build_on_one_of):
if not any(b_o in SUPPORTED_ARCHS for b_o in build_on_one_of):
raise ProjectValidationError(
str(
f"{error_prefix}: trying to build ROCK in one of "
f"{build_on_one_of}, but none of these build architectures is supported. "
f"Supported architectures: {list(_SUPPORTED_ARCHS.keys())}"
f"Supported architectures: {list(SUPPORTED_ARCHS.keys())}"
)
)

if build_target not in _SUPPORTED_ARCHS:
if build_target not in SUPPORTED_ARCHS:
raise ProjectValidationError(
str(
f"{error_prefix}: trying to build ROCK for target "
f"architecture {build_target}, which is not supported. "
f"Supported architectures: {list(_SUPPORTED_ARCHS.keys())}"
f"Supported architectures: {list(SUPPORTED_ARCHS.keys())}"
)
)

# The underlying build machine must be compatible
# with both build_on and build_for
# TODO: in the future, this may be removed
# as Rockcraft gains the ability to natively build
# for multiple architectures
build_for_compatible_uts = _SUPPORTED_ARCHS[
build_target
].compatible_uts_machine_archs
if _self_uts_machine not in build_for_compatible_uts:
raise ProjectValidationError(
str(
f"{error_prefix}: this machine's architecture ({_self_uts_machine}) "
"is not compatible with the ROCK's target architecture. Can only "
f"build a ROCK for {build_target} if the host is compatible with {build_for_compatible_uts}."
)
)

build_on_compatible_uts = list(
reduce(
operator.add,
map(
lambda m: _SUPPORTED_ARCHS[m].compatible_uts_machine_archs,
build_on_one_of,
),
)
)
if _self_uts_machine not in build_on_compatible_uts:
raise ProjectValidationError(
str(
f"{error_prefix}: this ROCK must be built on one of the "
f"following architectures: {build_on_compatible_uts}. "
f"This machine ({_self_uts_machine}) is not one of those."
)
)

# Add variant, if needed, and return sanitized platform
if build_target == "arm":
platform["build_for_variant"] = "v7"
elif build_target == "arm64":
platform["build_for_variant"] = "v8"

platforms[platform_label] = platform

return platforms
Expand Down Expand Up @@ -522,14 +409,12 @@ def get_build_plan(self) -> List[BuildInfo]:
for platform_entry, platform in self.platforms.items():
for build_for in platform.get("build_for") or [platform_entry]:
for build_on in platform.get("build_on") or [platform_entry]:
build_for_variant = platform.get("build_for_variant")
build_infos.append(
RockcraftBuildInfo(
BuildInfo(
platform=platform_entry,
build_on=build_on,
build_for=build_for,
base=base,
build_for_variant=build_for_variant,
)
)

Expand Down
28 changes: 18 additions & 10 deletions rockcraft/oci.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
from craft_parts.overlays import overlays

from rockcraft import errors
from rockcraft.architectures import SUPPORTED_ARCHS
from rockcraft.pebble import Pebble
from rockcraft.utils import get_snap_command_path

Expand Down Expand Up @@ -69,15 +70,14 @@ def from_docker_registry(
*,
image_dir: Path,
arch: str,
variant: Optional[str] = None,
) -> Tuple["Image", str]:
"""Obtain an image from a docker registry.
The image is fetched from the registry at ``REGISTRY_URL``.
:param image_name: The image to retrieve, in ``name:tag`` format.
:param image_dir: The directory to store local OCI images.
:param arch: The architecture of the Docker image to fetch.
:param arch: The architecture of the Docker image to fetch, in Debian format.
:param variant: The variant, if any, of the Docker image to fetch.
Expand All @@ -88,12 +88,16 @@ def from_docker_registry(

source_image = f"docker://{REGISTRY_URL}/{image_name}"
copy_params = ["--retry-times", str(MAX_DOWNLOAD_RETRIES)]

mapping = SUPPORTED_ARCHS[arch]

platform_params = [
"--override-arch",
arch,
mapping.go_arch,
]
if variant:
platform_params += ["--override-variant", variant]
if mapping.go_variant:
platform_params += ["--override-variant", mapping.go_variant]

_copy_image(
source_image,
f"oci:{image_target}",
Expand All @@ -109,13 +113,12 @@ def new_oci_image(
image_name: str,
image_dir: Path,
arch: str,
variant: Optional[str] = None,
) -> Tuple["Image", str]:
"""Create a new OCI image out of thin air.
:param image_name: The image to initiate, in ``name:tag`` format.
:param image_dir: The directory to store the local OCI image.
:param arch: The architecture of the OCI image to create.
:param arch: The architecture of the OCI image to create, in Debian format.
:param variant: The variant, if any, of the OCI image to create.
:returns: The new image object and it's corresponding source image
Expand All @@ -127,12 +130,17 @@ def new_oci_image(
_process_run(["umoci", "init", "--layout", image_target_no_tag])
_process_run(["umoci", "new", "--image", str(image_target)])

# Note: umoci's docs aren't clear on this but we assume that arch-related
# calls must use GOARCH-format, following the OCI spec.
mapping = SUPPORTED_ARCHS[arch]

# Unfortunately, umoci does not allow initializing an image
# with arch and variant. We can configure the arch via
# umoci config, but not the variant. Need to do it manually
_config_image(image_target, ["--architecture", arch, "--no-history"])
if variant:
_inject_architecture_variant(Path(image_target_no_tag), variant)
_config_image(image_target, ["--architecture", mapping.go_arch, "--no-history"])

if mapping.go_variant:
_inject_architecture_variant(Path(image_target_no_tag), mapping.go_variant)

# for new OCI images, the source image corresponds to the newly generated image
return (
Expand Down
2 changes: 0 additions & 2 deletions rockcraft/services/image.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,15 +72,13 @@ def _create_image_info(self) -> ImageInfo:
f"{project.base}:latest",
image_dir=image_dir,
arch=self._build_for,
variant=None, # TODO
)
else:
emit.progress(f"Retrieving base {project.base} for {build_for}")
base_image, source_image = oci.Image.from_docker_registry(
project.base,
image_dir=image_dir,
arch=self._build_for,
variant=None, # TODO
)
emit.progress(f"Retrieved base {project.base} for {build_for}")

Expand Down
Loading

0 comments on commit 340064d

Please sign in to comment.