From 5b0793c5ce9d3d7e21854c3d664defcb935a2f31 Mon Sep 17 00:00:00 2001 From: Yuval Peress Date: Tue, 10 Sep 2024 12:14:11 -0600 Subject: [PATCH] venv: Add support for virtual environment specs Adds handling for a venv key in the manifest. This key will then be used to provide virtual environment metadata to a west bootstrap command. The metadata includes: - A virtual environment name - A list of python requirement files, individual packages, constraints, and local directories. - A mapping of binary requirements and which URLs can be used to find them. These include architecture and OS specific mappings. Signed-off-by: Yuval Peress --- src/west/manifest-schema.yml | 72 ++++++++++ src/west/manifest.py | 261 +++++++++++++++++++++++++++++++++++ tests/test_manifest.py | 110 ++++++++++++++- 3 files changed, 442 insertions(+), 1 deletion(-) diff --git a/src/west/manifest-schema.yml b/src/west/manifest-schema.yml index 6982300f..ddd1c0bd 100644 --- a/src/west/manifest-schema.yml +++ b/src/west/manifest-schema.yml @@ -66,6 +66,78 @@ mapping: required: true type: str + # The venv key specifies requirements for the build environment for Zephyr + # West will handle merging any definitions into your binary dependency + # strings where allowed. + # Example: + # + # urls: + # "windows-*": + # url: "https://my.download.server/win/${arch}.zip" + # + # The above will match any architecture for the windows OS and generate a url + # replacing ${arch} with the current machine's underlying architecture. West + # will automatically provide a few definitions: + # - os: The current OS (one of 'windows', 'linux', or 'mac') + # - arch: The current architecture. This value is standardized such that + # x86_64 is represented as amd64 and aarch64 as arm64. All others match + # Python's `platform.machine()` + # - platform: A concatenation of ${os}-${arch} + venv: + required: false + type: map + mapping: + definitions: + required: false + type: map + mapping: + regex;(.*): + type: str + name: + required: false + type: str + bin-requirements: + required: false + type: map + mapping: + regex;(.*): + type: map + mapping: + definitions: + required: false + type: map + mapping: + regex;(.*): + type: str + urls: + required: true + type: map + mapping: + regex;(.*): + type: map + mapping: + url: + required: true + type: str + paths: + required: false + type: seq + sequence: + - type: str + py-requirements: + required: false + type: seq + sequence: + - type: map + mapping: + pattern: + required: true + type: str + type: + required: true + type: str + enum: ['directory', 'constraints', 'package', 'requirements'] + # The "projects" key specifies a sequence of "projects", each of which has a # remote, and may specify additional configuration. # diff --git a/src/west/manifest.py b/src/west/manifest.py index 57041eff..f56ec8ee 100644 --- a/src/west/manifest.py +++ b/src/west/manifest.py @@ -13,6 +13,7 @@ import logging import os from pathlib import PurePosixPath, Path +import platform import re import shlex import subprocess @@ -67,6 +68,64 @@ # Internal helpers # +# Mapping of the architectures from platofrm.machine() to a common standard +_ARCH_MAPPING = { + "x86_64": "amd64", + "aarch64": "arm64", +} + +def _get_global_defitions() -> Dict[str, str]: + """ + Get the default global 'os', 'arch', and 'platform' values. + + Returns: + A dictionary containing the keys 'os', 'arch', and 'platform' and their + current values. + """ + defs = {} + current_os = platform.system() + if current_os == "Windows": + defs["os"] = "windows" + elif current_os == "Darwin": + defs["os"] = "mac" + elif current_os == "Linux": + defs["os"] = "linux" + else: + raise RuntimeError("Unknown OS: " + current_os) + + arch = platform.machine() + defs["arch"] = _ARCH_MAPPING.get(arch, arch) + defs["platform"] = f"{defs['os']}-{defs['arch']}" + return defs + +def _replace_variables( + input_str: str, + defs: Dict[str, str], +) -> str: + """ + Given a set of definitions, find all instances in an input string of ${key} + and replace the 'key' with the corresponding definition. If a key is not + found in the defs dictionary, raise an exception + + Args: + input_str The input string to fill + defs The key/value mappings to use + Returns: + A new string with all ${...} resolved + """ + while True: + placeholders = re.findall(r"\$\{(.*?)\}", input_str) + if not placeholders: + return input_str + + key = placeholders[0] + input_str = re.sub(r"\$\{" + key + r"\}", defs[key], input_str) + +# Values that can be used in the venv string values via their keys. +# By default, west will include 'os', 'arch', and 'platform' (where +# platform is always '${os}-${arch}'). +GLOBAL_DEFINITIONS: Dict[str, str] = _get_global_defitions() + # The value of a west-commands as passed around during manifest # resolution. It can become a list due to resolving imports, even # though it's just a str in each individual file right now. @@ -89,6 +148,198 @@ # A list of group names belonging to a project, like ['foo', 'bar'] GroupsType = List[str] +class PyRequirementType(enum.Enum): + """ + Python requirements must be one of these: + - REQUIREMENTS: a requirements file + - PACKAGE: a single package string + - DIRECTORY: a local python package + - CONSTRAINTS: a constraints file + """ + + # A requirements file that can be passed to pip via a -r flag + REQUIREMENTS = 1 + # A single package that can be installed by pip + PACKAGE = 2 + # A constraints file that can be passed to pip via a -c flag + CONSTRAINTS = 3 + # A local directory containing a pakcage to install + DIRECTORY = 4 + + @staticmethod + def build(string: str) -> 'PyRequirementType': + """ + Convert a string to a PyRequirementType (if possible) + + Args: + string The string to convert + Returns: + PyRequirementType corresponding to the string + """ + for member in PyRequirementType: # Iterate over the enum members + if member.name.lower() == string.lower(): + return member + + raise RuntimeError( + f"Invalid Py requirement type '{string}'. " + + f"Must be one of: {[member.name.lower() for member in PyRequirementType]}" + ) + +class VenvPyRequirement(NamedTuple): + """ + Python requirements for the virtual environment + """ + + # The string pattern used to specify a file name, directory, or package + # name + pattern: str + + # The type of requirement represented by this object + type: PyRequirementType + + @staticmethod + def build(spec: Dict[str, str]) -> 'VenvPyRequirement': + """ + Convert a dictionary to a VenvPyRequirement (if possible). + + Args: + spec The dictionary to convert + Returns: + VenvPyRequirement representing the dictionary + """ + return VenvPyRequirement( + pattern=spec['pattern'], + type=PyRequirementType.build(spec['type']) + ) + + def as_dict(self) -> Dict: + return { + "pattern": self.pattern, + "type": self.type.name.lower(), + } + +class BinRequirementUrl(NamedTuple): + """ + A binary requirement that will be needed + """ + + # The URL used to fetch the requirement + url: str + + # One or more paths to be added to the PATH variable in order to make the + # binary discoverable + paths: List[str] + + @staticmethod + def build( + spec: Dict[str, Any], + defs: Dict[str, str] + ) -> 'BinRequirementUrl': + """ + Convert a URL spec and definitions to a requirement URL (if possible) + + Args: + spec The URL spec provided to the manifest. + defs A union of the global and local definitions + Returns: + A resolved binary URL requirement. + """ + url = _replace_variables(input_str=spec['url'], defs=defs) + paths = [ + _replace_variables(input_str=path, defs=defs) + for path in spec.get("paths", []) + ] + if not paths: + paths = ["."] + return BinRequirementUrl( + url=url, + paths=paths, + ) + +# Shorthand type for mapping a platform expression to a URL +BinRequirementUrls = Dict[str, BinRequirementUrl] + +class Venv(NamedTuple): + """ + Virtual environment used to build Zephyr. Includes both python and binary + requirements. + """ + + # The name of the virtual environment + name: str + + # Python requirements + py_requirements: List[VenvPyRequirement] + + # Binary requirements + bin_requirements: Dict[str, BinRequirementUrls] + + @staticmethod + def _resolve_bin_requirement_urls( + urls_spec: Dict[str, Any], + definitions: Dict[str, str], + ) -> BinRequirementUrls: + result: BinRequirementUrls = {} + for platform_key, url_spec in urls_spec.items(): + # Resolve the platform key + platform_key = platform_key.replace("*", ".*") + # Build the requirement URL object by resolving any definitions. + result[platform_key] = BinRequirementUrl.build( + spec=url_spec, + defs=definitions, + ) + return result + + @staticmethod + def build(spec: Dict[str, Any]) -> 'Venv': + """ + Construct a Venv from the manifest's 'venv' key. + + Args: + spec The value of the 'venv' key in the manifest + Returns: + A validate Venv object representing the virtual environment + """ + name = spec['name'] + top_level_definitions: Dict[str, str] = spec.get('definitions', {}) + py_requirements: List[VenvPyRequirement] = [ + VenvPyRequirement.build(py_requirement) + for py_requirement in spec.get('py-requirements', []) + ] + bin_requirements: Dict[str, BinRequirementUrls] = {} + for bin_req_name, bin_req_spec in spec.get('bin-requirements', {}).items(): + local_defs = top_level_definitions.copy() + local_defs.update(bin_req_spec.get("definitions", {})) + local_defs.update(GLOBAL_DEFINITIONS) + + bin_requirements[bin_req_name] = Venv._resolve_bin_requirement_urls( + urls_spec=bin_req_spec['urls'], + definitions=local_defs, + ) + return Venv( + name=name, + py_requirements=py_requirements, + bin_requirements=bin_requirements, + ) + + def as_dict(self) -> Dict: + result: Dict = { + "name": self.name, + "bin-requirements": {}, + "py-requirements": [], + } + for bin_req_name, req in self.bin_requirements.items(): + for platform_matcher, url in req.items(): + platform_key = platform_matcher.replace(".*", "*") + bin_requirements = result["bin-requirements"] + bin_req_entry = bin_requirements.setdefault(bin_req_name, {}) + bin_req_entry[platform_key] = url._asdict() + + for py_req in self.py_requirements: + result["py-requirements"].append(py_req.as_dict()) + + return result + class PFR(enum.Enum): # "Project filter result": internal type for expressing whether a # project has been explicitly made active or inactive via @@ -360,6 +611,8 @@ class _import_ctx(NamedTuple): # element. projects: Dict[str, 'Project'] + venv: Optional[Venv] + # The project filters we should apply while resolving imports. We # try to load this only once from the 'manifest.project-filter' # configuration option. @@ -1457,6 +1710,8 @@ def __init__(self, *, # All arguments are keyword-only. # ['-bar'], with filters imported from other manifests applied. self.group_filter: GroupFilterType = [] + self.venv: Optional[Venv] = None + # Initialize private state, some of which is also overwritten # later as needed. @@ -1601,6 +1856,8 @@ def _as_dict_helper( r['manifest'] = {} if self.group_filter: r['manifest']['group-filter'] = self.group_filter + if self.venv: + r['manifest']['venv'] = self.venv.as_dict() r['manifest']['projects'] = project_dicts r['manifest']['self'] = self.projects[MANIFEST_PROJECT_INDEX].as_dict() @@ -1883,6 +2140,7 @@ def _top_level_init(self, source_data, topdir, topdir_abspath, current_data = source_data current_repo_abspath = None project_filter: ProjectFilterType = [] + venv = None if topdir_abspath: config = config or Configuration(topdir=topdir_abspath) @@ -1929,6 +2187,7 @@ def get_option(option, default=None): config.get('manifest.project-filter')) return _import_ctx(projects={}, + venv=venv, project_filter=project_filter, group_filter_q=deque(), manifest_west_commands=[], @@ -2005,6 +2264,8 @@ def _load_validated(self) -> None: self._projects_by_name: Dict[str, Project] = {'manifest': mp} self._projects_by_name.update(self._ctx.projects) self._projects_by_rpath: Dict[Path, Project] = {} # resolved paths + if 'venv' in manifest_data: + self.venv = Venv.build(spec=manifest_data['venv']) if self.topdir: for p in self.projects: if TYPE_CHECKING: diff --git a/tests/test_manifest.py b/tests/test_manifest.py index bae75bf0..d5e3d8aa 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -29,7 +29,8 @@ from west.configuration import Configuration, ConfigFile, MalformedConfig # White box checks for the schema version. -from west.manifest import _VALID_SCHEMA_VERS +from west.manifest import _VALID_SCHEMA_VERS, VenvPyRequirement, \ + PyRequirementType, GLOBAL_DEFINITIONS, BinRequirementUrl from conftest import create_workspace, create_repo, checkout_branch, \ create_branch, add_commit, add_tag, rev_parse, GIT, check_proj_consistency @@ -2946,6 +2947,113 @@ def check(expected, group_filter, extra_filter=None): check((False, False, True), 'group-filter: [-ga]', extra_filter=['-gb']) +######################################### +# Manifest venv tests +# +# In schema version 1.3, "manifest: venv:" were added which enable +# extra information about both python and binary dependencies. + +def test_manifest_venv(): + m = M("""\ + projects: [] + venv: + name: ".venv" + definitions: + key0: "val0" + py-requirements: + - pattern: "scripts/requirements.txt" + type: "requirements" + - pattern: "pkg-name" + type: "package" + - pattern: "constraints.c" + type: "constraints" + - pattern: "path/to/py" + type: "directory" + bin-requirements: + bin_name0: + definitions: + key1: "val1" + urls: + linux-amd64: + url: "https://path/to/download/${platform}/${key1}" + windows-*: + url: "https://path/to/windows/download/${os}/${arch}/${key0}" + paths: ["win"] +""") + assert m.venv.name == ".venv" + assert m.venv.py_requirements == [ + VenvPyRequirement( + pattern="scripts/requirements.txt", + type=PyRequirementType.REQUIREMENTS, + ), + VenvPyRequirement( + pattern="pkg-name", + type=PyRequirementType.PACKAGE, + ), + VenvPyRequirement( + pattern="constraints.c", + type=PyRequirementType.CONSTRAINTS, + ), + VenvPyRequirement( + pattern="path/to/py", + type=PyRequirementType.DIRECTORY, + ), + ] + assert m.venv.bin_requirements == { + "bin_name0": { + "linux-amd64": BinRequirementUrl( + url=f"https://path/to/download/{GLOBAL_DEFINITIONS['platform']}/val1", + paths=["."], + ), + "windows-.*": BinRequirementUrl( + url=( + "https://path/to/windows/download/" + + f"{GLOBAL_DEFINITIONS['os']}/{GLOBAL_DEFINITIONS['arch']}/val0" + ), + paths=["win"], + ), + }, + } + assert m.as_dict() == { + 'manifest': { + 'projects': [], + 'self': {}, + 'venv': { + 'bin-requirements': { + 'bin_name0': { + 'linux-amd64': { + 'paths': ['.'], + 'url': 'https://path/to/download/linux-amd64/val1', + }, + 'windows-*': { + 'paths': ['win'], + 'url': 'https://path/to/windows/download/linux/amd64/val0', + }, + }, + }, + 'name': '.venv', + 'py-requirements': [ + { + 'pattern': 'scripts/requirements.txt', + 'type': 'requirements', + }, + { + 'pattern': 'pkg-name', + 'type': 'package', + }, + { + 'pattern': 'constraints.c', + 'type': 'constraints', + }, + { + 'pattern': 'path/to/py', + 'type': 'directory', + }, + ], + }, + }, + } + ######################################### # Manifest group-filter + import tests #