diff --git a/src/west/manifest-schema.yml b/src/west/manifest-schema.yml index 6982300f..ddd1c0bd 100644 --- a/src/west/manifest-schema.yml +++ b/src/west/manifest-schema.yml @@ -66,6 +66,78 @@ mapping: required: true type: str + # The venv key specifies requirements for the build environment for Zephyr + # West will handle merging any definitions into your binary dependency + # strings where allowed. + # Example: + # + # urls: + # "windows-*": + # url: "https://my.download.server/win/${arch}.zip" + # + # The above will match any architecture for the windows OS and generate a url + # replacing ${arch} with the current machine's underlying architecture. West + # will automatically provide a few definitions: + # - os: The current OS (one of 'windows', 'linux', or 'mac') + # - arch: The current architecture. This value is standardized such that + # x86_64 is represented as amd64 and aarch64 as arm64. All others match + # Python's `platform.machine()` + # - platform: A concatenation of ${os}-${arch} + venv: + required: false + type: map + mapping: + definitions: + required: false + type: map + mapping: + regex;(.*): + type: str + name: + required: false + type: str + bin-requirements: + required: false + type: map + mapping: + regex;(.*): + type: map + mapping: + definitions: + required: false + type: map + mapping: + regex;(.*): + type: str + urls: + required: true + type: map + mapping: + regex;(.*): + type: map + mapping: + url: + required: true + type: str + paths: + required: false + type: seq + sequence: + - type: str + py-requirements: + required: false + type: seq + sequence: + - type: map + mapping: + pattern: + required: true + type: str + type: + required: true + type: str + enum: ['directory', 'constraints', 'package', 'requirements'] + # The "projects" key specifies a sequence of "projects", each of which has a # remote, and may specify additional configuration. # diff --git a/src/west/manifest.py b/src/west/manifest.py index 57041eff..62b3d23d 100644 --- a/src/west/manifest.py +++ b/src/west/manifest.py @@ -13,6 +13,7 @@ import logging import os from pathlib import PurePosixPath, Path +import platform import re import shlex import subprocess @@ -67,6 +68,101 @@ # Internal helpers # +# Mapping of the architectures from platofrm.machine() to a common standard +# provided by CIPD (https://chrome-infra-packages.appspot.com/). While it's +# not the only naming scheme, it does allow for easy URL generation to a very +# robust package database. +_ARCH_MAPPING = { + "x86_64": "amd64", + "aarch64": "arm64", +} + +class GlobalDefitions(dict): + """ + Lazy constructed dictionary where special keys: os, arch, and platform are + only initialized when accessed. + """ + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._os_value: Union[None, str] = None + self._arch_value: Union[None, str] = None + + def to_dict(self) -> dict: + """ + Convert the definitions to a new dictionary + """ + result = dict(self) + result["os"] = self._os + result["arch"] = self._arch + result["platform"] = f"{self._os}-{self._arch}" + return result + + def __getitem__(self, key): + if key == "os": + return self._os + if key == "arch": + return self._arch + if key == "platform": + return f"{self._os}-{self._arch}" + return super().__getitem__(key) + + def __iter__(self): + yield "os" + yield "arch" + yield "platform" + yield from super().__iter__() + + @property + def _os(self) -> str: + if self._os_value is not None: + return self._os_value + current_os = platform.system() + if current_os == "Windows": + self._os_value = "windows" + elif current_os == "Darwin": + self._os_value = "mac" + elif current_os == "Linux": + self._os_value = "linux" + else: + raise RuntimeError("Unknown OS: " + current_os) + return self._os_value + + @property + def _arch(self) -> str: + if self._arch_value is not None: + return self._arch_value + arch = platform.machine() + self._arch_value = _ARCH_MAPPING.get(arch, arch) + return self._arch_value + +def _replace_variables( + input_str: str, + defs: Dict[str, str], +) -> str: + """ + Given a set of definitions, find all instances in an input string of ${key} + and replace the 'key' with the corresponding definition. If a key is not + found in the defs dictionary, raise an exception + + Args: + input_str The input string to fill + defs The key/value mappings to use + Returns: + A new string with all ${...} resolved + """ + while True: + placeholders = re.findall(r"\$\{(.*?)\}", input_str) + if not placeholders: + return input_str + + key = placeholders[0] + input_str = re.sub(r"\$\{" + key + r"\}", defs[key], input_str) + +# Values that can be used in the venv string values via their keys. +# By default, west will include 'os', 'arch', and 'platform' (where +# platform is always '${os}-${arch}'). +GLOBAL_DEFINITIONS: GlobalDefitions = GlobalDefitions() + # The value of a west-commands as passed around during manifest # resolution. It can become a list due to resolving imports, even # though it's just a str in each individual file right now. @@ -89,6 +185,198 @@ # A list of group names belonging to a project, like ['foo', 'bar'] GroupsType = List[str] +class PyRequirementType(enum.Enum): + """ + Python requirements must be one of these: + - REQUIREMENTS: a requirements file + - PACKAGE: a single package string + - DIRECTORY: a local python package + - CONSTRAINTS: a constraints file + """ + + # A requirements file that can be passed to pip via a -r flag + REQUIREMENTS = 1 + # A single package that can be installed by pip + PACKAGE = 2 + # A constraints file that can be passed to pip via a -c flag + CONSTRAINTS = 3 + # A local directory containing a pakcage to install + DIRECTORY = 4 + + @staticmethod + def build(string: str) -> 'PyRequirementType': + """ + Convert a string to a PyRequirementType (if possible) + + Args: + string The string to convert + Returns: + PyRequirementType corresponding to the string + """ + for member in PyRequirementType: # Iterate over the enum members + if member.name.lower() == string.lower(): + return member + + raise RuntimeError( + f"Invalid Py requirement type '{string}'. " + + f"Must be one of: {[member.name.lower() for member in PyRequirementType]}" + ) + +class VenvPyRequirement(NamedTuple): + """ + Python requirements for the virtual environment + """ + + # The string pattern used to specify a file name, directory, or package + # name + pattern: str + + # The type of requirement represented by this object + type: PyRequirementType + + @staticmethod + def build(spec: Dict[str, str]) -> 'VenvPyRequirement': + """ + Convert a dictionary to a VenvPyRequirement (if possible). + + Args: + spec The dictionary to convert + Returns: + VenvPyRequirement representing the dictionary + """ + return VenvPyRequirement( + pattern=spec['pattern'], + type=PyRequirementType.build(spec['type']) + ) + + def as_dict(self) -> Dict: + return { + "pattern": self.pattern, + "type": self.type.name.lower(), + } + +class BinRequirementUrl(NamedTuple): + """ + A binary requirement that will be needed + """ + + # The URL used to fetch the requirement + url: str + + # One or more paths to be added to the PATH variable in order to make the + # binary discoverable + paths: List[str] + + @staticmethod + def build( + spec: Dict[str, Any], + defs: Dict[str, str] + ) -> 'BinRequirementUrl': + """ + Convert a URL spec and definitions to a requirement URL (if possible) + + Args: + spec The URL spec provided to the manifest. + defs A union of the global and local definitions + Returns: + A resolved binary URL requirement. + """ + url = _replace_variables(input_str=spec['url'], defs=defs) + paths = [ + _replace_variables(input_str=path, defs=defs) + for path in spec.get("paths", []) + ] + if not paths: + paths = ["."] + return BinRequirementUrl( + url=url, + paths=paths, + ) + +# Shorthand type for mapping a platform expression to a URL +BinRequirementUrls = Dict[str, BinRequirementUrl] + +class Venv(NamedTuple): + """ + Virtual environment used to build Zephyr. Includes both python and binary + requirements. + """ + + # The name of the virtual environment + name: str + + # Python requirements + py_requirements: List[VenvPyRequirement] + + # Binary requirements + bin_requirements: Dict[str, BinRequirementUrls] + + @staticmethod + def _resolve_bin_requirement_urls( + urls_spec: Dict[str, Any], + definitions: Dict[str, str], + ) -> BinRequirementUrls: + result: BinRequirementUrls = {} + for platform_key, url_spec in urls_spec.items(): + # Resolve the platform key + platform_key = platform_key.replace("*", ".*") + # Build the requirement URL object by resolving any definitions. + result[platform_key] = BinRequirementUrl.build( + spec=url_spec, + defs=definitions, + ) + return result + + @staticmethod + def build(spec: Dict[str, Any]) -> 'Venv': + """ + Construct a Venv from the manifest's 'venv' key. + + Args: + spec The value of the 'venv' key in the manifest + Returns: + A validate Venv object representing the virtual environment + """ + name = spec['name'] + top_level_definitions: Dict[str, str] = spec.get('definitions', {}) + py_requirements: List[VenvPyRequirement] = [ + VenvPyRequirement.build(py_requirement) + for py_requirement in spec.get('py-requirements', []) + ] + bin_requirements: Dict[str, BinRequirementUrls] = {} + for bin_req_name, bin_req_spec in spec.get('bin-requirements', {}).items(): + local_defs = top_level_definitions.copy() + local_defs.update(bin_req_spec.get("definitions", {})) + local_defs.update(GLOBAL_DEFINITIONS.to_dict()) + + bin_requirements[bin_req_name] = Venv._resolve_bin_requirement_urls( + urls_spec=bin_req_spec['urls'], + definitions=local_defs, + ) + return Venv( + name=name, + py_requirements=py_requirements, + bin_requirements=bin_requirements, + ) + + def as_dict(self) -> Dict: + result: Dict = { + "name": self.name, + "bin-requirements": {}, + "py-requirements": [], + } + for bin_req_name, req in self.bin_requirements.items(): + for platform_matcher, url in req.items(): + platform_key = platform_matcher.replace(".*", "*") + bin_requirements = result["bin-requirements"] + bin_req_entry = bin_requirements.setdefault(bin_req_name, {}) + bin_req_entry[platform_key] = url._asdict() + + for py_req in self.py_requirements: + result["py-requirements"].append(py_req.as_dict()) + + return result + class PFR(enum.Enum): # "Project filter result": internal type for expressing whether a # project has been explicitly made active or inactive via @@ -360,6 +648,8 @@ class _import_ctx(NamedTuple): # element. projects: Dict[str, 'Project'] + venv: Optional[Venv] + # The project filters we should apply while resolving imports. We # try to load this only once from the 'manifest.project-filter' # configuration option. @@ -1457,6 +1747,8 @@ def __init__(self, *, # All arguments are keyword-only. # ['-bar'], with filters imported from other manifests applied. self.group_filter: GroupFilterType = [] + self.venv: Optional[Venv] = None + # Initialize private state, some of which is also overwritten # later as needed. @@ -1601,6 +1893,8 @@ def _as_dict_helper( r['manifest'] = {} if self.group_filter: r['manifest']['group-filter'] = self.group_filter + if self.venv: + r['manifest']['venv'] = self.venv.as_dict() r['manifest']['projects'] = project_dicts r['manifest']['self'] = self.projects[MANIFEST_PROJECT_INDEX].as_dict() @@ -1883,6 +2177,7 @@ def _top_level_init(self, source_data, topdir, topdir_abspath, current_data = source_data current_repo_abspath = None project_filter: ProjectFilterType = [] + venv = None if topdir_abspath: config = config or Configuration(topdir=topdir_abspath) @@ -1929,6 +2224,7 @@ def get_option(option, default=None): config.get('manifest.project-filter')) return _import_ctx(projects={}, + venv=venv, project_filter=project_filter, group_filter_q=deque(), manifest_west_commands=[], @@ -2005,6 +2301,8 @@ def _load_validated(self) -> None: self._projects_by_name: Dict[str, Project] = {'manifest': mp} self._projects_by_name.update(self._ctx.projects) self._projects_by_rpath: Dict[Path, Project] = {} # resolved paths + if 'venv' in manifest_data: + self.venv = Venv.build(spec=manifest_data['venv']) if self.topdir: for p in self.projects: if TYPE_CHECKING: diff --git a/tests/test_manifest.py b/tests/test_manifest.py index bae75bf0..97474b4a 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -29,7 +29,8 @@ from west.configuration import Configuration, ConfigFile, MalformedConfig # White box checks for the schema version. -from west.manifest import _VALID_SCHEMA_VERS +from west.manifest import _VALID_SCHEMA_VERS, VenvPyRequirement, \ + PyRequirementType, GLOBAL_DEFINITIONS, BinRequirementUrl from conftest import create_workspace, create_repo, checkout_branch, \ create_branch, add_commit, add_tag, rev_parse, GIT, check_proj_consistency @@ -2946,6 +2947,114 @@ def check(expected, group_filter, extra_filter=None): check((False, False, True), 'group-filter: [-ga]', extra_filter=['-gb']) +######################################### +# Manifest venv tests +# +# In schema version 1.3, "manifest: venv:" were added which enable +# extra information about both python and binary dependencies. + +def test_manifest_venv(): + m = M("""\ + projects: [] + venv: + name: ".venv" + definitions: + key0: "val0" + py-requirements: + - pattern: "scripts/requirements.txt" + type: "requirements" + - pattern: "pkg-name" + type: "package" + - pattern: "constraints.c" + type: "constraints" + - pattern: "path/to/py" + type: "directory" + bin-requirements: + bin_name0: + definitions: + key1: "val1" + urls: + linux-amd64: + url: "https://path/to/download/${platform}/${key1}" + windows-*: + url: "https://path/to/windows/download/${os}/${arch}/${key0}" + paths: ["win"] +""") + assert m.venv.name == ".venv" + assert m.venv.py_requirements == [ + VenvPyRequirement( + pattern="scripts/requirements.txt", + type=PyRequirementType.REQUIREMENTS, + ), + VenvPyRequirement( + pattern="pkg-name", + type=PyRequirementType.PACKAGE, + ), + VenvPyRequirement( + pattern="constraints.c", + type=PyRequirementType.CONSTRAINTS, + ), + VenvPyRequirement( + pattern="path/to/py", + type=PyRequirementType.DIRECTORY, + ), + ] + assert m.venv.bin_requirements == { + "bin_name0": { + "linux-amd64": BinRequirementUrl( + url=f"https://path/to/download/{GLOBAL_DEFINITIONS['platform']}/val1", + paths=["."], + ), + "windows-.*": BinRequirementUrl( + url=( + "https://path/to/windows/download/" + + f"{GLOBAL_DEFINITIONS['os']}/{GLOBAL_DEFINITIONS['arch']}/val0" + ), + paths=["win"], + ), + }, + } + venv_dict = m.as_dict()["manifest"]["venv"] + assert venv_dict["name"] == ".venv" + + bin_requirements = venv_dict["bin-requirements"] + assert len(bin_requirements) == 1 + + bin_name0 = bin_requirements["bin_name0"] + assert len(bin_name0) == 2 + + assert bin_name0["linux-amd64"] == { + "paths": ["."], + "url": f"https://path/to/download/{GLOBAL_DEFINITIONS['platform']}/val1", + } + assert bin_name0["windows-*"] == { + "paths": ["win"], + "url": ( + "https://path/to/windows/download/" + + f"{GLOBAL_DEFINITIONS['os']}/{GLOBAL_DEFINITIONS['arch']}/val0" + ), + } + + py_requirements = venv_dict["py-requirements"] + assert py_requirements == [ + { + "pattern": "scripts/requirements.txt", + "type": "requirements", + }, + { + "pattern": "pkg-name", + "type": "package", + }, + { + "pattern": "constraints.c", + "type": "constraints", + }, + { + "pattern": "path/to/py", + "type": "directory", + }, + ] + ######################################### # Manifest group-filter + import tests #