prototype implementation of PEP725 PURL mapping (#518)

grayskull/cli/stdout.py

@@ -8,6 +8,7 @@
 
 from grayskull.base.pkg_info import is_pkg_available
 from grayskull.cli import WIDGET_BAR_DOWNLOAD, CLIConfig
+from grayskull.utils import RE_PEP725_PURL
 
 
 def print_msg(msg: str):
@@ -78,6 +79,11 @@ def print_req(list_pkg):
                 pkg_name = pkg.replace("<{", "{{")
                 options = ""
                 colour = Fore.GREEN
+            elif RE_PEP725_PURL.match(pkg):
+                pkg_name = pkg
+                options = ""
+                all_missing_deps.add(pkg)
+                colour = Fore.YELLOW
             elif search_result:
                 pkg_name, options = search_result.groups()
                 if is_pkg_available(pkg_name):
@@ -102,7 +108,15 @@ def print_req(list_pkg):
         print_msg(f"{key.capitalize()} requirements (optional):")
         print_req(req_list)
 
-    print_msg(f"\n{Fore.RED}RED{Style.RESET_ALL}: Missing packages")
+    print_msg(
+        f"\n{Fore.RED}RED{Style.RESET_ALL}: Package names not available on conda-forge"
+    )
+    print_msg(
+        (
+            f"{Fore.YELLOW}YELLOW{Style.RESET_ALL}: "
+            "PEP-725 PURLs that did not map to known package"
+        )
+    )
     print_msg(f"{Fore.GREEN}GREEN{Style.RESET_ALL}: Packages available on conda-forge")
 
     if CLIConfig().list_missing_deps:

grayskull/strategy/py_base.py

@@ -27,6 +27,7 @@
 from grayskull.license.discovery import ShortLicense, search_license_file
 from grayskull.strategy.py_toml import get_all_toml_info
 from grayskull.utils import (
+    RE_PEP725_PURL,
     PyVer,
     get_vendored_dependencies,
     merge_dict_of_lists_item,
@@ -546,10 +547,12 @@ def clean_list_pkg(pkg, list_pkgs):
         return [p for p in list_pkgs if pkg != p.strip().split(" ", 1)[0]]
 
     for pkg in requirements["host"]:
-        pkg_name = RE_DEPS_NAME.match(pkg).group(0)
-        if pkg_name in PIN_PKG_COMPILER.keys():
-            requirements["run"] = clean_list_pkg(pkg_name, requirements["run"])
-            requirements["run"].append(PIN_PKG_COMPILER[pkg_name])
+        pkg_name_match = RE_DEPS_NAME.match(pkg)
+        if pkg_name_match:
+            pkg_name = pkg_name_match.group(0)
+            if pkg_name in PIN_PKG_COMPILER.keys():
+                requirements["run"] = clean_list_pkg(pkg_name, requirements["run"])
+                requirements["run"].append(PIN_PKG_COMPILER[pkg_name])
 
 
 def discover_license(metadata: dict) -> List[ShortLicense]:
@@ -733,6 +736,14 @@ def merge_setup_toml_metadata(setup_metadata: dict, pyproject_metadata: dict) ->
             setup_metadata.get("install_requires", []),
             pyproject_metadata["requirements"]["run"],
         )
+    # this is not a valid setup_metadata field, but we abuse it to pass it
+    # through to the conda recipe generator downstream. It's because setup.py
+    # does not have a notion of build vs. host requirements. It only has
+    # equivalents to host and run.
+    if pyproject_metadata["requirements"]["build"]:
+        setup_metadata["__build_requirements_placeholder"] = pyproject_metadata[
+            "requirements"
+        ]["build"]
     if pyproject_metadata["requirements"]["run_constrained"]:
         setup_metadata["requirements_run_constrained"] = pyproject_metadata[
             "requirements"
@@ -802,9 +813,8 @@ def ensure_pep440_in_req_list(list_req: List[str]) -> List[str]:
 
 
 def split_deps(deps: str) -> List[str]:
-    deps = deps.split(",")
     result = []
-    for d in deps:
+    for d in deps.split(","):
         constrain = ""
         for val in re.split(r"([><!=~^]+)", d):
             if not val:
@@ -817,9 +827,10 @@ def split_deps(deps: str) -> List[str]:
 
 
 def ensure_pep440(pkg: str) -> str:
-    if not pkg:
+    if not pkg or RE_PEP725_PURL.match(pkg):
         return pkg
-    if pkg.strip().startswith("<{") or pkg.strip().startswith("{{"):
+    pkg = pkg.strip()
+    if any([pkg.startswith(pattern) for pattern in ("<{", "{{")]):
         return pkg
     split_pkg = pkg.strip().split(" ")
     if len(split_pkg) <= 1:

grayskull/strategy/py_toml.py

@@ -246,6 +246,56 @@ def add_flit_metadata(metadata: dict, toml_metadata: dict) -> dict:
     return metadata
 
 
+def is_pep725_present(toml_metadata: dict):
+    return "external" in toml_metadata
+
+
+def get_pep725_mapping(purl: str):
+    """This function maps a PURL to the name in the conda ecosystem. It is expected
+    that this will be provided on a per-ecosystem basis (such as by conda-forge)"""
+
+    package_mapping = {
+        "virtual:compiler/c": "{{ compiler('c') }}",
+        "virtual:compiler/cpp": "{{ compiler('cxx') }}",
+        "virtual:compiler/fortran": "{{ compiler('fortran') }}",
+        "virtual:compiler/rust": "{{ compiler('rust') }}",
+        "virtual:interface/blas": "{{ blas }}",
+    }
+    return package_mapping.get(purl, purl)
+
+
+def add_pep725_metadata(metadata: dict, toml_metadata: dict):
+    if not is_pep725_present(toml_metadata):
+        return metadata
+
+    externals = toml_metadata["external"]
+    # each of these is a list of PURLs. For each one we find,
+    # we need to map it to the the conda ecosystem
+    requirements = metadata.get("requirements", {})
+    section_map = (
+        ("build", "build-requires"),
+        ("host", "host-requires"),
+        ("run", "dependencies"),
+    )
+    for conda_section, pep725_section in section_map:
+        requirements[conda_section] = [
+            get_pep725_mapping(purl) for purl in externals.get(pep725_section, [])
+        ]
+        # TODO: handle optional dependencies properly
+        optional_features = toml_metadata.get(f"optional-{pep725_section}", {})
+        for feature_name, feature_deps in optional_features.items():
+            requirements[conda_section].append(
+                f'# OPTIONAL dependencies from feature "{feature_name}"'
+            )
+            requirements[conda_section].extend(feature_deps)
+        if not requirements[conda_section]:
+            del requirements[conda_section]
+
+    if requirements:
+        metadata["requirements"] = requirements
+    return metadata
+
+
 def get_all_toml_info(path_toml: Union[Path, str]) -> dict:
     with open(path_toml, "rb") as f:
         toml_metadata = tomli.load(f)
@@ -288,5 +338,6 @@ def get_all_toml_info(path_toml: Union[Path, str]) -> dict:
 
     add_poetry_metadata(metadata, toml_metadata)
     add_flit_metadata(metadata, toml_metadata)
+    add_pep725_metadata(metadata, toml_metadata)
 
     return metadata

grayskull/strategy/pypi.py

@@ -111,6 +111,7 @@ def get_val(key):
         "requires_dist": requires_dist,
         "sdist_path": get_val("sdist_path"),
         "requirements_run_constrained": get_val("requirements_run_constrained"),
+        "__build_requirements_placeholder": get_val("__build_requirements_placeholder"),
     }
 
 
@@ -556,6 +557,8 @@ def extract_requirements(metadata: dict, config, recipe) -> Dict[str, List[str]]
     requires_dist = format_dependencies(metadata.get("requires_dist", []), name)
     setup_requires = metadata.get("setup_requires", [])
     host_req = format_dependencies(setup_requires or [], config.name)
+    build_requires = metadata.get("__build_requirements_placeholder", [])
+    build_req = format_dependencies(build_requires or [], config.name)
     if not requires_dist and not host_req and not metadata.get("requires_python"):
         if config.is_strict_cf:
             py_constrain = (
@@ -571,7 +574,9 @@ def extract_requirements(metadata: dict, config, recipe) -> Dict[str, List[str]]
 
     run_req = get_run_req_from_requires_dist(requires_dist, config)
     host_req = get_run_req_from_requires_dist(host_req, config)
-    build_req = [f"<{{ compiler('{c}') }}}}" for c in metadata.get("compilers", [])]
+    build_req = build_req or [
+        f"<{{ compiler('{c}') }}}}" for c in metadata.get("compilers", [])
+    ]
     if build_req:
         config.is_arch = True
 

grayskull/utils.py

@@ -24,6 +24,10 @@
 yaml.width = 600
 
 
+#  PURL fields               scheme      type           name
+RE_PEP725_PURL = re.compile(r"[a-z]+\:[\.a-z0-9_-]+\/[\.a-z0-9_-]+", re.IGNORECASE)
+
+
 @lru_cache(maxsize=10)
 def get_std_modules() -> List:
     from stdlib_list import stdlib_list
@@ -167,6 +171,9 @@ def format_dependencies(all_dependencies: List, name: str) -> List:
     re_remove_tags = re.compile(r"\s*(\[.*\])", re.DOTALL)
     re_remove_comments = re.compile(r"\s+#.*", re.DOTALL)
     for req in all_dependencies:
+        if RE_PEP725_PURL.match(req):
+            formatted_dependencies.append(req)
+            continue
         match_req = re_deps.match(req)
         deps_name = req
         if name is not None and deps_name.replace("-", "_") == name.replace("-", "_"):
@@ -220,11 +227,6 @@ def generate_recipe(
             copyfile(file_to_recipe, os.path.join(recipe_folder, name))
 
 
-def get_clean_yaml(recipe_yaml: CommentedMap) -> CommentedMap:
-    clean_yaml(recipe_yaml)
-    return add_new_lines_after_section(recipe_yaml)
-
-
 def add_new_lines_after_section(recipe_yaml: CommentedMap) -> CommentedMap:
     for section in recipe_yaml.keys():
         if section == "package":

tests/test_poetry.py → tests/test_py_toml.py

@@ -8,6 +8,8 @@
 from grayskull.main import generate_recipes_from_list, init_parser
 from grayskull.strategy.py_toml import (
     InvalidVersion,
+    add_flit_metadata,
+    add_pep725_metadata,
     add_poetry_metadata,
     encode_poetry_version,
     get_all_toml_info,
@@ -18,6 +20,13 @@
 )
 
 
+def test_add_flit_metadata():
+    metadata = {"build": {"entry_points": []}}
+    toml_metadata = {"tool": {"flit": {"scripts": {"key": "value"}}}}
+    result = add_flit_metadata(metadata, toml_metadata)
+    assert result == {"build": {"entry_points": ["key = value"]}}
+
+
 @pytest.mark.parametrize(
     "version, major, minor, patch",
     [
@@ -160,7 +169,7 @@ def test_poetry_langchain_snapshot(tmpdir):
     assert filecmp.cmp(snapshot_path, output_path, shallow=False)
 
 
-def test_get_constrained_dep_version_not_present():
+def test_poetry_get_constrained_dep_version_not_present():
     assert (
         get_constrained_dep(
             {"git": "https://codeberg.org/hjacobs/pytest-kind.git"}, "pytest-kind"
@@ -169,7 +178,7 @@ def test_get_constrained_dep_version_not_present():
     )
 
 
-def test_entrypoints():
+def test_poetry_entrypoints():
     poetry = {
         "requirements": {"host": ["setuptools"], "run": ["python"]},
         "build": {},
@@ -198,3 +207,25 @@ def test_entrypoints():
         },
         "test": {},
     }
+
+
+@pytest.mark.parametrize(
+    "conda_section, pep725_section",
+    [("build", "build-requires"), ("host", "host-requires"), ("run", "dependencies")],
+)
+@pytest.mark.parametrize(
+    "purl, purl_translated",
+    [
+        ("virtual:compiler/c", "{{ compiler('c') }}"),
+        ("pkg:alice/bob", "pkg:alice/bob"),
+    ],
+)
+def test_pep725_section_lookup(conda_section, pep725_section, purl, purl_translated):
+    toml_metadata = {
+        "external": {
+            pep725_section: [purl],
+        }
+    }
+    assert add_pep725_metadata({}, toml_metadata) == {
+        "requirements": {conda_section: [purl_translated]}
+    }