Generate recipe from project tree using pypa/build

grayskull/base/factory.py

@@ -7,6 +7,11 @@
 from souschef.recipe import Recipe
 
 from grayskull.strategy.cran import CranStrategy
+
+try:
+    from grayskull.strategy.py_build import PyBuild
+except ImportError:  # requires conda
+    PyBuild = None
 from grayskull.strategy.pypi import PypiStrategy
 
 
@@ -16,6 +21,9 @@ class GrayskullFactory(ABC):
         "cran": CranStrategy,
     }
 
+    if PyBuild:
+        REGISTERED_STRATEGY["pybuild"] = PyBuild
+
     @staticmethod
     def create_recipe(repo_type: str, config, pkg_name=None, sections_populate=None):
         if repo_type.lower() not in GrayskullFactory.REGISTERED_STRATEGY:

grayskull/config.py

@@ -42,6 +42,7 @@ class Configuration:
     is_arch: bool = False
     repo_github: Optional[str] = None
     from_local_sdist: bool = False
+    from_tree: bool = False
     local_sdist: Optional[str] = None
     missing_deps: set = field(default_factory=set)
     extras_require_test: Optional[str] = None

grayskull/license/discovery.py

@@ -62,6 +62,19 @@ def get_all_licenses_from_spdx() -> List:
     ]
 
 
+def _match_scrambled_exact(candidate, licenses):
+    """
+    Return license with rearranged word order only.
+
+    Fancy scorer confuses BSD-3-Clause with DEC-3-Clause.
+    """
+    bag = set(re.findall(r"\w+", candidate.lower()))
+    for license in licenses:
+        if bag == set(re.findall(r"\w+", license.lower())):
+            return license
+    return ""
+
+
 def match_license(name: str) -> dict:
     """Match if the given license name matches any license present on
     spdx.org
@@ -75,11 +88,16 @@ def match_license(name: str) -> dict:
     name = re.sub(r"\s+license\s*", "", name.strip(), flags=re.IGNORECASE)
     name = name.strip()
 
-    best_matches = process.extract(
-        name, _get_all_license_choice(all_licenses), scorer=partial_ratio
-    )
-    best_matches = process.extract(name, [lc for lc, *_ in best_matches])
-    spdx_license = best_matches[0]
+    exact_match = _match_scrambled_exact(name, _get_all_license_choice(all_licenses))
+    if exact_match:
+        best_matches = [(exact_match, 100, 0)]
+        spdx_license = best_matches[0]
+    else:
+        best_matches = process.extract(
+            name, _get_all_license_choice(all_licenses), scorer=partial_ratio
+        )
+        best_matches = process.extract(name, [lc for lc, *_ in best_matches])
+        spdx_license = best_matches[0]
 
     if spdx_license[1] < 100:
         # Prefer "-or-later" licenses over the "-only"

grayskull/main.py

@@ -14,7 +14,12 @@
 from grayskull.cli import CLIConfig
 from grayskull.cli.stdout import print_msg
 from grayskull.config import Configuration
-from grayskull.utils import generate_recipe, origin_is_github, origin_is_local_sdist
+from grayskull.utils import (
+    generate_recipe,
+    origin_is_github,
+    origin_is_local_sdist,
+    origin_is_tree,
+)
 
 init(autoreset=True)
 logging.basicConfig(format="%(levelname)s:%(message)s")
@@ -301,10 +306,13 @@ def generate_recipes_from_list(list_pkgs, args):
     for pkg_name in list_pkgs:
         logging.debug(f"Starting grayskull for pkg: {pkg_name}")
         from_local_sdist = origin_is_local_sdist(pkg_name)
+        from_tree = origin_is_tree(pkg_name)
         if origin_is_github(pkg_name):
             pypi_label = ""
         elif from_local_sdist:
             pypi_label = " (local)"
+        elif from_tree:
+            pypi_label = " (tree)"
         else:
             pypi_label = " (pypi)"
         print_msg(
@@ -322,6 +330,7 @@ def generate_recipes_from_list(list_pkgs, args):
                 url_pypi_metadata=args.url_pypi_metadata,
                 sections_populate=args.sections_populate,
                 from_local_sdist=from_local_sdist,
+                from_tree=from_tree,
                 extras_require_test=args.extras_require_test,
                 github_release_tag=args.github_release_tag,
                 extras_require_include=tuple(args.extras_require_include),
@@ -351,7 +360,9 @@ def create_python_recipe(pkg_name, sections_populate=None, **kwargs):
     config = Configuration(name=pkg_name, **kwargs)
     return (
         GrayskullFactory.create_recipe(
-            "pypi", config, sections_populate=sections_populate
+            "pybuild" if config.from_tree else "pypi",
+            config,
+            sections_populate=sections_populate,
         ),
         config,
     )

grayskull/strategy/py_build.py

@@ -0,0 +1,158 @@
+"""
+Use pypa/build to get project metadata from a checkout. Create a recipe suitable
+for inlinining into the first-party project source tree.
+"""
+
+import logging
+import tempfile
+from importlib.metadata import PathDistribution
+from pathlib import Path
+
+import build
+from conda.exceptions import InvalidMatchSpec
+from conda.models.match_spec import MatchSpec
+from packaging.requirements import Requirement
+from souschef.recipe import Recipe
+
+from grayskull.config import Configuration
+from grayskull.strategy.abstract_strategy import AbstractStrategy
+from grayskull.strategy.pypi import compose_test_section
+
+log = logging.getLogger(__name__)
+
+
+class PyBuild(AbstractStrategy):
+    @staticmethod
+    def fetch_data(recipe: Recipe, config: Configuration, sections=None):
+        project = build.ProjectBuilder(config.name)
+
+        recipe["source"]["path"] = "../"
+        # XXX relative to output. "git_url: ../" is also a good choice.
+
+        with tempfile.TemporaryDirectory(prefix="grayskull") as output:
+            build_system_requires = project.build_system_requires
+            requires_for_build = project.get_requires_for_build("wheel")
+            # If those are already installed, we can get the extras requirements
+            # without invoking pip e.g. setuptools_scm[toml]
+            print("Requires for build:", build_system_requires, requires_for_build)
+
+            # Example of finding extra dependencies for a distribution 'scm' (is
+            # there a dict API?) Subtract "has non-extra marker" dependencies from
+            # this set.
+            #
+            # Easier API is deprecated pkg_resources.Distribution().requires(extras=())
+            #
+            # for e in scm.metadata.get_all('provides-extra'):
+            #   print (e, [x for x in r if x.marker and x.marker.evaluate({'extra':e})])
+            #
+            #     docs [<Requirement('entangled-cli[rich]; extra == "docs"')>, <Requirement('mkdocs; extra == "docs"')>, <Requirement('mkdocs-entangled-plugin; extra == "docs"')>, <Requirement('mkdocs-material; extra == "docs"')>, <Requirement('mkdocstrings[python]; extra == "docs"')>, <Requirement('pygments; extra == "docs"')>]
+            #     rich [<Requirement('rich; extra == "rich"')>]
+            #     test [<Requirement('build; extra == "test"')>, <Requirement('pytest; extra == "test"')>, <Requirement('rich; extra == "test"')>, <Requirement('wheel; extra == "test"')>]
+            #     toml []
+
+            # build the project's metadata "dist-info" directory
+            metadata_path = Path(project.metadata_path(output_directory=output))
+
+            distribution = PathDistribution(metadata_path)
+
+            # real distribution name not pathname
+            config.name = distribution.name  # see also _normalized_name
+            config.version = distribution.version
+
+            # grayskull thought the name was the path. correct that.
+            if recipe[0] == '#% set name = "." %}':  # XXX fragile
+                # recipe[0] = x does not work
+                recipe._yaml._yaml_get_pre_comment()[0].value = (
+                    f'#% set name = "{config.name}" %}}\n'
+                    f'#% set version = "{config.version}" %}}\n'
+                )
+
+                recipe["package"]["version"] = "{{ version }}"
+            elif config.name not in str(recipe[0]):
+                log.warning("Package name not found in first line of recipe")
+
+            metadata = distribution.metadata
+
+            requires_python = metadata["requires-python"]
+            if requires_python:
+                requires_python = f"python { requires_python }"
+            else:
+                requires_python = "python"
+
+            recipe["requirements"]["host"] = [requires_python] + sorted(
+                (*build_system_requires, *requires_for_build)
+            )
+
+            requirements = [Requirement(r) for r in distribution.requires or []]
+            active_requirements = [
+                str(r).rsplit(";", 1)[0]
+                for r in requirements
+                if not r.marker or r.marker.evaluate()
+            ]
+            # XXX to normalize space between name and version, MatchSpec(r).spec
+            normalized_requirements = []
+            for requirement in active_requirements:
+                try:
+                    normalized_requirements.append(
+                        # MatchSpec uses a metaclass hiding its constructor from
+                        # the type checker
+                        MatchSpec(requirement).spec  # type: ignore
+                    )
+                except InvalidMatchSpec:
+                    log.warning("%s is not a valid MatchSpec", requirement)
+                    normalized_requirements.append(requirement)
+
+            # conda does support ~=3.0.0 "compatibility release" matches
+            recipe["requirements"]["run"] = [requires_python] + normalized_requirements
+            # includes extras as markers e.g. ; extra == 'testing'. Evaluate
+            # using Marker().
+
+            recipe["build"]["entry_points"] = [
+                f"{ep.name} = {ep.value}"
+                for ep in distribution.entry_points
+                if ep.group == "console_scripts"
+            ]
+
+            recipe["build"]["noarch"] = "python"
+            recipe["build"][
+                "script"
+            ] = "{{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation"
+            # XXX also --no-index?
+
+            # distribution.metadata.keys() for grayskull is
+            # Metadata-Version
+            # Name
+            # Version
+            # Summary
+            # Author-email
+            # License
+            # Project-URL
+            # Keywords
+            # Requires-Python
+            # Description-Content-Type
+            # License-File
+            # License-File
+            # Requires-Dist (many times)
+            # Provides-Extra (several times)
+            # Description or distribution.metadata.get_payload()
+
+            about = {
+                "summary": metadata["summary"],
+                "license": metadata["license"],
+                # there are two license-file in grayskull e.g.
+                "license_file": metadata["license-file"],
+            }
+            recipe["about"] = about
+
+            metadata_dict = dict(
+                metadata
+            )  # XXX not what compose_test_section expects at all
+            metadata_dict["name"] = config.name
+            metadata_dict["entry_points"] = [
+                f"{ep.name} = {ep.value}"
+                for ep in distribution.entry_points
+                if ep.group == "console_scripts"
+            ]
+            recipe["test"] = compose_test_section(metadata_dict, [])
+
+        # raise NotImplementedError()

grayskull/utils.py

@@ -109,6 +109,12 @@ def origin_is_local_sdist(name: str) -> bool:
     )
 
 
+def origin_is_tree(name: str) -> bool:
+    """Return True if it is a directory"""
+    path = Path(name)
+    return path.is_dir()
+
+
 def sha256_checksum(filename, block_size=65536):
     sha256 = hashlib.sha256()
     with open(filename, "rb") as f:

pyproject.toml

@@ -15,6 +15,7 @@ dynamic = ["version"]
 requires-python = ">=3.8"
 dependencies = [
     "beautifulsoup4",
+    "build",                  # conda install python-build
     "colorama",
     "conda-souschef >=2.2.3",
     "packaging >=21.3",
@@ -50,6 +51,7 @@ docs = [
     "mdit-py-plugins>=0.3.0",
 ]
 
+# both spellings of gray/grey
 [project.scripts]
 grayskull = "grayskull.main:main"
 greyskull = "grayskull.main:main"

tests/cli/test_cli_cmds.py

@@ -79,6 +79,7 @@ def test_change_pypi_url(mocker):
         url_pypi_metadata="http://url_pypi.com/abc",
         sections_populate=None,
         from_local_sdist=False,
+        from_tree=False,
         extras_require_test=None,
         github_release_tag=None,
         extras_require_include=tuple(),