diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d413d8a7a..2d5929313 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,6 +8,10 @@ on: branches: - "*" +defaults: + run: + shell: bash -l {0} + jobs: run: runs-on: ${{ matrix.os }} @@ -15,28 +19,40 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - py_ver: ["3.11", "3.12"] + py_ver: ["3.8", "3.9", "3.10", "3.11", "3.12"] env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.py_ver }} steps: - uses: actions/checkout@master - - uses: conda-incubator/setup-miniconda@v3.0.4 + - uses: mamba-org/setup-micromamba@v1 with: - auto-update-conda: true - channels: conda-forge,defaults - channel-priority: true - python-version: ${{ matrix.py_ver }} + #### + # https://github.com/mamba-org/setup-micromamba/issues/225 + micromamba-version: 1.5.10-0 + micromamba-binary-path: /home/runner/micromamba-bin-versioned/micromamba + #### environment-file: environment.yaml - activate-environment: gs + # Added an extra python to the create-args in order to bust the cache: + create-args: >- + python=${{ matrix.py_ver }} + python + cache-environment: true + - name: Install conda-recipe-manager if possible + # Possible when the Python version is >=3.11 + run: | + if [ $(python -c "import sys; print(sys.version_info[:2] >= (3,11))") = "True" ]; then + echo "Installing conda-recipe-manager" + micromamba install -y -c conda-forge conda-recipe-manager + else + echo "Skipping conda-recipe-manager installation" + fi - name: Conda info - shell: bash -l {0} run: | conda info --all conda list - name: Running doctests - shell: bash -l {0} run: | pytest grayskull \ -vv \ @@ -51,7 +67,6 @@ jobs: --junit-prefix=Linux-py${{ matrix.py_ver }}-serial - name: Running serial tests - shell: bash -l {0} run: | pytest tests \ -vv \ @@ -67,7 +82,6 @@ jobs: --junit-prefix=Linux-py${{ matrix.py_ver }}-serial - name: Running parallel tests - shell: bash -l {0} run: | pytest tests \ -vv \ diff --git a/environment.yaml b/environment.yaml index 65cf82b36..fae58036d 100644 --- a/environment.yaml +++ b/environment.yaml @@ -29,4 +29,3 @@ dependencies: - libcblas - beautifulsoup4 - semver >=3.0.0,<4.0.0 - - conda-recipe-manager >=0.2.0 diff --git a/grayskull/base/factory.py b/grayskull/base/factory.py index d78a5a630..813a878ec 100644 --- a/grayskull/base/factory.py +++ b/grayskull/base/factory.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import re from abc import ABC from pathlib import Path diff --git a/grayskull/license/discovery.py b/grayskull/license/discovery.py index 0fa4ec7dc..111152c66 100644 --- a/grayskull/license/discovery.py +++ b/grayskull/license/discovery.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import base64 import json import logging @@ -10,7 +12,6 @@ from pathlib import Path from subprocess import check_output from tempfile import mkdtemp -from typing import List, Optional, Tuple, Union import requests from colorama import Fore @@ -27,12 +28,12 @@ @dataclass class ShortLicense: name: str - path: Union[str, Path, None] + path: str | Path | None is_packaged: bool @lru_cache(maxsize=10) -def get_all_licenses_from_spdx() -> List: +def get_all_licenses_from_spdx() -> list: """Get all licenses available on spdx.org :return: List with all licenses information on spdx.org @@ -163,7 +164,7 @@ def get_short_license_id(name: str) -> str: return recipe_license["licenseId"] -def _get_license(license_id: str, all_licenses: List) -> dict: +def _get_license(license_id: str, all_licenses: list) -> dict: """Search for the license identification in all licenses received :param license_id: license identification @@ -175,7 +176,7 @@ def _get_license(license_id: str, all_licenses: List) -> dict: return one_license -def _get_all_names_from_api(one_license: dict) -> List: +def _get_all_names_from_api(one_license: dict) -> list: """Get the names and other names which each license has. :param one_license: License name @@ -191,7 +192,7 @@ def _get_all_names_from_api(one_license: dict) -> List: return list(result) -def get_other_names_from_opensource(license_spdx: str) -> List: +def get_other_names_from_opensource(license_spdx: str) -> list: lic = get_opensource_license(license_spdx) return [_license["name"] for _license in lic.get("other_names", [])] @@ -213,7 +214,7 @@ def read_licence_cache(): @lru_cache(maxsize=10) -def get_opensource_license_data() -> List: +def get_opensource_license_data() -> list: try: response = requests.get(url="https://api.opensource.org/licenses/", timeout=5) except requests.exceptions.RequestException: @@ -223,7 +224,7 @@ def get_opensource_license_data() -> List: return response.json() -def _get_all_license_choice(all_licenses: List) -> List: +def _get_all_license_choice(all_licenses: list) -> list: """Function responsible to get the whole licenses name :param all_licenses: list with all licenses @@ -237,11 +238,11 @@ def _get_all_license_choice(all_licenses: List) -> List: def search_license_file( folder_path: str, - git_url: Optional[str] = None, - version: Optional[str] = None, - license_name_metadata: Optional[str] = None, - folders_exclude_search: Tuple[str] = tuple(), -) -> List[ShortLicense]: + git_url: str | None = None, + version: str | None = None, + license_name_metadata: str | None = None, + folders_exclude_search: tuple[str] = tuple(), +) -> list[ShortLicense]: """Search for the license file. First it will try to find it in the given folder, after that it will search on the github api and for the last it will clone the repository and it will search for the license there. @@ -286,8 +287,8 @@ def search_license_file( @lru_cache(maxsize=13) def search_license_api_github( - github_url: str, version: Optional[str] = None, default: Optional[str] = "Other" -) -> Optional[ShortLicense]: + github_url: str, version: str | None = None, default: str | None = "Other" +) -> ShortLicense | None: """Search for the license asking in the github api :param github_url: GitHub URL @@ -315,7 +316,7 @@ def search_license_api_github( ) -def _get_api_github_url(github_url: str, version: Optional[str] = None) -> str: +def _get_api_github_url(github_url: str, version: str | None = None) -> str: """Try to presume the github url :param github_url: GitHub URL @@ -331,10 +332,10 @@ def _get_api_github_url(github_url: str, version: Optional[str] = None) -> str: def search_license_folder( - path: Union[str, Path], - default: Optional[str] = None, - folders_exclude_search: Tuple[str] = tuple(), -) -> List[ShortLicense]: + path: str | Path, + default: str | None = None, + folders_exclude_search: tuple[str] = tuple(), +) -> list[ShortLicense]: """Search for the license in the given folder :param path: Sdist folder @@ -366,10 +367,10 @@ def search_license_folder( def search_license_repo( git_url: str, - version: Optional[str], - default: Optional[str] = None, - folders_exclude_search: Tuple[str] = tuple(), -) -> Optional[List[ShortLicense]]: + version: str | None, + default: str | None = None, + folders_exclude_search: tuple[str] = tuple(), +) -> list[ShortLicense] | None: """Search for the license file in the given github repository :param git_url: GitHub URL @@ -399,7 +400,7 @@ def search_license_repo( ) -def _get_git_cmd(git_url: str, version: str, dest) -> List[str]: +def _get_git_cmd(git_url: str, version: str, dest) -> list[str]: """Return the full git command to clone the repository :param git_url: GitHub URL @@ -413,7 +414,7 @@ def _get_git_cmd(git_url: str, version: str, dest) -> List[str]: return git_cmd + [git_url, str(dest)] -def get_license_type(path_license: str, default: Optional[str] = None) -> Optional[str]: +def get_license_type(path_license: str, default: str | None = None) -> str | None: """Function tries to match the license with one of the models present in grayskull/license/data diff --git a/grayskull/main.py b/grayskull/main.py index 0413a571a..a01357081 100644 --- a/grayskull/main.py +++ b/grayskull/main.py @@ -1,9 +1,10 @@ +from __future__ import annotations + import argparse import logging import os import sys from pathlib import Path -from typing import List, Optional import requests from colorama import Fore, Style, init @@ -445,7 +446,7 @@ def create_r_recipe(pkg_name, sections_populate=None, **kwargs): ) -def add_extra_section(recipe, maintainers: Optional[List] = None): +def add_extra_section(recipe, maintainers: list | None = None): maintainers = maintainers or [get_git_current_user()] if "extra" in recipe: recipe["extra"]["recipe-maintainers"] = maintainers diff --git a/grayskull/strategy/cran.py b/grayskull/strategy/cran.py index 3ec559034..b831a2017 100644 --- a/grayskull/strategy/cran.py +++ b/grayskull/strategy/cran.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import os import re @@ -7,7 +9,6 @@ from copy import deepcopy from os.path import basename from tempfile import mkdtemp -from typing import Optional from urllib.request import Request, urlopen import requests @@ -235,7 +236,7 @@ def get_webpage(cran_url): return BeautifulSoup(html_page, features="html.parser") -def get_cran_index(cran_url: str, pkg_name: str, pkg_version: Optional[str] = None): +def get_cran_index(cran_url: str, pkg_name: str, pkg_version: str | None = None): """Fetch the entire CRAN index and store it.""" print_msg(f"Fetching main index from {cran_url}") diff --git a/grayskull/strategy/py_base.py b/grayskull/strategy/py_base.py index 0c78342d1..dcd9f1cac 100644 --- a/grayskull/strategy/py_base.py +++ b/grayskull/strategy/py_base.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import os import re @@ -11,7 +13,6 @@ from pathlib import Path from subprocess import check_output from tempfile import mkdtemp -from typing import Dict, List, Optional, Tuple, Union from urllib.parse import urlparse import requests @@ -41,7 +42,7 @@ PIN_PKG_COMPILER = {"numpy": "<{ pin_compatible('numpy') }}"} -def search_setup_root(path_folder: Union[Path, str]) -> Path: +def search_setup_root(path_folder: Path | str) -> Path: if setup_py := list(Path(path_folder).rglob("setup.py")): return setup_py[0] if setup_cfg := list(Path(path_folder).rglob("setup.cfg")): @@ -50,7 +51,7 @@ def search_setup_root(path_folder: Union[Path, str]) -> Path: return pyproject_toml[0] -def clean_deps_for_conda_forge(list_deps: List, py_ver_min: PyVer) -> List: +def clean_deps_for_conda_forge(list_deps: list, py_ver_min: PyVer) -> list: """Remove dependencies which conda-forge is not supporting anymore. For example Python 2.7, Python version less than 3.6""" re_delimiter = re.compile(r"#\s+\[py\s*(?:([<>=!]+))?\s*(\d+)\]\s*$", re.DOTALL) @@ -117,7 +118,7 @@ def parse_extra_metadata_to_selector(option: str, operation: str, value: str) -> return value_lower -def get_extra_from_requires_dist(string_parse: str) -> Union[List]: +def get_extra_from_requires_dist(string_parse: str) -> list: """Receives the extra metadata e parse it to get the option, operation and value. @@ -131,7 +132,7 @@ def get_extra_from_requires_dist(string_parse: str) -> Union[List]: ) -def get_name_version_from_requires_dist(string_parse: str) -> Tuple[str, str]: +def get_name_version_from_requires_dist(string_parse: str) -> tuple[str, str]: """Extract the name and the version from `requires_dist` present in PyPi`s metadata @@ -148,7 +149,7 @@ def get_name_version_from_requires_dist(string_parse: str) -> Tuple[str, str]: def generic_py_ver_to( metadata: dict, config: Configuration, is_selector: bool = False -) -> Optional[str]: # sourcery no-metrics +) -> str | None: # sourcery no-metrics """Generic function which abstract the parse of the requires_python present in the PyPi metadata. Basically it can generate the selectors for Python or the constrained version if it is a `noarch: python` python package""" @@ -344,7 +345,7 @@ def __fake_distutils_setup(*args, **kwargs): if not isinstance(kwargs, dict) or not kwargs: return - def _fix_list_requirements(key_deps: str) -> List: + def _fix_list_requirements(key_deps: str) -> list: """Fix when dependencies have lists inside of another sequence""" val_deps = kwargs.get(key_deps) if not val_deps: @@ -467,8 +468,8 @@ def __run_setup_py(path_setup: str, data_dist: dict, run_py=False, deps_installe def get_compilers( - requires_dist: List, sdist_metadata: dict, config: Configuration -) -> List: + requires_dist: list, sdist_metadata: dict, config: Configuration +) -> list: """Return which compilers are necessary""" compilers = set(sdist_metadata.get("compilers", [])) for pkg in requires_dist: @@ -482,10 +483,10 @@ def get_compilers( def get_py_multiple_selectors( - selectors: Dict[PyVer, bool], + selectors: dict[PyVer, bool], config: Configuration, is_selector: bool = False, -) -> List: +) -> list: """Get python selectors available. :param selectors: Dict with the Python version and if it is selected @@ -511,11 +512,11 @@ def get_py_multiple_selectors( return all_selector -def py_version_to_selector(pypi_metadata: dict, config) -> Optional[str]: +def py_version_to_selector(pypi_metadata: dict, config) -> str | None: return generic_py_ver_to(pypi_metadata, is_selector=True, config=config) -def py_version_to_limit_python(pypi_metadata: dict, config=None) -> Optional[str]: +def py_version_to_limit_python(pypi_metadata: dict, config=None) -> str | None: config = config or Configuration(pypi_metadata["name"]) result = generic_py_ver_to(pypi_metadata, is_selector=False, config=config) if not result and config.is_strict_cf: @@ -555,7 +556,7 @@ def clean_list_pkg(pkg, list_pkgs): requirements["run"].append(PIN_PKG_COMPILER[pkg_name]) -def discover_license(metadata: dict) -> List[ShortLicense]: +def discover_license(metadata: dict) -> list[ShortLicense]: """Based on the metadata this method will try to discover what is the right license for the package @@ -579,13 +580,13 @@ def discover_license(metadata: dict) -> List[ShortLicense]: ) -def get_test_entry_points(entry_points: Union[List, str]) -> List: +def get_test_entry_points(entry_points: list | str) -> list: if entry_points and isinstance(entry_points, str): entry_points = [entry_points] return [f"{ep.split('=')[0].strip()} --help" for ep in entry_points] -def get_test_imports(metadata: dict, default: Optional[str] = None) -> List: +def get_test_imports(metadata: dict, default: str | None = None) -> list: if default: default = default.replace("-", "_") if "packages" not in metadata or not metadata["packages"]: @@ -609,7 +610,7 @@ def get_test_imports(metadata: dict, default: Optional[str] = None) -> List: return result -def get_entry_points_from_sdist(sdist_metadata: dict) -> List: +def get_entry_points_from_sdist(sdist_metadata: dict) -> list: """Extract entry points from sdist metadata :param sdist_metadata: sdist metadata @@ -664,7 +665,7 @@ def get_entry_points_from_sdist(sdist_metadata: dict) -> List: return [] -def download_sdist_pkg(sdist_url: str, dest: str, name: Optional[str] = None): +def download_sdist_pkg(sdist_url: str, dest: str, name: str | None = None): """Download the sdist package :param sdist_url: sdist url @@ -678,17 +679,15 @@ def download_sdist_pkg(sdist_url: str, dest: str, name: Optional[str] = None): response = requests.get(sdist_url, allow_redirects=True, stream=True, timeout=5) response.raise_for_status() total_size = int(response.headers.get("Content-length", 0)) - with ( - manage_progressbar(max_value=total_size, prefix=f"{name} ") as bar, - open(dest, "wb") as pkg_file, - ): - progress_val = 0 - chunk_size = 512 - for chunk_data in response.iter_content(chunk_size=chunk_size): - if chunk_data: - pkg_file.write(chunk_data) - progress_val += chunk_size - bar.update(min(progress_val, total_size)) + with manage_progressbar(max_value=total_size, prefix=f"{name} ") as bar: + with open(dest, "wb") as pkg_file: + progress_val = 0 + chunk_size = 512 + for chunk_data in response.iter_content(chunk_size=chunk_size): + if chunk_data: + pkg_file.write(chunk_data) + progress_val += chunk_size + bar.update(min(progress_val, total_size)) def merge_deps_toml_setup(setup_deps: list, toml_deps: list) -> list: @@ -819,11 +818,11 @@ def get_sdist_metadata( return merge_setup_toml_metadata(metadata, pyproject_metadata) -def ensure_pep440_in_req_list(list_req: List[str]) -> List[str]: +def ensure_pep440_in_req_list(list_req: list[str]) -> list[str]: return [ensure_pep440(pkg) for pkg in list_req] -def split_deps(deps: str) -> List[str]: +def split_deps(deps: str) -> list[str]: result = [] for d in deps.split(","): constrain = "" diff --git a/grayskull/strategy/pypi.py b/grayskull/strategy/pypi.py index 9647473b8..9802c78c7 100644 --- a/grayskull/strategy/pypi.py +++ b/grayskull/strategy/pypi.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import itertools import json import logging @@ -6,7 +8,7 @@ from collections.abc import Mapping from pathlib import Path from tempfile import mkdtemp -from typing import Dict, Iterable, List, Optional +from typing import Iterable import requests from colorama import Fore @@ -121,7 +123,7 @@ def adjust_source_url_to_include_placeholders(url, version): return "/".join(url_split) -def get_url_filename(metadata: dict, default: Optional[str] = None) -> str: +def get_url_filename(metadata: dict, default: str | None = None) -> str: """Method responsible to get the filename and right extension to add to the pypi url @@ -164,7 +166,7 @@ def get_sdist_url_from_pypi(metadata: dict) -> str: return sdist_url["url"] -def skip_pypi_requirement(list_extra: List) -> bool: +def skip_pypi_requirement(list_extra: list) -> bool: """Test if it should skip the requirement :param list_extra: list with all extra requirements @@ -176,7 +178,7 @@ def skip_pypi_requirement(list_extra: List) -> bool: ) -def merge_requires_dist(pypi_metadata: dict, sdist_metadata: dict) -> List: +def merge_requires_dist(pypi_metadata: dict, sdist_metadata: dict) -> list: """Merge requirements metadata from pypi and sdist. :param pypi_metadata: pypi metadata @@ -296,7 +298,7 @@ def get_pypi_metadata(config: Configuration) -> dict: } -def get_run_req_from_requires_dist(requires_dist: List, config: Configuration) -> List: +def get_run_req_from_requires_dist(requires_dist: list, config: Configuration) -> list: """Get the run requirements looking for the `requires_dist` key present in the metadata""" run_req = [] @@ -320,7 +322,7 @@ def get_run_req_from_requires_dist(requires_dist: List, config: Configuration) - return run_req -def get_all_selectors_pypi(list_extra: List, config: Configuration) -> List: +def get_all_selectors_pypi(list_extra: list, config: Configuration) -> list: """Get the selectors looking for the pypi data :param list_extra: List of extra requirements from pypi @@ -487,7 +489,7 @@ def get_metadata(recipe, config) -> dict: } -def remove_all_inner_nones(metadata: Dict) -> Dict: +def remove_all_inner_nones(metadata: dict) -> dict: """Remove all inner None values from a dictionary.""" if not isinstance(metadata, Mapping): return metadata @@ -498,7 +500,7 @@ def remove_all_inner_nones(metadata: Dict) -> Dict: return metadata -def update_recipe(recipe: Recipe, config: Configuration, all_sections: List[str]): +def update_recipe(recipe: Recipe, config: Configuration, all_sections: list[str]): """Update one specific section.""" from souschef.section import Section @@ -541,7 +543,7 @@ def update_recipe(recipe: Recipe, config: Configuration, all_sections: List[str] def check_noarch_python_for_new_deps( - host_req: List, run_req: List, config: Configuration + host_req: list, run_req: list, config: Configuration ): if not config.is_arch: return @@ -559,7 +561,7 @@ def check_noarch_python_for_new_deps( config.is_arch = False -def extract_requirements(metadata: dict, config, recipe) -> Dict[str, List[str]]: +def extract_requirements(metadata: dict, config, recipe) -> dict[str, list[str]]: """Extract the requirements for `build`, `host` and `run`""" name = metadata["name"] requires_dist = format_dependencies(metadata.get("requires_dist", []), name) @@ -628,7 +630,7 @@ def extract_requirements(metadata: dict, config, recipe) -> Dict[str, List[str]] return result -def sort_reqs(reqs: Iterable[str], alphabetize: bool = False) -> List[str]: +def sort_reqs(reqs: Iterable[str], alphabetize: bool = False) -> list[str]: """Sort requirements. Put python first, then optionally sort alphabetically.""" reqs_list = list(reqs) @@ -644,8 +646,8 @@ def is_python(req: str) -> bool: def remove_selectors_pkgs_if_needed( - list_req: List, config_file: Optional[Path] = None -) -> List: + list_req: list, config_file: Path | None = None +) -> list: info_pkgs = _get_track_info_from_file(config_file or PYPI_CONFIG) re_selector = re.compile(r"\s+#\s+\[.*", re.DOTALL) result = [] @@ -657,7 +659,7 @@ def remove_selectors_pkgs_if_needed( return result -def extract_optional_requirements(metadata: dict, config) -> Dict[str, List[str]]: +def extract_optional_requirements(metadata: dict, config) -> dict[str, list[str]]: """Extract all optional requirements that are specified in the configuration""" keys = set() if config.extras_require_all: @@ -678,7 +680,7 @@ def extract_optional_requirements(metadata: dict, config) -> Dict[str, List[str] return result -def normalize_requirements_list(requirements: List[str], config) -> List[str]: +def normalize_requirements_list(requirements: list[str], config) -> list[str]: """Adapt requirements to PEP440, Conda and Conda-Forge""" requirements = solve_list_pkg_name(requirements, PYPI_CONFIG) requirements = ensure_pep440_in_req_list(requirements) @@ -689,7 +691,7 @@ def normalize_requirements_list(requirements: List[str], config) -> List[str]: return requirements -def compose_test_section(metadata: dict, test_requirements: List[str]) -> dict: +def compose_test_section(metadata: dict, test_requirements: list[str]) -> dict: test_imports = get_test_imports(metadata, metadata["name"]) test_requirements = ["pip"] + test_requirements test_commands = ["pip check"] diff --git a/grayskull/utils.py b/grayskull/utils.py index 2c9a998a8..fa423ea56 100644 --- a/grayskull/utils.py +++ b/grayskull/utils.py @@ -244,7 +244,15 @@ def upgrade_v0_recipe_to_v1(recipe_path: Path) -> None: JINJA plugin. :param recipe_path: Path to that contains the original recipe file to modify. """ - from conda_recipe_manager.parser.recipe_parser_convert import RecipeParserConvert + try: + from conda_recipe_manager.parser.recipe_parser_convert import ( + RecipeParserConvert, + ) + except ImportError as e: + raise ImportError( + "Please install conda-recipe-manager from conda-forge to enable " + "support for the V1 format. (Note that Python >=3.11 is required.)" + ) from e recipe_content: Final[str] = RecipeParserConvert.pre_process_recipe_text( recipe_path.read_text()