Add download (#81)

* Add download option to dump the sdist and pypi information along with the recipe

grayskull/__main__.py

@@ -25,6 +25,15 @@ def main(args=None):
     pypi_cmds.add_argument(
         "pypi_packages", nargs="+", help="Specify the PyPI packages name.", default=[]
     )
+    pypi_cmds.add_argument(
+        "--download",
+        "-d",
+        dest="download",
+        action="store_true",
+        default=False,
+        help="Download the sdist package and PyPI information in the same folder"
+        " the recipe is located.",
+    )
     pypi_cmds.add_argument(
         "--maintainers",
         "-m",
@@ -80,7 +89,9 @@ def main(args=None):
             f"{Fore.BLUE}{pkg_name} (pypi) {Fore.GREEN}####\n"
         )
         pkg_name, pkg_version = parse_pkg_name_version(pkg_name)
-        recipe = GrayskullFactory.create_recipe("pypi", pkg_name, pkg_version)
+        recipe = GrayskullFactory.create_recipe(
+            "pypi", pkg_name, pkg_version, download=args.download
+        )
         recipe.generate_recipe(args.output, mantainers=args.maintainers)
         print(
             f"\n{Fore.GREEN}#### Recipe generated on "

grayskull/base/extra.py

@@ -27,7 +27,6 @@ def get_git_current_user() -> str:
             f"Exception occurred when trying to recover user information from github."
             f" Exception: {err}"
         )
-        pass
     print(
         f"{Fore.LIGHTBLACK_EX}Using default recipe maintainer:"
         f" {Fore.LIGHTMAGENTA_EX}AddYourGitHubIdHere"

grayskull/base/factory.py

@@ -10,11 +10,15 @@ class GrayskullFactory(ABC):
     }
 
     @staticmethod
-    def create_recipe(repo_type: str, name: str, version: str = "") -> Union[PyPi]:
+    def create_recipe(
+        repo_type: str, name: str, version: str = "", **kwargs
+    ) -> Union[PyPi]:
         if repo_type.lower() not in GrayskullFactory.REGISTERED_CLASS:
             raise ValueError(
                 f"Recipe generator {repo_type.lower()} does not exist.\n"
                 f"Please inform a valid one."
                 f"{', '.join(GrayskullFactory.REGISTERED_CLASS.keys())}"
             )
-        return GrayskullFactory.REGISTERED_CLASS[repo_type.lower()](name, version)
+        return GrayskullFactory.REGISTERED_CLASS[repo_type.lower()](
+            name, version, **kwargs
+        )

grayskull/pypi/pypi.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import os
 import re
@@ -35,8 +36,13 @@ class PyPi(AbstractRecipeModel):
     RE_DEPS_NAME = re.compile(r"^\s*([\.a-zA-Z0-9_-]+)", re.MULTILINE)
     PIN_PKG_COMPILER = {"numpy": "<{ pin_compatible('numpy') }}"}
 
-    def __init__(self, name=None, version=None, force_setup=False):
-        self._force_setup = force_setup
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        version: Optional[str] = None,
+        download: bool = False,
+    ):
+        self._download = download
         self._setup_metadata = None
         self._is_arch = False
         super(PyPi, self).__init__(name=name, version=version)
@@ -71,9 +77,8 @@ def _download_sdist_pkg(sdist_url: str, dest: str):
                     progress_val += chunk_size
                     bar.update(min(progress_val, total_size))
 
-    @staticmethod
     @lru_cache(maxsize=10)
-    def _get_sdist_metadata(sdist_url: str, name: str) -> dict:
+    def _get_sdist_metadata(self, sdist_url: str, name: str) -> dict:
         """Method responsible to return the sdist metadata which is basically
         the metadata present in setup.py and setup.cfg
 
@@ -86,6 +91,8 @@ def _get_sdist_metadata(sdist_url: str, name: str) -> dict:
         path_pkg = os.path.join(temp_folder, pkg_name)
 
         PyPi._download_sdist_pkg(sdist_url=sdist_url, dest=path_pkg)
+        if self._download:
+            self.files_to_copy.append(path_pkg)
         log.debug(f"Unpacking {path_pkg} to {temp_folder}")
         shutil.unpack_archive(path_pkg, temp_folder)
         print(f"{Fore.LIGHTBLACK_EX}Recovering information from setup.py")
@@ -263,9 +270,8 @@ def __run_setup_py(path_setup: str, data_dist: dict, run_py=False):
             )
             PyPi._pip_install_dep(data_dist, err.name, pip_dir)
             PyPi.__run_setup_py(path_setup, data_dist, run_py)
-        except Exception as err:  # noqa
+        except Exception as err:
             log.debug(f"Exception when executing setup.py as script: {err}")
-            pass
         data_dist.update(
             PyPi._merge_sdist_metadata(
                 data_dist, PyPi._get_setup_cfg(os.path.dirname(str(path_setup)))
@@ -518,9 +524,8 @@ def _discover_license(metadata: dict) -> Optional[ShortLicense]:
         if short_license:
             return short_license
 
-    @staticmethod
     @lru_cache(maxsize=10)
-    def _get_pypi_metadata(name, version: Optional[str] = None) -> dict:
+    def _get_pypi_metadata(self, name, version: Optional[str] = None) -> dict:
         """Method responsible to communicate with the pypi api endpoints and
         get the whole metadata available for the specified package and version.
 
@@ -543,6 +548,13 @@ def _get_pypi_metadata(name, version: Optional[str] = None) -> dict:
             )
 
         metadata = metadata.json()
+        if self._download:
+            download_file = os.path.join(
+                str(mkdtemp(f"grayskull-pypi-metadata-{name}-")), "pypi.json"
+            )
+            with open(download_file, "w") as f:
+                json.dump(metadata, f, indent=4)
+            self.files_to_copy.append(download_file)
         info = metadata["info"]
         project_urls = info.get("project_urls") if info.get("project_urls") else {}
         log.info(f"Package: {name}=={info['version']}")

tests/cli/test_cli_cmds.py

@@ -12,12 +12,16 @@ def test_version(capsys):
 
 def test_pypi_cmd(tmpdir):
     out_folder = tmpdir.mkdir("out")
-    main(["pypi", "pytest=5.3.2", "-o", str(out_folder), "-m", "m1", "m2"])
+    main(
+        ["pypi", "pytest=5.3.2", "-o", str(out_folder), "-m", "m1", "m2", "--download"]
+    )
     pytest_folder = out_folder / "pytest"
     assert pytest_folder.isdir()
 
     recipe_file = pytest_folder / "meta.yaml"
     assert recipe_file.isfile()
+    assert (pytest_folder / "pypi.json").isfile()
+    assert (pytest_folder / "pytest-5.3.2.tar.gz").isfile()
 
 
 @pytest.mark.parametrize("option", ["--heman", "--shera"])

tests/test_pypi.py

@@ -38,7 +38,8 @@ def test_extract_pypi_requirements(pypi_metadata):
 
 
 def test_get_pypi_metadata(pypi_metadata):
-    metadata = PyPi._get_pypi_metadata(name="pytest", version="5.3.1")
+    recipe = PyPi(name="pytest", version="5.3.1")
+    metadata = recipe._get_pypi_metadata(name="pytest", version="5.3.1")
     assert metadata["name"] == "pytest"
     assert metadata["version"] == "5.3.1"
 
@@ -155,7 +156,8 @@ def test_get_sha256_from_pypi_metadata():
 
 
 def test_injection_distutils():
-    data = PyPi._get_sdist_metadata(
+    recipe = PyPi(name="hypothesis", version="5.5.1")
+    data = recipe._get_sdist_metadata(
         "https://pypi.io/packages/source/h/hypothesis/hypothesis-5.5.1.tar.gz",
         "hypothesis",
     )
@@ -171,7 +173,8 @@ def test_injection_distutils():
 
 
 def test_injection_distutils_pytest():
-    data = PyPi._get_sdist_metadata(
+    recipe = PyPi(name="pytest", version="5.3.2")
+    data = recipe._get_sdist_metadata(
         "https://pypi.io/packages/source/p/pytest/pytest-5.3.2.tar.gz", "pytest"
     )
     assert sorted(data["install_requires"]) == sorted(
@@ -195,16 +198,18 @@ def test_injection_distutils_pytest():
 
 
 def test_injection_distutils_compiler_gsw():
-    data = PyPi._get_sdist_metadata(
+    recipe = PyPi(name="gsw", version="3.3.1")
+    data = recipe._get_sdist_metadata(
         "https://pypi.io/packages/source/g/gsw/gsw-3.3.1.tar.gz", "gsw"
     )
     assert data.get("compilers") == ["c"]
     assert data["packages"] == ["gsw"]
 
 
 def test_merge_pypi_sdist_metadata():
-    pypi_metadata = PyPi._get_pypi_metadata(name="gsw", version="3.3.1")
-    sdist_metadata = PyPi._get_sdist_metadata(pypi_metadata["sdist_url"], "gsw")
+    recipe = PyPi(name="gsw", version="3.3.1")
+    pypi_metadata = recipe._get_pypi_metadata(name="gsw", version="3.3.1")
+    sdist_metadata = recipe._get_sdist_metadata(pypi_metadata["sdist_url"], "gsw")
     merged_data = PyPi._merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata)
     assert merged_data["compilers"] == ["c"]
     assert sorted(merged_data["setup_requires"]) == sorted(["numpy"])