Add wheel url support

default.nix

@@ -414,6 +414,7 @@ lib.makeScope pkgs.newScope (self: {
           nativeBuildInputs = inputAttrs.nativeBuildInputs ++ [
             hooks.removePathDependenciesHook
             hooks.removeGitDependenciesHook
+            hooks.removeWheelUrlDependenciesHook
           ];
         } // {
           pname = normalizePackageName pyProject.tool.poetry.name;

hooks/default.nix

@@ -6,7 +6,15 @@ let
   pythonSitePackages = python.sitePackages;
 
   nonOverlayedPython = pkgs.python3.pythonForBuild.withPackages (ps: [ ps.tomlkit ]);
-  makeRemoveSpecialDependenciesHook = { fields, kind }:
+  makeRemoveSpecialDependenciesHook =
+    { fields
+    , kind
+      /*
+       * A script that takes in --fields-to-remove <fields, nargs="*">, transforms
+       * stdin pyproject.toml onto stdout pyproject.toml
+       */
+    , pyprojectPatchScript ? "${./pyproject-without-special-deps.py}"
+    }:
     nonOverlayedPython.pkgs.callPackage
       (
         _:
@@ -18,7 +26,7 @@ let
               # because building of tomlkit and its dependencies also use these hooks.
               pythonPath = nonOverlayedPython.pkgs.makePythonPath [ nonOverlayedPython ];
               pythonInterpreter = nonOverlayedPython.interpreter;
-              pyprojectPatchScript = "${./pyproject-without-special-deps.py}";
+              inherit pyprojectPatchScript;
               inherit fields;
               inherit kind;
             };
@@ -42,6 +50,12 @@ in
     kind = "git";
   };
 
+  removeWheelUrlDependenciesHook = makeRemoveSpecialDependenciesHook {
+    fields = [ "url" ];
+    kind = "wheel-url";
+    pyprojectPatchScript = "${./pyproject-without-url-whl.py}";
+  };
+
   pipBuildHook =
     callPackage
       (

hooks/pyproject-without-url-whl.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# Patch out urls that have .whl
+
+import argparse
+import sys
+
+import tomlkit
+
+
+def main(input, output):
+    data = tomlkit.loads(input.read())
+
+    try:
+        deps = data["tool"]["poetry"]["dependencies"]
+    except KeyError:
+        pass
+    else:
+        for dep in deps.values():
+            if isinstance(dep, dict):
+                url = dep.get("url", None)
+                if url is None or not url.endswith("whl"):
+                    continue
+                dep["version"] = "*"
+                dep.pop("url", None)
+                dep.pop("develop", None)
+
+    output.write(tomlkit.dumps(data))
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser()
+    p.add_argument(
+        "-i",
+        "--input",
+        type=argparse.FileType("r"),
+        default=sys.stdin,
+        help="Location from which to read input TOML",
+    )
+    p.add_argument(
+        "-o",
+        "--output",
+        type=argparse.FileType("w"),
+        default=sys.stdout,
+        help="Location to write output TOML",
+    )
+    p.add_argument(
+        "-f",
+        "--fields-to-remove",
+        nargs="*",
+        help="The fields to remove from the dependency's TOML",
+    )
+
+    args = p.parse_args()
+    if args.fields_to_remove != ["url"]:
+        print(
+            f"WARN: url-whl ignoring fields_to_remove {args.fields_to_remove}",
+            file=sys.stderr,
+        )
+    main(args.input, args.output)

mk-poetry-dep.nix

@@ -124,6 +124,7 @@ pythonPackages.callPackage
       ++ lib.optionals (format == "pyproject") [
         hooks.removePathDependenciesHook
         hooks.removeGitDependenciesHook
+        hooks.removeWheelUrlDependenciesHook
         hooks.pipBuildHook
       ];
 

tests/default.nix

@@ -66,6 +66,7 @@ in
   mysqlclient = callTest ./mysqlclient { };
   jq = callTest ./jq { };
   ubersmith = callTest ./ubersmith { };
+  use-url-wheel = callTest ./use-url-wheel { };
   returns = callTest ./returns { };
   option = callTest ./option { };
   fastapi-utils = callTest ./fastapi-utils { };

tests/use-url-wheel/default.nix

@@ -0,0 +1,18 @@
+{ lib, poetry2nix, python3, stdenv }:
+let
+  args = {
+    python = python3;
+    projectDir = ./.;
+    preferWheels = true;
+  };
+  py = poetry2nix.mkPoetryPackages args;
+  app = poetry2nix.mkPoetryApplication args;
+  url_nix_store = py.python.pkgs.de-core-news-sm.src;
+  url_is_wheel = url_nix_store.isWheel or false;
+  is_wheel_attr_test = x: lib.warnIf (!stdenv.isLinux && !url_is_wheel)
+    "url should resolve to have src with .isWheel"
+    x;
+  is_wheel_test = x: assert lib.strings.hasSuffix "whl" url_nix_store; x;
+  app_builds = x: assert lib.isDerivation app; x;
+in
+lib.pipe app [ is_wheel_attr_test is_wheel_test app_builds ]

tests/use-url-wheel/pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "use-url-wheel"
+version = "0.1.0"
+description = "asserts that .whl via .url works"
+authors = [
+    "pegasust <[email protected]>",
+    "joopitz <[email protected]>"
+]
+license = "MIT"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+de_core_news_sm = {url = "https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.6.0/de_core_news_sm-3.6.0-py3-none-any.whl"}
+torch = "2.0.0"
+
+[tool.poetry.scripts]
+test = "whl_test.main:main"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[[tool.poetry.packages]]
+include = "whl_test"
+

tests/use-url-wheel/whl_test/main.py

@@ -0,0 +1,7 @@
+import de_core_news_sm
+import sys
+
+
+def main():
+    nlp = de_core_news_sm.load()
+    print(nlp("Dies ist ein Testsatz."))