build(nix): Package python scripts independently of main derivation

.devops/nix/package.nix

@@ -30,7 +30,6 @@
   useRocm ? config.rocmSupport,
   useVulkan ? false,
   llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake
-  gguf-py,
 }@inputs:
 
 let
@@ -151,7 +150,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
     # TODO: Package up each Python script or service appropriately.
     # If we were to migrate to buildPythonPackage and prepare the `pyproject.toml`,
     # we could make those *.py into setuptools' entrypoints
-    substituteInPlace ./*.py --replace "/usr/bin/env python" "${llama-python-base-with-gguf}/bin/python"
+    # substituteInPlace ./*.py --replace "/usr/bin/env python" "${llama-python-base-with-gguf}/bin/python"
   '';
 
   nativeBuildInputs =

.devops/nix/python-scripts.nix

@@ -0,0 +1,24 @@
+{
+  buildPythonPackage,
+  poetry-core,
+  breakpointHook,
+  python3Packages,
+  gguf-py
+}@inputs:
+
+buildPythonPackage {
+  pname = "llama-scripts";
+  src = ../../.;
+  version = "0.0.0";
+  pyproject = true;
+  nativeBuildInputs = [ poetry-core ];
+  projectDir = ../../.;
+  propagatedBuildInputs = with python3Packages; [
+    numpy
+    sentencepiece
+    transformers
+    protobuf
+    torchWithoutCuda
+    gguf-py
+  ];
+}

.devops/nix/scope.nix

@@ -3,6 +3,7 @@
   newScope,
   python3,
   llamaVersion ? "0.0.0",
+  poetry2nix,
 }:
 
 let
@@ -19,7 +20,6 @@ in
 
 lib.makeScope newScope (self: {
   inherit llamaVersion;
-  pp = python3.pkgs;
   gguf-py = self.callPackage ./package-gguf-py.nix {
     inherit
       buildPythonPackage
@@ -28,6 +28,9 @@ lib.makeScope newScope (self: {
       pytestCheckHook
       ;
   };
+  python-scripts = self.callPackage ./python-scripts.nix {
+    inherit buildPythonPackage poetry-core poetry2nix;
+  };
   llama-cpp = self.callPackage ./package.nix { };
   docker = self.callPackage ./docker.nix { };
   docker-min = self.callPackage ./docker.nix { interactive = false; };

convert-hf-to-gguf.py → bin/convert_hf_to_gguf.py

@@ -22,7 +22,7 @@
     sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
 import gguf
 
-from convert import HfVocab
+from bin.convert import HfVocab
 
 
 ###### MODEL DEFINITIONS ######

convert-llama-ggml-to-gguf.py → bin/convert_llama_ggml_to_gguf.py

@@ -352,7 +352,7 @@ def add_tensors(self, gguf_writer):
 
 
 def handle_metadata(cfg, hp):
-    import convert
+    import bin.convert as convert
     assert cfg.model_metadata_dir.is_dir(), 'Metadata dir is not a directory'
     hf_config_path   = cfg.model_metadata_dir / "config.json"
     orig_config_path = cfg.model_metadata_dir / "params.json"

flake.nix

@@ -109,11 +109,14 @@
         # Cf. https://nixos.org/manual/nix/unstable/command-ref/new-cli/nix3-flake.html?highlight=flake#flake-format
         flake.overlays.default = (
           final: prev: {
-            llamaPackages = final.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
+            llamaPackages = final.callPackage .devops/nix/scope.nix {
+              inherit llamaVersion;
+            };
             inherit (final.llamaPackages) llama-cpp;
           }
         );
 
+        debug = true;
         systems = [
           "aarch64-darwin"
           "aarch64-linux"
@@ -132,6 +135,7 @@
             ...
           }:
           {
+            debug = true;
             # Unlike `.#packages`, legacyPackages may contain values of
             # arbitrary types (including nested attrsets) and may even throw
             # exceptions. This attribute isn't recursed into by `nix flake
@@ -141,9 +145,18 @@
             # access them as `nix build .#llamaPackages.${scriptName}` using
             # the same path you would with an overlay.
             legacyPackages = {
-              llamaPackages = pkgs.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
-              llamaPackagesCuda = pkgsCuda.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
-              llamaPackagesRocm = pkgsRocm.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
+              llamaPackages = pkgs.callPackage .devops/nix/scope.nix {
+                inherit llamaVersion;
+                poetry2nix = import inputs.poetry2nix { pkgs = pkgs; };
+              };
+              llamaPackagesCuda = pkgsCuda.callPackage .devops/nix/scope.nix {
+                inherit llamaVersion;
+                poetry2nix = import inputs.poetry2nix { pkgs = pkgs; };
+              };
+              llamaPackagesRocm = pkgsRocm.callPackage .devops/nix/scope.nix {
+                inherit llamaVersion;
+                poetry2nix = import inputs.poetry2nix { pkgs = pkgs; };
+              };
             };
 
             # We don't use the overlay here so as to avoid making too many instances of nixpkgs,

gguf-py/pyproject.toml

@@ -4,9 +4,9 @@ version = "0.7.0"
 description = "Read and write ML models in GGUF for GGML"
 authors = ["GGML <[email protected]>"]
 packages = [
-    {include = "gguf"},
-    {include = "gguf/py.typed"},
-    {include = "scripts"},
+    { include = "gguf" },
+    { include = "gguf/py.typed" },
+    { include = "scripts" },
 ]
 readme = "README.md"
 homepage = "https://ggml.ai"