From 0126788271b4c6cbf57afb251f6fdd0c6dc098f2 Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Thu, 22 Feb 2024 23:55:55 +0530
Subject: [PATCH 01/20] style: format with nixfmt/rfc101-style

---
 .devops/nix/devshells.nix         |  11 +-
 .devops/nix/nixpkgs-instances.nix |  18 +-
 .devops/nix/package.nix           | 416 +++++++++++++++---------------
 .devops/nix/scope.nix             |  16 +-
 flake.nix                         |   4 +-
 5 files changed, 227 insertions(+), 238 deletions(-)

diff --git a/.devops/nix/devshells.nix b/.devops/nix/devshells.nix
index 1862f0f085100..9895d803f041f 100644
--- a/.devops/nix/devshells.nix
+++ b/.devops/nix/devshells.nix
@@ -2,12 +2,9 @@
   perSystem =
     { config, lib, ... }:
     {
-      devShells =
-        lib.concatMapAttrs
-          (name: package: {
-            ${name} = package.passthru.shell;
-            ${name + "-extra"} = package.passthru.shell-extra;
-          })
-          config.packages;
+      devShells = lib.concatMapAttrs (name: package: {
+        ${name} = package.passthru.shell;
+        ${name + "-extra"} = package.passthru.shell-extra;
+      }) config.packages;
     };
 }
diff --git a/.devops/nix/nixpkgs-instances.nix b/.devops/nix/nixpkgs-instances.nix
index 4a2f81c4bfd04..90d683a713aa1 100644
--- a/.devops/nix/nixpkgs-instances.nix
+++ b/.devops/nix/nixpkgs-instances.nix
@@ -26,16 +26,14 @@
           config.cudaSupport = true;
           config.allowUnfreePredicate =
             p:
-            builtins.all
-              (
-                license:
-                license.free
-                || builtins.elem license.shortName [
-                  "CUDA EULA"
-                  "cuDNN EULA"
-                ]
-              )
-              (p.meta.licenses or [ p.meta.license ]);
+            builtins.all (
+              license:
+              license.free
+              || builtins.elem license.shortName [
+                "CUDA EULA"
+                "cuDNN EULA"
+              ]
+            ) (p.meta.licenses or [ p.meta.license ]);
         };
         # Ensure dependencies use ROCm consistently
         pkgsRocm = import inputs.nixpkgs {
diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index cfffac257b75b..e065004af31f1 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -20,12 +20,14 @@
   vulkan-loader,
   curl,
   shaderc,
-  useBlas ? builtins.all (x: !x) [
-    useCuda
-    useMetalKit
-    useRocm
-    useVulkan
-  ] && blas.meta.available,
+  useBlas ?
+    builtins.all (x: !x) [
+      useCuda
+      useMetalKit
+      useRocm
+      useVulkan
+    ]
+    && blas.meta.available,
   useCuda ? config.cudaSupport,
   useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin,
   useMpi ? false, # Increases the runtime closure size by ~700M
@@ -38,7 +40,7 @@
   # otherwise we get libstdc++ errors downstream.
   effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
   enableStatic ? effectiveStdenv.hostPlatform.isStatic,
-  precompileMetalShaders ? false
+  precompileMetalShaders ? false,
 }@inputs:
 
 let
@@ -63,9 +65,9 @@ let
   pnameSuffix =
     strings.optionalString (suffices != [ ])
       "-${strings.concatMapStringsSep "-" strings.toLower suffices}";
-  descriptionSuffix =
-    strings.optionalString (suffices != [ ])
-      ", accelerated with ${strings.concatStringsSep ", " suffices}";
+  descriptionSuffix = strings.optionalString (
+    suffices != [ ]
+  ) ", accelerated with ${strings.concatStringsSep ", " suffices}";
 
   executableSuffix = effectiveStdenv.hostPlatform.extensions.executable;
 
@@ -76,41 +78,37 @@ let
   #
   # TODO: Package up each Python script or service appropriately, by making
   # them into "entrypoints"
-  llama-python = python3.withPackages (
-    ps: [
-      ps.numpy
-      ps.sentencepiece
-    ]
-  );
+  llama-python = python3.withPackages (ps: [
+    ps.numpy
+    ps.sentencepiece
+  ]);
 
   # TODO(Green-Sky): find a better way to opt-into the heavy ml python runtime
-  llama-python-extra = python3.withPackages (
-    ps: [
-      ps.numpy
-      ps.sentencepiece
-      ps.tiktoken
-      ps.torchWithoutCuda
-      ps.transformers
-
-      # server bench
-      ps.matplotlib
-
-      # server tests
-      ps.openai
-      ps.behave
-      ps.prometheus-client
-
-      # for examples/pydantic-models-to-grammar-examples.py
-      ps.docstring-parser
-      ps.pydantic
-
-      # for scripts/compare-llama-bench.py
-      ps.gitpython
-      ps.tabulate
-    ]
-  );
-
-  xcrunHost = runCommand "xcrunHost" {} ''
+  llama-python-extra = python3.withPackages (ps: [
+    ps.numpy
+    ps.sentencepiece
+    ps.tiktoken
+    ps.torchWithoutCuda
+    ps.transformers
+
+    # server bench
+    ps.matplotlib
+
+    # server tests
+    ps.openai
+    ps.behave
+    ps.prometheus-client
+
+    # for examples/pydantic-models-to-grammar-examples.py
+    ps.docstring-parser
+    ps.pydantic
+
+    # for scripts/compare-llama-bench.py
+    ps.gitpython
+    ps.tabulate
+  ]);
+
+  xcrunHost = runCommand "xcrunHost" { } ''
     mkdir -p $out/bin
     ln -s /usr/bin/xcrun $out/bin
   '';
@@ -145,178 +143,174 @@ let
   ];
 in
 
-effectiveStdenv.mkDerivation (
-  finalAttrs: {
-    pname = "llama-cpp${pnameSuffix}";
-    version = llamaVersion;
-
-    # Note: none of the files discarded here are visible in the sandbox or
-    # affect the output hash. This also means they can be modified without
-    # triggering a rebuild.
-    src = lib.cleanSourceWith {
-      filter =
-        name: type:
-        let
-          noneOf = builtins.all (x: !x);
-          baseName = baseNameOf name;
-        in
-        noneOf [
-          (lib.hasSuffix ".nix" name) # Ignore *.nix files when computing outPaths
-          (lib.hasSuffix ".md" name) # Ignore *.md changes whe computing outPaths
-          (lib.hasPrefix "." baseName) # Skip hidden files and directories
-          (baseName == "flake.lock")
-        ];
-      src = lib.cleanSource ../../.;
-    };
-
-    postPatch = ''
-      substituteInPlace ./ggml/src/ggml-metal.m \
-        --replace '[bundle pathForResource:@"ggml-metal" ofType:@"metal"];' "@\"$out/bin/ggml-metal.metal\";"
-      substituteInPlace ./ggml/src/ggml-metal.m \
-        --replace '[bundle pathForResource:@"default" ofType:@"metallib"];' "@\"$out/bin/default.metallib\";"
-    '';
-
-    # With PR#6015 https://github.com/ggerganov/llama.cpp/pull/6015,
-    # `default.metallib` may be compiled with Metal compiler from XCode
-    # and we need to escape sandbox on MacOS to access Metal compiler.
-    # `xcrun` is used find the path of the Metal compiler, which is varible
-    # and not on $PATH
-    # see https://github.com/ggerganov/llama.cpp/pull/6118 for discussion
-    __noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders;
-
-    nativeBuildInputs =
-      [
-        cmake
-        ninja
-        pkg-config
-        git
-      ]
-      ++ optionals useCuda [
-        cudaPackages.cuda_nvcc
-        autoAddDriverRunpath
-      ]
-      ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [
-        glibc.static
-      ] ++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [
-        xcrunHost
+effectiveStdenv.mkDerivation (finalAttrs: {
+  pname = "llama-cpp${pnameSuffix}";
+  version = llamaVersion;
+
+  # Note: none of the files discarded here are visible in the sandbox or
+  # affect the output hash. This also means they can be modified without
+  # triggering a rebuild.
+  src = lib.cleanSourceWith {
+    filter =
+      name: type:
+      let
+        noneOf = builtins.all (x: !x);
+        baseName = baseNameOf name;
+      in
+      noneOf [
+        (lib.hasSuffix ".nix" name) # Ignore *.nix files when computing outPaths
+        (lib.hasSuffix ".md" name) # Ignore *.md changes whe computing outPaths
+        (lib.hasPrefix "." baseName) # Skip hidden files and directories
+        (baseName == "flake.lock")
       ];
+    src = lib.cleanSource ../../.;
+  };
+
+  postPatch = ''
+    substituteInPlace ./ggml/src/ggml-metal.m \
+      --replace '[bundle pathForResource:@"ggml-metal" ofType:@"metal"];' "@\"$out/bin/ggml-metal.metal\";"
+    substituteInPlace ./ggml/src/ggml-metal.m \
+      --replace '[bundle pathForResource:@"default" ofType:@"metallib"];' "@\"$out/bin/default.metallib\";"
+  '';
 
-    buildInputs =
-      optionals effectiveStdenv.isDarwin darwinBuildInputs
-      ++ optionals useCuda cudaBuildInputs
-      ++ optionals useMpi [ mpi ]
-      ++ optionals useRocm rocmBuildInputs
-      ++ optionals useBlas [ blas ]
-      ++ optionals useVulkan vulkanBuildInputs
-      ++ optionals enableCurl [ curl ];
-
-    cmakeFlags =
-      [
-        (cmakeBool "LLAMA_BUILD_SERVER" true)
-        (cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
-        (cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
-        (cmakeBool "LLAMA_CURL" enableCurl)
-        (cmakeBool "GGML_NATIVE" false)
-        (cmakeBool "GGML_BLAS" useBlas)
-        (cmakeBool "GGML_CUDA" useCuda)
-        (cmakeBool "GGML_HIPBLAS" useRocm)
-        (cmakeBool "GGML_METAL" useMetalKit)
-        (cmakeBool "GGML_VULKAN" useVulkan)
-        (cmakeBool "GGML_STATIC" enableStatic)
-      ]
-      ++ optionals useCuda [
-        (
-          with cudaPackages.flags;
-          cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
-            builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
-          )
-        )
-      ]
-      ++ optionals useRocm [
-        (cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
-        (cmakeFeature "CMAKE_HIP_ARCHITECTURES" (builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets))
-      ]
-      ++ optionals useMetalKit [
-        (lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
-        (cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
-      ];
+  # With PR#6015 https://github.com/ggerganov/llama.cpp/pull/6015,
+  # `default.metallib` may be compiled with Metal compiler from XCode
+  # and we need to escape sandbox on MacOS to access Metal compiler.
+  # `xcrun` is used find the path of the Metal compiler, which is varible
+  # and not on $PATH
+  # see https://github.com/ggerganov/llama.cpp/pull/6118 for discussion
+  __noChroot = effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders;
 
-    # Environment variables needed for ROCm
-    env = optionals useRocm {
-      ROCM_PATH = "${rocmPackages.clr}";
-      HIP_DEVICE_LIB_PATH = "${rocmPackages.rocm-device-libs}/amdgcn/bitcode";
-    };
+  nativeBuildInputs =
+    [
+      cmake
+      ninja
+      pkg-config
+      git
+    ]
+    ++ optionals useCuda [
+      cudaPackages.cuda_nvcc
 
-    # TODO(SomeoneSerge): It's better to add proper install targets at the CMake level,
-    # if they haven't been added yet.
-    postInstall = ''
-      mkdir -p $out/include
-      cp $src/include/llama.h $out/include/
-    '';
-
-    # Define the shells here, but don't add in the inputsFrom to avoid recursion.
-    passthru = {
-      inherit
-        useBlas
-        useCuda
-        useMetalKit
-        useMpi
-        useRocm
-        useVulkan
-        ;
-
-      shell = mkShell {
-        name = "shell-${finalAttrs.finalPackage.name}";
-        description = "contains numpy and sentencepiece";
-        buildInputs = [ llama-python ];
-        inputsFrom = [ finalAttrs.finalPackage ];
-        shellHook = ''
-          addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib effectiveStdenv.cc.cc}/lib"
-        '';
-      };
-
-      shell-extra = mkShell {
-        name = "shell-extra-${finalAttrs.finalPackage.name}";
-        description = "contains numpy, sentencepiece, torchWithoutCuda, and transformers";
-        buildInputs = [ llama-python-extra ];
-        inputsFrom = [ finalAttrs.finalPackage ];
-      };
-    };
+      autoAddDriverRunpath
+    ]
+    ++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
+    ++ optionals (effectiveStdenv.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
+
+  buildInputs =
+    optionals effectiveStdenv.isDarwin darwinBuildInputs
+    ++ optionals useCuda cudaBuildInputs
+    ++ optionals useMpi [ mpi ]
+    ++ optionals useRocm rocmBuildInputs
+    ++ optionals useBlas [ blas ]
+    ++ optionals useVulkan vulkanBuildInputs
+    ++ optionals enableCurl [ curl ];
+
+  cmakeFlags =
+    [
+      (cmakeBool "LLAMA_BUILD_SERVER" true)
+      (cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
+      (cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
+      (cmakeBool "LLAMA_CURL" enableCurl)
+      (cmakeBool "GGML_NATIVE" false)
+      (cmakeBool "GGML_BLAS" useBlas)
+      (cmakeBool "GGML_CUDA" useCuda)
+      (cmakeBool "GGML_HIPBLAS" useRocm)
+      (cmakeBool "GGML_METAL" useMetalKit)
+      (cmakeBool "GGML_VULKAN" useVulkan)
+      (cmakeBool "GGML_STATIC" enableStatic)
+    ]
+    ++ optionals useCuda [
+      (
+        with cudaPackages.flags;
+        cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
+          builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
+        )
+      )
+    ]
+    ++ optionals useRocm [
+      (cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
+      (cmakeFeature "CMAKE_HIP_ARCHITECTURES" (builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets))
+    ]
+    ++ optionals useMetalKit [
+      (lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
+      (cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
+    ];
+
+  # Environment variables needed for ROCm
+  env = optionals useRocm {
+    ROCM_PATH = "${rocmPackages.clr}";
+    HIP_DEVICE_LIB_PATH = "${rocmPackages.rocm-device-libs}/amdgcn/bitcode";
+  };
+
+  # TODO(SomeoneSerge): It's better to add proper install targets at the CMake level,
+  # if they haven't been added yet.
+  postInstall = ''
+    mkdir -p $out/include
+    cp $src/include/llama.h $out/include/
+  '';
 
-    meta = {
-      # Configurations we don't want even the CI to evaluate. Results in the
-      # "unsupported platform" messages. This is mostly a no-op, because
-      # cudaPackages would've refused to evaluate anyway.
-      badPlatforms = optionals useCuda lib.platforms.darwin;
-
-      # Configurations that are known to result in build failures. Can be
-      # overridden by importing Nixpkgs with `allowBroken = true`.
-      broken = (useMetalKit && !effectiveStdenv.isDarwin);
-
-      description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}";
-      homepage = "https://github.com/ggerganov/llama.cpp/";
-      license = lib.licenses.mit;
-
-      # Accommodates `nix run` and `lib.getExe`
-      mainProgram = "llama-cli";
-
-      # These people might respond, on the best effort basis, if you ping them
-      # in case of Nix-specific regressions or for reviewing Nix-specific PRs.
-      # Consider adding yourself to this list if you want to ensure this flake
-      # stays maintained and you're willing to invest your time. Do not add
-      # other people without their consent. Consider removing people after
-      # they've been unreachable for long periods of time.
-
-      # Note that lib.maintainers is defined in Nixpkgs, but you may just add
-      # an attrset following the same format as in
-      # https://github.com/NixOS/nixpkgs/blob/f36a80e54da29775c78d7eff0e628c2b4e34d1d7/maintainers/maintainer-list.nix
-      maintainers = with lib.maintainers; [
-        philiptaron
-        SomeoneSerge
-      ];
+  # Define the shells here, but don't add in the inputsFrom to avoid recursion.
+  passthru = {
+    inherit
+      useBlas
+      useCuda
+      useMetalKit
+      useMpi
+      useRocm
+      useVulkan
+      ;
+
+    shell = mkShell {
+      name = "shell-${finalAttrs.finalPackage.name}";
+      description = "contains numpy and sentencepiece";
+      buildInputs = [ llama-python ];
+      inputsFrom = [ finalAttrs.finalPackage ];
+      shellHook = ''
+        addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib effectiveStdenv.cc.cc}/lib"
+      '';
+    };
 
-      # Extend `badPlatforms` instead
-      platforms = lib.platforms.all;
+    shell-extra = mkShell {
+      name = "shell-extra-${finalAttrs.finalPackage.name}";
+      description = "contains numpy, sentencepiece, torchWithoutCuda, and transformers";
+      buildInputs = [ llama-python-extra ];
+      inputsFrom = [ finalAttrs.finalPackage ];
     };
-  }
-)
+  };
+
+  meta = {
+    # Configurations we don't want even the CI to evaluate. Results in the
+    # "unsupported platform" messages. This is mostly a no-op, because
+    # cudaPackages would've refused to evaluate anyway.
+    badPlatforms = optionals useCuda lib.platforms.darwin;
+
+    # Configurations that are known to result in build failures. Can be
+    # overridden by importing Nixpkgs with `allowBroken = true`.
+    broken = (useMetalKit && !effectiveStdenv.isDarwin);
+
+    description = "Inference of LLaMA model in pure C/C++${descriptionSuffix}";
+    homepage = "https://github.com/ggerganov/llama.cpp/";
+    license = lib.licenses.mit;
+
+    # Accommodates `nix run` and `lib.getExe`
+    mainProgram = "llama-cli";
+
+    # These people might respond, on the best effort basis, if you ping them
+    # in case of Nix-specific regressions or for reviewing Nix-specific PRs.
+    # Consider adding yourself to this list if you want to ensure this flake
+    # stays maintained and you're willing to invest your time. Do not add
+    # other people without their consent. Consider removing people after
+    # they've been unreachable for long periods of time.
+
+    # Note that lib.maintainers is defined in Nixpkgs, but you may just add
+    # an attrset following the same format as in
+    # https://github.com/NixOS/nixpkgs/blob/f36a80e54da29775c78d7eff0e628c2b4e34d1d7/maintainers/maintainer-list.nix
+    maintainers = with lib.maintainers; [
+      philiptaron
+      SomeoneSerge
+    ];
+
+    # Extend `badPlatforms` instead
+    platforms = lib.platforms.all;
+  };
+})
diff --git a/.devops/nix/scope.nix b/.devops/nix/scope.nix
index 78530c9e8a230..d9fc1fdf01cad 100644
--- a/.devops/nix/scope.nix
+++ b/.devops/nix/scope.nix
@@ -8,12 +8,10 @@
 # because it allows users to apply overlays later using `overrideScope'`.
 # Cf. https://noogle.dev/f/lib/makeScope
 
-lib.makeScope newScope (
-  self: {
-    inherit llamaVersion;
-    llama-cpp = self.callPackage ./package.nix { };
-    docker = self.callPackage ./docker.nix { };
-    docker-min = self.callPackage ./docker.nix { interactive = false; };
-    sif = self.callPackage ./sif.nix { };
-  }
-)
+lib.makeScope newScope (self: {
+  inherit llamaVersion;
+  llama-cpp = self.callPackage ./package.nix { };
+  docker = self.callPackage ./docker.nix { };
+  docker-min = self.callPackage ./docker.nix { interactive = false; };
+  sif = self.callPackage ./sif.nix { };
+})
diff --git a/flake.nix b/flake.nix
index c69637d111784..4c75cbbcc757b 100644
--- a/flake.nix
+++ b/flake.nix
@@ -145,7 +145,9 @@
             # the same path you would with an overlay.
             legacyPackages = {
               llamaPackages = pkgs.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
-              llamaPackagesWindows = pkgs.pkgsCross.mingwW64.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
+              llamaPackagesWindows = pkgs.pkgsCross.mingwW64.callPackage .devops/nix/scope.nix {
+                inherit llamaVersion;
+              };
               llamaPackagesCuda = pkgsCuda.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
               llamaPackagesRocm = pkgsRocm.callPackage .devops/nix/scope.nix { inherit llamaVersion; };
             };

From c3bc2f6ddf83e0f4f80c1f75207aafa71b70e98b Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Thu, 22 Feb 2024 21:02:44 +0530
Subject: [PATCH 02/20] build(nix): Package gguf-py

---
 .devops/nix/package.nix | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index e065004af31f1..f5a6c0d7d9d73 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -70,6 +70,24 @@ let
   ) ", accelerated with ${strings.concatStringsSep ", " suffices}";
 
   executableSuffix = effectiveStdenv.hostPlatform.extensions.executable;
+  
+  gguf-py = python3.pkgs.buildPythonPackage rec {
+    pname = "gguf";
+    version = "0.0.0";
+    pyproject = true;
+    nativeBuildInputs = with python3.pkgs; [ poetry-core ];
+    # buildInputs = [
+    #   python3.pkgs.poetry-core
+    # ];
+    propagatedBuildInputs = with python3.pkgs; [ numpy ];
+    src = lib.cleanSource ../../gguf-py;
+    doCheck = false;
+    meta = with lib; {
+      description = "Python package for writing binary files in the GGU format";
+      license = licenses.mit;
+      maintainers = [ maintainers.philiptaron ];
+    };
+  };
 
   # TODO: package the Python in this repository in a Nix-like way.
   # It'd be nice to migrate to buildPythonPackage, as well as ensure this repo
@@ -81,6 +99,7 @@ let
   llama-python = python3.withPackages (ps: [
     ps.numpy
     ps.sentencepiece
+    gguf-py
   ]);
 
   # TODO(Green-Sky): find a better way to opt-into the heavy ml python runtime

From f363d308a4f5947e05dfb8ccbb601d028a73a248 Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Thu, 22 Feb 2024 22:44:35 +0530
Subject: [PATCH 03/20] build(nix): Refactor to new scope for gguf-py

---
 .devops/nix/package-gguf-py.nix | 20 ++++++++++++++++++++
 .devops/nix/package.nix         | 22 +++-------------------
 .devops/nix/scope.nix           |  1 +
 3 files changed, 24 insertions(+), 19 deletions(-)
 create mode 100644 .devops/nix/package-gguf-py.nix

diff --git a/.devops/nix/package-gguf-py.nix b/.devops/nix/package-gguf-py.nix
new file mode 100644
index 0000000000000..944ebd700b449
--- /dev/null
+++ b/.devops/nix/package-gguf-py.nix
@@ -0,0 +1,20 @@
+{
+  lib,
+  llamaVersion,
+  python3,
+}@inputs:
+
+python3.pkgs.buildPythonPackage rec {
+  pname = "gguf";
+  version = llamaVersion;
+  pyproject = true;
+  nativeBuildInputs = with python3.pkgs; [ poetry-core ];
+  propagatedBuildInputs = with python3.pkgs; [ numpy ];
+  src = lib.cleanSource ../../gguf-py;
+  doCheck = false;
+  meta = with lib; {
+    description = "Python package for writing binary files in the GGUF format";
+    license = licenses.mit;
+    maintainers = [ maintainers.ditsuke ];
+  };
+}
diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index f5a6c0d7d9d73..6f797431ddc1b 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -30,7 +30,8 @@
     && blas.meta.available,
   useCuda ? config.cudaSupport,
   useMetalKit ? stdenv.isAarch64 && stdenv.isDarwin,
-  useMpi ? false, # Increases the runtime closure size by ~700M
+  # Increases the runtime closure size by ~700M
+  useMpi ? false,
   useRocm ? config.rocmSupport,
   enableCurl ? true,
   useVulkan ? false,
@@ -41,6 +42,7 @@
   effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
   enableStatic ? effectiveStdenv.hostPlatform.isStatic,
   precompileMetalShaders ? false,
+  gguf-py,
 }@inputs:
 
 let
@@ -70,24 +72,6 @@ let
   ) ", accelerated with ${strings.concatStringsSep ", " suffices}";
 
   executableSuffix = effectiveStdenv.hostPlatform.extensions.executable;
-  
-  gguf-py = python3.pkgs.buildPythonPackage rec {
-    pname = "gguf";
-    version = "0.0.0";
-    pyproject = true;
-    nativeBuildInputs = with python3.pkgs; [ poetry-core ];
-    # buildInputs = [
-    #   python3.pkgs.poetry-core
-    # ];
-    propagatedBuildInputs = with python3.pkgs; [ numpy ];
-    src = lib.cleanSource ../../gguf-py;
-    doCheck = false;
-    meta = with lib; {
-      description = "Python package for writing binary files in the GGU format";
-      license = licenses.mit;
-      maintainers = [ maintainers.philiptaron ];
-    };
-  };
 
   # TODO: package the Python in this repository in a Nix-like way.
   # It'd be nice to migrate to buildPythonPackage, as well as ensure this repo
diff --git a/.devops/nix/scope.nix b/.devops/nix/scope.nix
index d9fc1fdf01cad..eaf7d7a27f1ea 100644
--- a/.devops/nix/scope.nix
+++ b/.devops/nix/scope.nix
@@ -10,6 +10,7 @@
 
 lib.makeScope newScope (self: {
   inherit llamaVersion;
+  gguf-py = self.callPackage ./package-gguf-py.nix { };
   llama-cpp = self.callPackage ./package.nix { };
   docker = self.callPackage ./docker.nix { };
   docker-min = self.callPackage ./docker.nix { interactive = false; };

From 0b8ddf8694d15a055ddc9e7a3474a349d44abc8e Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Fri, 23 Feb 2024 15:12:13 +0530
Subject: [PATCH 04/20] build(nix): Exclude gguf-py from devShells

---
 .devops/nix/package.nix | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index 6f797431ddc1b..ff0f2188bc57c 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -72,6 +72,7 @@ let
   ) ", accelerated with ${strings.concatStringsSep ", " suffices}";
 
   executableSuffix = effectiveStdenv.hostPlatform.extensions.executable;
+  mapToPythonPackages = ps: packages: map (package: ps.${package}) packages;
 
   # TODO: package the Python in this repository in a Nix-like way.
   # It'd be nice to migrate to buildPythonPackage, as well as ensure this repo
@@ -266,7 +267,10 @@ effectiveStdenv.mkDerivation (finalAttrs: {
     shell = mkShell {
       name = "shell-${finalAttrs.finalPackage.name}";
       description = "contains numpy and sentencepiece";
-      buildInputs = [ llama-python ];
+      buildInputs = [
+        python3.withPackages
+        (ps: mapToPythonPackages ps llama-python-base-deps)
+      ];
       inputsFrom = [ finalAttrs.finalPackage ];
       shellHook = ''
         addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib effectiveStdenv.cc.cc}/lib"
@@ -276,7 +280,10 @@ effectiveStdenv.mkDerivation (finalAttrs: {
     shell-extra = mkShell {
       name = "shell-extra-${finalAttrs.finalPackage.name}";
       description = "contains numpy, sentencepiece, torchWithoutCuda, and transformers";
-      buildInputs = [ llama-python-extra ];
+      buildInputs = [
+        python3.withPackages
+        (ps: mapToPythonPackages ps llama-python-full-deps)
+      ];
       inputsFrom = [ finalAttrs.finalPackage ];
     };
   };

From 11e581b45db1fddf8633bbf40ae996f8402ed62a Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Fri, 23 Feb 2024 23:01:19 +0530
Subject: [PATCH 05/20] build(nix): Refactor gguf-py derivation to take in
 exact deps

---
 .devops/nix/package-gguf-py.nix | 10 ++++++----
 .devops/nix/scope.nix           | 11 ++++++++++-
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/.devops/nix/package-gguf-py.nix b/.devops/nix/package-gguf-py.nix
index 944ebd700b449..ee73150d4099d 100644
--- a/.devops/nix/package-gguf-py.nix
+++ b/.devops/nix/package-gguf-py.nix
@@ -1,15 +1,17 @@
 {
   lib,
   llamaVersion,
-  python3,
+  numpy,
+  poetry-core,
+  buildPythonPackage,
 }@inputs:
 
-python3.pkgs.buildPythonPackage rec {
+buildPythonPackage {
   pname = "gguf";
   version = llamaVersion;
   pyproject = true;
-  nativeBuildInputs = with python3.pkgs; [ poetry-core ];
-  propagatedBuildInputs = with python3.pkgs; [ numpy ];
+  nativeBuildInputs = [ poetry-core ];
+  propagatedBuildInputs = [ numpy ];
   src = lib.cleanSource ../../gguf-py;
   doCheck = false;
   meta = with lib; {
diff --git a/.devops/nix/scope.nix b/.devops/nix/scope.nix
index eaf7d7a27f1ea..abef51fd35ce0 100644
--- a/.devops/nix/scope.nix
+++ b/.devops/nix/scope.nix
@@ -1,16 +1,25 @@
 {
   lib,
   newScope,
+  python3,
   llamaVersion ? "0.0.0",
 }:
 
+let
+  pythonPackages = python3.pkgs;
+  buildPythonPackage = pythonPackages.buildPythonPackage;
+  numpy = pythonPackages.numpy;
+  poetry-core = pythonPackages.poetry-core;
+in
+
 # We're using `makeScope` instead of just writing out an attrset
 # because it allows users to apply overlays later using `overrideScope'`.
 # Cf. https://noogle.dev/f/lib/makeScope
 
 lib.makeScope newScope (self: {
   inherit llamaVersion;
-  gguf-py = self.callPackage ./package-gguf-py.nix { };
+  pp = python3.pkgs;
+  gguf-py = self.callPackage ./package-gguf-py.nix { inherit buildPythonPackage numpy poetry-core; };
   llama-cpp = self.callPackage ./package.nix { };
   docker = self.callPackage ./docker.nix { };
   docker-min = self.callPackage ./docker.nix { interactive = false; };

From 4b124fb2a4783a3d4931945fe59b2a5a8a499a64 Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Sun, 25 Feb 2024 17:14:33 +0530
Subject: [PATCH 06/20] build(nix): Enable pytestCheckHook and
 pythonImportsCheck for gguf-py

---
 .devops/nix/package-gguf-py.nix |  8 +++++++-
 .devops/nix/scope.nix           | 10 +++++++++-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/.devops/nix/package-gguf-py.nix b/.devops/nix/package-gguf-py.nix
index ee73150d4099d..24b42d446f334 100644
--- a/.devops/nix/package-gguf-py.nix
+++ b/.devops/nix/package-gguf-py.nix
@@ -4,6 +4,7 @@
   numpy,
   poetry-core,
   buildPythonPackage,
+  pytestCheckHook,
 }@inputs:
 
 buildPythonPackage {
@@ -13,7 +14,12 @@ buildPythonPackage {
   nativeBuildInputs = [ poetry-core ];
   propagatedBuildInputs = [ numpy ];
   src = lib.cleanSource ../../gguf-py;
-  doCheck = false;
+  pythonImportsCheck = [
+    "numpy"
+    "gguf"
+  ];
+  nativeCheckInputs = [ pytestCheckHook ];
+  doCheck = true;
   meta = with lib; {
     description = "Python package for writing binary files in the GGUF format";
     license = licenses.mit;
diff --git a/.devops/nix/scope.nix b/.devops/nix/scope.nix
index abef51fd35ce0..4babfa44875af 100644
--- a/.devops/nix/scope.nix
+++ b/.devops/nix/scope.nix
@@ -10,6 +10,7 @@ let
   buildPythonPackage = pythonPackages.buildPythonPackage;
   numpy = pythonPackages.numpy;
   poetry-core = pythonPackages.poetry-core;
+  pytestCheckHook = pythonPackages.pytestCheckHook;
 in
 
 # We're using `makeScope` instead of just writing out an attrset
@@ -19,7 +20,14 @@ in
 lib.makeScope newScope (self: {
   inherit llamaVersion;
   pp = python3.pkgs;
-  gguf-py = self.callPackage ./package-gguf-py.nix { inherit buildPythonPackage numpy poetry-core; };
+  gguf-py = self.callPackage ./package-gguf-py.nix {
+    inherit
+      buildPythonPackage
+      numpy
+      poetry-core
+      pytestCheckHook
+      ;
+  };
   llama-cpp = self.callPackage ./package.nix { };
   docker = self.callPackage ./docker.nix { };
   docker-min = self.callPackage ./docker.nix { interactive = false; };

From 10a49b9999fc7029ab5d6f213fa4bde48d449a75 Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Sun, 25 Feb 2024 18:38:36 +0530
Subject: [PATCH 07/20] build(python): Package python scripts with
 pyproject.toml

---
 __init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 __init__.py

diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d

From 51056d932c31a278b13d4e985e470744f00bbf8b Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Tue, 27 Feb 2024 00:50:01 +0530
Subject: [PATCH 08/20] chore: Cleanup

---
 .devops/nix/scope.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.devops/nix/scope.nix b/.devops/nix/scope.nix
index 4babfa44875af..6480fcaefa93c 100644
--- a/.devops/nix/scope.nix
+++ b/.devops/nix/scope.nix
@@ -28,6 +28,7 @@ lib.makeScope newScope (self: {
       pytestCheckHook
       ;
   };
+  python-scripts = self.callPackage ./python-scripts.nix { inherit buildPythonPackage poetry-core; };
   llama-cpp = self.callPackage ./package.nix { };
   docker = self.callPackage ./docker.nix { };
   docker-min = self.callPackage ./docker.nix { interactive = false; };

From ef2dae9249f556116fd34e5d93a213c77507d7ff Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Tue, 27 Feb 2024 00:51:12 +0530
Subject: [PATCH 09/20] dev(nix): Break up python/C devShells

---
 .devops/nix/devshells.nix      |  1 -
 .devops/nix/package.nix        | 56 ++--------------------------------
 .devops/nix/python-scripts.nix | 42 +++++++++++++++++++++++++
 flake.nix                      |  1 +
 4 files changed, 46 insertions(+), 54 deletions(-)
 create mode 100644 .devops/nix/python-scripts.nix

diff --git a/.devops/nix/devshells.nix b/.devops/nix/devshells.nix
index 9895d803f041f..858c5bea91e04 100644
--- a/.devops/nix/devshells.nix
+++ b/.devops/nix/devshells.nix
@@ -4,7 +4,6 @@
     {
       devShells = lib.concatMapAttrs (name: package: {
         ${name} = package.passthru.shell;
-        ${name + "-extra"} = package.passthru.shell-extra;
       }) config.packages;
     };
 }
diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index ff0f2188bc57c..824908fbb3b31 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -6,6 +6,7 @@
   mkShell,
   runCommand,
   cmake,
+  gcc,
   ninja,
   pkg-config,
   git,
@@ -74,44 +75,6 @@ let
   executableSuffix = effectiveStdenv.hostPlatform.extensions.executable;
   mapToPythonPackages = ps: packages: map (package: ps.${package}) packages;
 
-  # TODO: package the Python in this repository in a Nix-like way.
-  # It'd be nice to migrate to buildPythonPackage, as well as ensure this repo
-  # is PEP 517-compatible, and ensure the correct .dist-info is generated.
-  # https://peps.python.org/pep-0517/
-  #
-  # TODO: Package up each Python script or service appropriately, by making
-  # them into "entrypoints"
-  llama-python = python3.withPackages (ps: [
-    ps.numpy
-    ps.sentencepiece
-    gguf-py
-  ]);
-
-  # TODO(Green-Sky): find a better way to opt-into the heavy ml python runtime
-  llama-python-extra = python3.withPackages (ps: [
-    ps.numpy
-    ps.sentencepiece
-    ps.tiktoken
-    ps.torchWithoutCuda
-    ps.transformers
-
-    # server bench
-    ps.matplotlib
-
-    # server tests
-    ps.openai
-    ps.behave
-    ps.prometheus-client
-
-    # for examples/pydantic-models-to-grammar-examples.py
-    ps.docstring-parser
-    ps.pydantic
-
-    # for scripts/compare-llama-bench.py
-    ps.gitpython
-    ps.tabulate
-  ]);
-
   xcrunHost = runCommand "xcrunHost" { } ''
     mkdir -p $out/bin
     ln -s /usr/bin/xcrun $out/bin
@@ -268,21 +231,8 @@ effectiveStdenv.mkDerivation (finalAttrs: {
       name = "shell-${finalAttrs.finalPackage.name}";
       description = "contains numpy and sentencepiece";
       buildInputs = [
-        python3.withPackages
-        (ps: mapToPythonPackages ps llama-python-base-deps)
-      ];
-      inputsFrom = [ finalAttrs.finalPackage ];
-      shellHook = ''
-        addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib effectiveStdenv.cc.cc}/lib"
-      '';
-    };
-
-    shell-extra = mkShell {
-      name = "shell-extra-${finalAttrs.finalPackage.name}";
-      description = "contains numpy, sentencepiece, torchWithoutCuda, and transformers";
-      buildInputs = [
-        python3.withPackages
-        (ps: mapToPythonPackages ps llama-python-full-deps)
+        cmake
+        gcc
       ];
       inputsFrom = [ finalAttrs.finalPackage ];
     };
diff --git a/.devops/nix/python-scripts.nix b/.devops/nix/python-scripts.nix
new file mode 100644
index 0000000000000..fa0d606542b87
--- /dev/null
+++ b/.devops/nix/python-scripts.nix
@@ -0,0 +1,42 @@
+{
+  lib,
+  stdenv,
+  buildPythonPackage,
+  poetry-core,
+  breakpointHook,
+  mkShell,
+  python3Packages,
+  gguf-py,
+}@inputs:
+
+let
+  llama-python-deps = with python3Packages; [
+    numpy
+    sentencepiece
+    transformers
+    protobuf
+    torchWithoutCuda
+    gguf-py
+  ];
+in
+
+buildPythonPackage ({
+  pname = "llama-scripts";
+  src = ../../.;
+  version = "0.0.0";
+  pyproject = true;
+  nativeBuildInputs = [ poetry-core ];
+  projectDir = ../../.;
+  propagatedBuildInputs = llama-python-deps;
+
+  passthru = {
+    shell = mkShell {
+      name = "shell-python-scripts";
+      description = "contains numpy and sentencepiece";
+      buildInputs = llama-python-deps;
+      shellHook = ''
+        addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib stdenv.cc.cc}/lib"
+      '';
+    };
+  };
+})
diff --git a/flake.nix b/flake.nix
index 4c75cbbcc757b..26a2588169101 100644
--- a/flake.nix
+++ b/flake.nix
@@ -159,6 +159,7 @@
                 default = config.legacyPackages.llamaPackages.llama-cpp;
                 vulkan = config.packages.default.override { useVulkan = true; };
                 windows = config.legacyPackages.llamaPackagesWindows.llama-cpp;
+                python-scripts = config.legacyPackages.llamaPackages.python-scripts;
               }
               // lib.optionalAttrs pkgs.stdenv.isLinux {
                 cuda = config.legacyPackages.llamaPackagesCuda.llama-cpp;

From d7b57766195fca342ca7d15a26038be8efc0cf48 Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Tue, 27 Feb 2024 00:52:51 +0530
Subject: [PATCH 10/20] build(python): Relax pytorch version constraint

Nix has an older version
---
 .devops/nix/package-gguf-py.nix | 4 +++-
 .devops/nix/python-scripts.nix  | 1 +
 .devops/nix/scope.nix           | 4 ++++
 gguf-py/pyproject.toml          | 1 +
 pyproject.toml                  | 2 +-
 5 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.devops/nix/package-gguf-py.nix b/.devops/nix/package-gguf-py.nix
index 24b42d446f334..dad2b41ab7fde 100644
--- a/.devops/nix/package-gguf-py.nix
+++ b/.devops/nix/package-gguf-py.nix
@@ -2,6 +2,8 @@
   lib,
   llamaVersion,
   numpy,
+  tqdm,
+  sentencepiece,
   poetry-core,
   buildPythonPackage,
   pytestCheckHook,
@@ -12,7 +14,7 @@ buildPythonPackage {
   version = llamaVersion;
   pyproject = true;
   nativeBuildInputs = [ poetry-core ];
-  propagatedBuildInputs = [ numpy ];
+  propagatedBuildInputs = [ numpy tqdm sentencepiece ];
   src = lib.cleanSource ../../gguf-py;
   pythonImportsCheck = [
     "numpy"
diff --git a/.devops/nix/python-scripts.nix b/.devops/nix/python-scripts.nix
index fa0d606542b87..119fee99b3cfe 100644
--- a/.devops/nix/python-scripts.nix
+++ b/.devops/nix/python-scripts.nix
@@ -17,6 +17,7 @@ let
     protobuf
     torchWithoutCuda
     gguf-py
+    tqdm
   ];
 in
 
diff --git a/.devops/nix/scope.nix b/.devops/nix/scope.nix
index 6480fcaefa93c..9d353f15dedbf 100644
--- a/.devops/nix/scope.nix
+++ b/.devops/nix/scope.nix
@@ -9,6 +9,8 @@ let
   pythonPackages = python3.pkgs;
   buildPythonPackage = pythonPackages.buildPythonPackage;
   numpy = pythonPackages.numpy;
+  tqdm = pythonPackages.tqdm;
+  sentencepiece = pythonPackages.sentencepiece;
   poetry-core = pythonPackages.poetry-core;
   pytestCheckHook = pythonPackages.pytestCheckHook;
 in
@@ -24,6 +26,8 @@ lib.makeScope newScope (self: {
     inherit
       buildPythonPackage
       numpy
+      tqdm
+      sentencepiece
       poetry-core
       pytestCheckHook
       ;
diff --git a/gguf-py/pyproject.toml b/gguf-py/pyproject.toml
index eea381e5a6b92..33cfe26b7fe30 100644
--- a/gguf-py/pyproject.toml
+++ b/gguf-py/pyproject.toml
@@ -23,6 +23,7 @@ python = ">=3.8"
 numpy = ">=1.17"
 tqdm = ">=4.27"
 pyyaml = ">=5.1"
+sentencepiece = ">=0.1.98,<=0.2.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"
diff --git a/pyproject.toml b/pyproject.toml
index 25e2e20b24896..84e71de6def38 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,7 +17,7 @@ classifiers = [
 [tool.poetry.dependencies]
 python = ">=3.9"
 numpy = "^1.25.0"
-sentencepiece = ">=0.1.98,<0.2.0"
+sentencepiece = ">=0.1.98,<=0.2.0"
 transformers = ">=4.35.2,<5.0.0"
 protobuf = ">=4.21.0,<5.0.0"
 gguf = { path = "./gguf-py" }

From a62a2c21b3a53a313e2c73332f54b9ff89ffb41a Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Tue, 27 Feb 2024 11:47:51 +0530
Subject: [PATCH 11/20] chore: Move cmake to nativeBuildInputs for devShell

---
 .devops/nix/package.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index 824908fbb3b31..c7f95d3dd8279 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -230,9 +230,8 @@ effectiveStdenv.mkDerivation (finalAttrs: {
     shell = mkShell {
       name = "shell-${finalAttrs.finalPackage.name}";
       description = "contains numpy and sentencepiece";
-      buildInputs = [
+      nativeBuildInputs = [
         cmake
-        gcc
       ];
       inputsFrom = [ finalAttrs.finalPackage ];
     };

From b7611bfb51fecda0ffb857d5796f4ea3333d22dc Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Sun, 10 Mar 2024 21:58:10 +0530
Subject: [PATCH 12/20] fmt: Reconcile formatting with rebase

---
 .devops/nix/package.nix | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index c7f95d3dd8279..7576bca21d967 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -230,9 +230,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
     shell = mkShell {
       name = "shell-${finalAttrs.finalPackage.name}";
       description = "contains numpy and sentencepiece";
-      nativeBuildInputs = [
-        cmake
-      ];
+      nativeBuildInputs = [ cmake ];
       inputsFrom = [ finalAttrs.finalPackage ];
     };
   };

From 8e911d79fb6830c72cdf93d63a0d199b991156be Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Mon, 8 Jul 2024 17:27:06 +0530
Subject: [PATCH 13/20] style: nix fmt

---
 .devops/nix/package-gguf-py.nix | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.devops/nix/package-gguf-py.nix b/.devops/nix/package-gguf-py.nix
index dad2b41ab7fde..42d3531900392 100644
--- a/.devops/nix/package-gguf-py.nix
+++ b/.devops/nix/package-gguf-py.nix
@@ -14,7 +14,11 @@ buildPythonPackage {
   version = llamaVersion;
   pyproject = true;
   nativeBuildInputs = [ poetry-core ];
-  propagatedBuildInputs = [ numpy tqdm sentencepiece ];
+  propagatedBuildInputs = [
+    numpy
+    tqdm
+    sentencepiece
+  ];
   src = lib.cleanSource ../../gguf-py;
   pythonImportsCheck = [
     "numpy"

From 4a71d86d85f64aa39cd7b3ae2f7949d8e2c05b7c Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Mon, 8 Jul 2024 17:29:02 +0530
Subject: [PATCH 14/20] cleanup: Remove unncessary __init__.py

---
 __init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 __init__.py

diff --git a/__init__.py b/__init__.py
deleted file mode 100644
index e69de29bb2d1d..0000000000000

From 03b2afea2456ccdadfc236936a3b375d9e15e3e6 Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Mon, 8 Jul 2024 23:33:43 +0530
Subject: [PATCH 15/20] chore: Suggestions from review

- Filter out non-source files from llama-scripts flake derivation
- Clean up unused closure
- Remove scripts devShell
---
 .devops/nix/devshells.nix      | 11 ++++++++---
 .devops/nix/package.nix        | 14 ++++++--------
 .devops/nix/python-scripts.nix | 21 ++++++++++++++++++---
 3 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/.devops/nix/devshells.nix b/.devops/nix/devshells.nix
index 858c5bea91e04..b7eed5c74e202 100644
--- a/.devops/nix/devshells.nix
+++ b/.devops/nix/devshells.nix
@@ -2,8 +2,13 @@
   perSystem =
     { config, lib, ... }:
     {
-      devShells = lib.concatMapAttrs (name: package: {
-        ${name} = package.passthru.shell;
-      }) config.packages;
+      devShells = lib.pipe (config.packages) [
+        (lib.concatMapAttrs
+        (name: package: {
+          ${name} = package.passthru.shell or null;
+        }))
+        (lib.filterAttrs (name: value: value != null))
+      ];
     };
 }
+
diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index 7576bca21d967..b6548e1bb27e5 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -73,7 +73,6 @@ let
   ) ", accelerated with ${strings.concatStringsSep ", " suffices}";
 
   executableSuffix = effectiveStdenv.hostPlatform.extensions.executable;
-  mapToPythonPackages = ps: packages: map (package: ps.${package}) packages;
 
   xcrunHost = runCommand "xcrunHost" { } ''
     mkdir -p $out/bin
@@ -121,14 +120,13 @@ effectiveStdenv.mkDerivation (finalAttrs: {
     filter =
       name: type:
       let
-        noneOf = builtins.all (x: !x);
-        baseName = baseNameOf name;
+        any = builtins.any (x: x);
+        baseName = builtins.baseNameOf name;
       in
-      noneOf [
-        (lib.hasSuffix ".nix" name) # Ignore *.nix files when computing outPaths
-        (lib.hasSuffix ".md" name) # Ignore *.md changes whe computing outPaths
-        (lib.hasPrefix "." baseName) # Skip hidden files and directories
-        (baseName == "flake.lock")
+      any [
+        (lib.hasSuffix ".py" name)
+        (baseName == "README.md")
+        (baseName == "pyproject.toml")
       ];
     src = lib.cleanSource ../../.;
   };
diff --git a/.devops/nix/python-scripts.nix b/.devops/nix/python-scripts.nix
index 119fee99b3cfe..4b1a598f8601c 100644
--- a/.devops/nix/python-scripts.nix
+++ b/.devops/nix/python-scripts.nix
@@ -23,12 +23,27 @@ in
 
 buildPythonPackage ({
   pname = "llama-scripts";
-  src = ../../.;
   version = "0.0.0";
   pyproject = true;
+
+  # NOTE: The files filtered out here are not visible in the build sandbox, neither
+  # do they affect the output hash. They can be modified without triggering a rebuild.
+  src = lib.cleanSourceWith {
+    filter =
+      name: type:
+      let
+        any = builtins.any (x: x);
+        baseName = builtins.baseNameOf name;
+      in
+      any [
+        (lib.hasSuffix ".py" name)
+        (baseName == "README.md")
+        (baseName == "pyproject.toml")
+      ];
+    src = lib.cleanSource ../../.;
+  };
   nativeBuildInputs = [ poetry-core ];
-  projectDir = ../../.;
-  propagatedBuildInputs = llama-python-deps;
+  dependencies = llama-python-deps;
 
   passthru = {
     shell = mkShell {

From cd8da29059ad655ce8e1d3b01567bab44d1ae8d1 Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Tue, 9 Jul 2024 01:43:44 +0530
Subject: [PATCH 16/20] revert: Bad changes

---
 .devops/nix/package.nix | 13 +++++++------
 .devops/nix/scope.nix   |  1 -
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index b6548e1bb27e5..99a82f797272a 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -120,13 +120,14 @@ effectiveStdenv.mkDerivation (finalAttrs: {
     filter =
       name: type:
       let
-        any = builtins.any (x: x);
-        baseName = builtins.baseNameOf name;
+        noneOf = builtins.all (x: !x);
+        baseName = baseNameOf name;
       in
-      any [
-        (lib.hasSuffix ".py" name)
-        (baseName == "README.md")
-        (baseName == "pyproject.toml")
+      noneOf [
+        (lib.hasSuffix ".nix" name) # Ignore *.nix files when computing outPaths
+        (lib.hasSuffix ".md" name) # Ignore *.md changes whe computing outPaths
+        (lib.hasPrefix "." baseName) # Skip hidden files and directories
+        (baseName == "flake.lock")
       ];
     src = lib.cleanSource ../../.;
   };
diff --git a/.devops/nix/scope.nix b/.devops/nix/scope.nix
index 9d353f15dedbf..9e846c5af8ef7 100644
--- a/.devops/nix/scope.nix
+++ b/.devops/nix/scope.nix
@@ -21,7 +21,6 @@ in
 
 lib.makeScope newScope (self: {
   inherit llamaVersion;
-  pp = python3.pkgs;
   gguf-py = self.callPackage ./package-gguf-py.nix {
     inherit
       buildPythonPackage

From 06a8547a20d4cf48aad46b863ec7e12b101ba4ed Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Tue, 20 Aug 2024 02:57:02 +0530
Subject: [PATCH 17/20] dev: Simplify devShells, restore the -extra devShell

---
 .devops/nix/devshells.nix      | 52 ++++++++++++++++++++++++++++------
 .devops/nix/package.nix        | 19 -------------
 .devops/nix/python-scripts.nix | 32 +++++++++++++--------
 3 files changed, 63 insertions(+), 40 deletions(-)

diff --git a/.devops/nix/devshells.nix b/.devops/nix/devshells.nix
index b7eed5c74e202..255ca018dea5f 100644
--- a/.devops/nix/devshells.nix
+++ b/.devops/nix/devshells.nix
@@ -1,14 +1,48 @@
+{ inputs, ... }:
+
 {
   perSystem =
-    { config, lib, ... }:
     {
-      devShells = lib.pipe (config.packages) [
-        (lib.concatMapAttrs
-        (name: package: {
-          ${name} = package.passthru.shell or null;
-        }))
-        (lib.filterAttrs (name: value: value != null))
-      ];
+      config,
+      lib,
+      system,
+      ...
+    }:
+    {
+      devShells =
+        let
+          pkgs = import inputs.nixpkgs { inherit system; };
+          stdenv = pkgs.stdenv;
+          scripts = config.packages.python-scripts;
+        in
+        lib.pipe (config.packages) [
+          (lib.concatMapAttrs (
+            name: package: {
+              ${name} = pkgs.mkShell {
+                name = "${name}";
+                inputsFrom = [ package ];
+                shellHook = ''
+                  echo "Entering ${name} devShell"
+                '';
+              };
+              "${name}-extra" =
+                if (name == "python-scripts") then
+                  null
+                else
+                  pkgs.mkShell {
+                    name = "${name}-extra";
+                    inputsFrom = [
+                      package
+                      scripts
+                    ];
+                    shellHook = ''
+                      echo "Entering ${name} devShell"
+                      addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib stdenv.cc.cc}/lib"
+                    '';
+                  };
+            }
+          ))
+          (lib.filterAttrs (name: value: value != null))
+        ];
     };
 }
-
diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index 99a82f797272a..7b5c77cf2689a 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -215,25 +215,6 @@ effectiveStdenv.mkDerivation (finalAttrs: {
     cp $src/include/llama.h $out/include/
   '';
 
-  # Define the shells here, but don't add in the inputsFrom to avoid recursion.
-  passthru = {
-    inherit
-      useBlas
-      useCuda
-      useMetalKit
-      useMpi
-      useRocm
-      useVulkan
-      ;
-
-    shell = mkShell {
-      name = "shell-${finalAttrs.finalPackage.name}";
-      description = "contains numpy and sentencepiece";
-      nativeBuildInputs = [ cmake ];
-      inputsFrom = [ finalAttrs.finalPackage ];
-    };
-  };
-
   meta = {
     # Configurations we don't want even the CI to evaluate. Results in the
     # "unsupported platform" messages. This is mostly a no-op, because
diff --git a/.devops/nix/python-scripts.nix b/.devops/nix/python-scripts.nix
index 4b1a598f8601c..392e9ffe41bf5 100644
--- a/.devops/nix/python-scripts.nix
+++ b/.devops/nix/python-scripts.nix
@@ -3,7 +3,6 @@
   stdenv,
   buildPythonPackage,
   poetry-core,
-  breakpointHook,
   mkShell,
   python3Packages,
   gguf-py,
@@ -18,6 +17,25 @@ let
     torchWithoutCuda
     gguf-py
     tqdm
+
+    # for scripts/compare-llama-bench.py
+    gitpython
+    tabulate
+
+    # for examples/pydantic-models-to-grammar-examples.py
+    docstring-parser
+    pydantic
+
+  ];
+
+  llama-python-test-deps = with python3Packages; [
+    # Server bench
+    matplotlib
+
+    # server tests
+    openai
+    behave
+    prometheus-client
   ];
 in
 
@@ -43,16 +61,6 @@ buildPythonPackage ({
     src = lib.cleanSource ../../.;
   };
   nativeBuildInputs = [ poetry-core ];
+  nativeCheckInputs = llama-python-test-deps;
   dependencies = llama-python-deps;
-
-  passthru = {
-    shell = mkShell {
-      name = "shell-python-scripts";
-      description = "contains numpy and sentencepiece";
-      buildInputs = llama-python-deps;
-      shellHook = ''
-        addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib stdenv.cc.cc}/lib"
-      '';
-    };
-  };
 })

From dcee4754e50d8024f6a684c9bf6e0e81972dcecd Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Tue, 20 Aug 2024 03:05:48 +0530
Subject: [PATCH 18/20] build(nix): Add pyyaml for gguf-py

---
 .devops/nix/package-gguf-py.nix | 2 ++
 .devops/nix/scope.nix           | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/.devops/nix/package-gguf-py.nix b/.devops/nix/package-gguf-py.nix
index 42d3531900392..083151fbca0a8 100644
--- a/.devops/nix/package-gguf-py.nix
+++ b/.devops/nix/package-gguf-py.nix
@@ -4,6 +4,7 @@
   numpy,
   tqdm,
   sentencepiece,
+  pyyaml,
   poetry-core,
   buildPythonPackage,
   pytestCheckHook,
@@ -18,6 +19,7 @@ buildPythonPackage {
     numpy
     tqdm
     sentencepiece
+    pyyaml
   ];
   src = lib.cleanSource ../../gguf-py;
   pythonImportsCheck = [
diff --git a/.devops/nix/scope.nix b/.devops/nix/scope.nix
index 9e846c5af8ef7..478e8c4228afa 100644
--- a/.devops/nix/scope.nix
+++ b/.devops/nix/scope.nix
@@ -11,6 +11,7 @@ let
   numpy = pythonPackages.numpy;
   tqdm = pythonPackages.tqdm;
   sentencepiece = pythonPackages.sentencepiece;
+  pyyaml = pythonPackages.pyyaml;
   poetry-core = pythonPackages.poetry-core;
   pytestCheckHook = pythonPackages.pytestCheckHook;
 in
@@ -28,6 +29,7 @@ lib.makeScope newScope (self: {
       tqdm
       sentencepiece
       poetry-core
+      pyyaml
       pytestCheckHook
       ;
   };

From fa0e7c9d5b4016233b437353a60f152c416041e1 Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Fri, 30 Aug 2024 16:26:10 +0530
Subject: [PATCH 19/20] chore: Remove some unused bindings

---
 .devops/nix/package-gguf-py.nix | 2 +-
 .devops/nix/package.nix         | 9 +--------
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/.devops/nix/package-gguf-py.nix b/.devops/nix/package-gguf-py.nix
index 083151fbca0a8..cca2f36a5bd4d 100644
--- a/.devops/nix/package-gguf-py.nix
+++ b/.devops/nix/package-gguf-py.nix
@@ -8,7 +8,7 @@
   poetry-core,
   buildPythonPackage,
   pytestCheckHook,
-}@inputs:
+}:
 
 buildPythonPackage {
   pname = "gguf";
diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix
index 7b5c77cf2689a..5d7d7ea5ae2d0 100644
--- a/.devops/nix/package.nix
+++ b/.devops/nix/package.nix
@@ -3,14 +3,11 @@
   glibc,
   config,
   stdenv,
-  mkShell,
   runCommand,
   cmake,
-  gcc,
   ninja,
   pkg-config,
   git,
-  python3,
   mpi,
   blas,
   cudaPackages,
@@ -43,8 +40,7 @@
   effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
   enableStatic ? effectiveStdenv.hostPlatform.isStatic,
   precompileMetalShaders ? false,
-  gguf-py,
-}@inputs:
+}:
 
 let
   inherit (lib)
@@ -52,7 +48,6 @@ let
     cmakeFeature
     optionals
     strings
-    versionOlder
     ;
 
   stdenv = throw "Use effectiveStdenv instead";
@@ -72,8 +67,6 @@ let
     suffices != [ ]
   ) ", accelerated with ${strings.concatStringsSep ", " suffices}";
 
-  executableSuffix = effectiveStdenv.hostPlatform.extensions.executable;
-
   xcrunHost = runCommand "xcrunHost" { } ''
     mkdir -p $out/bin
     ln -s /usr/bin/xcrun $out/bin

From bd574ba4872ab216283030300a0c14d8d1bf463a Mon Sep 17 00:00:00 2001
From: ditsuke <ditsuke@protonmail.com>
Date: Fri, 30 Aug 2024 16:39:43 +0530
Subject: [PATCH 20/20] dev: Add tiktoken to -extra devShells

---
 .devops/nix/devshells.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.devops/nix/devshells.nix b/.devops/nix/devshells.nix
index 255ca018dea5f..bfd304af14dcd 100644
--- a/.devops/nix/devshells.nix
+++ b/.devops/nix/devshells.nix
@@ -35,6 +35,10 @@
                       package
                       scripts
                     ];
+                    # Extra packages that *may* be used by some scripts
+                    packages = [
+                        pkgs.python3Packages.tiktoken
+                    ];
                     shellHook = ''
                       echo "Entering ${name} devShell"
                       addToSearchPath "LD_LIBRARY_PATH" "${lib.getLib stdenv.cc.cc}/lib"