comments

vllm-project · Dec 13, 2024 · cf9f0ff · cf9f0ff
1 parent 4a63261
commit cf9f0ff
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 1 deletion.
diff --git a/tests/e2e/vLLM/configs/kv_cache_deepseek.yaml b/tests/e2e/vLLM/configs/kv_cache_deepseek.yaml
@@ -1,6 +1,6 @@
 cadence: "nightly"
 test_type: "regression"
-model: deepseek-ai/DeepSeek-Coder-V2-Instruct
+model: deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct
 recipe: tests/e2e/vLLM/recipes/kv_cache/default.yaml
 dataset_id: HuggingFaceH4/ultrachat_200k
 dataset_split: train_sft

diff --git a/tests/e2e/vLLM/recipes/kv_cache/gptq.yaml b/tests/e2e/vLLM/recipes/kv_cache/gptq.yaml
@@ -0,0 +1,18 @@
+quant_stage:
+  quant_modifiers:
+    QuantizationModifier:
+      kv_cache_scheme:
+        {num_bits: 8, type: float, symmetric: true, strategy: tensor}
+    GPTQModifier:
+      sequential_update: false
+      ignore: ["lm_head"]
+      config_groups:
+          group_0:
+              weights:
+                  num_bits: 4
+                  type: "int"
+                  symmetric: true
+                  strategy: "channel"
+                  actorder: False
+              targets: ["Linear"]
+