update_cpu_offload

vllm-project · Dec 15, 2024 · ee2e01c · ee2e01c
1 parent 540d4b2
commit ee2e01c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/examples/big_models_with_accelerate/mult_gpus_int8_device_map.py b/examples/big_models_with_accelerate/mult_gpus_int8_device_map.py
@@ -13,7 +13,7 @@
 # reserve_for_hessians=True reserves memory which is required by
 # GPTQModifier and SparseGPTModifier
 device_map = calculate_offload_device_map(
-    MODEL_ID, num_gpus=2, reserve_for_hessians=True, torch_dtype=torch.bfloat16
+    MODEL_ID, num_gpus=1, reserve_for_hessians=True, torch_dtype=torch.bfloat16
 )
 
 model = AutoModelForCausalLM.from_pretrained(