Big hammer for Mixtral sliding window config

arcee-ai · Dec 18, 2023 · 66523c3 · 66523c3
1 parent 2939acd
commit 66523c3
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/mergekit/scripts/mixtral_moe.py b/mergekit/scripts/mixtral_moe.py
@@ -162,6 +162,7 @@ def build(
     out_cfg = MixtralConfig(**base_cfg.to_dict())
     out_cfg.architectures = ["MixtralForCausalLM"]
     out_cfg.num_local_experts = len(config.experts)
+    out_cfg.sliding_window = None
     out_cfg.save_pretrained(out_path)
 
     if (out_cfg.num_local_experts & (out_cfg.num_local_experts - 1)) != 0: