..

mosaicml · Aug 15, 2024 · 409ed15 · 409ed15
1 parent 83cbd98
commit 409ed15
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/llmfoundry/models/layers/attention.py b/llmfoundry/models/layers/attention.py
@@ -202,7 +202,6 @@ def scaled_multihead_dot_product_attention(
             window_mask,
             diagonal=-sliding_window_size,
         )
-        window_mask = window_mask[-s_q:, -s_k:]
         window_mask = ~window_mask
         attn_weight = attn_weight.masked_fill(
             window_mask.view(1, 1, s_q, s_k),