From 409ed1513779af95229ab64e914997a1a6dffaed Mon Sep 17 00:00:00 2001 From: Shashank Rajput Date: Wed, 14 Aug 2024 23:23:04 -0700 Subject: [PATCH] .. --- llmfoundry/models/layers/attention.py | 1 - 1 file changed, 1 deletion(-) diff --git a/llmfoundry/models/layers/attention.py b/llmfoundry/models/layers/attention.py index 12a0e5e3f2..4eac3d2105 100644 --- a/llmfoundry/models/layers/attention.py +++ b/llmfoundry/models/layers/attention.py @@ -202,7 +202,6 @@ def scaled_multihead_dot_product_attention( window_mask, diagonal=-sliding_window_size, ) - window_mask = window_mask[-s_q:, -s_k:] window_mask = ~window_mask attn_weight = attn_weight.masked_fill( window_mask.view(1, 1, s_q, s_k),