supprot fa-2 + right padding + forward

huggingface · Oct 18, 2023 · df8ebb2 · df8ebb2
1 parent 5a73316
commit df8ebb2
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/transformers/models/mistral/modeling_mistral.py b/src/transformers/models/mistral/modeling_mistral.py
@@ -876,6 +876,7 @@ def forward(
             padding_mask is not None
             and hasattr(self.config, "_flash_attn_2_enabled")
             and self.config._flash_attn_2_enabled
+            and past_key_values is not None
         ):
             is_padding_right = padding_mask[:, -1].sum().item() != batch_size
             if is_padding_right: