Added doc for attention_head_size

Zyphra · Sep 27, 2024 · a109b3f · a109b3f
1 parent 75554d8
commit a109b3f
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/transformers/models/zamba/configuration_zamba.py b/src/transformers/models/zamba/configuration_zamba.py
@@ -44,6 +44,8 @@ class ZambaConfig(PretrainedConfig):
             model has a output word embedding layer.
         hidden_size (`int`, *optional*, defaults to 3712):
             Dimension of the hidden representations.
+        attention_hidden_size (`int`, *optional*, defaults to `None`):
+            Dimension of the hidden representations of the inputs to the Attention layer.
         intermediate_size (`int`, *optional*, defaults to 14848):
             Dimension of the MLP representations.
         num_hidden_layers (`int`, *optional*, defaults to 76):