Update llmfoundry/models/utils/tp_strategy.py

Co-authored-by: Mihir Patel <[email protected]>
mosaicml · Sep 26, 2024 · 92719e7 · 92719e7
1 parent 39a92ad
commit 92719e7
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/llmfoundry/models/utils/tp_strategy.py b/llmfoundry/models/utils/tp_strategy.py
@@ -14,7 +14,7 @@
 def ffn_tp_strategy(model: ComposerModel) -> dict[str, ParallelStyle]:
     TP_LAYERS = {'up_proj', 'down_proj'}
 
-    # validate that all TP_LAYERS are in model
+    # Validate that all TP_LAYERS are in model
     tp_layers_in_model = {
         layer for layer in TP_LAYERS for name, _ in model.named_modules()
         if layer in name