diff --git a/examples/benchmarks/bert/src/hf_bert.py b/examples/benchmarks/bert/src/hf_bert.py
index 28c7d966..3cdd74c3 100644
--- a/examples/benchmarks/bert/src/hf_bert.py
+++ b/examples/benchmarks/bert/src/hf_bert.py
@@ -101,8 +101,8 @@ def create_hf_bert_mlm(pretrained_model_name: str = 'bert-base-uncased',
         tokenizer = None
 
     metrics = [
-        LanguageCrossEntropy(ignore_index=-100,
-                             vocab_size=model.config.vocab_size),
+        # vocab_size no longer in composer
+        LanguageCrossEntropy(ignore_index=-100),
         MaskedAccuracy(ignore_index=-100)
     ]
     return HuggingFaceModel(model=model,
diff --git a/examples/benchmarks/bert/src/mosaic_bert.py b/examples/benchmarks/bert/src/mosaic_bert.py
index e28b425b..8436783c 100644
--- a/examples/benchmarks/bert/src/mosaic_bert.py
+++ b/examples/benchmarks/bert/src/mosaic_bert.py
@@ -119,6 +119,7 @@ def create_mosaic_bert_mlm(pretrained_model_name: str = 'bert-base-uncased',
             pretrained_model_name)
 
     metrics = [
+        # vocab size no longer arg in composer
         LanguageCrossEntropy(ignore_index=-100),
         MaskedAccuracy(ignore_index=-100)
     ]