From 9025b833d06127eda230f745b2c706e98da2f0a1 Mon Sep 17 00:00:00 2001 From: Mihir Patel Date: Tue, 3 Oct 2023 11:45:13 -0400 Subject: [PATCH] Add flag to disable train metrics (#642) * free mem * lint * lint --- llmfoundry/models/hf/hf_causal_lm.py | 8 ++++---- llmfoundry/models/mpt/modeling_mpt.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/llmfoundry/models/hf/hf_causal_lm.py b/llmfoundry/models/hf/hf_causal_lm.py index d5ef2435f9..ce398a8b2d 100644 --- a/llmfoundry/models/hf/hf_causal_lm.py +++ b/llmfoundry/models/hf/hf_causal_lm.py @@ -65,10 +65,7 @@ def __init__(self, om_model_config: Union[DictConfig, nn.Module], tokenizer: PreTrainedTokenizerBase): # set up training and eval metrics - train_metrics = [ - LanguageCrossEntropy(), - LanguagePerplexity(), - ] + train_metrics = [LanguageCrossEntropy(), LanguagePerplexity()] eval_metrics = [ LanguageCrossEntropy(), LanguagePerplexity(), @@ -92,6 +89,9 @@ def __init__(self, om_model_config: Union[DictConfig, 'which is not significantly slower and not compatible with the LLM foundry training code, rather than the code release by MosaicML.' ) + if not om_model_config.get('use_train_metrics', True): + train_metrics = [] + # load the model config trust_remote_code = om_model_config.get('trust_remote_code', True) use_auth_token = om_model_config.get('use_auth_token', False) diff --git a/llmfoundry/models/mpt/modeling_mpt.py b/llmfoundry/models/mpt/modeling_mpt.py index b1dff15398..cd162195b6 100644 --- a/llmfoundry/models/mpt/modeling_mpt.py +++ b/llmfoundry/models/mpt/modeling_mpt.py @@ -694,7 +694,9 @@ def __init__( hf_config = MPTConfig.from_dict(resolved_om_model_config) model = MPTForCausalLM(hf_config) - train_metrics = [LanguageCrossEntropy(), LanguagePerplexity()] + use_train_metrics = om_model_config.get('use_train_metrics', True) + train_metrics = [LanguageCrossEntropy(), + LanguagePerplexity()] if use_train_metrics else [] eval_metrics = [ LanguageCrossEntropy(), LanguagePerplexity(),