From 103db99766a4bfb25613c1464eb51406c2d5f175 Mon Sep 17 00:00:00 2001 From: Matthew Ding Date: Mon, 16 Sep 2024 13:40:15 -0700 Subject: [PATCH] yapf --- llmfoundry/command_utils/data_prep/convert_text_to_mds.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/llmfoundry/command_utils/data_prep/convert_text_to_mds.py b/llmfoundry/command_utils/data_prep/convert_text_to_mds.py index 39b7178219..668f42a679 100644 --- a/llmfoundry/command_utils/data_prep/convert_text_to_mds.py +++ b/llmfoundry/command_utils/data_prep/convert_text_to_mds.py @@ -89,11 +89,10 @@ def __iter__(self) -> Iterable[dict[str, NDArray]]: buffer += iids while len(buffer) >= self.max_length: concat_sample = buffer[:self.max_length] - buffer = buffer[self.max_length: - ] if self.should_wrap else [] + buffer = buffer[self. + max_length:] if self.should_wrap else [] yield { - 'tokens': - np.asarray(concat_sample, dtype=np.int32), + 'tokens': np.asarray(concat_sample, dtype=np.int32), } first_chunk = False