Update scripts/data_prep/convert_text_to_mds.py

mosaicml · Sep 13, 2023 · e2d772b · e2d772b
1 parent 5b8cee5
commit e2d772b
Showing 1 changed file with 0 additions and 5 deletions.
diff --git a/scripts/data_prep/convert_text_to_mds.py b/scripts/data_prep/convert_text_to_mds.py
@@ -196,17 +196,12 @@ def download_and_convert(
         columns = {'tokens': 'bytes'}
 
         print(f'Converting to MDS format...')
-        total_tokens_bytes = 0
         with MDSWriter(out=output_folder,
                        columns=columns,
                        max_mds_writer_workers=max_mds_writer_workers,
                        compression=compression) as out:
             for sample in tqdm(dataset):
-                total_tokens_bytes += len(sample['tokens'])
                 out.write(sample)
-        total_tokens = total_tokens_bytes / 8
-        print('tokens', total_tokens_bytes, total_tokens)
-        return total_tokens
 
 
 def is_remote_path(path: str) -> bool: