update

mosaicml · Aug 14, 2024 · 8de4a00 · 8de4a00
1 parent cd6bb2e
commit 8de4a00
Show file tree

Hide file tree

Showing 3 changed files with 1,108 additions and 1,058 deletions.
diff --git a/llmfoundry/utils/__init__.py b/llmfoundry/utils/__init__.py
@@ -55,21 +55,23 @@
     import_file,
     save_registry,
 )
+from llmfoundry.utils.validation_utils import (
+    check_HF_datasets,
+    convert_text_to_mds,
+    create_om_cfg,
+    integrity_check,
+    is_hf_dataset_path,
+    is_uc_delta_table,
+    parse_args,
+    plot_hist,
+    token_counts_with_collate,
+)
 from llmfoundry.utils.warnings import (
     ExperimentalWarning,
     VersionedDeprecationWarning,
     experimental_class,
     experimental_function,
 )
-from llmfoundry.utils.validation_utils import (check_HF_datasets,
-                                               convert_text_to_mds,
-                                               create_om_cfg,
-                                               integrity_check,
-                                               is_hf_dataset_path,
-                                               is_uc_delta_table,
-                                               parse_args,
-                                               token_counts,
-                                               token_counts_with_collate)
 
 config_transforms.register(
     'update_batch_size_info',
@@ -127,6 +129,6 @@
     'is_hf_dataset_path',
     'is_uc_delta_table',
     'parse_args',
-    'token_counts',
-    'token_counts_with_collate'
+    'token_counts_with_collate',
+    'plot_hist',
 ]