From 488c11511049919f6ceaaae606430c4890315af1 Mon Sep 17 00:00:00 2001 From: xiaohanzhangcmu Date: Wed, 14 Aug 2024 15:50:31 -0700 Subject: [PATCH] update --- llmfoundry/utils/validation_utils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/llmfoundry/utils/validation_utils.py b/llmfoundry/utils/validation_utils.py index a6ce687e54..b7e3fd25a5 100644 --- a/llmfoundry/utils/validation_utils.py +++ b/llmfoundry/utils/validation_utils.py @@ -8,13 +8,11 @@ import tempfile from argparse import Namespace from concurrent.futures import ProcessPoolExecutor -from typing import Any, Mapping, Optional, Tuple, Union, cast +from typing import Any, Mapping, Optional, Tuple, Union import torch from composer.utils import ( - ObjectStore, maybe_create_object_store_from_uri, - parse_uri, ) from datasets import get_dataset_split_names from huggingface_hub import dataset_info @@ -29,7 +27,6 @@ get_task_args, is_already_processed, is_remote_path, - write_done_file, ) from llmfoundry.utils import build_tokenizer from llmfoundry.utils.data_prep_utils import ( @@ -37,7 +34,6 @@ merge_shard_groups, ) from llmfoundry.utils.exceptions import ( - DatasetTooSmallError, InputFolderMissingDataError, OutputFolderNotEmptyError, )