Skip to content

Commit

Permalink
updated StreamingTextDataset and StreamingFinetuningDataset with new …
Browse files Browse the repository at this point in the history
…streaming args, bumped streaming version
  • Loading branch information
snarayan21 committed Sep 15, 2023
1 parent 411d92d commit 65a936c
Showing 1 changed file with 1 addition and 2 deletions.
3 changes: 1 addition & 2 deletions tests/test_dataloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def get_abs_data_path(data_local: str):
return os.path.join(os.getcwd(), data_local)


def build_mock_ft_streaming_dataset(data_path, split):
def build_mock_ft_streaming_dataset(data_path: str, split: str):
columns = {'prompt': 'str', 'response': 'str'}

dataset = [{
Expand Down Expand Up @@ -435,7 +435,6 @@ def test_finetuning_dataloader_custom_split_remote(


def test_finetuning_dataloader_streaming(tmp_path: pathlib.Path):
tokenizer_name = 'gpt2'
max_seq_len = 2048

remote_path = os.path.join(tmp_path, 'remote')
Expand Down

0 comments on commit 65a936c

Please sign in to comment.