Skip to content

Commit

Permalink
Catch multiple slashes in source dataset into one slash (#1697)
Browse files Browse the repository at this point in the history
Co-authored-by: Vincent Chen <[email protected]>
  • Loading branch information
KuuCi and Vincent Chen authored Dec 13, 2024
1 parent a27c720 commit c494017
Showing 1 changed file with 3 additions and 1 deletion.
4 changes: 3 additions & 1 deletion llmfoundry/utils/config_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import os
import warnings
from dataclasses import dataclass, fields
from pathlib import Path
from typing import (
Any,
Callable,
Expand Down Expand Up @@ -703,6 +704,8 @@ def _process_data_source(
true_split (str): The split of the dataset to be added (i.e. train or eval)
data_paths (List[Tuple[str, str, str]]): A list of tuples formatted as (data type, path, split)
"""
if source_dataset_path:
source_dataset_path = str(Path(source_dataset_path))
# Check for Delta table
if source_dataset_path and len(source_dataset_path.split('.')) == 3:
data_paths.append(('delta_table', source_dataset_path, true_split))
Expand Down Expand Up @@ -788,7 +791,6 @@ def log_dataset_uri(cfg: dict[str, Any]) -> None:

# Map data source types to their respective MLFlow DataSource.
for dataset_type, path, split in data_paths:

if dataset_type in dataset_source_mapping:
source_class = dataset_source_mapping[dataset_type]
if dataset_type == 'delta_table':
Expand Down

0 comments on commit c494017

Please sign in to comment.