use pathlib instead

mosaicml · Dec 13, 2024 · d453c76 · d453c76
1 parent 1f3be0b
commit d453c76
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/llmfoundry/utils/config_utils.py b/llmfoundry/utils/config_utils.py
@@ -6,9 +6,9 @@
 import logging
 import math
 import os
-import re
 import warnings
 from dataclasses import dataclass, fields
+from pathlib import Path
 from typing import (
     Any,
     Callable,
@@ -705,7 +705,7 @@ def _process_data_source(
         data_paths (List[Tuple[str, str, str]]): A list of tuples formatted as (data type, path, split)
     """
     if source_dataset_path:
-        source_dataset_path = re.sub(r'/+', '/', source_dataset_path)
+        source_dataset_path = str(Path(source_dataset_path))
     # Check for Delta table
     if source_dataset_path and len(source_dataset_path.split('.')) == 3:
         data_paths.append(('delta_table', source_dataset_path, true_split))