diff --git a/open_lm/datapreprocess/ray/tokenize_shuffle.py b/open_lm/datapreprocess/ray/tokenize_shuffle.py index f0e350f2..b3eb0ff3 100644 --- a/open_lm/datapreprocess/ray/tokenize_shuffle.py +++ b/open_lm/datapreprocess/ray/tokenize_shuffle.py @@ -496,7 +496,7 @@ def glob_files(path, suffixes): def write_manifest(jsonl_lines, args): "Write manifest to provided output path." - output_path = os.path.join(args.output.strip("/"), "manifest.jsonl") + output_path = os.path.join(args.output.rstrip("/"), "manifest.jsonl") if output_path.startswith("s3://"): # Use boto3 for S3 paths