Change 16 to 32

mosaicml · Sep 18, 2023 · e1997b4 · e1997b4
1 parent c576a4d
commit e1997b4
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/scripts/data_prep/convert_text_to_mds.py b/scripts/data_prep/convert_text_to_mds.py
@@ -88,7 +88,7 @@ def parse_args() -> Namespace:
         '--processes',
         type=int,
         required=False,
-        default=min(max(psutil.cpu_count() - 2, 1), 16),
+        default=min(max(psutil.cpu_count() - 2, 1), 32),
         help=
         'The number of processes to use to download and convert the dataset',
     )