mk_repo_file: Parallelize fetch

Fixes nix-community#169.

scripts/mk_repo_file.py

@@ -8,6 +8,8 @@
 import argparse
 import copy
 import json
+import multiprocessing
+import multiprocessing.pool
 import os
 import re
 import shutil
@@ -118,12 +120,17 @@ def make_repo_file(url: str, ref: str,
             if callback is not None:
                 callback(data)
 
-    for relpath, p in data.items():
+    pool = multiprocessing.pool.ThreadPool(16)
+    cb_lock = multiprocessing.Lock()
+
+    def process_item(item: Tuple[str, ProjectInfoDict]):
+        relpath, p = item
+
         if len(include_prefix) > 0 and (not any(relpath.startswith(p) for p in include_prefix)):
-            continue
+            return
 
         if relpath in exclude_path:
-            continue
+            return
 
         for project, rev in override_project_revs.items():
             # We have to iterate over the whole output since we don't save
@@ -164,7 +171,7 @@ def make_repo_file(url: str, ref: str,
             # Used cached copies if available
             if (p['rev'], fetch_submodules) in revInfo:
                 p.update(cast(ProjectInfoDict, revInfo.get((p['rev'], fetch_submodules), {})))
-                continue
+                return
 
             p_url = get_mirrored_url(p['url'])
             found_treehash = False
@@ -177,7 +184,7 @@ def make_repo_file(url: str, ref: str,
                     p.update(cast(ProjectInfoDict, treeInfo.get((p['tree'], fetch_submodules), {})))
                     found_treehash = True
             if found_treehash:
-                continue
+                return
 
             # Fetch information. Use revisionExpr if it is a tag so we use the
             # tag in the name of the nix derivation instead of the revision
@@ -191,8 +198,11 @@ def make_repo_file(url: str, ref: str,
 
             add_to_cache(p)
 
-            if callback is not None:
-                callback(data)
+            with cb_lock:
+                if callback is not None:
+                    callback(data)
+
+    pool.map(process_item, data.items())
 
     # Save at the end as well!
     if callback is not None: