Parallelise import scanning

Author: Peter554

src/grimp/application/usecases.py

@@ -3,6 +3,8 @@
 """
 from typing import Dict, Sequence, Set, Type, Union, cast
 
+from joblib import Parallel, delayed, parallel_config  # type: ignore
+
 from ..application.ports import caching
 from ..application.ports.filesystem import AbstractFileSystem
 from ..application.ports.graph import ImportGraph
@@ -137,11 +139,22 @@ def _scan_packages(
             else:
                 imports_by_module_file[module_file] = direct_imports
 
-    # TODO Parallelise this part.
-    for module_file in modules_files_to_scan.difference(imports_by_module_file):
-        imports_by_module_file[module_file] = import_scanner.scan_for_imports(
-            module_file.module, exclude_type_checking_imports=exclude_type_checking_imports
-        )
+    remaining_modules_files_to_scan = list(
+        modules_files_to_scan.difference(imports_by_module_file)
+    )
+    # n_jobs=-1 will create one job per CPU.
+    with parallel_config(n_jobs=-1):
+        for module_file, direct_imports in zip(
+            remaining_modules_files_to_scan,
+            Parallel()(
+                delayed(import_scanner.scan_for_imports)(
+                    module_file.module,
+                    exclude_type_checking_imports=exclude_type_checking_imports,
+                )
+                for module_file in remaining_modules_files_to_scan
+            ),
+        ):
+            imports_by_module_file[module_file] = direct_imports
 
     imports_by_module: Dict[Module, Set[DirectImport]] = {
         k.module: v for k, v in imports_by_module_file.items()