Skip to content

Commit cf2b9c2

Browse files
Peter554seddonym
authored andcommitted
fixup: use multiprocessing.Pool instead of joblib
1 parent 678e800 commit cf2b9c2

2 files changed

Lines changed: 7 additions & 14 deletions

File tree

pyproject.toml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ authors = [
1616
]
1717
requires-python = ">=3.9"
1818
dependencies = [
19-
"joblib~=1.4",
2019
"typing-extensions>=3.10.0.0",
2120
]
2221
classifiers = [

src/grimp/application/usecases.py

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@
66
import multiprocessing
77
import math
88

9-
from joblib import Parallel, delayed, parallel_config # type: ignore
10-
11-
129
from ..application.ports import caching
1310
from ..application.ports.filesystem import AbstractFileSystem
1411
from ..application.ports.graph import ImportGraph
@@ -227,17 +224,14 @@ def _scan_imports(
227224
n_chunks = min(N_CPUS, max_n_chunks)
228225

229226
chunks = _create_chunks(list(module_files), n_chunks=n_chunks)
230-
with parallel_config(n_jobs=n_chunks):
231-
import_scanning_jobs = Parallel()(
232-
delayed(_scan_chunk)(
233-
import_scanner=import_scanner,
234-
exclude_type_checking_imports=exclude_type_checking_imports,
235-
chunk=chunk,
236-
)
237-
for chunk in chunks
227+
with multiprocessing.Pool(n_chunks) as pool:
228+
import_scanning_jobs = pool.starmap(
229+
_scan_chunk,
230+
[(import_scanner, exclude_type_checking_imports, chunk) for chunk in chunks],
238231
)
239-
for chunk_imports_by_module_file in import_scanning_jobs:
240-
imports_by_module_file.update(chunk_imports_by_module_file)
232+
for chunk_imports_by_module_file in import_scanning_jobs:
233+
imports_by_module_file.update(chunk_imports_by_module_file)
234+
241235
return imports_by_module_file
242236

243237

0 commit comments

Comments
 (0)