Pass a RealFileSystem in

Author: seddonym

rust/src/lib.rs

@@ -18,11 +18,14 @@ use pyo3::types::{IntoPyDict, PyDict, PyFrozenSet, PyList, PySet, PyString, PyTu
 use rayon::prelude::*;
 use rustc_hash::FxHashSet;
 use std::collections::HashSet;
+use std::fs;
+use std::io;
 
 #[pymodule]
 fn _rustgrimp(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_wrapped(wrap_pyfunction!(parse_imported_objects_from_code))?;
     m.add_class::<GraphWrapper>()?;
+    m.add_class::<RealFileSystem>()?;
     m.add("ModuleNotPresent", py.get_type::<ModuleNotPresent>())?;
     m.add("NoSuchContainer", py.get_type::<NoSuchContainer>())?;
     m.add(
@@ -640,3 +643,20 @@ struct Route {
     middle: Vec<String>,
     tails: Vec<String>,
 }
+
+
+#[pyclass]
+struct RealFileSystem {}
+
+
+#[pymethods]
+impl RealFileSystem {
+    #[new]
+    fn new() -> Self {
+        RealFileSystem {}
+    }
+
+    fn read(&self, filename: &str) -> Result<String, io::Error> {
+        fs::read_to_string(filename)
+    }
+}

src/grimp/application/usecases.py

@@ -8,7 +8,7 @@
 import joblib  # type: ignore
 
 from ..application.ports import caching
-from ..application.ports.filesystem import AbstractFileSystem
+from ..application.ports.filesystem import AbstractFileSystem, FileSystem2
 from ..application.ports.graph import ImportGraph
 from ..application.ports.importscanner import AbstractImportScanner
 from ..application.ports.modulefinder import AbstractModuleFinder, FoundPackage, ModuleFile
@@ -61,6 +61,7 @@ def build_graph(
     """
 
     file_system: AbstractFileSystem = settings.FILE_SYSTEM
+    file_system_2: FileSystem2 | None = settings.FILE_SYSTEM_2
 
     found_packages = _find_packages(
         file_system=file_system,
@@ -70,6 +71,7 @@ def build_graph(
     imports_by_module = _scan_packages(
         found_packages=found_packages,
         file_system=file_system,
+        file_system_2=file_system_2,
         include_external_packages=include_external_packages,
         exclude_type_checking_imports=exclude_type_checking_imports,
         cache_dir=cache_dir,
@@ -115,6 +117,7 @@ def _validate_package_names_are_strings(
 def _scan_packages(
     found_packages: Set[FoundPackage],
     file_system: AbstractFileSystem,
+    file_system_2: FileSystem2 | None,
     include_external_packages: bool,
     exclude_type_checking_imports: bool,
     cache_dir: Union[str, Type[NotSupplied], None],
@@ -146,6 +149,7 @@ def _scan_packages(
             _scan_imports(
                 remaining_module_files_to_scan,
                 file_system=file_system,
+                file_system_2=file_system_2,
                 found_packages=found_packages,
                 include_external_packages=include_external_packages,
                 exclude_type_checking_imports=exclude_type_checking_imports,
@@ -214,14 +218,20 @@ def _scan_imports(
     module_files: Collection[ModuleFile],
     *,
     file_system: AbstractFileSystem,
+    file_system_2: FileSystem2 | None,
     found_packages: Set[FoundPackage],
     include_external_packages: bool,
     exclude_type_checking_imports: bool,
 ) -> Dict[ModuleFile, Set[DirectImport]]:
-    chunks = _create_chunks(module_files)
+    if file_system_2:
+        # Multiprocessing is not supported - just do one chunk.
+        chunks = (module_files,)
+    else:
+        chunks = _create_chunks(module_files)
     return _scan_chunks(
         chunks,
         file_system,
+        file_system_2,
         found_packages,
         include_external_packages,
         exclude_type_checking_imports,
@@ -259,12 +269,14 @@ def _decide_number_of_processes(number_of_module_files: int) -> int:
 def _scan_chunks(
     chunks: Collection[Collection[ModuleFile]],
     file_system: AbstractFileSystem,
+    file_system_2: FileSystem2 | None,
     found_packages: Set[FoundPackage],
     include_external_packages: bool,
     exclude_type_checking_imports: bool,
 ) -> Dict[ModuleFile, Set[DirectImport]]:
     import_scanner: AbstractImportScanner = settings.IMPORT_SCANNER_CLASS(
         file_system=file_system,
+        file_system_2=file_system_2,
         found_packages=found_packages,
         include_external_packages=include_external_packages,
     )

src/grimp/main.py

@@ -9,10 +9,12 @@
 from .adaptors.timing import SystemClockTimer
 from .application.config import settings
 from .application.usecases import build_graph
+from grimp import _rustgrimp as rust  # type: ignore[attr-defined]
 
 settings.configure(
     MODULE_FINDER=ModuleFinder(),
     FILE_SYSTEM=FileSystem(),
+    FILE_SYSTEM_2=rust.RealFileSystem(),
     IMPORT_SCANNER_CLASS=ImportScanner,
     IMPORT_GRAPH_CLASS=ImportGraph,
     PACKAGE_FINDER=ImportLibPackageFinder(),

tests/unit/application/test_usecases.py

@@ -45,7 +45,11 @@ def test_happy_path(self, include_external_packages):
         class FakePackageFinder(BaseFakePackageFinder):
             directory_map = {"mypackage": "/path/to/mypackage"}
 
-        with override_settings(FILE_SYSTEM=file_system, PACKAGE_FINDER=FakePackageFinder()):
+        with override_settings(
+            FILE_SYSTEM=file_system,
+            FILE_SYSTEM_2=None,
+            PACKAGE_FINDER=FakePackageFinder(),
+        ):
             graph = usecases.build_graph(
                 "mypackage", include_external_packages=include_external_packages
             )
@@ -200,6 +204,7 @@ class FakeModuleFinder(BaseFakeModuleFinder):
 
         with override_settings(
             FILE_SYSTEM=FakeFileSystem(),
+            FILE_SYSTEM_2=None,
             PACKAGE_FINDER=FakePackageFinder(),
             MODULE_FINDER=FakeModuleFinder(),
         ), patch.object(os, "environ", fake_environ):