feat!: concurrency refactor, postprocessors reorganisation, and v7 packaging slices

.gitignore

@@ -1,3 +1,5 @@
 __pycache__/
 dist/
 .claude/
+
+.vscode/

CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## 8.0.0 - 2026-03-20
+
+### Breaking
+
+- `kg_build` postprocessor subprocess entry points renamed:
+  - `runner.py` → `oneshot.py`
+  - `worker.py` → `persistent.py`
+- `SubprocessPostprocessor` split into `OneshotPostprocessor` and `PersistentPostprocessor`; any host code referencing the old class name must be updated.
+- `PostprocessorService` is now profile-agnostic; profile resolution no longer happens inside the service.
+- `utils.get_me` / `utils.reset_me` module files renamed to `_get_me.py` / `_reset_me.py`; direct submodule imports (not recommended) must be updated.
+
+### Added
+
+- `ShaclValidationService` — runs SHACL validation in a dedicated process pool via `PreparedShaclValidator`, wired into `ProfileImportProtocol`.
+- Separate pool-size settings for postprocessors and SHACL validation.
+- In-process postprocessor runtime (`inprocess`) for single-process execution.
+- SHACL process-pool queue-wait and execution-time tracking in timing logs.
+- `morph_kgc` subprocess pool for true RML-mapping parallelism, bypassing `pyparsing` lock contention and the GIL.
+  - Configurable pool size via `morph_kgc_pool_size` / `MORPH_KGC_POOL_SIZE`.
+  - Subprocess queue-wait tracked separately in timing logs.
+- `PostprocessorResult` dataclass — replaces implicit tuple return from postprocessing stage.
+- `ImportAnnotationPostprocessor` and `RootIdReconcilerPostprocessor` extracted as named processors.
+- `first_level_subjects` graph utility helper.
+- Slice verification tooling extended with `run_slice_smoke_imports.py` and `run_slice_tests.py`.
+
+### Changed
+
+- Postprocessors reorganised into `postprocessors/` subpackage (`processors/`, `PostprocessorService`, loader helpers).
+- `ProfileImportProtocol.__init__` decomposed into focused `_init_*` factory methods; class surface significantly reduced.
+- `morph_kgc` RML mapping stage runs in subprocess pool instead of a thread executor.
+- SHACL validation and postprocessors offloaded to dedicated thread/process pools; ingestion runs in an executor to avoid blocking the event loop.
+- Persistent `ApiClient` reused across requests instead of one per graph; `ApiClient` is closed on protocol shutdown.
+- Lazy-export guards remapped to modules with real third-party dependencies so `ModuleNotFoundError` fires correctly when an extra is absent.
+- `python-liquid` added to the `workflow` extra (required by `graph.ttl_liquid`).
+
 ## 7.0.0 - 2026-03-15
 
 ### Breaking

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wordlift-sdk"
-version = "7.0.1"
+version = "8.0.0"
 description = "Python toolkit for orchestrating WordLift imports and structured data workflows."
 authors = ["David Riccitelli <david@wordlift.io>"]
 readme = "README.md"
@@ -81,6 +81,7 @@ workflow = [
     "pandas",
     "playwright",
     "pydantic-core",
+    "python-liquid",
     "rdflib",
     "tqdm",
 ]

tests/kg_build/test_id_allocator.py → tests/kg_build/postprocessors/processors/test_id_allocator.py

@@ -2,8 +2,11 @@
 
 from rdflib import Graph, Literal, RDF, URIRef
 
-import wordlift_sdk.kg_build.id_allocator as id_allocator_module
-from wordlift_sdk.kg_build.id_allocator import IdAllocator, normalize_slug
+import wordlift_sdk.kg_build.postprocessors.processors.id_allocator as id_allocator_module
+from wordlift_sdk.kg_build.postprocessors.processors.id_allocator import (
+    IdAllocator,
+    normalize_slug,
+)
 
 
 def _graph(subject: URIRef) -> Graph:

tests/kg_build/test_kg_build_id_generator.py → tests/kg_build/postprocessors/processors/test_id_generator.py

@@ -3,7 +3,9 @@
 from rdflib import Graph, Literal, RDF, URIRef
 from rdflib.namespace import XSD
 
-from wordlift_sdk.kg_build.id_generator import CanonicalIdGenerator
+from wordlift_sdk.kg_build.postprocessors.processors.id_generator import (
+    CanonicalIdGenerator,
+)
 from wordlift_sdk.kg_build.iri_lookup import IriLookup
 from wordlift_sdk.kg_build.id_policy import DEFAULT_ID_POLICY, IdPolicy
 

tests/kg_build/test_id_postprocessor.py → tests/kg_build/postprocessors/processors/test_id_postprocessor.py

@@ -4,7 +4,9 @@
 
 from rdflib import Graph, Literal, RDF, URIRef
 
-from wordlift_sdk.kg_build.id_postprocessor import CanonicalIdsPostprocessor
+from wordlift_sdk.kg_build.postprocessors.processors.id_postprocessor import (
+    CanonicalIdsPostprocessor,
+)
 
 
 def test_id_postprocessor_no_dataset_uri_returns_original_graph() -> None:

tests/kg_build/test_postprocessor_runner_helpers.py → tests/kg_build/postprocessors/test_oneshot_helpers.py

@@ -4,7 +4,7 @@
 
 from rdflib import Graph, Literal, URIRef
 
-from wordlift_sdk.kg_build import postprocessor_runner as runner
+from wordlift_sdk.kg_build.postprocessors import oneshot as runner
 
 
 def test_load_class_variants(monkeypatch) -> None:

tests/kg_build/test_postprocessor_runner_main.py → tests/kg_build/postprocessors/test_oneshot_main.py

@@ -6,7 +6,7 @@
 
 from rdflib import Graph, Literal, URIRef
 
-from wordlift_sdk.kg_build import postprocessor_runner as runner
+from wordlift_sdk.kg_build.postprocessors import oneshot as runner
 
 
 def _graph() -> Graph:

tests/kg_build/test_postprocessor_worker.py → tests/kg_build/postprocessors/test_persistent.py

@@ -7,7 +7,7 @@
 
 from rdflib import Graph, Literal, URIRef
 
-from wordlift_sdk.kg_build import postprocessor_worker as worker
+from wordlift_sdk.kg_build.postprocessors import persistent as worker
 
 
 def _graph() -> Graph:

tests/kg_build/test_postprocessors.py → tests/kg_build/postprocessors/test_postprocessors.py

@@ -12,19 +12,22 @@
 import pytest
 from rdflib import Dataset, Graph, Literal, URIRef
 
-from wordlift_sdk.kg_build.postprocessor_runner import (
+from wordlift_sdk.kg_build.postprocessors.oneshot import (
     _build_context,
     _read_graph_nquads,
 )
 from wordlift_sdk.kg_build.postprocessors import (
     LoadedPostprocessor,
     PostprocessorContext,
     PostprocessorSpec,
-    SubprocessPostprocessor,
-    _build_runner_payload,
     close_loaded_postprocessors,
     load_postprocessors_for_profile,
 )
+from wordlift_sdk.kg_build.postprocessors.graph_io import _build_runner_payload
+from wordlift_sdk.kg_build.postprocessors.subprocess import (
+    OneshotSubprocessPostprocessor,
+    PersistentSubprocessPostprocessor,
+)
 
 PROJECT_ROOT = Path(__file__).resolve().parents[2]
 _current_pythonpath = os.environ.get("PYTHONPATH", "")
@@ -162,8 +165,8 @@ class = "test_pp:ProfileTwo"
 
     first = loaded[0].handler
     second = loaded[1].handler
-    assert isinstance(second, SubprocessPostprocessor)
-    assert isinstance(first, SubprocessPostprocessor)
+    assert isinstance(second, OneshotSubprocessPostprocessor)
+    assert isinstance(first, OneshotSubprocessPostprocessor)
     assert first.spec.python == "/profile/python"
     assert first.spec.timeout_seconds == 17
     assert first.spec.keep_temp_on_error is True
@@ -190,7 +193,7 @@ class = "test_pp:BaseOne"
     assert [item.name for item in loaded] == ["test_pp:BaseOne"]
 
     first = loaded[0].handler
-    assert isinstance(first, SubprocessPostprocessor)
+    assert isinstance(first, OneshotSubprocessPostprocessor)
     assert first.spec.python == "/base/python"
     assert first.spec.timeout_seconds == 11
     assert first.spec.keep_temp_on_error is False
@@ -219,8 +222,7 @@ class = "test_pp:ProfileOne"
         runtime="persistent",
     )
     assert len(loaded) == 1
-    assert isinstance(loaded[0].handler, SubprocessPostprocessor)
-    assert loaded[0].handler.runtime == "persistent"
+    assert isinstance(loaded[0].handler, PersistentSubprocessPostprocessor)
 
 
 def test_subprocess_execution_and_nquads_exchange(tmp_path: Path) -> None:
@@ -249,7 +251,7 @@ def process_graph(self, graph, context):
         enabled=True,
         keep_temp_on_error=False,
     )
-    processor = SubprocessPostprocessor(spec=spec, root_dir=root)
+    processor = OneshotSubprocessPostprocessor(spec=spec, root_dir=root)
 
     output = processor.process_graph(_sample_graph(), _sample_context())
     assert output is not None
@@ -291,11 +293,7 @@ def process_graph(self, graph, context):
         enabled=True,
         keep_temp_on_error=False,
     )
-    processor = SubprocessPostprocessor(
-        spec=spec,
-        root_dir=root,
-        runtime="persistent",
-    )
+    processor = PersistentSubprocessPostprocessor(spec=spec, root_dir=root)
 
     first = processor.process_graph(_sample_graph(), _sample_context())
     second = processor.process_graph(_sample_graph(), _sample_context())
@@ -351,7 +349,12 @@ def process_graph(self, graph, context):
         enabled=True,
         keep_temp_on_error=False,
     )
-    processor = SubprocessPostprocessor(spec=spec, root_dir=root, runtime=runtime)
+    cls = (
+        PersistentSubprocessPostprocessor
+        if runtime == "persistent"
+        else OneshotSubprocessPostprocessor
+    )
+    processor = cls(spec=spec, root_dir=root)
     try:
         output = processor.process_graph(
             _sample_graph(),
@@ -405,7 +408,12 @@ def process_graph(self, graph, context):
         enabled=True,
         keep_temp_on_error=False,
     )
-    processor = SubprocessPostprocessor(spec=spec, root_dir=root, runtime=runtime)
+    cls = (
+        PersistentSubprocessPostprocessor
+        if runtime == "persistent"
+        else OneshotSubprocessPostprocessor
+    )
+    processor = cls(spec=spec, root_dir=root)
     try:
         output = processor.process_graph(
             _sample_graph(),
@@ -471,7 +479,7 @@ def process_graph(self, graph, context):
         [
             sys.executable,
             "-m",
-            "wordlift_sdk.kg_build.postprocessor_runner",
+            "wordlift_sdk.kg_build.postprocessors.oneshot",
             "--class",
             "test_pp:AddRunnerTriple",
             "--input-graph",
@@ -517,7 +525,7 @@ def process_graph(self, graph, context):
         enabled=True,
         keep_temp_on_error=False,
     )
-    processor = SubprocessPostprocessor(spec=spec, root_dir=root)
+    processor = OneshotSubprocessPostprocessor(spec=spec, root_dir=root)
 
     with pytest.raises(subprocess.TimeoutExpired):
         processor.process_graph(_sample_graph(), _sample_context())
@@ -543,11 +551,7 @@ def process_graph(self, graph, context):
         enabled=True,
         keep_temp_on_error=False,
     )
-    processor = SubprocessPostprocessor(
-        spec=spec,
-        root_dir=root,
-        runtime="persistent",
-    )
+    processor = PersistentSubprocessPostprocessor(spec=spec, root_dir=root)
 
     with pytest.raises(subprocess.TimeoutExpired):
         processor.process_graph(_sample_graph(), _sample_context())
@@ -571,7 +575,7 @@ def process_graph(self, graph, context):
         enabled=True,
         keep_temp_on_error=True,
     )
-    processor = SubprocessPostprocessor(spec=spec, root_dir=root)
+    processor = OneshotSubprocessPostprocessor(spec=spec, root_dir=root)
 
     with pytest.raises(RuntimeError):
         processor.process_graph(_sample_graph(), _sample_context())
@@ -607,7 +611,7 @@ def process_graph(self, graph, context):
         enabled=True,
         keep_temp_on_error=True,
     )
-    processor = SubprocessPostprocessor(spec=spec, root_dir=root)
+    processor = OneshotSubprocessPostprocessor(spec=spec, root_dir=root)
     secret = "top-secret-key"
 
     with pytest.raises(RuntimeError):
@@ -683,7 +687,7 @@ def test_subprocess_uses_inherited_environment_without_pythonpath_injection(
         enabled=True,
         keep_temp_on_error=False,
     )
-    processor = SubprocessPostprocessor(spec=spec, root_dir=root)
+    processor = OneshotSubprocessPostprocessor(spec=spec, root_dir=root)
     captured: dict[str, object] = {}
 
     def fake_run(*args, **kwargs):