Merge branch 'main' into remove-attrs

Author: tobiasraabe

.github/workflows/update-plugin-list.yml

@@ -37,7 +37,7 @@ jobs:
         run: python scripts/update_plugin_list.py
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725
+        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
         with:
           commit-message: '[automated] Update plugin list'
           author: 'Tobias Raabe <tobiasraabe@users.noreply.github.com>'

.pre-commit-config.yaml

@@ -25,12 +25,12 @@ repos:
     -   id: python-no-log-warn
     -   id: text-unicode-replacement-char
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.10
+    rev: v0.14.14
     hooks:
       - id: ruff-format
       - id: ruff-check
 -   repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.9.18
+    rev: 0.9.28
     hooks:
       - id: uv-lock
 -   repo: https://github.com/executablebooks/mdformat
@@ -54,12 +54,12 @@ repos:
         ]
         files: (docs/.)
 -   repo: https://github.com/kynan/nbstripout
-    rev: 0.8.2
+    rev: 0.9.0
     hooks:
     -   id: nbstripout
         exclude: (docs)
 -   repo: https://github.com/crate-ci/typos
-    rev: typos-dict-v0.13.13
+    rev: v1
     hooks:
     -   id: typos
         exclude: (\.ipynb)

CHANGELOG.md

@@ -7,7 +7,11 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
 
 ## Unreleased
 
-- Removed the direct dependency on attrs and migrated internal models to dataclasses.
+- {pull}`744` Removed the direct dependency on attrs and migrated internal models to
+  dataclasses.
+- {pull}`766` moves runtime profiling persistence from SQLite to a JSON snapshot plus
+  append-only journal in `.pytask/`, keeping runtime data resilient to crashes and
+  compacted on normal build exits.
 
 ## 0.5.8 - 2025-12-30
 

justfile

@@ -12,7 +12,7 @@ test-cov *FLAGS:
 
 # Run type checking
 typing:
-    uv run --group typing --group test --isolated ty check src/ tests/
+    uv run --group typing --group test --isolated ty check
 
 # Run linting
 lint:

pyproject.toml

@@ -23,6 +23,7 @@ dynamic = ["version"]
 dependencies = [
     "click>=8.1.8,!=8.2.0",
     "click-default-group>=1.2.4",
+    "msgspec>=0.18.6",
     "networkx>=2.4.0",
     "optree>=0.9.0",
     "packaging>=23.0.0",
@@ -74,7 +75,7 @@ test = [
     "pytest-xdist>=3.6.1",
     "syrupy>=4.5.0",
     "aiohttp>=3.11.0",                                                        # For HTTPPath tests.
-    "coiled>=1.42.0",
+    "coiled>=1.42.0; python_version < '3.14'",
     "pygraphviz>=1.12;platform_system=='Linux'",
 ]
 typing = ["ty>=0.0.8"]
@@ -177,6 +178,7 @@ include = [
 unused-ignore-comment = "ignore"
 
 [tool.ty.src]
+include = ["src", "tests"]
 exclude = ["src/_pytask/_hashlib.py"]
 
 [tool.ty.terminal]

src/_pytask/build.py

@@ -5,7 +5,6 @@
 import json
 import sys
 from contextlib import suppress
-from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Any
 from typing import Literal
@@ -16,8 +15,7 @@
 from _pytask.capture_utils import CaptureMethod
 from _pytask.capture_utils import ShowCapture
 from _pytask.click import ColoredCommand
-from _pytask.config_utils import find_project_root_and_config
-from _pytask.config_utils import read_config
+from _pytask.config_utils import normalize_programmatic_config
 from _pytask.console import console
 from _pytask.dag import create_dag
 from _pytask.exceptions import CollectionError
@@ -30,13 +28,12 @@
 from _pytask.pluginmanager import hookimpl
 from _pytask.pluginmanager import storage
 from _pytask.session import Session
-from _pytask.shared import parse_paths
-from _pytask.shared import to_list
 from _pytask.traceback import Traceback
 
 if TYPE_CHECKING:
     from collections.abc import Callable
     from collections.abc import Iterable
+    from pathlib import Path
     from typing import NoReturn
 
     from _pytask.node_protocols import PTask
@@ -66,7 +63,7 @@ def pytask_unconfigure(session: Session) -> None:
     path.write_text(json.dumps(HashPathCache._cache))
 
 
-def build(  # noqa: C901, PLR0912, PLR0913, PLR0915
+def build(  # noqa: PLR0913
     *,
     capture: Literal["fd", "no", "sys", "tee-sys"] | CaptureMethod = CaptureMethod.FD,
     check_casing_of_paths: bool = True,
@@ -225,47 +222,14 @@ def build(  # noqa: C901, PLR0912, PLR0913, PLR0915
 
         # If someone called the programmatic interface, we need to do some parsing.
         if "command" not in raw_config:
-            raw_config["command"] = "build"
             # Add defaults from cli.
             from _pytask.cli import DEFAULTS_FROM_CLI  # noqa: PLC0415
 
-            raw_config = {**DEFAULTS_FROM_CLI, **raw_config}
-
-            paths_value = raw_config["paths"]
-            # Convert tuple to list since parse_paths expects Path | list[Path]
-            if isinstance(paths_value, tuple):
-                paths_value = list(paths_value)
-            if not isinstance(paths_value, (Path, list)):
-                msg = f"paths must be Path or list, got {type(paths_value)}"
-                raise TypeError(msg)  # noqa: TRY301
-            # Cast is justified - we validated at runtime
-            raw_config["paths"] = parse_paths(cast("Path | list[Path]", paths_value))
-
-            if raw_config["config"] is not None:
-                config_value = raw_config["config"]
-                if not isinstance(config_value, (str, Path)):
-                    msg = f"config must be str or Path, got {type(config_value)}"
-                    raise TypeError(msg)  # noqa: TRY301
-                raw_config["config"] = Path(config_value).resolve()
-                raw_config["root"] = raw_config["config"].parent
-            else:
-                (
-                    raw_config["root"],
-                    raw_config["config"],
-                ) = find_project_root_and_config(raw_config["paths"])
-
-            if raw_config["config"] is not None:
-                config_from_file = read_config(raw_config["config"])
-
-                if "paths" in config_from_file:
-                    paths = config_from_file["paths"]
-                    paths = [
-                        raw_config["config"].parent.joinpath(path).resolve()
-                        for path in to_list(paths)
-                    ]
-                    config_from_file["paths"] = paths
-
-                raw_config = {**raw_config, **config_from_file}
+            raw_config = normalize_programmatic_config(
+                raw_config,
+                command="build",
+                defaults_from_cli=cast("dict[str, Any]", DEFAULTS_FROM_CLI),
+            )
 
         config_ = pm.hook.pytask_configure(pm=pm, raw_config=raw_config)
 

src/_pytask/capture.py

@@ -156,7 +156,7 @@ def __init__(self, other: TextIO) -> None:
         self._other = other
         super().__init__()
 
-    def write(self, s: str) -> int:  # ty: ignore[invalid-method-override]
+    def write(self, s: str) -> int:
         super().write(s)
         return self._other.write(s)
 

src/_pytask/config_utils.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import enum
 import os
 import sys
 from pathlib import Path
@@ -12,6 +13,7 @@
 import click
 
 from _pytask.shared import parse_paths
+from _pytask.shared import to_list
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -22,7 +24,12 @@
     import tomli as tomllib  # ty: ignore[unresolved-import]
 
 
-__all__ = ["find_project_root_and_config", "read_config", "set_defaults_from_config"]
+__all__ = [
+    "find_project_root_and_config",
+    "normalize_programmatic_config",
+    "read_config",
+    "set_defaults_from_config",
+]
 
 
 def set_defaults_from_config(
@@ -78,6 +85,73 @@ def set_defaults_from_config(
     return context.params["config"]
 
 
+def normalize_programmatic_config(
+    raw_config: dict[str, Any],
+    *,
+    command: str,
+    defaults_from_cli: dict[str, Any],
+) -> dict[str, Any]:
+    """Normalize programmatic raw_config inputs."""
+    raw_config = {**defaults_from_cli, **raw_config}
+    raw_config["command"] = command
+    raw_config["paths"] = _normalize_paths_value(raw_config["paths"])
+    _normalize_config_value(raw_config)
+
+    if raw_config["config"] is not None:
+        config_from_file = read_config(raw_config["config"])
+
+        if "paths" in config_from_file:
+            paths = config_from_file["paths"]
+            paths = [
+                raw_config["config"].parent.joinpath(path).resolve()
+                for path in to_list(paths)
+            ]
+            config_from_file["paths"] = paths
+
+        raw_config = {**raw_config, **config_from_file}
+
+    return raw_config
+
+
+def _normalize_paths_value(paths_value: Any) -> list[Path]:
+    """Normalize paths from programmatic inputs."""
+    if isinstance(paths_value, tuple):
+        paths_value = list(paths_value)
+    if not isinstance(paths_value, (Path, list)):
+        msg = f"paths must be Path or list, got {type(paths_value)}"
+        raise TypeError(msg)
+    # Cast is justified - we validated at runtime
+    return parse_paths(cast("Path | list[Path]", paths_value))
+
+
+def _normalize_config_value(raw_config: dict[str, Any]) -> None:
+    """Normalize config value from programmatic inputs."""
+    config_value = raw_config["config"]
+    if _is_click_sentinel(config_value):
+        config_value = None
+        raw_config["config"] = None
+    if config_value is not None:
+        if not isinstance(config_value, (str, Path)):
+            msg = f"config must be str or Path, got {type(config_value)}"
+            raise TypeError(msg)
+        raw_config["config"] = Path(config_value).resolve()
+        raw_config["root"] = raw_config["config"].parent
+    else:
+        raw_config["root"], raw_config["config"] = find_project_root_and_config(
+            raw_config["paths"]
+        )
+
+
+def _is_click_sentinel(value: Any) -> bool:
+    """Return True if value looks like Click's Sentinel.UNSET."""
+    return (
+        isinstance(value, enum.Enum)
+        and value.name == "UNSET"
+        and value.__class__.__name__ == "Sentinel"
+        and value.__class__.__module__ == "click._utils"
+    )
+
+
 def find_project_root_and_config(
     paths: Sequence[Path] | None,
 ) -> tuple[Path, Path | None]:

src/_pytask/dag_command.py

@@ -16,8 +16,7 @@
 from _pytask.click import EnumChoice
 from _pytask.compat import check_for_optional_program
 from _pytask.compat import import_optional_dependency
-from _pytask.config_utils import find_project_root_and_config
-from _pytask.config_utils import read_config
+from _pytask.config_utils import normalize_programmatic_config
 from _pytask.console import console
 from _pytask.dag import create_dag
 from _pytask.exceptions import CollectionError
@@ -28,9 +27,7 @@
 from _pytask.pluginmanager import hookimpl
 from _pytask.pluginmanager import storage
 from _pytask.session import Session
-from _pytask.shared import parse_paths
 from _pytask.shared import reduce_names_of_multiple_nodes
-from _pytask.shared import to_list
 from _pytask.traceback import Traceback
 
 
@@ -150,47 +147,14 @@ def build_dag(raw_config: dict[str, Any]) -> nx.DiGraph:
 
         # If someone called the programmatic interface, we need to do some parsing.
         if "command" not in raw_config:
-            raw_config["command"] = "dag"
             # Add defaults from cli.
             from _pytask.cli import DEFAULTS_FROM_CLI  # noqa: PLC0415
 
-            raw_config = {**DEFAULTS_FROM_CLI, **raw_config}  # ty: ignore[invalid-assignment]
-
-            paths_value = raw_config["paths"]
-            # Convert tuple to list since parse_paths expects Path | list[Path]
-            if isinstance(paths_value, tuple):
-                paths_value = list(paths_value)
-            if not isinstance(paths_value, (Path, list)):
-                msg = f"paths must be Path or list, got {type(paths_value)}"
-                raise TypeError(msg)  # noqa: TRY301
-            # Cast is justified - we validated at runtime
-            raw_config["paths"] = parse_paths(cast("Path | list[Path]", paths_value))
-
-            if raw_config["config"] is not None:
-                config_value = raw_config["config"]
-                if not isinstance(config_value, (str, Path)):
-                    msg = f"config must be str or Path, got {type(config_value)}"
-                    raise TypeError(msg)  # noqa: TRY301
-                raw_config["config"] = Path(config_value).resolve()
-                raw_config["root"] = raw_config["config"].parent
-            else:
-                (
-                    raw_config["root"],
-                    raw_config["config"],
-                ) = find_project_root_and_config(raw_config["paths"])
-
-            if raw_config["config"] is not None:
-                config_from_file = read_config(raw_config["config"])
-
-                if "paths" in config_from_file:
-                    paths = config_from_file["paths"]
-                    paths = [
-                        raw_config["config"].parent.joinpath(path).resolve()
-                        for path in to_list(paths)
-                    ]
-                    config_from_file["paths"] = paths
-
-                raw_config = {**raw_config, **config_from_file}
+            raw_config = normalize_programmatic_config(
+                raw_config,
+                command="dag",
+                defaults_from_cli=cast("dict[str, Any]", DEFAULTS_FROM_CLI),
+            )
 
         config = pm.hook.pytask_configure(pm=pm, raw_config=raw_config)
 

src/_pytask/database.py

@@ -14,11 +14,14 @@
 @hookimpl
 def pytask_parse_config(config: dict[str, Any]) -> None:
     """Parse the configuration."""
+    database_url = config["database_url"]
     # Set default.
-    if not config["database_url"]:
+    if not database_url:
         config["database_url"] = make_url(
             f"sqlite:///{config['root'].joinpath('.pytask').as_posix()}/pytask.sqlite3"
         )
+    elif isinstance(database_url, str):
+        config["database_url"] = make_url(database_url)
 
     if (
         config["database_url"].drivername == "sqlite"