Add task log capture and export (closes #169)

Author: tobiasraabe

docs/source/_static/md/commands/build-options.md

@@ -1,8 +1,8 @@
-| Command                                   | Description                                                   |
-| ----------------------------------------- | ------------------------------------------------------------- |
-| [`build`](../../../commands/build.md)     | Collect tasks, execute them and report the results.           |
-| [`clean`](../../../commands/clean.md)     | Clean the provided paths by removing files unknown to pytask. |
-| [`collect`](../../../commands/collect.md) | Collect tasks and report information about them.              |
-| [`dag`](../../../commands/dag.md)         | Create a visualization of the directed acyclic graph.         |
-| [`markers`](../../../commands/markers.md) | Show all registered markers.                                  |
-| [`profile`](../../../commands/profile.md) | Show information about resource consumption.                  |
+| Command                 | Description                                                   |
+| ----------------------- | ------------------------------------------------------------- |
+| [`build`](build.md)     | Collect tasks, execute them and report the results.           |
+| [`clean`](clean.md)     | Clean the provided paths by removing files unknown to pytask. |
+| [`collect`](collect.md) | Collect tasks and report information about them.              |
+| [`dag`](dag.md)         | Create a visualization of the directed acyclic graph.         |
+| [`markers`](markers.md) | Show all registered markers.                                  |
+| [`profile`](profile.md) | Show information about resource consumption.                  |

docs/source/_static/md/commands/command-list.md

@@ -10,11 +10,14 @@ default.
 If the task fails, the output is shown along with the traceback to help you track down
 the error.
 
-## Default stdout/stderr/stdin capturing behavior
+## Default stdout/stderr/logging/stdin capturing behavior
 
 Any output sent to `stdout` and `stderr` is captured during task execution. pytask
 displays it only if the task fails in addition to the traceback.
 
+Log records emitted with Python's `logging` module are also captured during task
+execution and shown in their own report section for failing tasks.
+
 In addition, `stdin` is set to a "null" object which will fail on attempts to read from
 it because it is rarely desired to wait for interactive input when running automated
 tasks.
@@ -46,6 +49,15 @@ $ pytask --capture=tee-sys   # combines 'sys' and '-s', capturing sys.stdout/std
                              # and passing it along to the actual sys.stdout/stderr
 ```
 
+## Controlling captured log output
+
+Use `--show-capture=log` to only display captured log records for failing tasks or
+`--show-capture=all` to display logs together with captured `stdout` and `stderr`.
+
+You can also export task logs to a file with `--log-file` and customize the formatting
+with `--log-format`, `--log-date-format`, `--log-file-format`, and
+`--log-file-date-format`.
+
 ## Using print statements for debugging
 
 One primary benefit of the default capturing of stdout/stderr output is that you can use

docs/source/tutorials/capturing_output.md

@@ -0,0 +1,144 @@
+"""Run a small demo project for log capture and export.
+
+Usage
+-----
+uv run python scripts/demo_logging_capture.py
+
+The script creates a temporary demo project, runs a few `pytask` commands against it,
+prints their output, and leaves the files on disk for inspection.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import tempfile
+import textwrap
+from pathlib import Path
+
+
+def main() -> None:
+    repo_root = Path(__file__).resolve().parents[1]
+    demo_root = Path(tempfile.mkdtemp(prefix="pytask-logging-demo-"))
+    _write_line(f"Demo project: {demo_root}")
+    _write_line()
+
+    _write_demo_project(demo_root)
+
+    _run_demo_case(
+        repo_root=repo_root,
+        title="Show only captured logs for the failing task",
+        args=[
+            str(demo_root),
+            "--log-level=INFO",
+            "--log-date-format=%H:%M:%S",
+            "--log-format=%(asctime)s %(levelname)-8s %(name)s:%(message)s",
+            "--show-capture=log",
+        ],
+    )
+
+    _run_demo_case(
+        repo_root=repo_root,
+        title="Show logs, stdout, and stderr, and export logs to a file",
+        args=[
+            str(demo_root),
+            "--force",
+            "--log-level=INFO",
+            "--log-date-format=%H:%M:%S",
+            "--show-capture=all",
+            "--log-file=build.log",
+            "--log-format=%(asctime)s %(levelname)-8s %(name)s:%(message)s",
+        ],
+    )
+
+    log_file = demo_root.joinpath("build.log")
+    if log_file.exists():
+        _write_line("=" * 80)
+        _write_line(f"Exported log file: {log_file}")
+        _write_line("=" * 80)
+        _write(log_file.read_text())
+        _write_line()
+
+    _write_line("=" * 80)
+    _write_line("You can rerun the demo project manually with commands like:")
+    _write_line(f"  cd {repo_root}")
+    _write_line(
+        "  uv run python -m pytask "
+        f"{demo_root} --log-level=INFO --show-capture=all --log-file=build.log"
+    )
+    _write_line("=" * 80)
+
+
+def _write_demo_project(demo_root: Path) -> None:
+    source = """
+    from __future__ import annotations
+
+    import logging
+    import sys
+    from pathlib import Path
+
+    logger = logging.getLogger("demo")
+
+
+    def task_prepare_report(produces=Path("report.txt")):
+        logger.info("preparing report.txt")
+        produces.write_text("report created\\n")
+
+
+    def task_publish_report(
+        depends_on=Path("report.txt"), produces=Path("published.txt")
+    ):
+        logger.warning("publishing report is about to fail")
+        print("stdout from task_publish_report")
+        sys.stderr.write("stderr from task_publish_report\\n")
+        raise RuntimeError("simulated publish failure")
+    """
+    demo_root.joinpath("task_logging_demo.py").write_text(textwrap.dedent(source))
+
+
+def _run_demo_case(*, repo_root: Path, title: str, args: list[str]) -> None:
+    command = [sys.executable, "-m", "pytask", *args]
+    rendered_command = " ".join(command)
+
+    _write_line("=" * 80)
+    _write_line(title)
+    _write_line(rendered_command)
+    _write_line("=" * 80)
+
+    result = subprocess.run(
+        command,
+        cwd=repo_root,
+        check=False,
+        text=True,
+        capture_output=True,
+        env=os.environ | {"PYTHONIOENCODING": "utf-8"},
+    )
+
+    if result.stdout:
+        _write(result.stdout)
+        if not result.stdout.endswith("\n"):
+            _write_line()
+    if result.stderr:
+        _write_line("-" * 80)
+        _write_line("stderr")
+        _write_line("-" * 80)
+        _write(result.stderr)
+        if not result.stderr.endswith("\n"):
+            _write_line()
+
+    _write_line("-" * 80)
+    _write_line(f"exit code: {result.returncode}")
+    _write_line()
+
+
+def _write(text: str) -> None:
+    sys.stdout.write(text)
+
+
+def _write_line(text: str = "") -> None:
+    sys.stdout.write(f"{text}\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/demo_logging_capture.py

@@ -85,8 +85,16 @@ def build(  # noqa: PLR0913
     paths: Path | Iterable[Path] = (),
     pdb: bool = False,
     pdb_cls: str = "",
+    log_date_format: str = "%H:%M:%S",
+    log_file: Path | str | None = None,
+    log_file_date_format: str | None = None,
+    log_file_format: str | None = None,
+    log_file_level: int | str | None = None,
+    log_file_mode: Literal["w", "a"] = "w",
+    log_format: str = "%(levelname)-8s %(name)s:%(filename)s:%(lineno)d %(message)s",
+    log_level: int | str | None = None,
     s: bool = False,
-    show_capture: Literal["no", "stdout", "stderr", "all"]
+    show_capture: Literal["no", "stdout", "stderr", "log", "all"]
     | ShowCapture = ShowCapture.ALL,
     show_errors_immediately: bool = False,
     show_locals: bool = False,
@@ -148,9 +156,26 @@ def build(  # noqa: PLR0913
     pdb_cls : str, default=""
         Start a custom debugger on errors. For example:
         ``--pdbcls=IPython.terminal.debugger:TerminalPdb``
+    log_date_format : str, default="%H:%M:%S"
+        The date format used for captured logs.
+    log_file : Path | str | None, default=None
+        A path to a file where logs from executed tasks should be written.
+    log_file_date_format : str | None, default=None
+        The date format used for exported logs. Falls back to ``log_date_format``.
+    log_file_format : str | None, default=None
+        The format used for exported logs. Falls back to ``log_format``.
+    log_file_level : int | str | None, default=None
+        The level of messages written to the log file. Falls back to ``log_level``.
+    log_file_mode : Literal["w", "a"], default="w"
+        The file mode used for the exported log file.
+    log_format : str, default="%(levelname)-8s %(name)s:%(filename)s:%(lineno)d "
+        "%(message)s"
+        The format used for captured logs.
+    log_level : int | str | None, default=None
+        The level of messages to capture. If not set, the logger configuration is used.
     s : bool, default=False
         Shortcut for ``capture="no"``.
-    show_capture : Literal["no", "stdout", "stderr", "all"] | ShowCapture
+    show_capture : Literal["no", "stdout", "stderr", "log", "all"] | ShowCapture
         Choose which captured output should be shown for failed tasks.
     show_errors_immediately : bool, default=False
         Show errors with tracebacks as soon as the task fails.
@@ -202,6 +227,14 @@ def build(  # noqa: PLR0913
             "paths": paths,
             "pdb": pdb,
             "pdb_cls": pdb_cls,
+            "log_date_format": log_date_format,
+            "log_file": log_file,
+            "log_file_date_format": log_file_date_format,
+            "log_file_format": log_file_format,
+            "log_file_level": log_file_level,
+            "log_file_mode": log_file_mode,
+            "log_format": log_format,
+            "log_level": log_level,
             "s": s,
             "show_capture": show_capture,
             "show_errors_immediately": show_errors_immediately,

src/_pytask/build.py

@@ -9,6 +9,7 @@ class ShowCapture(enum.Enum):
     NO = "no"
     STDOUT = "stdout"
     STDERR = "stderr"
+    LOG = "log"
     ALL = "all"