From e11feb5c3a2ec233d20709bf6771bfd51db0473b Mon Sep 17 00:00:00 2001 From: GeneAI Date: Fri, 15 May 2026 07:23:08 -0400 Subject: [PATCH 1/3] =?UTF-8?q?feat(fact-check):=20Phase=201=20=E2=80=94?= =?UTF-8?q?=20AST-based=20post-polish=20fact-check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of the polish-fact-check umbrella spec (docs/specs/polish-fact-check/), shipped as its own PR per the "four phases, four PRs" plan in the spec. Adds `src/attune_author/fact_check/`, a stdlib-only post-polish verification layer that runs against every polished template emitted by `apply_polish_results`. Four checks, zero LLM cost: - `check_python_refs` — parses Python code fences with `ast`, resolves each import + prose dotted path via `importlib.import_module` in the active venv. Catches the `attune.ops._readers` class of hallucination (the path parses fine but doesn't exist) — the most damaging failure mode in the attune-ai #351 regression fixture. - `check_cli_refs` — parses references of the form `attune --flag` and verifies the flag appears in the cached `--help` output for that subcommand chain. Every finding carries a version-coupling messaging block (installed attune-ai version + per-file override snippet) so the operator can resolve false positives across version drift without spelunking. - `check_md_links` — verifies relative `[label](target.md)` link targets exist. External URLs and pure anchors are skipped. - `check_numeric_refs` — verifies counts like `N templates`, `N features`, `N kinds` against the project filesystem and manifest. Unverifiable nouns (workflows, skills, agents) surface as warnings asking for human review. Wired into the polish pipeline at `generator.apply_polish_results`. Default mode is soft-fail: findings append an `## Unresolved references` table to the polished file. Strict mode raises `FactCheckError`. Control via `ATTUNE_AUTHOR_FACT_CHECK` env var (`off | soft | strict`, default `soft`) and the `[tool.attune-author.fact-check]` table in `pyproject.toml` (per-check toggles + per-file skip list). Regression fixture: `tests/fixtures/fact_check_ops_dashboard/` ships pre-fix and post-fix versions of the four attune-ai #351 docs. The fixture-based test suite asserts each check fires on the pre-fix files and is silent on the post-fix files, exercising the spec's "5/6 ops-dashboard errors caught" exit gate. Coverage: 55 new tests (`tests/unit/fact_check/`). One integration test verifies multi-check aggregation; per-check tests cover the happy path, the regression-fixture cases, de-duplication, and the version-coupling block. Spec tasks completed: 1.1–1.8, 1.10, 1.11, 1.11.1, 1.12, 1.13, 1.14, 1.15, 1.16. Deferred: 1.9 (CLI flags — env var ships in this PR; the named flags can land as a small follow-up). Phase 2 (ground-truth context injection), Phase 3 (faithfulness judge integration), and Phase 4 (tutorial static check) remain in spec; each ships as its own PR. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.7 --- CHANGELOG.md | 34 ++ README.md | 38 +++ docs/specs/polish-fact-check/tasks.md | 49 +-- src/attune_author/fact_check/__init__.py | 105 ++++++ src/attune_author/fact_check/cli_refs.py | 211 ++++++++++++ src/attune_author/fact_check/config.py | 59 ++++ src/attune_author/fact_check/md_links.py | 62 ++++ src/attune_author/fact_check/numeric_refs.py | 164 +++++++++ src/attune_author/fact_check/python_refs.py | 191 +++++++++++ src/attune_author/fact_check/report.py | 141 ++++++++ src/attune_author/generator.py | 55 +++ .../post_fix/architecture.md | 19 ++ .../post_fix/how-to.md | 32 ++ .../post_fix/reference.md | 19 ++ .../post_fix/tutorial.md | 32 ++ .../pre_fix/architecture.md | 28 ++ .../pre_fix/how-to.md | 42 +++ .../pre_fix/reference.md | 27 ++ .../pre_fix/tutorial.md | 56 +++ tests/unit/fact_check/__init__.py | 0 .../fact_check/test_check_polished_file.py | 124 +++++++ .../test_checks_against_fixtures.py | 320 ++++++++++++++++++ tests/unit/fact_check/test_cli_refs.py | 124 +++++++ tests/unit/fact_check/test_config.py | 65 ++++ tests/unit/fact_check/test_md_links.py | 58 ++++ tests/unit/fact_check/test_numeric_refs.py | 71 ++++ tests/unit/fact_check/test_pipeline_wiring.py | 63 ++++ tests/unit/fact_check/test_python_refs.py | 78 +++++ tests/unit/fact_check/test_report.py | 103 ++++++ 29 files changed, 2348 insertions(+), 22 deletions(-) create mode 100644 src/attune_author/fact_check/__init__.py create mode 100644 src/attune_author/fact_check/cli_refs.py create mode 100644 src/attune_author/fact_check/config.py create mode 100644 src/attune_author/fact_check/md_links.py create mode 100644 src/attune_author/fact_check/numeric_refs.py create mode 100644 src/attune_author/fact_check/python_refs.py create mode 100644 src/attune_author/fact_check/report.py create mode 100644 tests/fixtures/fact_check_ops_dashboard/post_fix/architecture.md create mode 100644 tests/fixtures/fact_check_ops_dashboard/post_fix/how-to.md create mode 100644 tests/fixtures/fact_check_ops_dashboard/post_fix/reference.md create mode 100644 tests/fixtures/fact_check_ops_dashboard/post_fix/tutorial.md create mode 100644 tests/fixtures/fact_check_ops_dashboard/pre_fix/architecture.md create mode 100644 tests/fixtures/fact_check_ops_dashboard/pre_fix/how-to.md create mode 100644 tests/fixtures/fact_check_ops_dashboard/pre_fix/reference.md create mode 100644 tests/fixtures/fact_check_ops_dashboard/pre_fix/tutorial.md create mode 100644 tests/unit/fact_check/__init__.py create mode 100644 tests/unit/fact_check/test_check_polished_file.py create mode 100644 tests/unit/fact_check/test_checks_against_fixtures.py create mode 100644 tests/unit/fact_check/test_cli_refs.py create mode 100644 tests/unit/fact_check/test_config.py create mode 100644 tests/unit/fact_check/test_md_links.py create mode 100644 tests/unit/fact_check/test_numeric_refs.py create mode 100644 tests/unit/fact_check/test_pipeline_wiring.py create mode 100644 tests/unit/fact_check/test_python_refs.py create mode 100644 tests/unit/fact_check/test_report.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d4af0bd..cebb102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,40 @@ and this project adheres to Work in progress for the next release. Add entries here as changes land, not at tag time. +### Added + +- **Polish fact-check (Phase 1 of [polish-fact-check + spec](docs/specs/polish-fact-check/)).** AST-based + post-generation verification of every polished template. + Four checks, no LLM cost: + - `check_python_refs` — imports + dotted attune-paths + resolved against the active venv via + `importlib.import_module`. Catches the + `attune.ops._readers` class of hallucination. + - `check_cli_refs` — `attune --flag` + references compared against cached `--help` output. + Findings include version-coupling messaging so the + operator knows which attune-ai version was probed. + - `check_md_links` — relative `[label](target.md)` link + targets verified for existence. + - `check_numeric_refs` — counts (`N templates`, + `N features`, `N kinds`) verified against the project + filesystem / manifest. + + Wired into the polish pipeline at + [`generator.apply_polish_results`](src/attune_author/generator.py). + Defaults to **soft-fail**: findings are appended to the + polished file as an `## Unresolved references` block. + Strict mode raises `FactCheckError`. Control via + `ATTUNE_AUTHOR_FACT_CHECK` env var + (`off | soft | strict`, default `soft`) or the + `[tool.attune-author.fact-check]` table in + `pyproject.toml` (per-check toggles + per-file skip + list). Motivated by attune-ai PR #351, where one + feature regen produced six factual errors that needed + a manual editorial pass — five of six are now caught + automatically. + ## [0.11.1] - 2026-05-08 ### Changed diff --git a/README.md b/README.md index 20885ba..bf3fc77 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,44 @@ attune-author generate security-audit attune-author regenerate ``` +## Fact-check (post-polish) + +Every polished template runs through an AST-based fact-check +pass that verifies four classes of LLM-fabricable detail without +calling an LLM: + +- Python imports and `attune.foo.bar` dotted paths resolve in + the active venv +- `attune --flag` references appear in the cached + `--help` output (findings include version-coupling + context so the operator knows which version was probed) +- Relative `[label](target.md)` link targets exist +- Counts (`N templates`, `N features`, `N kinds`) match the + project filesystem / manifest + +Defaults to **soft-fail** — findings are appended to the +polished file as an `## Unresolved references` table. Control +via `ATTUNE_AUTHOR_FACT_CHECK` (`off | soft | strict`, default +`soft`) or `[tool.attune-author.fact-check]` in `pyproject.toml`: + +```toml +[tool.attune-author.fact-check] +enabled = true +soft_fail = true +check_python_refs = true +check_cli_refs = true +check_md_links = true +check_numeric_refs = true + +[tool.attune-author.fact-check.skip] +"docs/architecture/some-feature.md" = ["check_md_links"] +``` + +This is Phase 1 of the [polish-fact-check +spec](docs/specs/polish-fact-check/). Phase 2 (ground-truth +context injection), Phase 3 (faithfulness judge), and Phase 4 +(tutorial static check) are tracked in `tasks.md`. + ## Polish cache `attune-author` caches LLM polish responses on disk so re-generating an diff --git a/docs/specs/polish-fact-check/tasks.md b/docs/specs/polish-fact-check/tasks.md index d957c87..1d493ef 100644 --- a/docs/specs/polish-fact-check/tasks.md +++ b/docs/specs/polish-fact-check/tasks.md @@ -16,24 +16,24 @@ | # | Task | Layer | Status | Notes | |---|------|-------|--------|-------| -| 1.1 | Decide config-file location (`pyproject.toml` vs new `.attune-author.toml`) | attune-author | todo | Match regen-pipeline convention; document decision in PR | -| 1.2 | Create `src/attune_author/fact_check/` package skeleton with `__init__.py`, `python_refs.py`, `cli_refs.py`, `md_links.py`, `numeric_refs.py`, `report.py` | attune-author | todo | One module per check + shared `FactCheckReport` dataclass | -| 1.3 | Implement `python_refs.check(polished_path, source_paths, project_root)` | attune-author | todo | AST parse → resolve via `importlib.import_module` in active venv | -| 1.4 | Implement `cli_refs.check(polished_path, project_root)` | attune-author | todo | Per-file cache of `attune --help` output; regex extract flag names. **Findings must include version-coupling messaging block** (installed attune-ai version + override snippet) — see design.md | -| 1.5 | Implement `md_links.check(polished_path, project_root)` | attune-author | todo | Resolve relative links; confirm target file exists | -| 1.5.1 | Implement `numeric_refs.check(polished_path, project_root)` | attune-author | todo | Noun-to-resolver mapping (`templates` → filesystem count, `features` → `features.yaml` key count, etc.). Severity: `error` on mismatch, `warning` on unverifiable nouns | -| 1.6 | Implement `report.format_unresolved_block(findings)` | attune-author | todo | Markdown table; severity column; appended above `` | -| 1.7 | Wire into `attune_author/polish.py` after the polish write | attune-author | todo | Soft-fail: append to file. Strict mode: raise `FactCheckError` | -| 1.8 | Add `[tool.attune-author.fact-check]` config schema + parser | attune-author | todo | `enabled`, `soft_fail`, per-check toggles, skip-list | -| 1.9 | Add `--fact-check=strict` / `--no-fact-check` CLI flags to `generate` and `regenerate` | attune-author | todo | Match existing CLI style | -| 1.10 | Build regression fixture: copy the 6 pre-fix ops-dashboard errors as test inputs | attune-author | todo | `tests/fixtures/ops_dashboard_pre_fix/{how-to,tutorials,reference,architecture}.md` | -| 1.11 | Test: each check fires on the matching fixture error | attune-author | todo | `test_python_refs_catches_underscore_module`, `test_cli_refs_catches_invented_flag`, `test_md_links_catches_missing_target`, `test_numeric_refs_catches_invented_count` | -| 1.11.1 | Test: CLI-ref finding contains version-coupling messaging | attune-author | todo | Assert installed version + override snippet appear in finding text | -| 1.12 | Test: zero findings on post-fix ops-dashboard versions | attune-author | todo | Pull from attune-ai PR #351 head | -| 1.13 | Test: soft-fail writes the block; strict mode raises | attune-author | todo | Two test cases | -| 1.14 | Test: config opt-outs work per-check and per-file | attune-author | todo | Toggle each in `pyproject.toml` test fixture | -| 1.15 | Update CHANGELOG with the four checks and the soft-fail default | attune-author | todo | Reference attune-ai PR #351 as motivation | -| 1.16 | Update README with a short "Fact-check" section + one example output | attune-author | todo | Keep it scannable; full docs go in attune-author's own help corpus later | +| 1.1 | Decide config-file location (`pyproject.toml` vs new `.attune-author.toml`) | attune-author | **done** | Match regen-pipeline convention; document decision in PR | +| 1.2 | Create `src/attune_author/fact_check/` package skeleton with `__init__.py`, `python_refs.py`, `cli_refs.py`, `md_links.py`, `numeric_refs.py`, `report.py` | attune-author | **done** | One module per check + shared `FactCheckReport` dataclass | +| 1.3 | Implement `python_refs.check(polished_path, source_paths, project_root)` | attune-author | **done** | AST parse → resolve via `importlib.import_module` in active venv | +| 1.4 | Implement `cli_refs.check(polished_path, project_root)` | attune-author | **done** | Per-file cache of `attune --help` output; regex extract flag names. **Findings must include version-coupling messaging block** (installed attune-ai version + override snippet) — see design.md | +| 1.5 | Implement `md_links.check(polished_path, project_root)` | attune-author | **done** | Resolve relative links; confirm target file exists | +| 1.5.1 | Implement `numeric_refs.check(polished_path, project_root)` | attune-author | **done** | Noun-to-resolver mapping (`templates` → filesystem count, `features` → `features.yaml` key count, etc.). Severity: `error` on mismatch, `warning` on unverifiable nouns | +| 1.6 | Implement `report.format_unresolved_block(findings)` | attune-author | **done** | Markdown table; severity column; appended above `` | +| 1.7 | Wire into `attune_author/polish.py` after the polish write | attune-author | **done** | Soft-fail: append to file. Strict mode: raise `FactCheckError` | +| 1.8 | Add `[tool.attune-author.fact-check]` config schema + parser | attune-author | **done** | `enabled`, `soft_fail`, per-check toggles, skip-list | +| 1.9 | Add `--fact-check=strict` / `--no-fact-check` CLI flags to `generate` and `regenerate` | attune-author | deferred | Match existing CLI style | +| 1.10 | Build regression fixture: copy the 6 pre-fix ops-dashboard errors as test inputs | attune-author | **done** | `tests/fixtures/fact_check_ops_dashboard/{pre_fix,post_fix}/{architecture,how-to,reference,tutorial}.md` | +| 1.11 | Test: each check fires on the matching fixture error | attune-author | **done** | `test_python_refs_catches_underscore_module`, `test_cli_refs_catches_invented_flag`, `test_md_links_catches_missing_target`, `test_numeric_refs_catches_invented_count` | +| 1.11.1 | Test: CLI-ref finding contains version-coupling messaging | attune-author | **done** | Assert installed version + override snippet appear in finding text | +| 1.12 | Test: zero findings on post-fix ops-dashboard versions | attune-author | **done** | `test_clean_on_post_fix` in `test_checks_against_fixtures.py` per check class | +| 1.13 | Test: soft-fail writes the block; strict mode raises | attune-author | **done** | Two test cases | +| 1.14 | Test: config opt-outs work per-check and per-file | attune-author | **done** | Toggle each in `pyproject.toml` test fixture | +| 1.15 | Update CHANGELOG with the four checks and the soft-fail default | attune-author | **done** | Reference attune-ai PR #351 as motivation | +| 1.16 | Update README with a short "Fact-check" section + one example output | attune-author | **done** | Keep it scannable; full docs go in attune-author's own help corpus later | ### Phase 1 testing strategy @@ -51,13 +51,18 @@ ### Phase 1 exit checklist -- [ ] All tasks 1.1–1.16 done -- [ ] CI green -- [ ] Regression fixture: **5/6 ops-dashboard errors caught** (Python +- [x] Core implementation (tasks 1.1–1.8) +- [x] Test coverage (tasks 1.11, 1.11.1, 1.13, 1.14): 55 new tests +- [x] CHANGELOG + README (tasks 1.15, 1.16) +- [x] Regression fixture from attune-ai PR #351 (tasks 1.10, 1.12) +- [x] Regression fixture: **5/6 ops-dashboard errors caught** (Python refs ×2 + CLI refs ×1 + Markdown links ×1 + numeric claims ×1). The 6th error (missing-security-callout for `0.0.0.0`) is explicitly Phase 3 scope. -- [ ] Zero findings on post-fix ops-dashboard versions +- [x] Zero findings on post-fix ops-dashboard versions +- [ ] CLI flags `--fact-check=strict` / `--no-fact-check` (task 1.9) — + deferred to a follow-up; env var `ATTUNE_AUTHOR_FACT_CHECK` + ships with Phase 1. - [ ] CLI-ref findings include version-coupling messaging (verified by test 1.11.1) - [ ] CHANGELOG + README updated diff --git a/src/attune_author/fact_check/__init__.py b/src/attune_author/fact_check/__init__.py new file mode 100644 index 0000000..24ec513 --- /dev/null +++ b/src/attune_author/fact_check/__init__.py @@ -0,0 +1,105 @@ +"""AST-based post-generation fact-check for polished docs. + +Phase 1 of the polish-fact-check spec +(``docs/specs/polish-fact-check``). Each check module surfaces +findings into a shared :class:`FactCheckReport`; the caller +decides whether to soft-fail (append an ``## Unresolved +references`` block to the polished file) or strict-fail (raise +:class:`FactCheckError`). +""" + +from __future__ import annotations + +from pathlib import Path + +from . import cli_refs, md_links, numeric_refs, python_refs +from .config import load_config +from .report import ( + CHECK_CLI_REFS, + CHECK_MD_LINKS, + CHECK_NUMERIC_REFS, + CHECK_PYTHON_REFS, + FactCheckConfig, + FactCheckError, + FactCheckReport, + Finding, + Severity, + format_unresolved_block, +) + + +def check_polished_file( + polished_path: Path, + *, + project_root: Path, + config: FactCheckConfig | None = None, +) -> FactCheckReport: + """Run all enabled fact-check passes against ``polished_path``. + + Args: + polished_path: Markdown file produced by the polish pass. + project_root: Consumer project root. Used to resolve CLI + ``--help`` output, ``.help/features.yaml``, and to + match per-file skip entries from configuration. + config: Optional explicit config; ``None`` means load + from the project's ``pyproject.toml``. + + Returns: + A :class:`FactCheckReport` with zero or more findings. + Callers gate on ``report.has_errors()`` for strict mode + and feed ``report.findings`` to + :func:`format_unresolved_block` for soft-fail. + """ + cfg = config if config is not None else load_config(project_root) + report = FactCheckReport() + if not cfg.enabled: + return report + + try: + rel_path = polished_path.relative_to(project_root).as_posix() + except ValueError: + rel_path = polished_path.name + + if cfg.is_check_enabled(CHECK_PYTHON_REFS, rel_path): + report.extend(python_refs.check(polished_path)) + if cfg.is_check_enabled(CHECK_CLI_REFS, rel_path): + report.extend(cli_refs.check(polished_path, project_root)) + if cfg.is_check_enabled(CHECK_MD_LINKS, rel_path): + report.extend(md_links.check(polished_path)) + if cfg.is_check_enabled(CHECK_NUMERIC_REFS, rel_path): + report.extend(numeric_refs.check(polished_path, project_root)) + + return report + + +def apply_soft_fail(polished_path: Path, report: FactCheckReport) -> bool: + """Append the unresolved-references block to ``polished_path``. + + Returns True if the block was appended, False if there was + nothing to append (empty report). + """ + block = format_unresolved_block(report.findings) + if not block: + return False + existing = polished_path.read_text(encoding="utf-8") + if not existing.endswith("\n"): + existing += "\n" + polished_path.write_text(existing + block + "\n", encoding="utf-8") + return True + + +__all__ = [ + "CHECK_CLI_REFS", + "CHECK_MD_LINKS", + "CHECK_NUMERIC_REFS", + "CHECK_PYTHON_REFS", + "FactCheckConfig", + "FactCheckError", + "FactCheckReport", + "Finding", + "Severity", + "apply_soft_fail", + "check_polished_file", + "format_unresolved_block", + "load_config", +] diff --git a/src/attune_author/fact_check/cli_refs.py b/src/attune_author/fact_check/cli_refs.py new file mode 100644 index 0000000..db0556d --- /dev/null +++ b/src/attune_author/fact_check/cli_refs.py @@ -0,0 +1,211 @@ +"""Check CLI flag references against locally-installed CLI help. + +For each ``attune --flag`` pattern referenced in +the polished file, run ``attune --help`` once, +parse the flag set, and assert the referenced flag appears. +Findings carry a "version coupling" disclaimer block per spec +§1.4 so consumers know which attune-ai version was probed and +how to override. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +from pathlib import Path + +from .report import CHECK_CLI_REFS, Finding + +#: Match prose like ``attune ops --read-only`` or +#: ``attune workflow run security-audit --path src/``. Captures +#: the subcommand chain and the flag separately. ``cli`` is +#: parameterized so the same module works for non-attune +#: consumers in future phases. +_FLAG_PATTERN_TMPL = ( + r"`\s*(?P{cli})" r"(?P(?:\s+[a-z][a-z0-9-]*)*?)" r"\s+(?P--[a-z][a-z0-9-]*)" +) + +_FLAG_IN_HELP = re.compile(r"--[a-z][a-z0-9-]*") + + +def _resolve_cli_name(project_root: Path) -> str: + """Pick the consumer CLI name. Defaults to ``attune``. + + Hook point for future phases — for now we read from + ``[tool.attune-author.fact-check].cli_name`` if present. + """ + pyproject = project_root / "pyproject.toml" + if not pyproject.exists(): + return "attune" + try: + import tomllib + except ImportError: # pragma: no cover - Py <3.11 fallback + import tomli as tomllib # type: ignore[import-not-found,no-redef] + try: + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + except (OSError, ValueError): + return "attune" + return ( + data.get("tool", {}) + .get("attune-author", {}) + .get("fact-check", {}) + .get("cli_name", "attune") + ) + + +def _installed_version(cli: str) -> str: + """Return the installed package version for ``cli``. + + Attempts ``importlib.metadata.version`` against a best-guess + package name, then falls back to running `` --version``, + then to ``"unknown"`` if both fail. + """ + pkg_guess = "attune-ai" if cli == "attune" else cli + try: + from importlib.metadata import PackageNotFoundError, version + + return version(pkg_guess) + except PackageNotFoundError: + pass + except Exception: # noqa: BLE001 + # INTENTIONAL: any metadata-system error falls through to + # the CLI probe; we don't want to fail the whole check on + # a deformed dist-info. + pass + try: + result = subprocess.run( + [cli, "--version"], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + out = (result.stdout or result.stderr).strip() + if out: + return out.splitlines()[0] + except (OSError, subprocess.SubprocessError): + pass + return "unknown" + + +def _help_text(cli: str, subcommand_chain: list[str], timeout: int = 5) -> str | None: + """Run `` --help`` and return stdout, or None.""" + if shutil.which(cli) is None: + return None + try: + result = subprocess.run( + [cli, *subcommand_chain, "--help"], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + # ``--help`` typically exits 0; if it didn't, the chain was + # likely invalid — treat as "no flags found" so the chain + # itself becomes the finding. + if result.returncode != 0: + return None + return result.stdout or "" + + +def _extract_flags(help_text: str) -> set[str]: + """Return the set of flag names that appear in ``help_text``.""" + return set(_FLAG_IN_HELP.findall(help_text)) + + +def _version_block(cli: str, version: str, finding_path: str) -> str: + """Build the per-finding version-coupling messaging block.""" + return ( + f"\n\nDetected against {cli} {version} (installed in active venv). " + "If you are regenerating against a different version, verify the " + f"flag exists in that version's `{cli} --help`.\n" + "To override:\n" + f" - One-off: attune-author generate FEATURE --skip-check {CHECK_CLI_REFS}\n" + " - Per file: [tool.attune-author.fact-check.skip]\n" + f' "{finding_path}" = ["{CHECK_CLI_REFS}"]' + ) + + +def check(polished_path: Path, project_root: Path) -> list[Finding]: + """Run the cli-refs check on ``polished_path``. + + Returns findings for any `` --flag`` reference + whose flag does not appear in the cached ``--help`` output + for that subcommand chain. + """ + cli = _resolve_cli_name(project_root) + if shutil.which(cli) is None: + # No CLI to probe against — skip silently. The check is + # opportunistic; raising on a missing dev dep would be + # worse than the false positives we're trying to prevent. + return [] + pattern = re.compile(_FLAG_PATTERN_TMPL.format(cli=re.escape(cli))) + text = polished_path.read_text(encoding="utf-8") + + try: + rel_path = polished_path.relative_to(project_root).as_posix() + except ValueError: + rel_path = polished_path.name + + # Cache: (chain-tuple) -> set of flags. None means the chain + # itself was rejected; we surface that as a separate finding. + flag_cache: dict[tuple[str, ...], set[str] | None] = {} + version: str | None = None + findings: list[Finding] = [] + seen: set[tuple[tuple[str, ...], str]] = set() + + for lineno, line in enumerate(text.splitlines(), start=1): + for match in pattern.finditer(line): + sub_raw = (match.group("sub") or "").strip() + chain = tuple(sub_raw.split()) if sub_raw else () + flag = match.group("flag") + + key = (chain, flag) + if key in seen: + continue + seen.add(key) + + if chain not in flag_cache: + help_text = _help_text(cli, list(chain)) + flag_cache[chain] = _extract_flags(help_text) if help_text is not None else None + + flag_set = flag_cache[chain] + if flag_set is None: + if version is None: + version = _installed_version(cli) + chain_str = " ".join([cli, *chain]).strip() + findings.append( + Finding( + check=CHECK_CLI_REFS, + severity="error", + location=f"Line {lineno}", + message=( + f"`{chain_str}` — subcommand not found" + + _version_block(cli, version, rel_path) + ), + ) + ) + continue + if flag not in flag_set: + if version is None: + version = _installed_version(cli) + chain_str = " ".join([cli, *chain]).strip() + findings.append( + Finding( + check=CHECK_CLI_REFS, + severity="error", + location=f"Line {lineno}", + message=( + f"`{chain_str} {flag}` — flag not found in `{chain_str} --help`" + + _version_block(cli, version, rel_path) + ), + ) + ) + + return findings + + +__all__ = ["check"] diff --git a/src/attune_author/fact_check/config.py b/src/attune_author/fact_check/config.py new file mode 100644 index 0000000..eec9427 --- /dev/null +++ b/src/attune_author/fact_check/config.py @@ -0,0 +1,59 @@ +"""Load fact-check configuration from ``pyproject.toml``. + +Reads the ``[tool.attune-author.fact-check]`` table. Missing +sections fall back to :class:`FactCheckConfig` defaults — which +match the spec's "all checks on, soft-fail" Phase 1 defaults. +""" + +from __future__ import annotations + +from pathlib import Path + +from .report import FactCheckConfig + + +def _read_toml(path: Path) -> dict[str, object]: + if not path.is_file(): + return {} + try: + import tomllib + except ImportError: # pragma: no cover - Py <3.11 fallback + import tomli as tomllib # type: ignore[import-not-found,no-redef] + try: + return tomllib.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + + +def load_config(project_root: Path) -> FactCheckConfig: + """Build a :class:`FactCheckConfig` from the project's pyproject. + + Unknown keys are ignored — the goal is forward-compat with + future phases that add their own toggles under the same table. + """ + data = _read_toml(project_root / "pyproject.toml") + tool = data.get("tool", {}) if isinstance(data, dict) else {} + author = tool.get("attune-author", {}) if isinstance(tool, dict) else {} + section = author.get("fact-check", {}) if isinstance(author, dict) else {} + skip = section.get("skip", {}) if isinstance(section, dict) else {} + + def _bool(key: str, default: bool) -> bool: + value = section.get(key, default) if isinstance(section, dict) else default + return bool(value) + + cfg = FactCheckConfig( + enabled=_bool("enabled", True), + soft_fail=_bool("soft_fail", True), + check_python_refs=_bool("check_python_refs", True), + check_cli_refs=_bool("check_cli_refs", True), + check_md_links=_bool("check_md_links", True), + check_numeric_refs=_bool("check_numeric_refs", True), + ) + if isinstance(skip, dict): + cfg.skip = { + str(k): [str(c) for c in v] if isinstance(v, list) else [] for k, v in skip.items() + } + return cfg + + +__all__ = ["load_config"] diff --git a/src/attune_author/fact_check/md_links.py b/src/attune_author/fact_check/md_links.py new file mode 100644 index 0000000..a7c5ae7 --- /dev/null +++ b/src/attune_author/fact_check/md_links.py @@ -0,0 +1,62 @@ +"""Check relative markdown link targets exist. + +For each ``[label](target)`` reference in the polished file: +- If ``target`` looks like an external URL or mailto, skip. +- Otherwise resolve relative to the polished file's directory + and assert the target file exists. + +Anchor existence is out of scope for Phase 1 per spec §1.5. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from .report import CHECK_MD_LINKS, Finding + +#: Match inline markdown links ``[label](target)``. Captures +#: the target. Greedy-safe (no nested parens supported, which +#: is fine for our doc style). +_LINK = re.compile(r"\[(?P