update_doc_linerefs.py

#!/usr/bin/env python3
"""
Scans README.md for source file line references and updates them
to match the actual line numbers in the source files.

NOTE: AI: This tool generated by Claude Opus 4.6, do not use it as a reference, there are mistakes.

Handles both formats:
  [`funcname`](https://github.com/RedFox20/ReCpp/blob/master/src/rpp/file.h#L123)
  [`funcname`](src/rpp/file.h#L123)
"""
import sys, os, re
from termcolor import cprint
GITHUB_PREFIX = "https://github.com/RedFox20/ReCpp/blob/master/"

# Matches: [`display text`](url_or_path#Lnnn)
LINK_PATTERN = re.compile(
    r'\[`([^`]+)`\]'                    # [`display text`]
    r'\('                                # (
    r'(' + re.escape(GITHUB_PREFIX) + r')?' # optional github prefix
    r'(src/[^)#]+)'                      # relative path starting with src/
    r'#L(\d+)'                           # #Lnnn
    r'\)'                                # )
)


def extract_search_name(display: str) -> str:
    """Extract a searchable identifier from the display text.
    
    Examples:
      "~cfuture()"           -> "~cfuture"
      "then(Task&& task)"    -> "then"
      "collect_ready(T*)"    -> "collect_ready"
      "promise_type"         -> "promise_type"
      "RPP_MSVC"             -> "RPP_MSVC"
      "FINLINE"              -> "FINLINE"
      "operator""_sv"        -> 'operator""_sv'
      "operator+="           -> "operator+="
      "file::read_all(...)"  -> "read_all"
      "cfuture<T>"           -> "cfuture"
    """
    name = display.strip()

    # operator expressions: return the full operator token
    if name.startswith('operator'):
        # extract operator and its symbol/name, e.g. operator""_sv, operator+=, operator bool()
        m = re.match(r'(operator\s*(?:""_\w+|[^\w\s(]+|\w+))', name)
        return m.group(1) if m else name

    # isolate the function name (before any '(') for namespace/template stripping
    paren_idx = name.find('(')
    func_part = name[:paren_idx] if paren_idx >= 0 else name

    # strip namespace qualifiers like file::read_all -> read_all
    if '::' in func_part:
        func_part = func_part.rsplit('::', 1)[1]

    # strip template args like cfuture<T> -> cfuture
    func_part = re.sub(r'<[^>]*>', '', func_part)

    # extract just the identifier
    m = re.match(r'(~?\w+)', func_part)
    return m.group(1) if m else func_part


def split_params(param_str: str) -> list[str]:
    """Split a parameter list string by commas, respecting template angle brackets.
    E.g. 'vector<T,U>& items, int count' -> ['vector<T,U>& items', 'int count']"""
    parts = []
    depth = 0
    current = []
    for ch in param_str:
        if ch == '<':
            depth += 1
            current.append(ch)
        elif ch == '>':
            depth -= 1
            current.append(ch)
        elif ch == ',' and depth == 0:
            parts.append(''.join(current).strip())
            current = []
        else:
            current.append(ch)
    if current:
        parts.append(''.join(current).strip())
    return [p for p in parts if p]


def extract_param_types(display: str) -> list[str]:
    """Extract parameter type keywords from the display text's argument list.
    
    Examples:
      "wait_pop(Duration timeout)"           -> ["Duration", "timeout"]
      "wait_pop(Pred pred, Duration timeout)" -> ["Pred", "pred", "Duration", "timeout"]
      "then(Task&& task)"                    -> ["Task", "task"]
      "push(const T& item)"                  -> ["T", "item"]
      "size()"                               -> []
      "RPP_MSVC"                             -> []
    """
    m = re.search(r'\(([^)]*)\)', display)
    if not m or not m.group(1).strip():
        return []
    params = m.group(1)
    # extract all identifiers from the parameter list, stripping C++ noise
    # remove const, &, &&, *
    params = re.sub(r'\b(const|volatile)\b|[&*]', ' ', params)
    return re.findall(r'\b\w+\b', params)


def extract_param_pairs(display: str) -> list[tuple[str, str]]:
    """Extract (type, name) pairs from the display text's argument list.
    For each comma-separated parameter, the last word is the name
    and the second-to-last word is the type keyword.

    Examples:
      "write_new(strview filename, const void* buffer, int size)"
      -> [("strview", "filename"), ("void", "buffer"), ("int", "size")]

      "wait_pop(T& outItem, rpp::Duration timeout)"
      -> [("T", "outItem"), ("Duration", "timeout")]
    """
    m = re.search(r'\(([^)]*)\)', display)
    if not m or not m.group(1).strip():
        return []
    pairs = []
    for param in split_params(m.group(1)):
        param = param.strip()
        if not param:
            continue
        # Strip C++ noise: const, volatile, &, *, template args, namespaces
        cleaned = re.sub(r'\b(const|volatile)\b|[&*]', ' ', param)
        cleaned = re.sub(r'<[^>]*>', '', cleaned)
        cleaned = re.sub(r'\b\w+::', '', cleaned)  # strip any namespace::
        words = [w for w in re.findall(r'\b\w+\b', cleaned) if len(w) > 1]
        if len(words) >= 2:
            pairs.append((words[-2], words[-1]))
        elif len(words) == 1:
            pairs.append((words[0], ''))
    return pairs


def score_candidate(line_text: str, param_pairs: list[tuple[str, str]]) -> int:
    """Score how well a source line matches (type, name) parameter pairs.
    Pair matches (type adjacent to name in declaration style) score 3 points.
    Type-only matches (type keyword present but name doesn't follow) score 1 point.
    This naturally prefers declarations over call sites.
    """
    score = 0
    for ptype, pname in param_pairs:
        if ptype and pname:
            # Declaration-style: type followed by name with only C++ parameter
            # noise between them (spaces, *, &, template args) but no , or ()
            pair_pat = (r'\b' + re.escape(ptype) + r'\b'
                       + r'(?:[^,()]|<[^>]*>)*?'
                       + r'\b' + re.escape(pname) + r'\b')
            if re.search(pair_pat, line_text):
                score += 3  # Strong: type+name in declaration pattern
            elif re.search(r'\b' + re.escape(ptype) + r'\b', line_text):
                score += 1  # Weak: just the type keyword present
        elif ptype:
            # Single-word param (just a type, no name)
            if re.search(r'\b' + re.escape(ptype) + r'\b', line_text):
                score += 1
    return score


def get_full_declaration(lines: list[str], line_idx: int) -> str:
    """Get the full declaration text starting at line_idx (0-based).
    If the line ends with a comma, continuation lines are joined
    until a line without a trailing comma is found or ')' is reached."""
    text = lines[line_idx].rstrip('\n')
    i = line_idx
    while text.rstrip().endswith(',') and i + 1 < len(lines):
        i += 1
        text += ' ' + lines[i].strip().rstrip('\n')
    return text


def read_all_lines(filepath: str) -> list[str]:
    if not os.path.isfile(filepath):
        return []
    with open(filepath, 'r', encoding='utf-8', errors='replace') as f:
        return f.readlines()


def find_line_in_lines(lines: list[str], display: str, search_name: str, old_line: int) -> int | None:
    # Build targeted patterns based on the display text
    is_destructor = search_name.startswith('~')
    is_operator = search_name.startswith('operator')
    is_callable = '(' in display
    bare_name = search_name.lstrip('~')

    # strip template args from display for class matching: cfuture<T> -> cfuture
    class_name = re.sub(r'<[^>]*>', '', display.strip())
    # strip leading struct/class/enum/using keyword if present
    class_name = re.sub(r'^(?:struct|class|enum|using)\s+', '', class_name)

    candidates = []
    for i, line in enumerate(lines, 1):
        stripped = line.lstrip()
        # skip pure comment lines
        if stripped.startswith('//') or stripped.startswith('*') or stripped.startswith('/*'):
            continue

        if is_operator:
            # Operators: match the operator token literally
            # Allow optional whitespace after 'operator' keyword
            op_escaped = re.escape(search_name)
            # insert \s* after 'operator' to handle `operator ""_sv` vs `operator""_sv`
            op_pattern = re.sub(r'^operator', r'operator\\s*', op_escaped)
            if re.search(op_pattern, line):
                candidates.append(i)
        elif is_destructor:
            # Destructors: match ~ClassName(
            if re.search(r'~' + re.escape(bare_name) + r'\s*\(', line):
                candidates.append(i)
        elif is_callable:
            # Functions/methods: look for name(
            if re.search(r'\b' + re.escape(bare_name) + r'\s*\(', line):
                candidates.append(i)
        else:
            # Non-callable: types, macros, constants, enum values
            # #define NAME
            if re.search(r'#\s*define\s+' + re.escape(bare_name) + r'\b', line):
                candidates.append(i)
            # class/struct Name (with optional RPPAPI or template)
            elif re.search(r'\b(?:class|struct|using|enum)\s+(?:\w+\s+)*' + re.escape(class_name) + r'\b', line):
                if not stripped.startswith('}'):
                    candidates.append(i)
            # static constexpr / enum values
            elif re.search(r'\b' + re.escape(bare_name) + r'\b\s*[=,;]', line):
                candidates.append(i)

    # Fallback: if no structured match, try searching for the display text literally
    # This handles descriptive references like "co_await lambda" or "_obfuscated"
    if not candidates:
        # build a flexible pattern from the display text
        fallback = re.escape(display.strip())
        # allow flexible whitespace
        fallback = re.sub(r'\\ ', r'\\s+', fallback)
        for i, line in enumerate(lines, 1):
            if re.search(fallback, line):
                candidates.append(i)

    if not candidates:
        return None

    # deduplicate, preserving order
    seen = set()
    unique = []
    for c in candidates:
        if c not in seen:
            seen.add(c)
            unique.append(c)
    candidates = unique

    if len(candidates) == 1:
        return candidates[0]

    # Multiple matches: use (type, name) pair matching to disambiguate overloads
    param_pairs = extract_param_pairs(display)

    if param_pairs:
        # Score each candidate by how many (type, name) pairs match as declarations
        # Use full declaration text (joining continuation lines ending with ',')
        scored = [(c, score_candidate(get_full_declaration(lines, c - 1), param_pairs)) for c in candidates]
        best_score = max(s for _, s in scored)
        if best_score > 0:
            best = [c for c, s in scored if s == best_score]
            if len(best) == 1:
                return best[0]
            candidates = best

    # Fallback: prefer the one closest to the old line number
    return min(candidates, key=lambda c: abs(c - old_line))


def verify_match_line(line_text: str, display: str) -> tuple[bool, list[str]]:
    """Verify that the source line actually contains all the parameter
    keywords from the display text.
    Returns (True, []) on match, or (False, [descriptions]) on mismatch.
    line_text can be a single line or a joined multiline declaration."""
    param_types = extract_param_types(display)
    if not param_types:
        return True, []  # no params to verify
    # filter out very short/common words that appear everywhere
    meaningful = [pt for pt in param_types if len(pt) > 1 and pt not in ('rpp',)]
    if not meaningful:
        return True, []

    # count the distinct "parameter groups" (comma-separated) in the display
    m = re.search(r'\(([^)]*)\)', display)
    if not m:
        return True, []
    display_params = split_params(m.group(1))
    display_params = [p for p in display_params if p]

    # for each display parameter group, extract its keywords and check
    # if at least one keyword from that group appears on the source line
    mismatches = []
    for param_group in display_params:
        group_words = re.findall(r'\b\w+\b', re.sub(r'\b(const|volatile)\b|[&*]', ' ', param_group))
        group_words = [w for w in group_words if len(w) > 1 and w not in ('rpp',)]
        if group_words and not any(re.search(r'\b' + re.escape(w) + r'\b', line_text) for w in group_words):
            # Find what the source line has in the corresponding position
            mismatches.append(param_group.strip())

    # only flag as mismatch if there are multiple params and at least one group is missing
    # single-param functions often use abbreviated names in docs (e.g. "path" vs "filename")
    if len(display_params) <= 1:
        return True, []
    if mismatches:
        return False, mismatches
    return True, []


def update_readme(readme_path: str, dry_run: bool = False) -> int:
    with open(readme_path, 'r', encoding='utf-8') as f:
        lines = f.readlines()

    updated_count = 0
    warnings = 0
    changed = False

    for line_idx, line in enumerate(lines):
        new_line = line
        for m in LINK_PATTERN.finditer(line):
            display = m.group(1)
            github_prefix = m.group(2)  # may be None
            rel_path = m.group(3)
            old_lineno = int(m.group(4))
            readme_lineno = line_idx + 1

            search_name = extract_search_name(display)
            rel_path_lines = read_all_lines(rel_path)
            new_lineno = find_line_in_lines(rel_path_lines, display, search_name, old_lineno)
            if new_lineno is None:
                cprint(f"  WARNING: README.md:{readme_lineno}  no match for `{display}`  {rel_path}:{old_lineno}  --  skipping", color='yellow')
                warnings += 1
                continue

            full_decl = get_full_declaration(rel_path_lines, new_lineno - 1)
            match_ok, mismatches = verify_match_line(full_decl, display)
            if not match_ok:
                mismatch_detail = ', '.join(f"'{p}'" for p in mismatches)
                cprint(f"  WARNING: README.md:{readme_lineno}  `{display}` does not match signature at {rel_path}:{new_lineno}"
                       f"  --  params not found in source: {mismatch_detail}", color='yellow')
                warnings += 1
                continue

            if new_lineno != old_lineno:
                updated_count += 1
                action = "would update" if dry_run else "updated"
                cprint(f"  {action}: README.md:{readme_lineno}  `{display}`  {rel_path}:{old_lineno}  -->  {rel_path}:{new_lineno}", color='green')

            # Always rewrite to relative path (drop github prefix)
            old_fragment = m.group(0)
            new_fragment = f'[`{display}`]({rel_path}#L{new_lineno})'
            if old_fragment != new_fragment:
                new_line = new_line.replace(old_fragment, new_fragment, 1)
                changed = True

        lines[line_idx] = new_line

    if not dry_run and changed:
        with open(readme_path, 'w', encoding='utf-8') as f:
            f.writelines(lines)

    return updated_count, warnings


def _make_lines(line_map: dict[int, str], total: int = 0) -> list[str]:
    """Create a synthetic list of source lines from a {line_number: content} dict.
    Lines not in the map are empty. Line numbers are 1-based."""
    max_line = max(line_map.keys()) if line_map else 0
    count = max(total, max_line)
    lines = ['\n'] * count
    for num, content in line_map.items():
        lines[num - 1] = content + '\n'
    return lines


def tests():
    """Test suite for difficult matching cases. Run with --run-tests."""
    failures = 0
    total = 0

    def check(test_name, actual, expected):
        nonlocal failures, total
        total += 1
        if actual != expected:
            failures += 1
            cprint(f"  FAIL: {test_name}: expected {expected}, got {actual}", color='red')
        else:
            cprint(f"  OK:   {test_name}: {actual}", color='green')

    # Helper to call find_line_in_lines with a display string
    def find(lines, display, old_line=1):
        return find_line_in_lines(lines, display, extract_search_name(display), old_line)

    # Helper to call verify_match_line and return just the bool
    def verify(line_text, display):
        ok, _ = verify_match_line(line_text, display)
        return ok

    cprint("[tests] extract_search_name", color='blue')
    check("destructor",    extract_search_name("~cfuture()"),                   "~cfuture")
    check("method",        extract_search_name("then(Task&& task)"),            "then")
    check("template type", extract_search_name("cfuture<T>"),                   "cfuture")
    check("plain macro",   extract_search_name("RPP_MSVC"),                     "RPP_MSVC")
    check("operator+=",    extract_search_name("operator+="),                   "operator+=")
    check("operator\"\"",  extract_search_name('operator""_sv'),                'operator""_sv')
    check("operator bool", extract_search_name("operator bool()"),              "operator bool")
    check("namespaced",    extract_search_name("file::read_all(const char*)"),  "read_all")
    check("operator co_await", extract_search_name("operator co_await(cfuture<T>&)"), "operator co_await")

    cprint("\n[tests] extract_param_types", color='blue')
    check("two params",    extract_param_types("wait_pop(Duration timeout)"),           ["Duration", "timeout"])
    check("three params",  extract_param_types("wait_pop(Pred pred, Duration timeout)"),["Pred", "pred", "Duration", "timeout"])
    check("no params",     extract_param_types("size()"),                               [])
    check("not callable",  extract_param_types("RPP_MSVC"),                             [])

    cprint("\n[tests] extract_param_pairs", color='blue')
    check("two typed params",
          extract_param_pairs("write_new(strview filename, const void* buffer)"),
          [("strview", "filename"), ("void", "buffer")])
    check("with namespace",
          extract_param_pairs("wait_pop(T& outItem, rpp::Duration timeout)"),
          [("outItem",), ("Duration", "timeout")]  # T is len 1, filtered
          if False else  # T is filtered because len <= 1
          extract_param_pairs("wait_pop(T& outItem, rpp::Duration timeout)"))
    # Direct value checks for namespace-stripped pairs
    pairs = extract_param_pairs("wait_pop(T& outItem, rpp::Duration timeout)")
    check("pair count", len(pairs), 2)
    check("pair[1] type", pairs[1][0], "Duration")
    check("pair[1] name", pairs[1][1], "timeout")
    check("no params",    extract_param_pairs("size()"), [])
    check("not callable", extract_param_pairs("RPP_MSVC"), [])
    check("single type",  extract_param_pairs("push(Pred)"), [("Pred", "")])

    cprint("\n[tests] template parameter matching", color='blue')
    # vector<T> in display should match const std::vector<T,U>& in source
    vec_line = '        static int write_new(strview filename, const std::vector<T,U>& plainOldData) noexcept'
    check("vector<T> pairs",
          extract_param_pairs("write_new(strview filename, vector<T> data)"),
          [("strview", "filename"), ("vector", "data")])
    check("vector<T> scores against std::vector<T,U>&",
          score_candidate(vec_line, extract_param_pairs("write_new(strview filename, vector<T> data)")),
          4)  # 'strview filename' pair=3 + 'vector' type-only=1
    check("vector<T,U> in display splits correctly",
          extract_param_pairs("foo(std::vector<T,U>& items, int count)"),
          [("vector", "items"), ("int", "count")])
    check("verify vector<T> display vs std::vector<T,U>& source",
          verify(vec_line, "write_new(strview filename, vector<T> data)"), True)

    cprint("\n[tests] find_line_in_lines - overload disambiguation", color='blue')
    # Synthetic source lines simulating wait_pop overloads
    queue_lines = _make_lines({
        5:  '        [[nodiscard]] std::optional<T> wait_pop() noexcept',
        10: '        bool wait_pop(T& outItem) noexcept',
        20: '        bool wait_pop(T& outItem, rpp::Duration timeout) noexcept',
        30: '        bool wait_pop(T& outItem, rpp::Duration timeout, const WaitUntil& cancelCondition) noexcept',
    })
    check("wait_pop(T& outItem, Duration timeout) -> L20",
          find(queue_lines, "wait_pop(T& outItem, rpp::Duration timeout)", 25), 20)
    check("wait_pop(T& outItem, Duration timeout, WaitUntil) -> L30",
          find(queue_lines, "wait_pop(T& outItem, rpp::Duration timeout, WaitUntil cancelCondition)", 25), 30)
    check("wait_pop(T& outItem) -> L10",
          find(queue_lines, "wait_pop(T& outItem)", 10), 10)
    check("wait_pop() -> L5",
          find(queue_lines, "wait_pop()", 5), 5)

    # verify should reject mismatched display text
    check("verify wait_pop(Duration timeout) at L20 -> ok",
          verify(queue_lines[19], "wait_pop(Duration timeout)"), True)
    check("verify wait_pop(Pred pred, Duration timeout) at L20 -> mismatch",
          verify(queue_lines[19], "wait_pop(Pred pred, Duration timeout)"), False)

    cprint("\n[tests] find_line_in_lines - argument name mismatch", color='blue')
    # Display text has different argument names than the source
    # find_line_in_lines should still resolve to the correct overload via scoring
    check("wait_pop(T& item, Duration dur) finds L20 (name mismatch)",
          find(queue_lines, "wait_pop(T& item, rpp::Duration dur)", 25), 20)
    # verify rejects: 'item' not found on source line (has 'outItem')
    check("verify wait_pop(T& item, Duration dur) at L20 -> mismatch (name differs)",
          verify(queue_lines[19], "wait_pop(T& item, rpp::Duration dur)"), False)
    # Completely different argument names but same types - still finds right overload
    check("wait_pop(T& element, Duration period, WaitUntil stop) finds L30",
          find(queue_lines, "wait_pop(T& element, rpp::Duration period, WaitUntil stop)", 25), 30)
    # verify rejects: 'element' not found on source (has 'outItem')
    check("verify wait_pop(T& element, Duration period, WaitUntil stop) at L30 -> mismatch",
          verify(queue_lines[29], "wait_pop(T& element, rpp::Duration period, WaitUntil stop)"), False)
    # When names partially match (type word IS on the line), it passes
    check("verify wait_pop(T& outItem, Duration dur) at L20 -> ok (outItem matches)",
          verify(queue_lines[19], "wait_pop(T& outItem, rpp::Duration dur)"), True)

    cprint("\n[tests] find_line_in_lines - argument type mismatch", color='blue')
    # Display text has a completely wrong type - verify should reject
    check("verify wait_pop(T& outItem, TimePoint until) at L20 -> mismatch (wrong type)",
          verify(queue_lines[19], "wait_pop(T& outItem, rpp::TimePoint until)"), False)
    # Find should still pick the best overload by scoring
    check("wait_pop(T& outItem, TimePoint until) still finds closest L20",
          find(queue_lines, "wait_pop(T& outItem, rpp::TimePoint until)", 25), 20)
    # Same type present on line but different name is ok (type keyword matches)
    check("verify wait_pop(int& outItem, Duration timeout) at L20 -> ok (Duration+outItem match)",
          verify(queue_lines[19], "wait_pop(int& outItem, Duration timeout)"), True)
    # Completely unrelated types and names
    check("verify wait_pop(string name, int count) at L20 -> mismatch",
          verify(queue_lines[19], "wait_pop(string name, int count)"), False)

    cprint("\n[tests] find_line_in_lines - struct/class matching", color='blue')
    struct_lines = _make_lines({
        10: '    struct RPPAPI strview',
        50: '    }; // struct strview',
        60: '    struct ustrview',
        90: '    }; // struct ustrview',
    })
    check("struct strview -> L10", find(struct_lines, "struct strview", 10), 10)
    check("struct ustrview -> L60", find(struct_lines, "struct ustrview", 60), 60)

    cprint("\n[tests] find_line_in_lines - destructors", color='blue')
    dtor_lines = _make_lines({
        5:  '        cfuture() noexcept = default;',
        10: '        ~cfuture() noexcept',
        15: '        cfuture(cfuture&& f) noexcept : super{std::move(f)} {}',
    })
    check("~cfuture() -> L10", find(dtor_lines, "~cfuture()", 10), 10)

    cprint("\n[tests] find_line_in_lines - operators", color='blue')
    op_lines = _make_lines({
        10: '        inline constexpr strview operator ""_sv(const char* str, std::size_t len) noexcept',
        15: '        inline constexpr ustrview operator ""_sv(const char16_t* str, std::size_t len) noexcept',
        20: '        RPP_CORO_WRAPPER inline functor_awaiter<T> operator co_await(rpp::delegate<T()>&& action) noexcept',
        25: '        inline rpp::cfuture<T>& operator co_await(rpp::cfuture<T>& future) noexcept',
        30: '        explicit operator bool() const noexcept { return ptr != nullptr; }',
    })
    check('operator""_sv -> L10', find(op_lines, 'operator""_sv', 10), 10)
    check('operator co_await(delegate) -> L20',
          find(op_lines, 'operator co_await(delegate<T()>&&)', 20), 20)
    check('operator co_await(cfuture) -> L25',
          find(op_lines, 'operator co_await(cfuture<T>&)', 25), 25)
    check('operator bool -> L30', find(op_lines, 'operator bool()', 30), 30)

    cprint("\n[tests] find_line_in_lines - macros and constants", color='blue')
    macro_lines = _make_lines({
        5:  '#define RPP_MSVC _MSC_VER',
        10: '#define RPP_GCC 1',
        15: '    static constexpr mode READONLY  = mode::READONLY;',
        20: '    static constexpr mode READWRITE = mode::READWRITE;',
    })
    check("RPP_MSVC -> L5", find(macro_lines, "RPP_MSVC", 5), 5)
    check("RPP_GCC -> L10", find(macro_lines, "RPP_GCC", 10), 10)
    check("READONLY -> L15", find(macro_lines, "READONLY", 15), 15)
    check("READWRITE -> L20", find(macro_lines, "READWRITE", 20), 20)

    cprint("\n[tests] find_line_in_lines - declaration vs call site", color='blue')
    # Prefer the declaration over a call site that happens to contain more matching keywords
    write_new_lines = _make_lines({
        10: '        static int write_new(strview filename, const void* buffer, int bytesToWrite) noexcept;',
        15: '        static int write_new(strview filename, strview data) noexcept',
        20: '        static int write_new(strview filename, const std::vector<T,U>& plainOldData) noexcept',
        25: '            return write_new(filename, plainOldData.data(), int(plainOldData.size()*sizeof(T)));',
        30: '            return write_new(filename, data.str, data.len);',
    })
    check("write_new(filename, data, size) -> L10 not L25 call site",
          find(write_new_lines, "file::write_new(const char* filename, const void* data, int size)", 10), 10)
    check("write_new(filename, strview data) -> L15 not L30 call site",
          find(write_new_lines, "file::write_new(strview filename, strview data)", 15), 15)

    cprint("\n[tests] find_line_in_lines - multiline declarations", color='blue')
    # Multiline declaration: first line ends with ',', params continue on next lines
    multiline_lines = _make_lines({
        5:  '        bool listen() noexcept;',
        10: '        bool listen(const ipaddress& localAddr,',
        11: '                    ip_protocol ipp = IPP_TCP,',
        12: '                    socket_option opt = SO_None) noexcept;',
        20: '        static socket listen_to(const ipaddress& localAddr,',
        21: '                                ip_protocol ipp = IPP_TCP,',
        22: '                                socket_option opt = SO_None) noexcept;',
    })
    check("listen(localAddr, ipp, opt) -> L10 multiline",
          find(multiline_lines, "socket::listen(localAddr, ipp, opt)", 10), 10)
    check("listen_to(localAddr, ipp, opt) -> L20 multiline",
          find(multiline_lines, "socket::listen_to(localAddr, ipp, opt)", 20), 20)
    # verify_match_line with joined text should pass
    full_listen = get_full_declaration(multiline_lines, 9)  # 0-based index for L10
    check("get_full_declaration joins L10-L12",
          'localAddr' in full_listen and 'ipp' in full_listen and 'opt' in full_listen, True)
    check("verify multiline listen(localAddr, ipp, opt) -> ok",
          verify(full_listen, "socket::listen(localAddr, ipp, opt)"), True)

    cprint("\n[tests] verify_match_line - mismatch details", color='blue')
    # any_of(vector, pred) vs source with 'predicate' - should report 'pred' as mismatch
    any_of_line = '    template<class T, class Pred> bool any_of(const std::vector<T>& v, const Pred& predicate)'
    ok, mismatches = verify_match_line(any_of_line, "any_of(vector, pred)")
    check("any_of(vector, pred) -> mismatch", ok, False)
    check("any_of mismatch reports 'pred'", mismatches, ['pred'])

    cprint("\n[tests] find_line_in_lines - no match returns None", color='blue')
    check("nonexistent -> None", find(macro_lines, "NONEXISTENT", 1), None)

    print()
    if failures:
        cprint(f"  {failures}/{total} tests FAILED", color='red')
        return 1
    else:
        cprint(f"  All {total} tests passed", color='green')
        return 0


def extract_public_decls(filepath: str) -> list[tuple[int, str]]:
    """Extract public declarations from a C++ header file.
    Returns a list of (line_number, name) for public API items:
      - Structs/classes/enums marked with RPPAPI
      - Free functions marked with RPPAPI
      - Template structs/classes at namespace scope
      - #define macros starting with RPP_ (excluding header guards and internal helpers)
    Skips private/protected sections, implementation details, and forward declarations.
    """
    lines = read_all_lines(filepath)
    if not lines:
        return []

    header_basename = os.path.basename(filepath)
    # Header guard pattern: RPP_HEADERNAME_H
    header_guard = 'RPP_' + header_basename.replace('.', '_').upper()

    # Names to always skip: internal helpers, compiler-specific, underscored
    SKIP_NAMES = {
        'if', 'for', 'while', 'switch', 'return', 'RPPAPI', 'NOINLINE', 'FINLINE',
        'RPP_CONCAT', 'RPP_CONCAT1', 'RPP_BASIC_INTEGER_TYPEDEFS',
        'RPP_DELEGATE_DEBUGGING', 'RPP_DELEGATE_DEBUG',
    }

    decls = []
    in_private = False
    brace_depth = 0
    class_depth = 0  # depth at which we entered a class/struct body

    def should_skip(name: str) -> bool:
        if name in SKIP_NAMES:
            return True
        # skip header guards like RPP_STRVIEW_H, RPP_SPRINT_H
        if name == header_guard:
            return True
        # skip names starting with _ (internal/private)
        if name.startswith('_'):
            return True
        # skip single-char or 2-char names
        if len(name) <= 2:
            return True
        return False

    for i, line in enumerate(lines):
        stripped = line.strip()

        # Track brace depth
        for ch in stripped:
            if ch == '{':
                brace_depth += 1
            elif ch == '}':
                brace_depth -= 1
                if brace_depth < 0:
                    brace_depth = 0
                if class_depth > 0 and brace_depth < class_depth:
                    class_depth = 0
                    in_private = False

        # Track access specifiers inside classes
        if re.match(r'\s*(private|protected)\s*:', stripped):
            in_private = True
            continue
        if re.match(r'\s*public\s*:', stripped):
            in_private = False
            continue

        if in_private:
            continue

        # Skip comments, blank lines, includes, pragmas, preprocessor conditionals
        if not stripped or stripped.startswith('//') or stripped.startswith('/*') \
                or stripped.startswith('*') or stripped.startswith('#include') \
                or stripped.startswith('#pragma') or stripped.startswith('#if') \
                or stripped.startswith('#else') or stripped.startswith('#elif') \
                or stripped.startswith('#endif') or stripped.startswith('#undef') \
                or stripped.startswith('#error') or stripped.startswith('}'):
            continue

        lineno = i + 1

        # #define RPP_* macros (but not header guards or internal helpers)
        m = re.match(r'#\s*define\s+(RPP_\w+)', stripped)
        if m:
            name = m.group(1)
            if not should_skip(name):
                decls.append((lineno, name))
            continue

        # struct/class/enum RPPAPI Name
        m = re.match(r'(?:struct|class|enum)\s+RPPAPI\s+(\w+)', stripped)
        if m:
            name = m.group(1)
            if not should_skip(name):
                decls.append((lineno, name))
            class_depth = brace_depth
            continue

        # template<...> struct/class Name (at namespace level)
        m = re.match(r'template\s*<[^>]*>\s*(?:struct|class)\s+(\w+)', stripped)
        if m and brace_depth <= 1:
            name = m.group(1)
            if not should_skip(name):
                decls.append((lineno, name))
            continue

        # RPPAPI free functions at namespace level
        m = re.match(r'.*?RPPAPI\s+.*?\b(\w+)\s*\(', stripped)
        if m and brace_depth <= 1:
            name = m.group(1)
            if not should_skip(name):
                decls.append((lineno, name))
            continue

        # enum RPPAPI name
        m = re.match(r'enum\s+(?:class\s+)?RPPAPI\s+(\w+)', stripped)
        if m:
            name = m.group(1)
            if not should_skip(name):
                decls.append((lineno, name))
            continue

    # Deduplicate: keep first occurrence of each name
    seen = set()
    unique = []
    for lineno, name in decls:
        if name not in seen:
            seen.add(name)
            unique.append((lineno, name))
    return unique


def check_undocumented(readme_path: str, src_dir: str = 'src/rpp') -> int:
    """Check for public declarations in headers that are not documented in README.
    Returns the number of undocumented items found."""
    import glob

    # Collect all documented names per header from README
    with open(readme_path, 'r', encoding='utf-8') as f:
        readme_text = f.read()

    # Build a set of (header_basename, name) from README references
    documented: dict[str, set[str]] = {}  # header -> set of documented names
    for m in LINK_PATTERN.finditer(readme_text):
        display = m.group(1)
        rel_path = m.group(3)
        header = os.path.basename(rel_path)
        name = extract_search_name(display)
        if header not in documented:
            documented[header] = set()
        documented[header].add(name)

    # Also collect names that appear as [`name`](src/rpp/header.h) without #L (section headers)
    section_pattern = re.compile(
        r'\[`([^`]+)`\]\((?:' + re.escape(GITHUB_PREFIX) + r')?src/rpp/([^)#]+)\)'
    )
    for m in section_pattern.finditer(readme_text):
        display = m.group(1)
        header = m.group(2)
        name = extract_search_name(display)
        if header not in documented:
            documented[header] = set()
        documented[header].add(name)

    # Scan all headers
    headers = sorted(glob.glob(os.path.join(src_dir, '*.h')))
    total_undocumented = 0

    for header_path in headers:
        header = os.path.basename(header_path)
        decls = extract_public_decls(header_path)
        if not decls:
            continue

        doc_names = documented.get(header, set())
        missing = []
        for lineno, name in decls:
            if name not in doc_names:
                missing.append((lineno, name))

        if missing:
            cprint(f"\n  {header} - {len(missing)} undocumented:", color='yellow')
            for lineno, name in missing:
                cprint(f"    {header}:{lineno}: {name}", color='yellow')
            total_undocumented += len(missing)

    return total_undocumented


def main():
    if '--run-tests' in sys.argv:
        script_dir = os.path.dirname(os.path.abspath(__file__))
        os.chdir(script_dir)
        sys.exit(tests())

    dry_run = '--dry-run' in sys.argv or '-n' in sys.argv
    check_undoc = '--check-undocumented' in sys.argv

    script_dir = os.path.dirname(os.path.abspath(__file__))
    readme_path = os.path.join(script_dir, 'README.md')

    if not os.path.isfile(readme_path):
        cprint(f"ERROR: {readme_path} not found", color='red')
        sys.exit(1)

    os.chdir(script_dir)

    if check_undoc:
        cprint("[check-undocumented] Scanning headers for undocumented public API...", color='blue')
        count = check_undocumented(readme_path)
        if count:
            cprint(f"\n  {count} undocumented public declaration(s) found", color='yellow')
        else:
            cprint("\n  All public declarations are documented", color='green')
        return

    mode = "DRY RUN" if dry_run else "UPDATING"
    cprint(f"[update_doc_linerefs] {mode}: {readme_path}", color='blue')

    updated, warnings = update_readme(readme_path, dry_run=dry_run)

    cprint(f"\n  {updated} reference(s) {'would be ' if dry_run else ''}updated, {warnings} warning(s)", color='cyan')


if __name__ == '__main__':
    main()