dotfiles/scripts/verify-quotes.py

#!/usr/bin/env python3
"""verify-quotes.py — check a package's blockquotes against the records it cites.

WHY THIS EXISTS, AND WHY IT IS THREE-VALUED (REVIEWED-⟨N⟩ condition 1, PENDING-124).

The inline predecessor of this script reported `verified` for a REVIEWED-104 blockquote
that was a NORMALIZED RECONSTRUCTION, not the placed text: `will not be`→`won't be`,
`does not say`→`doesn't say`, `wrong.`→`wrong:`, `CONDITION:`→`So:`, two separate blocks
spliced into one, and — the part that mattered — THE CLOSING SENTENCE DROPPED. That
sentence was the one that answered the package's own Q2 without asking.

A two-valued verifier said `verified`. The honest report was `matched after
normalization`. It was a two-valued verifier inside a package arguing that verifiers
whose subject can be absent must be three-valued, which is the defect the doctrine
describes, committed by the instrument checking the doctrine's own evidence.

  exact       byte-for-byte in a cited source
  re-wrapped  identical after collapsing WHITESPACE only — a record stored with hard
              line wraps quoted as one line. Content identical; safe.
  normalized  found only after folding emphasis/contractions/punctuation — REPORT IT,
              because THIS is where a reconstruction hides
  not-found   in no cited source

  ⚠ re-wrapped and normalized were one tier in the first version, which cried wolf on
  every correctly-copied quote. Splitting them is the same lesson the fleet summary
  learned: a warning that fires on the safe case stops being read.

DIAGNOSIS THE SCRIPT CANNOT MAKE, so the author must: BOTH citation errors in that
package had one cause — quoting the ADVISORY message and attributing it to the PLACED
record. They are different documents. Placement can add, cut or re-word, so quoting the
advisory systematically loses whatever the act of placing contributed. When a passage
comes from a relayed message rather than a file, it is not a quotation of the record and
must be labelled advisory.

A blockquote that is the author's own proposed text is not a quotation claim at all and
is excluded from assessment — mark it with `<!-- own-text -->` on the line before. An
unmarked block that is not found is reported, never silently skipped.

Usage:  python3 verify-quotes.py PACKAGE.md SOURCE [SOURCE ...]
Exit:   0 all assessed blocks exact · 1 any not-found · 3 any normalized-only
"""
import re
import sys
from pathlib import Path

EXACT, REWRAPPED, NORMALIZED, NOT_FOUND = "exact", "rewrapped", "normalized", "not-found"


def fold(s: str) -> str:
    """Normalization deliberately AGGRESSIVE: the wider it folds, the more it will
    classify as `normalized` rather than `exact`, and that is the reporting we want."""
    s = re.sub(r"[*_`]", "", s)
    s = (s.replace("—", "-").replace("–", "-")
           .replace("’", "'").replace("‘", "'")
           .replace("“", '"').replace("”", '"'))
    for long, short in [("will not", "wont"), ("won't", "wont"),
                        ("does not", "doesnt"), ("doesn't", "doesnt"),
                        ("cannot", "cant"), ("can't", "cant"),
                        ("is not", "isnt"), ("isn't", "isnt")]:
        s = s.replace(long, short)
    s = re.sub(r"[^\w\s']", " ", s)
    return re.sub(r"\s+", " ", s).strip().lower()


def blocks(text: str):
    """Contiguous blockquote runs, with the `own-text` marker honoured."""
    out, cur, own = [], [], False
    for line in text.splitlines():
        if line.strip().lower() == "<!-- own-text -->":
            own = True
            continue
        if line.startswith(">"):
            cur.append(line.lstrip(">").strip())
        elif cur:
            out.append((" ".join(cur).strip(), own)); cur, own = [], False
        elif line.strip():
            own = False
    if cur:
        out.append((" ".join(cur).strip(), own))
    return [(b, o) for b, o in out if b]


def ws(s: str) -> str:
    """Whitespace ONLY. A record stored with hard line wraps is byte-different from the
    same text quoted as one line, and calling that a `normalized match` cries wolf — the
    same failure as a fleet summary that never varies. Re-wrapping is safe; re-wording
    is not, and they must not share a verdict."""
    import re as _re
    return _re.sub(r"\s+", " ", s).strip()


def classify(quote: str, sources: dict[str, str]):
    for name, text in sources.items():
        if quote and quote in text:
            return EXACT, name
    wq = ws(quote)
    for name, text in sources.items():
        if wq and wq in ws(text):
            return REWRAPPED, name
    fq = fold(quote)
    for name, text in sources.items():
        if fq and fq in fold(text):
            return NORMALIZED, name
    return NOT_FOUND, None


def main(argv):
    if len(argv) < 2:
        print(__doc__); return 2
    pkg = Path(argv[0]).read_text(encoding="utf-8")
    sources = {}
    for p in argv[1:]:
        path = Path(p).expanduser()
        if path.exists():
            sources[path.name] = path.read_text(encoding="utf-8")
        else:
            print(f"  ⚠ source unreadable, NOT searched: {p}")
    counts = {EXACT: 0, REWRAPPED: 0, NORMALIZED: 0, NOT_FOUND: 0, "own-text": 0}
    print(f"  checking {len(blocks(pkg))} blockquote(s) against {len(sources)} source(s)\n")
    for q, own in blocks(pkg):
        if own:
            counts["own-text"] += 1
            print(f"  [own-text ] {q[:66]}…")
            continue
        state, where = classify(q, sources)
        counts[state] += 1
        tag = {EXACT: "exact    ", REWRAPPED: "re-wrapped", NORMALIZED: "NORMALIZED", NOT_FOUND: "NOT FOUND"}[state]
        print(f"  [{tag}] {q[:60]}…" + (f"   <- {where}" if where else ""))
        if state == NORMALIZED:
            print("               ^ reconstruction, not the placed text. Diff it before relying on it.")
    print(f"\n  exact {counts[EXACT]} · re-wrapped {counts[REWRAPPED]} (content identical) · "
          f"normalized {counts[NORMALIZED]} · not-found {counts[NOT_FOUND]} · "
          f"own-text {counts['own-text']} (unassessed by design)")
    if counts[NOT_FOUND]:
        return 1
    if counts[NORMALIZED]:
        return 3
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))