dotfiles/scripts/governance-drift-check.py

#!/usr/bin/env python3
"""
governance-drift-check — report state claims in ~/CLAUDE.md that the substrate contradicts.

Reports. Does not correct. Detection needs no authorization; correction does
(Constitutional Constraint #1). This exists so that a stale governance document
is *visible* rather than *misleading* — Constitutional Constraint #4, honest
degradation, applied to the governance document itself.

Every check carries a POSITIVE CONTROL in the same run: an absence is not
evidence until the instrument is shown capable of detecting presence.
(Ratified 2026-07-27 as an epistemic standard, jurist Q2.)

Built 2026-07-27 on steward authorization. Exit code is always 0 — this is a
report, not a gate.
"""

import datetime
import json
import os
import re
import signal
import sys
from datetime import date
from pathlib import Path

# Report cleanly when the reader closes early (`| head`), rather than tracebacking.
try:
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
except (AttributeError, ValueError):  # not POSIX, or not on the main thread
    pass

HOME = Path.home()
CLAUDE_MD = HOME / "dotfiles" / "CLAUDE.md"
MONTHS = ("january february march april may june july august september "
          "october november december").split()

findings: list[str] = []
controls: list[tuple[str, bool]] = []   # (label, passed)


def control(label: str, passed: bool) -> bool:
    controls.append((label, passed))
    return passed


# ---------------------------------------------------------------- load
if not CLAUDE_MD.exists():
    print(f"drift-check: CANNOT RUN — {CLAUDE_MD} not found")
    sys.exit(0)

text = CLAUDE_MD.read_text()
lines = text.split("\n")


# ------------------------------------------------- 1. referenced paths
# Backticked paths that look like filesystem locations.
raw = set(re.findall(r"`(~[^`\s]+|/Users/[^`]+?)`", text))
paths = {p for p in raw if ("/" in p) and not p.endswith(("`",))}

control("path-check instrument reaches the filesystem", CLAUDE_MD.exists())
for p in sorted(paths):
    expanded = Path(os.path.expanduser(p.strip().rstrip(".,;")))
    if not expanded.exists():
        ln = next((i + 1 for i, l in enumerate(lines) if p in l), None)
        findings.append(f"L{ln}: path does not resolve — {p}")


# ------------------------------------- 2. named MCP tools / servers
# Tool names the document instructs the executor to call.
tool_names = set(re.findall(r"`(kg_\w+|diary_\w+|find_tunnels|traverse)`", text))
configured: set[str] = set()
for cfg in (HOME / ".claude/settings.json", HOME / ".claude/settings.local.json",
            HOME / ".claude.json", HOME / ".mcp.json"):
    if cfg.exists():
        try:
            configured |= set((json.loads(cfg.read_text()).get("mcpServers") or {}).keys())
        except Exception:
            pass

control("MCP config readable (>=1 server found somewhere)", bool(configured))
if tool_names and not any("mempal" in s.lower() for s in configured):
    ln = next((i + 1 for i, l in enumerate(lines) if "kg_query" in l), None)
    findings.append(
        f"L{ln}: {len(tool_names)} named tools do not resolve — "
        f"{', '.join(sorted(tool_names))} (no mempalace MCP server configured; "
        f"servers present: {sorted(configured) or 'none'})"
    )


# ------------------------------------------------- 3. hooks claimed to fire
hook_claims = re.findall(r"(Stop and PreCompact hooks fire|PreCompact hook)", text)
if hook_claims:
    live: set[str] = set()
    for cfg in (HOME / ".claude/settings.json", HOME / ".claude/settings.local.json"):
        if cfg.exists():
            try:
                h = json.loads(cfg.read_text()).get("hooks") or {}
                live |= {k for k, v in h.items() if v}
            except Exception:
                pass
    control("hooks config readable (>=1 hook event configured)", bool(live))
    missing = [e for e in ("Stop", "PreCompact") if e not in live]
    if missing:
        ln = next((i + 1 for i, l in enumerate(lines) if "PreCompact" in l), None)
        findings.append(
            f"L{ln}: document claims these hooks fire, but they are unconfigured — "
            f"{', '.join(missing)} (configured: {sorted(live) or 'none'})"
        )


# ---------------------------------------------------- 4. expired horizons
today = date.today()
horizon_re = re.compile(r"through\s+(?:the\s+)?end\s+of\s+(\w+)\s+(\d{4})", re.I)
control("horizon regex matches a known-present phrase",
        bool(horizon_re.search(text)) or "through end of" not in text.lower())
for i, line in enumerate(lines, 1):
    for m in horizon_re.finditer(line):
        month, year = m.group(1).lower(), int(m.group(2))
        if month in MONTHS:
            mi = MONTHS.index(month) + 1
            if (year, mi) < (today.year, today.month):
                findings.append(
                    f"L{i}: horizon expired — \"{m.group(0)}\" "
                    f"(elapsed {(today.year - year) * 12 + today.month - mi} months)"
                )


# ------------------------------------------------ 5. structural integrity
if not text.endswith("\n"):
    findings.append(f"L{len(lines)}: no terminal newline "
                    f"(wc -l undercounts by 1 — breaks line-referenced patches)")
for i, line in enumerate(lines, 1):
    # `||` inside a line carrying table pipes = two rows fused onto one line.
    # Do NOT require the line to start with `|`: the real instance (L243) began
    # mid-sentence, which is exactly how the fusion hides.
    if "||" in line and line.count("|") >= 3:
        findings.append(f"L{i}: table rows fused on one line (`||`) — row will not render")
    if re.match(r"^\s+\|", line):
        findings.append(f"L{i}: table row has stray leading whitespace — breaks the table")
    if re.match(r"^#{2,4} .*\s+$", line) and i < len(lines) and lines[i].startswith("#"):
        findings.append(f"L{i}: empty heading stub immediately followed by a heading "
                        f"— orphans the section beneath it")


# --------------------------------------------- 6. doctrine-ID integrity
# Doctrine units may carry a stable id in an HTML comment: `<!-- D:memory.check-first -->`.
# Invisible in prose, parseable by tools, and IN the canonical — so there is no second
# version to drift (L110). Skills cite an id instead of paraphrasing the rule; this
# section catches a citation whose doctrine has been reworded away, and duplicate ids.
# Dormant until the first id exists; the controls below run either way, so "nothing
# reported" means "checked and clean", not "never looked".
DOCTRINE_DEF = re.compile(r"<!--\s*(D:[a-z0-9][a-z0-9-]*\.[a-z0-9][a-z0-9-]*)\s*-->")
DOCTRINE_REF = re.compile(r"\b(D:[a-z0-9][a-z0-9-]*\.[a-z0-9][a-z0-9-]*)\b")

defined: dict[str, int] = {}
for i, line in enumerate(lines, 1):
    for did in DOCTRINE_DEF.findall(line):
        if did in defined:
            findings.append(f"L{i}: duplicate doctrine id {did} "
                            f"(also L{defined[did]}) — a citation cannot resolve it")
        else:
            defined[did] = i

# Citing surface: the skills, which are the intended consumers. Deliberately NOT
# PENDING.md — drafts there legitimately quote ids that do not exist yet.
skills_dir = HOME / ".claude" / "skills"
scanned = 0
if skills_dir.is_dir():
    for f in sorted(skills_dir.rglob("*.md")):
        try:
            if f.stat().st_size > 200_000:
                continue
            body = f.read_text(errors="replace")
        except OSError:
            continue
        scanned += 1
        for ref in sorted(set(DOCTRINE_REF.findall(body))):
            if ref not in defined:
                findings.append(
                    f"{f.relative_to(HOME)}: cites doctrine id {ref}, which "
                    f"~/CLAUDE.md does not define — the rule was reworded or removed")

control("doctrine-id parser detects a definition",
        bool(DOCTRINE_DEF.findall("x <!-- D:memory.check-first --> y")))
control("doctrine-id parser detects a citation",
        DOCTRINE_REF.findall("see D:memory.check-first here") == ["D:memory.check-first"])
control("doctrine-id parser rejects a non-id",
        not DOCTRINE_DEF.findall("<!-- D:nodot -->")
        and not DOCTRINE_REF.findall("D:NoDot.Caps"))
control("doctrine-id citing surface is reachable",
        (not skills_dir.is_dir()) or scanned > 0)


# ------------------------------------------ 7. register integrity (REVIEWED)
# EARNED 2026-08-07, from a real loss. An amendment block was placed OVER the
# record it amends: the original `## REVIEWED-87 — PENDING-99 — …` entry was
# replaced by `## REVIEWED-87 — AMENDMENT 2026-08-07`, leaving the amendment's
# own `**Amends:** REVIEWED-87` line pointing at a record no longer in the file.
# The content was recoverable from git and the underlying jurist ruling was
# filed separately, so nothing was lost — but NOTHING DETECTED IT. It surfaced
# because a diff was read by hand, and the tell was a deletion count on what
# should have been a pure append.
#
# The class: a register whose entries can silently replace one another cannot be
# trusted to answer "what was ruled under N", which is the register's whole job.
# This is `removing-a-claim-is-not-removing-the-reliance` at the governance
# layer — the amendment's dependency on the original survived the original's
# removal, and became invisible.
#
# Detection only, like every check here: REVIEWED.md is [ESCALATE], the
# steward's hand (Constitutional Constraint #1).
REVIEWED_MD = HOME / "dotfiles" / "REVIEWED.md"

RE_HEAD = re.compile(r"^##\s+REVIEWED-(\d+)\s*[—-]\s*(.*)$", re.M)
RE_AMENDS = re.compile(r"\*\*Amends:\*\*\s*REVIEWED-(\d+)")


def register_findings(text: str, label: str) -> list[str]:
    """Every amendment must sit ALONGSIDE the record it amends, never replace it."""
    out: list[str] = []
    heads = RE_HEAD.findall(text)
    originals = {n for n, rest in heads
                 if not rest.strip().upper().startswith("AMENDMENT")}
    for n, rest in heads:
        if rest.strip().upper().startswith("AMENDMENT") and n not in originals:
            out.append(f"{label}: '## REVIEWED-{n} — AMENDMENT' exists with no "
                       f"un-amended REVIEWED-{n} entry — the amendment replaced "
                       f"the record it amends")
    for n in sorted(set(RE_AMENDS.findall(text))):
        if n not in originals:
            out.append(f"{label}: a block declares '**Amends:** REVIEWED-{n}' but "
                       f"no REVIEWED-{n} entry exists in the file")
    return out


reg_findings: list[str] = []
if REVIEWED_MD.exists():
    reg_findings = register_findings(REVIEWED_MD.read_text(errors="replace"),
                                     "REVIEWED.md")

# Controls. The third is the one that matters and is the lesson of the day:
# a check that has never fired on a known-bad input is unestablished, so the
# instrument is run against a synthetic reproduction of the actual failure.
_GOOD = ("## REVIEWED-87 — PENDING-99 — original\n**Date:** 2026-08-05\n\n"
         "## REVIEGH\n\n## REVIEWED-87 — AMENDMENT 2026-08-07\n"
         "**Amends:** REVIEWED-87 (x).\n")
_BAD = ("## REVIEWED-87 — AMENDMENT 2026-08-07\n"
        "**Amends:** REVIEWED-87 (x).\n")
control("register parser finds REVIEWED headings", len(RE_HEAD.findall(_GOOD)) == 2)
control("register check passes a correctly JOINED amendment",
        not register_findings(_GOOD, "t"))
control("register check DETECTS an amendment that replaced its record  "
        "[reproduces the 2026-08-07 loss]",
        len(register_findings(_BAD, "t")) == 2)
control("register file is reachable", REVIEWED_MD.exists())


# ------------------------------------------ 8. deferred decisions, and their triggers
# EARNED 2026-08-07. The jurist settlement of 2026-05-16 deferred TEI-native
# authoring "until Cluster A's MD-with-sidecar form is operational". Cluster A
# became operational, the condition was met, and NOBODY LOOKED — it surfaced
# months later, by accident, while reading an unrelated document. The steward's
# stated reason for settling it that day was not the format question but the
# forgetting: "I abhor deferring so many things and then forgetting them."
#
# A deferral is a claim: "not yet". When its trigger fires, the substrate
# contradicts that claim — which is exactly what this instrument detects. So a
# deferred decision declares a MACHINE-CHECKABLE trigger and this checks it,
# rather than relying on anyone to remember.
#
#   <!-- DEFERRED-DECISION: <slug>
#        since: YYYY-MM-DD
#        owner: steward | jurist | executor
#        trigger: glob <pattern> | path-exists <path> | date <YYYY-MM-DD> | manual
#        discriminator: <where the deciding evidence is written down> -->
#
# `manual` never auto-fires and is listed rather than checked — an honest way to
# record a deferral whose condition genuinely cannot be mechanised, instead of
# inventing a proxy. Proxies are what failed here: the old trigger stood in for
# "behavioural evidence on high-fidelity sources" and came true without it.
DEFERRED_RE = re.compile(
    r"<!--\s*DEFERRED-DECISION:\s*([a-z0-9][a-z0-9-]*)\s*\n(.*?)-->", re.S)

SCAN_ROOTS = [HOME / "_Dev", HOME / "dotfiles"]
TRANSCRIPTS = HOME / ".claude/projects/-Users-davidglidden"
# Governance packages live outside the */docs/** convention the scan was written for,
# so a deferral filed there was invisible to this check. Found 2026-08-07 while wiring
# PENDING-112's falsifier: the mechanism existed, and the one place it most needed to
# reach was the one place it did not look.
EXTRA_SCAN_GLOBS = [(HOME / "dotfiles", "claude/governance/**/*.md")]
deferrals: list[dict] = []


def _repo_root(p: Path) -> Path:
    for parent in [p] + list(p.parents):
        if (parent / ".git").exists():
            return parent
    return p.parent


def parse_deferrals(text: str, src: Path) -> list[dict]:
    out = []
    for slug, body in DEFERRED_RE.findall(text):
        fields = dict(re.findall(r"^\s*([a-z-]+):\s*(.+?)\s*$", body, re.M))
        out.append({"slug": slug, "file": src, "root": _repo_root(src),
                    "trigger": fields.get("trigger", "manual"),
                    "owner": fields.get("owner", "?"),
                    "since": fields.get("since", "?")})
    return out


def trigger_fired(d: dict) -> bool | None:
    """True = condition met (decision is due). None = not mechanically checkable."""
    kind, _, arg = d["trigger"].partition(" ")
    arg = arg.strip()
    if kind == "glob":
        return any(d["root"].glob(arg))
    if kind == "path-exists":
        return Path(os.path.expanduser(arg)).exists() if arg.startswith(("~", "/")) \
            else (d["root"] / arg).exists()
    if kind == "date":
        return datetime.date.today().isoformat() >= arg
    if kind == "transcripts":
        # Session-count trigger. Added 2026-08-07 for PENDING-112's pre-registered
        # 20-session falsifier, which the jurist required be BINDING rather than a
        # disclosed intention. A date would have been a proxy — sessions run at wildly
        # variable rates — and this block's own comment records that proxies are what
        # failed last time. Counting transcripts encodes the real condition.
        try:
            return len(list(TRANSCRIPTS.glob("*.jsonl"))) >= int(arg)
        except (ValueError, OSError):
            return None
    return None


_scan_targets = [(r, "*/docs/**/*.md") for r in SCAN_ROOTS] + EXTRA_SCAN_GLOBS
_seen_files: set = set()
for root, pattern in _scan_targets:
    if not root.is_dir():
        continue
    for f in root.glob(pattern):
        if f in _seen_files:
            continue
        _seen_files.add(f)
        try:
            if f.stat().st_size > 400_000:
                continue
            body = f.read_text(errors="replace")
        except OSError:
            continue
        if "DEFERRED-DECISION:" in body:
            deferrals.extend(parse_deferrals(body, f))

fired = [d for d in deferrals if trigger_fired(d) is True]
manual = [d for d in deferrals if trigger_fired(d) is None]

_T_OK = ("<!-- DEFERRED-DECISION: tei-native\n     since: 2026-08-07\n"
         "     owner: steward\n     trigger: date 2000-01-01\n-->")
_T_WAIT = _T_OK.replace("date 2000-01-01", "date 2999-01-01")
_p_ok = parse_deferrals(_T_OK, CLAUDE_MD)
_p_wait = parse_deferrals(_T_WAIT, CLAUDE_MD)
control("deferred-decision parser reads a well-formed block",
        len(_p_ok) == 1 and _p_ok[0]["slug"] == "tei-native")
control("deferred-decision parser rejects a non-block",
        not parse_deferrals("<!-- DEFERRED: nope -->", CLAUDE_MD))
# transcripts-trigger controls: it must fire on a threshold already passed and stay
# silent on one that has not. An absence is not evidence until the instrument is shown
# capable of detecting presence — and this trigger carries a standing obligation.
control("transcripts trigger fires on a passed threshold",
        trigger_fired({"trigger": "transcripts 1", "root": HOME}) is True)
control("transcripts trigger silent on an unreached threshold",
        trigger_fired({"trigger": "transcripts 999999", "root": HOME}) is False)
control("governance dir is inside the deferral scan",
        any("claude/governance" in str(p) for p in _seen_files)
        or not (HOME / "dotfiles/claude/governance").is_dir())
control("trigger evaluator FIRES on a met condition",
        _p_ok and trigger_fired(_p_ok[0]) is True)
control("trigger evaluator does NOT fire on an unmet condition  "
        "[the discriminating half]",
        _p_wait and trigger_fired(_p_wait[0]) is False)
control("deferred-decision scan surface is reachable",
        any(r.is_dir() for r in SCAN_ROOTS))


# ------------------------------------------------------------- report
failed_controls = [lbl for lbl, ok in controls if not ok]
if failed_controls:
    print("⚠ drift-check: INSTRUMENT NOT VERIFIED — treat results as unestablished")
    for lbl in failed_controls:
        print(f"    failed control: {lbl}")
    print()

if not findings:
    print(f"✓ governance drift-check: CLAUDE.md clean "
          f"({len(controls)}/{len(controls)} controls passed, {len(paths)} paths verified)")
else:
    print(f"⚑ governance drift-check: {len(findings)} claim(s) in ~/CLAUDE.md "
          f"contradicted by substrate")
    for f in findings:
        print(f"    {f}")
    print("\n  Correction requires [ESCALATE] (Constitutional Constraint #1). "
          "This report is detection only.")

# Register integrity is reported SEPARATELY. Folding it into the count above
# would make that line's own claim false — it says "claim(s) in ~/CLAUDE.md",
# and these are findings about a different file.
if reg_findings:
    print(f"\n⚑ register integrity: {len(reg_findings)} broken amendment link(s) "
          f"in ~/REVIEWED.md")
    for f in reg_findings:
        print(f"    {f}")
    print("\n  An amendment must sit alongside the record it amends, never replace it.")
    print("  Correction requires [ESCALATE] — REVIEWED.md is the steward's hand.")
elif REVIEWED_MD.exists():
    n_am = sum(1 for _, r in RE_HEAD.findall(REVIEWED_MD.read_text(errors="replace"))
               if r.strip().upper().startswith("AMENDMENT"))
    print(f"✓ register integrity: every amendment link resolves "
          f"({n_am} amendment(s) checked)")

# Deferred decisions: a fired trigger is a decision that has come DUE, not a defect.
if deferrals:
    if fired:
        print(f"\n⏰ deferred decisions: {len(fired)} of {len(deferrals)} have COME DUE")
        for d in fired:
            print(f"    {d['slug']} — owner: {d['owner']}, deferred since {d['since']}")
            print(f"       trigger MET: {d['trigger']}")
            print(f"       {d['file'].relative_to(HOME)}")
        print("\n  A deferral is the claim 'not yet'. These triggers say otherwise.")
    else:
        waiting = len(deferrals) - len(manual)
        print(f"✓ deferred decisions: {len(deferrals)} tracked, none due "
              f"({waiting} checkable, {len(manual)} manual-only)")

sys.exit(0)