diff --git a/claude/governance/fool/test_discrimination.py b/claude/governance/fool/test_discrimination.py new file mode 100755 index 0000000..812057a --- /dev/null +++ b/claude/governance/fool/test_discrimination.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +""" +Discrimination gate — the answer to a class, not to an instance. + +THE CLASS + Four times in three days a passing check certified a property of the CODE + while claiming a property of the RESULT, and every one was found by a person + looking at the output rather than by any check: + + · vignette — a field colour bound to a class no element carried + · trial 03 — `degraded: null` on a run that produced no answer at all + · splitter — five defects, all found by contact with a real document + · §3.3 — a FALSE PASS on a package whose Part VII is a collected + limitations section + + Each check tested a predicate NECESSARY but not SUFFICIENT for the property. + Quotes-present is a subset of inference-survives. Answer-non-empty is a subset + of answer-produced. No-heading-says-"limitations" is a subset of no-collected- + limitations-section. + +WHY THE EXISTING POSITIVE CONTROLS DID NOT CATCH IT + Because the fixtures were derived from the CHECK ("what makes this regex + fail?") instead of from the PROPERTY ("what makes this claim false?"). A + control built out of the check's own vocabulary inherits its blind spot by + construction — the same shape as the recorded drift-pattern that a control + built by EXTRACTION leaks by construction. + +THE GATE + A check must return DIFFERENT verdicts on two REAL artifacts, one known to + have the property and one known to lack it. Same verdict on both means the + check has discriminated nothing, however many synthetic fixtures it passes. + + Real artifacts, not synthetic — a synthetic negative is written by the same + hand as the check and shares its assumptions. + +Usage: ./test_discrimination.py +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from reduce import FORBIDDEN_HEADING_RE, TAGGABLE, split_spans # noqa: E402 + +GOV = Path(__file__).resolve().parent.parent + +# Two real governance documents, and the ground truth about them established by +# READING, not by running the check. +# +# RULING — has no section collecting the author's own caveats. Its +# qualifications are woven into the determinations themselves. +# PACKAGE — Part VII, "Disconfirming evidence, which the steward specifically +# asked to be carried", collects them. Trial 03 is the evidence that +# this matters: the model located that section, classified it as +# author-named limitation, and skipped its contents wholesale. +RULING = GOV / "skill-harvest-fix-lane-JURIST-RULING-2026-08-01.md" +PACKAGE = GOV / "differently-biased-checkers-JURIST-PACKAGE-2026-08-01.md" + +# The pattern as it shipped, before Reduction 02. Kept verbatim so the gate can be +# shown catching a defect that really occurred, rather than one invented for it. +PATTERN_BEFORE_FIX = re.compile( + r"limitation|caveat|assumption|what this does not|open question", re.IGNORECASE +) + +failures: list[str] = [] + + +def flags_collected_section(pattern: re.Pattern, doc: Path) -> bool: + """Run the §3.3 screen over a document's headings.""" + text = doc.read_text(encoding="utf-8") + for sp in split_spans(text): + if sp["kind"] == "heading" and sp["kind"] in TAGGABLE: + if pattern.search(text[sp["start"]:sp["end"]]): + return True + return False + + +def discriminates(pattern: re.Pattern, label: str) -> bool: + has = flags_collected_section(pattern, PACKAGE) # ground truth: HAS one + lacks = flags_collected_section(pattern, RULING) # ground truth: LACKS one + verdict = has and not lacks + print(f" {label}") + print(f" package (has a collected section) → flagged={has}") + print(f" ruling (has none) → flagged={lacks}") + print(f" discriminates: {verdict}") + return verdict + + +for p in (RULING, PACKAGE): + if not p.is_file(): + failures.append(f"real artifact missing: {p.name} — the gate did not run") + +if not failures: + print("§3.3 — collected-limitations screen, against two real documents:\n") + + # 1. The gate must FAIL the pattern that actually shipped. If it passes this, + # the gate cannot catch the defect it was built for and is worthless. + if discriminates(PATTERN_BEFORE_FIX, "as shipped, before Reduction 02:"): + failures.append( + "THE GATE IS BLIND: the pre-fix pattern discriminated, but it is known " + "to have FALSE-PASSED the package. The gate proves nothing." + ) + else: + print(" → correctly REJECTED. The gate catches the real defect.\n") + + # 2. The current pattern must pass. + if not discriminates(FORBIDDEN_HEADING_RE, "current:"): + failures.append("current §3.3 pattern does not discriminate on real documents") + else: + print(" → accepted.\n") + + # 3. And the limit is stated rather than implied: a section titled only + # "Part VII" defeats any wordlist. Passing this gate is not a §2a verdict. + bare = re.compile(r"^## Part VII$", re.MULTILINE) + print(" residue, stated: a heading naming no topic defeats every wordlist —") + print(f" '## Part VII' matched by the current screen: " + f"{bool(FORBIDDEN_HEADING_RE.search('## Part VII'))}") + print(" so §3.3 is a SCREEN. §2a compliance stays in Kernel §4 judgement.") + assert bare # keep the illustration honest about what it is + +if failures: + print(f"\nDISCRIMINATION GATE FAILED — {len(failures)}:") + for f in failures: + print(f" - {f}") + sys.exit(1) + +print("\nDiscrimination gate passed, and shown rejecting the pattern that shipped.")