David F Glidden
da321177e9
Filed in trial 04's tool review, now closed. The guard reported UNTAGGED
SCRATCHPAD ... "Do not grade this as the checker's findings" for both of the two
situations it can see, and they are opposite:
trial 03 — deliberation that ran into the CEILING. No answer ever existed. VOID,
and the absence of findings is NOT restraint.
trial 04 — deliberation that COMPLETED. The answer follows the scratchpad in the
same file. Perfectly gradeable once extracted. NOT void.
Collapsing them would have thrown away six good runs; not distinguishing them
would have graded trial 03's silence as restraint. The guard now branches on
hit_token_ceiling and says which case it is.
Controls added for all four shapes, including the two the trials actually
produced and a clean answer that merely hit the ceiling — truncation is reported
separately and is not a scratchpad problem.
The guard does NOT auto-extract the embedded answer. A heuristic split would be a
new failure mode in the instrument whose entire job is to not silently mis-report
what it has. It flags; a person extracts.
134 lines
5.1 KiB
Python
Executable File
134 lines
5.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Positive control for the degraded-run guard.
|
|
|
|
An absence is not evidence until the instrument is shown capable of detecting
|
|
presence. The old guard PASSED trial 03 — 2,944 words of untagged deliberation,
|
|
truncated at the token ceiling, recorded as `degraded: null`. So the test is not
|
|
"does the new guard run"; it is "does the new guard catch THE ACTUAL OUTPUT that
|
|
defeated the old one", and does it stay quiet on output that is genuinely fine.
|
|
|
|
Runs anywhere — imports no mlx. Usage: ./test_degraded_guard.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from run_trial import UNTAGGED_SCRATCHPAD_RE, split_reasoning # noqa: E402
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
TRIAL_03 = HERE / "runs" / "trial-03-20260802T144136Z.answer.md"
|
|
|
|
# A real answer to this prompt. Must NOT trip the guard — a guard that fires on
|
|
# everything detects nothing.
|
|
CLEAN_ANSWER = """**Assumption:** The separation-of-powers analogy maps validly.
|
|
|
|
**Where relied upon:** Part III asserts it as a historical premise.
|
|
|
|
**What must be true:** That the conditions enabling checks in human political
|
|
systems are present in this configuration."""
|
|
|
|
# The failure mode in miniature, for when the trial-03 artefact is not present.
|
|
SYNTHETIC_SCRATCHPAD = """Here's a thinking process:
|
|
|
|
1. **Analyze User Input:** The task is to identify claims the document relies on."""
|
|
|
|
# Near-misses that must stay quiet: deliberation words appearing in a real answer.
|
|
NEAR_MISSES = [
|
|
"The document's reasoning about correlated misses is never demonstrated.",
|
|
# Caught a real false positive in the first version of the guard: a bare
|
|
# `okay` plus any deliberation word within 80 characters.
|
|
"Okay is not a word this document uses, but its approach to falsification is.",
|
|
"**Assumption 1:** the author's thinking process is treated as transparent.",
|
|
"I will not restate what Part VII already names as its own limitation.",
|
|
"Here's the assumption the argument needs: that the analogy holds.",
|
|
]
|
|
|
|
failures: list[str] = []
|
|
|
|
|
|
def check(name: str, got: bool, want: bool, detail: str = "") -> None:
|
|
if got != want:
|
|
failures.append(f"{name}: expected {want}, got {got}. {detail}")
|
|
print(f" FAIL {name}")
|
|
else:
|
|
print(f" ok {name}")
|
|
|
|
|
|
print("Positive control — the artefact that defeated the old guard:")
|
|
if TRIAL_03.is_file():
|
|
text = TRIAL_03.read_text(encoding="utf-8")
|
|
reasoning, answer = split_reasoning(text)
|
|
check("trial-03: no <think> tag found", reasoning is None, True)
|
|
check(
|
|
"trial-03: untagged scratchpad DETECTED",
|
|
bool(UNTAGGED_SCRATCHPAD_RE.match(answer)),
|
|
True,
|
|
"This is the exact output the old guard passed as degraded:null.",
|
|
)
|
|
else:
|
|
print(f" SKIP {TRIAL_03.name} not present — running synthetic only")
|
|
failures.append(
|
|
"trial-03 artefact absent: the positive control did not run against real "
|
|
"output. Treat the guard as UNVERIFIED against the case it was built for."
|
|
)
|
|
|
|
print("\nSynthetic scratchpad:")
|
|
check(
|
|
"synthetic scratchpad detected",
|
|
bool(UNTAGGED_SCRATCHPAD_RE.match(SYNTHETIC_SCRATCHPAD)),
|
|
True,
|
|
)
|
|
|
|
print("\nNegative controls — must stay quiet:")
|
|
check("clean answer not flagged", bool(UNTAGGED_SCRATCHPAD_RE.match(CLEAN_ANSWER)), False)
|
|
for i, text in enumerate(NEAR_MISSES):
|
|
check(f"near-miss {i}", bool(UNTAGGED_SCRATCHPAD_RE.match(text)), False, repr(text[:50]))
|
|
|
|
print("\nThe distinction trials 03 and 04 paid for — deliberation is not one case:")
|
|
# Replicates the guard's branch logic without importing mlx-dependent code.
|
|
def classify(answer: str, reasoning, generated_tokens, ceiling):
|
|
untagged = reasoning is None and bool(UNTAGGED_SCRATCHPAD_RE.match(answer))
|
|
hit = generated_tokens is not None and generated_tokens >= ceiling - 2
|
|
if untagged and hit:
|
|
return "VOID"
|
|
if untagged:
|
|
return "EMBEDDED"
|
|
return "OK"
|
|
|
|
check(
|
|
"trial 03 shape (deliberation + ceiling) → VOID",
|
|
classify(SYNTHETIC_SCRATCHPAD, None, 4096, 4096), "VOID",
|
|
"this is the run where no answer ever existed",
|
|
)
|
|
check(
|
|
"trial 04 shape (deliberation, completed) → EMBEDDED, not void",
|
|
classify(SYNTHETIC_SCRATCHPAD, None, 4428, 12000), "EMBEDDED",
|
|
"collapsing this into VOID would have discarded six good runs",
|
|
)
|
|
check(
|
|
"clean answer, completed → OK",
|
|
classify(CLEAN_ANSWER, None, 900, 12000), "OK",
|
|
)
|
|
check(
|
|
"clean answer that hit the ceiling → not EMBEDDED",
|
|
classify(CLEAN_ANSWER, None, 12000, 12000), "OK",
|
|
"truncation is reported separately; it is not a scratchpad problem",
|
|
)
|
|
|
|
print("\nTagged output still splits correctly:")
|
|
r, a = split_reasoning("<think>deliberating</think>\nThe answer.")
|
|
check("reasoning extracted", r == "deliberating", True)
|
|
check("answer extracted", a == "The answer.", True)
|
|
|
|
if failures:
|
|
print(f"\nINSTRUMENT NOT VERIFIED — {len(failures)} failure(s):")
|
|
for f in failures:
|
|
print(f" - {f}")
|
|
sys.exit(1)
|
|
|
|
print("\nAll checks passed. The guard catches the case that defeated its predecessor.")
|