David F Glidden
eda11e559b
Trial 03 ran and produced nothing gradeable. Recorded as VOID rather than
omitted, because an absent row reads as a trial not attempted.
Two independent failures, both found by reading the output, neither by a check,
and every check passed:
1. The harness certified a run with no answer. Qwen emitted its scratchpad as
plain prose ('Here's a thinking process:', zero <think> tags), so the tag
regex reported reasoning_present:false and recorded all 2,944 words of
deliberation as the ANSWER; the token ceiling then cut it off mid-sentence
before the answer began. degraded:null. The guard tested the STRING for
emptiness while its field claimed a property of the RESULT — which is the
previous session's open question, answered by the instrument built to audit
instruments. Trial 02 had listed the inline-scratchpad problem as Open; the
harness closed it assuming inline meant tagged.
2. Worse: the design forbade the region it was measuring. The self-exemption
axis lives in Part VII; the anti-echo constraint added in trial 02 tells the
reader to skip author-named limitations, and the scratchpad shows the model
reaching Part VII and leaving it, citing that constraint. Silence about
self-reference is indistinguishable from obedience. The axis was unmeasurable
by construction, independent of the truncation. Trial 02's fix and trial 03's
document were each sound alone; their interaction was not.
Guard now reports every degradation, not the first: empty answer, untagged
scratchpad, and token-ceiling truncation. reasoning_present renamed
think_tag_found — it was a claim about a regex wearing the name of a claim about
the model. test_degraded_guard.py is a positive control that runs against the
actual trial-03 artefact, not a synthetic one; it caught a false positive in the
first version of my own guard (a bare 'okay' matched a legitimate sentence).
The false-positive control STILL has never been run. Two attempts, two unrelated
causes — the obstacle is the instrument and the design, not the model.
103 lines
3.9 KiB
Python
Executable File
103 lines
3.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Positive control for the degraded-run guard.
|
|
|
|
An absence is not evidence until the instrument is shown capable of detecting
|
|
presence. The old guard PASSED trial 03 — 2,944 words of untagged deliberation,
|
|
truncated at the token ceiling, recorded as `degraded: null`. So the test is not
|
|
"does the new guard run"; it is "does the new guard catch THE ACTUAL OUTPUT that
|
|
defeated the old one", and does it stay quiet on output that is genuinely fine.
|
|
|
|
Runs anywhere — imports no mlx. Usage: ./test_degraded_guard.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from run_trial import UNTAGGED_SCRATCHPAD_RE, split_reasoning # noqa: E402
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
TRIAL_03 = HERE / "runs" / "trial-03-20260802T144136Z.answer.md"
|
|
|
|
# A real answer to this prompt. Must NOT trip the guard — a guard that fires on
|
|
# everything detects nothing.
|
|
CLEAN_ANSWER = """**Assumption:** The separation-of-powers analogy maps validly.
|
|
|
|
**Where relied upon:** Part III asserts it as a historical premise.
|
|
|
|
**What must be true:** That the conditions enabling checks in human political
|
|
systems are present in this configuration."""
|
|
|
|
# The failure mode in miniature, for when the trial-03 artefact is not present.
|
|
SYNTHETIC_SCRATCHPAD = """Here's a thinking process:
|
|
|
|
1. **Analyze User Input:** The task is to identify claims the document relies on."""
|
|
|
|
# Near-misses that must stay quiet: deliberation words appearing in a real answer.
|
|
NEAR_MISSES = [
|
|
"The document's reasoning about correlated misses is never demonstrated.",
|
|
# Caught a real false positive in the first version of the guard: a bare
|
|
# `okay` plus any deliberation word within 80 characters.
|
|
"Okay is not a word this document uses, but its approach to falsification is.",
|
|
"**Assumption 1:** the author's thinking process is treated as transparent.",
|
|
"I will not restate what Part VII already names as its own limitation.",
|
|
"Here's the assumption the argument needs: that the analogy holds.",
|
|
]
|
|
|
|
failures: list[str] = []
|
|
|
|
|
|
def check(name: str, got: bool, want: bool, detail: str = "") -> None:
|
|
if got != want:
|
|
failures.append(f"{name}: expected {want}, got {got}. {detail}")
|
|
print(f" FAIL {name}")
|
|
else:
|
|
print(f" ok {name}")
|
|
|
|
|
|
print("Positive control — the artefact that defeated the old guard:")
|
|
if TRIAL_03.is_file():
|
|
text = TRIAL_03.read_text(encoding="utf-8")
|
|
reasoning, answer = split_reasoning(text)
|
|
check("trial-03: no <think> tag found", reasoning is None, True)
|
|
check(
|
|
"trial-03: untagged scratchpad DETECTED",
|
|
bool(UNTAGGED_SCRATCHPAD_RE.match(answer)),
|
|
True,
|
|
"This is the exact output the old guard passed as degraded:null.",
|
|
)
|
|
else:
|
|
print(f" SKIP {TRIAL_03.name} not present — running synthetic only")
|
|
failures.append(
|
|
"trial-03 artefact absent: the positive control did not run against real "
|
|
"output. Treat the guard as UNVERIFIED against the case it was built for."
|
|
)
|
|
|
|
print("\nSynthetic scratchpad:")
|
|
check(
|
|
"synthetic scratchpad detected",
|
|
bool(UNTAGGED_SCRATCHPAD_RE.match(SYNTHETIC_SCRATCHPAD)),
|
|
True,
|
|
)
|
|
|
|
print("\nNegative controls — must stay quiet:")
|
|
check("clean answer not flagged", bool(UNTAGGED_SCRATCHPAD_RE.match(CLEAN_ANSWER)), False)
|
|
for i, text in enumerate(NEAR_MISSES):
|
|
check(f"near-miss {i}", bool(UNTAGGED_SCRATCHPAD_RE.match(text)), False, repr(text[:50]))
|
|
|
|
print("\nTagged output still splits correctly:")
|
|
r, a = split_reasoning("<think>deliberating</think>\nThe answer.")
|
|
check("reasoning extracted", r == "deliberating", True)
|
|
check("answer extracted", a == "The answer.", True)
|
|
|
|
if failures:
|
|
print(f"\nINSTRUMENT NOT VERIFIED — {len(failures)} failure(s):")
|
|
for f in failures:
|
|
print(f" - {f}")
|
|
sys.exit(1)
|
|
|
|
print("\nAll checks passed. The guard catches the case that defeated its predecessor.")
|