From da321177e98a24ec078684970edfc3918e73b0e7 Mon Sep 17 00:00:00 2001 From: David F Glidden Date: Sun, 2 Aug 2026 19:10:49 +0200 Subject: [PATCH] [FIX] Degraded guard: deliberation is two cases, not one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Filed in trial 04's tool review, now closed. The guard reported UNTAGGED SCRATCHPAD ... "Do not grade this as the checker's findings" for both of the two situations it can see, and they are opposite: trial 03 — deliberation that ran into the CEILING. No answer ever existed. VOID, and the absence of findings is NOT restraint. trial 04 — deliberation that COMPLETED. The answer follows the scratchpad in the same file. Perfectly gradeable once extracted. NOT void. Collapsing them would have thrown away six good runs; not distinguishing them would have graded trial 03's silence as restraint. The guard now branches on hit_token_ceiling and says which case it is. Controls added for all four shapes, including the two the trials actually produced and a clean answer that merely hit the ceiling — truncation is reported separately and is not a scratchpad problem. The guard does NOT auto-extract the embedded answer. A heuristic split would be a new failure mode in the instrument whose entire job is to not silently mis-report what it has. It flags; a person extracts. --- claude/governance/fool/run_trial.py | 40 ++++++++++++++----- claude/governance/fool/test_degraded_guard.py | 31 ++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/claude/governance/fool/run_trial.py b/claude/governance/fool/run_trial.py index 3338888..be929dc 100644 --- a/claude/governance/fool/run_trial.py +++ b/claude/governance/fool/run_trial.py @@ -321,22 +321,44 @@ def main() -> None: "Do not grade it as restraint." ) - if reasoning is None and UNTAGGED_SCRATCHPAD_RE.match(answer): - problems.append( - "UNTAGGED SCRATCHPAD: the output opens as deliberation about the task, " - "and no tag was emitted, so it was recorded as the ANSWER. " - "reasoning_present=false here means 'no tag was found', NOT 'the model " - "did not deliberate'. Do not grade this as the checker's findings." - ) - ceiling = sampling["max_tokens"] - if generated_tokens is not None and generated_tokens >= ceiling - 2: + hit_ceiling = generated_tokens is not None and generated_tokens >= ceiling - 2 + untagged_scratchpad = reasoning is None and bool(UNTAGGED_SCRATCHPAD_RE.match(answer)) + + if hit_ceiling: problems.append( f"TOKEN CEILING: generation stopped at the max_tokens limit " f"(~{generated_tokens} of {ceiling}). The output is CUT OFF, not " f"complete. Anything absent from it may simply never have been reached." ) + # Trials 03 and 04 are the two halves of this distinction, and the first + # version of the guard collapsed them. Trial 03: untagged deliberation that ran + # into the ceiling, so no answer ever existed and the run was VOID. Trial 04: + # untagged deliberation that COMPLETED, so the answer follows the scratchpad in + # the same file and the run is perfectly gradeable once it is extracted. + # + # Reporting both as "do not grade this" would have thrown away six good runs. + # Reporting both as fine would have graded trial 03's silence as restraint. + if untagged_scratchpad and hit_ceiling: + problems.append( + "VOID — DELIBERATION, THEN TRUNCATION: the output opens as deliberation " + "about the task, no tag was emitted, and generation hit the " + "ceiling before any answer began. NOTHING here is a finding, and the " + "absence of findings is NOT restraint. Re-run with a higher ceiling." + ) + elif untagged_scratchpad: + problems.append( + "ANSWER EMBEDDED: the output opens as deliberation and no tag " + "was emitted, so the scratchpad was recorded as the ANSWER — but " + "generation COMPLETED, so the real answer follows it in the same file. " + "This run is NOT void. Extract the answer before grading, and do not " + "count the deliberation as findings." + ) + + # `think_tag_found: false` says a regex found no tag. It has never said the + # model did not deliberate, and trial 04 is the proof: 2,944 words of it. + degraded: str | None = "\n".join(problems) if problems else None if degraded: print(f"\n*** DEGRADED RUN ***\n{degraded}\n", file=sys.stderr) diff --git a/claude/governance/fool/test_degraded_guard.py b/claude/governance/fool/test_degraded_guard.py index 62cdc59..6763606 100755 --- a/claude/governance/fool/test_degraded_guard.py +++ b/claude/governance/fool/test_degraded_guard.py @@ -88,6 +88,37 @@ check("clean answer not flagged", bool(UNTAGGED_SCRATCHPAD_RE.match(CLEAN_ANSWER for i, text in enumerate(NEAR_MISSES): check(f"near-miss {i}", bool(UNTAGGED_SCRATCHPAD_RE.match(text)), False, repr(text[:50])) +print("\nThe distinction trials 03 and 04 paid for — deliberation is not one case:") +# Replicates the guard's branch logic without importing mlx-dependent code. +def classify(answer: str, reasoning, generated_tokens, ceiling): + untagged = reasoning is None and bool(UNTAGGED_SCRATCHPAD_RE.match(answer)) + hit = generated_tokens is not None and generated_tokens >= ceiling - 2 + if untagged and hit: + return "VOID" + if untagged: + return "EMBEDDED" + return "OK" + +check( + "trial 03 shape (deliberation + ceiling) → VOID", + classify(SYNTHETIC_SCRATCHPAD, None, 4096, 4096), "VOID", + "this is the run where no answer ever existed", +) +check( + "trial 04 shape (deliberation, completed) → EMBEDDED, not void", + classify(SYNTHETIC_SCRATCHPAD, None, 4428, 12000), "EMBEDDED", + "collapsing this into VOID would have discarded six good runs", +) +check( + "clean answer, completed → OK", + classify(CLEAN_ANSWER, None, 900, 12000), "OK", +) +check( + "clean answer that hit the ceiling → not EMBEDDED", + classify(CLEAN_ANSWER, None, 12000, 12000), "OK", + "truncation is reported separately; it is not a scratchpad problem", +) + print("\nTagged output still splits correctly:") r, a = split_reasoning("deliberating\nThe answer.") check("reasoning extracted", r == "deliberating", True)