diff --git a/claude/governance/fool/run_trial.py b/claude/governance/fool/run_trial.py index 3338888..be929dc 100644 --- a/claude/governance/fool/run_trial.py +++ b/claude/governance/fool/run_trial.py @@ -321,22 +321,44 @@ def main() -> None: "Do not grade it as restraint." ) - if reasoning is None and UNTAGGED_SCRATCHPAD_RE.match(answer): - problems.append( - "UNTAGGED SCRATCHPAD: the output opens as deliberation about the task, " - "and no tag was emitted, so it was recorded as the ANSWER. " - "reasoning_present=false here means 'no tag was found', NOT 'the model " - "did not deliberate'. Do not grade this as the checker's findings." - ) - ceiling = sampling["max_tokens"] - if generated_tokens is not None and generated_tokens >= ceiling - 2: + hit_ceiling = generated_tokens is not None and generated_tokens >= ceiling - 2 + untagged_scratchpad = reasoning is None and bool(UNTAGGED_SCRATCHPAD_RE.match(answer)) + + if hit_ceiling: problems.append( f"TOKEN CEILING: generation stopped at the max_tokens limit " f"(~{generated_tokens} of {ceiling}). The output is CUT OFF, not " f"complete. Anything absent from it may simply never have been reached." ) + # Trials 03 and 04 are the two halves of this distinction, and the first + # version of the guard collapsed them. Trial 03: untagged deliberation that ran + # into the ceiling, so no answer ever existed and the run was VOID. Trial 04: + # untagged deliberation that COMPLETED, so the answer follows the scratchpad in + # the same file and the run is perfectly gradeable once it is extracted. + # + # Reporting both as "do not grade this" would have thrown away six good runs. + # Reporting both as fine would have graded trial 03's silence as restraint. + if untagged_scratchpad and hit_ceiling: + problems.append( + "VOID — DELIBERATION, THEN TRUNCATION: the output opens as deliberation " + "about the task, no tag was emitted, and generation hit the " + "ceiling before any answer began. NOTHING here is a finding, and the " + "absence of findings is NOT restraint. Re-run with a higher ceiling." + ) + elif untagged_scratchpad: + problems.append( + "ANSWER EMBEDDED: the output opens as deliberation and no tag " + "was emitted, so the scratchpad was recorded as the ANSWER — but " + "generation COMPLETED, so the real answer follows it in the same file. " + "This run is NOT void. Extract the answer before grading, and do not " + "count the deliberation as findings." + ) + + # `think_tag_found: false` says a regex found no tag. It has never said the + # model did not deliberate, and trial 04 is the proof: 2,944 words of it. + degraded: str | None = "\n".join(problems) if problems else None if degraded: print(f"\n*** DEGRADED RUN ***\n{degraded}\n", file=sys.stderr) diff --git a/claude/governance/fool/test_degraded_guard.py b/claude/governance/fool/test_degraded_guard.py index 62cdc59..6763606 100755 --- a/claude/governance/fool/test_degraded_guard.py +++ b/claude/governance/fool/test_degraded_guard.py @@ -88,6 +88,37 @@ check("clean answer not flagged", bool(UNTAGGED_SCRATCHPAD_RE.match(CLEAN_ANSWER for i, text in enumerate(NEAR_MISSES): check(f"near-miss {i}", bool(UNTAGGED_SCRATCHPAD_RE.match(text)), False, repr(text[:50])) +print("\nThe distinction trials 03 and 04 paid for — deliberation is not one case:") +# Replicates the guard's branch logic without importing mlx-dependent code. +def classify(answer: str, reasoning, generated_tokens, ceiling): + untagged = reasoning is None and bool(UNTAGGED_SCRATCHPAD_RE.match(answer)) + hit = generated_tokens is not None and generated_tokens >= ceiling - 2 + if untagged and hit: + return "VOID" + if untagged: + return "EMBEDDED" + return "OK" + +check( + "trial 03 shape (deliberation + ceiling) → VOID", + classify(SYNTHETIC_SCRATCHPAD, None, 4096, 4096), "VOID", + "this is the run where no answer ever existed", +) +check( + "trial 04 shape (deliberation, completed) → EMBEDDED, not void", + classify(SYNTHETIC_SCRATCHPAD, None, 4428, 12000), "EMBEDDED", + "collapsing this into VOID would have discarded six good runs", +) +check( + "clean answer, completed → OK", + classify(CLEAN_ANSWER, None, 900, 12000), "OK", +) +check( + "clean answer that hit the ceiling → not EMBEDDED", + classify(CLEAN_ANSWER, None, 12000, 12000), "OK", + "truncation is reported separately; it is not a scratchpad problem", +) + print("\nTagged output still splits correctly:") r, a = split_reasoning("deliberating\nThe answer.") check("reasoning extracted", r == "deliberating", True)