From b678d2f57b0b308eabffcaa7b48cb3f78cb10166 Mon Sep 17 00:00:00 2001 From: David F Glidden Date: Sun, 2 Aug 2026 16:42:57 +0200 Subject: [PATCH] [FIX] fool harness: record mlx version correctly + self-hash; trial-03 pre-run addendum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two instrument defects, both of the class the harness was built to prevent — a probe that could not look reporting a value that reads like a result: - environment() read mlx.__version__, which does not exist (only mlx.core.__version__). Every run record would have said mlx_version "unknown" for an installed, versioned package, losing the one field that makes trial 03 comparable to trial 02. It is MLX 0.31.2, identical. - git_revision() returns null whenever the harness runs outside its repo, which is always — it must run on the machine holding the model. The prompt and input were hashed; the instrument itself was not. Now self-hashed. The pre-registration addendum is committed BEFORE the run produced output, so the ordering is checkable rather than asserted. It records: the 'unruled' premise expiring at REVIEWED-86 (12:13, 32 min after the pre-registration was written) and why the ordering favours the ground truth; the contamination that CANNOT be removed, since the amended doctrine is in the executor's auto-loaded context and I am therefore not a blind grader; the (a)/anti-echo collision resolved against my own convenience before output existed; and the seed. Ground truth (a)-(e) is unrevised and will not be revised. --- claude/governance/fool/run_trial.py | 60 ++++++++++++++----- .../fool/trial-03-PREREGISTRATION.md | 45 ++++++++++++++ 2 files changed, 91 insertions(+), 14 deletions(-) diff --git a/claude/governance/fool/run_trial.py b/claude/governance/fool/run_trial.py index 06663b9..f3c6fa1 100644 --- a/claude/governance/fool/run_trial.py +++ b/claude/governance/fool/run_trial.py @@ -75,31 +75,62 @@ def read_text(path: Path) -> str: return path.read_text(encoding="utf-8") +def _package_version(dist_name: str, module_name: str, attr_module: str | None = None) -> str: + """ + Resolve a package version without depending on a top-level __version__. + + `mlx` has no `mlx.__version__` — only `mlx.core.__version__` — so the naive + getattr fallback silently records "unknown" for an installed, versioned + package. That is the exact confusion this harness exists to prevent: a probe + that could not look, reporting a value that reads like a result. Installed + metadata is asked first; the module attribute is the fallback; and a genuine + absence is reported as absence, distinguishably. + """ + try: + import importlib.metadata as md + + return md.version(dist_name) + except Exception: + pass + try: + import importlib + + mod = importlib.import_module(attr_module or module_name) + v = getattr(mod, "__version__", None) + if v: + return str(v) + return "INSTALLED, VERSION UNDETERMINED" + except Exception as exc: # pragma: no cover - environment probe + return f"UNAVAILABLE ({exc.__class__.__name__})" + + def environment() -> dict: """Capture enough of the machine to make a later re-run comparable.""" - env = { + return { "host": socket.gethostname(), "user": getpass.getuser(), "platform": platform.platform(), "machine": platform.machine(), "python": sys.version.split()[0], - "mlx_version": None, - "mlx_lm_version": None, + "mlx_version": _package_version("mlx", "mlx", attr_module="mlx.core"), + "mlx_lm_version": _package_version("mlx-lm", "mlx_lm"), } - try: - import mlx.core # noqa: F401 - import mlx - env["mlx_version"] = getattr(mlx, "__version__", "unknown") - except Exception as exc: # pragma: no cover - environment probe - env["mlx_version"] = f"UNAVAILABLE ({exc.__class__.__name__})" - try: - import mlx_lm - env["mlx_lm_version"] = getattr(mlx_lm, "__version__", "unknown") +def harness_sha256() -> str: + """ + Hash this file into every run record. + + `git_revision()` returns None whenever the harness is executed outside its + repository — which is the normal case, because it must run on the machine + holding the model. A run record whose harness field is null cannot be traced + to the code that produced it. The prompt and input are already hashed; the + instrument itself was not. + """ + try: + return sha256(Path(__file__).resolve().read_text(encoding="utf-8")) except Exception as exc: # pragma: no cover - env["mlx_lm_version"] = f"UNAVAILABLE ({exc.__class__.__name__})" - return env + return f"UNAVAILABLE ({exc.__class__.__name__})" def git_revision() -> str | None: @@ -282,6 +313,7 @@ def main() -> None: }, "environment": environment(), "harness_git_rev": git_revision(), + "harness_sha256": harness_sha256(), } (RUNS_DIR / f"{slug}.json").write_text( diff --git a/claude/governance/fool/trial-03-PREREGISTRATION.md b/claude/governance/fool/trial-03-PREREGISTRATION.md index 4e735bb..a93af82 100644 --- a/claude/governance/fool/trial-03-PREREGISTRATION.md +++ b/claude/governance/fool/trial-03-PREREGISTRATION.md @@ -71,6 +71,51 @@ And per the standing caveat: the grade below will be assigned by the executor, w errors are among those being graded. The findings will be individually checkable; the grade will not be independent. +## Addendum, written DURING the run and BEFORE any output was seen + +*(Run launched 2026-08-02 ~13:0x; model still loading; the output file was empty when +each item below was written. Recorded here rather than in the write-up precisely because +its whole value is that it precedes the result.)* + +**1. The "unruled" premise above expired 32 minutes after it was written.** It was true at +11:41. At **12:13** the steward placed **REVIEWED-86**, design-gating this doctrine with two +required conditions. The ordering is the part that matters and it is favourable: ground truth +(a)–(e) was fixed **before** the jurist ruled, so the jurist's conditions are an *independent* +check on the ground-truth list rather than a source of it. (a)–(e) will **not** be revised. +Grading order is therefore fixed: **grade against (a)–(e) first and write it down, then read +REVIEWED-86 and compare.** + +**Contamination that cannot be removed, stated plainly:** the *amended* doctrine text now sits +in `~/CLAUDE.md`, which is loaded into the executor's context automatically. I have therefore +already seen the jurist's conditions in their applied form. I am not a blind grader and must +not be described as one. What survives clean is the timestamp on (a)–(e), which is checkable +in git. + +**2. Ground-truth item (a) collides with the anti-echo clause, and the collision must be +resolved now rather than conveniently.** Part VII §1 of the document already states *"The +doctrine's own second consequence is the sharpest argument against the arrangement that +produced it."* So the first half of (a) is **author-named** and, per the prompt's own +instruction, is **not a finding**. Ruling, fixed before output: + +- Reporting that consequence 2 indicts the jurist–executor pair → **ECHO**, not a hit. +- Reporting that the package **never resolves what should change** as a result — that it + proposes the doctrine anyway and leaves its own configuration in place — → **HIT on (a)**. +- This makes (a) *harder* to score than the other four, not easier. Recorded because the + temptation afterwards will run the other way. + +**3. Seed declared before the run: `20260802`.** The pre-registration left the seed +unspecified, which would have meant `None` — non-deterministic, and a reproduction of the exact +irreproducibility this harness exists to end. Fixing a seed changes nothing pre-registered +(input, prompt, absence of steer, thinking ON). + +**4. Two harness edits made before the run, both environment-recording only, neither touching +the trial design.** `mlx.__version__` does not exist — only `mlx.core.__version__` — so the +probe would have recorded `"unknown"` for an installed, versioned package, losing the single +field that makes trial 03 comparable to trial 02. It is **MLX 0.31.2, identical to trial 02**. +Second: the harness now hashes *itself* into the run record, because `git_revision()` returns +null whenever the harness runs outside its repository — which is always, since it must run on +the machine holding the model. + ## Result *(To be filled after the run. Empty until then — deliberately.)*