[FIX] fool harness: record mlx version correctly + self-hash; trial-03 pre-run addendum
Two instrument defects, both of the class the harness was built to prevent — a probe that could not look reporting a value that reads like a result: - environment() read mlx.__version__, which does not exist (only mlx.core.__version__). Every run record would have said mlx_version "unknown" for an installed, versioned package, losing the one field that makes trial 03 comparable to trial 02. It is MLX 0.31.2, identical. - git_revision() returns null whenever the harness runs outside its repo, which is always — it must run on the machine holding the model. The prompt and input were hashed; the instrument itself was not. Now self-hashed. The pre-registration addendum is committed BEFORE the run produced output, so the ordering is checkable rather than asserted. It records: the 'unruled' premise expiring at REVIEWED-86 (12:13, 32 min after the pre-registration was written) and why the ordering favours the ground truth; the contamination that CANNOT be removed, since the amended doctrine is in the executor's auto-loaded context and I am therefore not a blind grader; the (a)/anti-echo collision resolved against my own convenience before output existed; and the seed. Ground truth (a)-(e) is unrevised and will not be revised.
This commit is contained in:
@@ -75,31 +75,62 @@ def read_text(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _package_version(dist_name: str, module_name: str, attr_module: str | None = None) -> str:
|
||||
"""
|
||||
Resolve a package version without depending on a top-level __version__.
|
||||
|
||||
`mlx` has no `mlx.__version__` — only `mlx.core.__version__` — so the naive
|
||||
getattr fallback silently records "unknown" for an installed, versioned
|
||||
package. That is the exact confusion this harness exists to prevent: a probe
|
||||
that could not look, reporting a value that reads like a result. Installed
|
||||
metadata is asked first; the module attribute is the fallback; and a genuine
|
||||
absence is reported as absence, distinguishably.
|
||||
"""
|
||||
try:
|
||||
import importlib.metadata as md
|
||||
|
||||
return md.version(dist_name)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
import importlib
|
||||
|
||||
mod = importlib.import_module(attr_module or module_name)
|
||||
v = getattr(mod, "__version__", None)
|
||||
if v:
|
||||
return str(v)
|
||||
return "INSTALLED, VERSION UNDETERMINED"
|
||||
except Exception as exc: # pragma: no cover - environment probe
|
||||
return f"UNAVAILABLE ({exc.__class__.__name__})"
|
||||
|
||||
|
||||
def environment() -> dict:
|
||||
"""Capture enough of the machine to make a later re-run comparable."""
|
||||
env = {
|
||||
return {
|
||||
"host": socket.gethostname(),
|
||||
"user": getpass.getuser(),
|
||||
"platform": platform.platform(),
|
||||
"machine": platform.machine(),
|
||||
"python": sys.version.split()[0],
|
||||
"mlx_version": None,
|
||||
"mlx_lm_version": None,
|
||||
"mlx_version": _package_version("mlx", "mlx", attr_module="mlx.core"),
|
||||
"mlx_lm_version": _package_version("mlx-lm", "mlx_lm"),
|
||||
}
|
||||
try:
|
||||
import mlx.core # noqa: F401
|
||||
import mlx
|
||||
|
||||
env["mlx_version"] = getattr(mlx, "__version__", "unknown")
|
||||
except Exception as exc: # pragma: no cover - environment probe
|
||||
env["mlx_version"] = f"UNAVAILABLE ({exc.__class__.__name__})"
|
||||
try:
|
||||
import mlx_lm
|
||||
|
||||
env["mlx_lm_version"] = getattr(mlx_lm, "__version__", "unknown")
|
||||
def harness_sha256() -> str:
|
||||
"""
|
||||
Hash this file into every run record.
|
||||
|
||||
`git_revision()` returns None whenever the harness is executed outside its
|
||||
repository — which is the normal case, because it must run on the machine
|
||||
holding the model. A run record whose harness field is null cannot be traced
|
||||
to the code that produced it. The prompt and input are already hashed; the
|
||||
instrument itself was not.
|
||||
"""
|
||||
try:
|
||||
return sha256(Path(__file__).resolve().read_text(encoding="utf-8"))
|
||||
except Exception as exc: # pragma: no cover
|
||||
env["mlx_lm_version"] = f"UNAVAILABLE ({exc.__class__.__name__})"
|
||||
return env
|
||||
return f"UNAVAILABLE ({exc.__class__.__name__})"
|
||||
|
||||
|
||||
def git_revision() -> str | None:
|
||||
@@ -282,6 +313,7 @@ def main() -> None:
|
||||
},
|
||||
"environment": environment(),
|
||||
"harness_git_rev": git_revision(),
|
||||
"harness_sha256": harness_sha256(),
|
||||
}
|
||||
|
||||
(RUNS_DIR / f"{slug}.json").write_text(
|
||||
|
||||
Reference in New Issue
Block a user