{ "lab_artifact_version": 1, "run_id": "article-to-verdict-association__keyed-v2__bcde0d5383b5__008af826__interrupted", "experiment_id": "article-to-verdict-association", "candidate": { "candidate_id": "keyed-v2", "kind": "preserved-version", "description": "Proposed contract: every verdict names its article, the id set must match exactly, and a non-stop finish_reason is a failure.", "declared_protocol": "keyed-v2", "source_path": "backend/lab/contract/versions/keyed_v2.py", "source_sha256": "bcde0d5383b54e71df8341cc58cd5b9d5cd505b402333f58f51bd21822b35641", "source_bytes": 6390, "transcribed_from": "modelled on backend/app/services/ranking_contract.py:131", "patch": "diff --git a/backend/lab/contract/versions/positional_v0.py b/backend/lab/contract/versions/keyed_v2.py\n--- a/backend/lab/contract/versions/positional_v0.py\n+++ b/backend/lab/contract/versions/keyed_v2.py\n@@ -1,18 +1,26 @@\n-\"\"\"Historical behaviour, transcribed from `origin/main`.\n+\"\"\"The proposed contract: every verdict identifies its own article.\n \n-Source: backend/app/services/openai_service.py, score_articles_batch, the\n-`normalized` loop. Verbatim semantics:\n+Modelled on the id-keyed validation Daily already ships in\n+`backend/app/services/ranking_contract.py:131` (`validate_judgments`), which is\n+the pattern this repository already trusts for the S7 ranking path:\n \n- if len(results_list) != len(articles):\n- logger.warning(\"... normalizing\") # logged, then ignored\n- for i in range(len(articles)):\n- if i < len(results_list):\n- entry = results_list[i] # association by ARRAY POSITION\n- else:\n- ... {\"relevant\": False, \"score\": 0.0, \"reason\": \"scoring incomplete\"}\n+ packs = {e.article_id: e for e in evidence}\n+ ids = [v.article_id for v in values]\n+ if len(ids) != len(set(ids)) or set(ids) != set(packs):\n+ raise ValueError('ranker must return exact article ID set')\n \n-This version is preserved so the experiment can measure the defect rather than\n-describe it. It is not a control: it is what production does today.\n+One set-equality check covers duplicate, unknown and missing ids at once, and\n+refuses to salvage a partial answer. The same shape is reproduced here, plus\n+the truncation check `ranking_provider.py:296` performs and\n+`score_articles_batch` does not:\n+\n+ if choice.get('finish_reason') != 'stop':\n+ raise ProviderFailure('incomplete_response')\n+\n+What this version does NOT claim: that it ranks better. Association is a\n+correctness property, and correctness is all that is under test. Relevance\n+quality under this protocol is unmeasured — it needs recordings that do not\n+exist, because no budgeted keyed run has been made.\n \"\"\"\n \n from __future__ import annotations\n@@ -54,45 +62,88 @@ import json\n from typing import Any\n \n \n-VERSION_ID = \"positional-v0\"\n-PROTOCOL = \"positional-v0\"\n+VERSION_ID = \"keyed-v2\"\n+PROTOCOL = \"keyed-v2\"\n \n \n def parse(articles: list[dict[str, Any]], response: dict[str, Any]) -> dict[str, Any]:\n- if response.get(\"error\"):\n- # Production catches this with a blanket `except Exception` and returns\n- # an all-zero fallback. Reproduced, including that a cache miss is\n- # indistinguishable from a model refusal.\n- return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n+ error = response.get(\"error\")\n+ if error:\n+ if error == \"no_recording\":\n+ return refuse(\"no_recording\", \"no recorded response for this request\")\n+ if error in {\"timeout\", \"cancelled\"}:\n+ return refuse(error, error)\n+ if error == \"budget_exceeded\":\n+ return refuse(\"retries_exhausted\", error)\n+ return refuse(\"retries_exhausted\", str(error))\n+\n+ # A truncated completion is a failure, not a shorter answer. Production\n+ # never reads finish_reason, which is why 33 recorded responses that stopped\n+ # at the output ceiling were indistinguishable from malformed ones.\n+ finish_reason = response.get(\"finish_reason\")\n+ if finish_reason is not None and finish_reason != \"stop\":\n+ return refuse(\"truncated_response\", f\"finish_reason={finish_reason}\")\n \n content = response.get(\"content\")\n if content is None:\n- return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n+ return refuse(\"retries_exhausted\", \"no content returned\")\n \n try:\n result = json.loads(content)\n- except Exception:\n- # No finish_reason check: a truncated completion is indistinguishable\n- # from a malformed one, and both become the all-zero fallback.\n- return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n-\n- results_list = result.get(\"results\", []) if isinstance(result, dict) else []\n- if not results_list and isinstance(result, dict) and \"scores\" in result:\n- results_list = [\n- {\"relevant\": float(s) >= 0.5, \"score\": float(s), \"reason\": \"\"}\n- for s in result[\"scores\"]\n- ]\n+ except Exception as exc:\n+ return refuse(\"malformed_json\", str(exc))\n+\n+ if not isinstance(result, dict) or not isinstance(result.get(\"results\"), list):\n+ return refuse(\"unexpected_shape\", \"expected an object with a results array\")\n+\n+ entries = result[\"results\"]\n+ expected = {a[\"id\"]: a for a in articles}\n+\n+ seen: list[str] = []\n+ for entry in entries:\n+ if not isinstance(entry, dict):\n+ return refuse(\"invalid_type\", f\"entry is {type(entry).__name__}\")\n+ article_id = entry.get(\"article_id\", entry.get(\"id\"))\n+ if not isinstance(article_id, str):\n+ return refuse(\"invalid_type\", \"article_id missing or not a string\")\n+ seen.append(article_id)\n+\n+ # Duplicate, unknown and missing are distinguished for the operator even\n+ # though any one of them is fatal. `validate_judgments` collapses all three\n+ # into one message; the Lab separates them so a counterexample can name the\n+ # exact failure, then refuses just as hard.\n+ if len(seen) != len(set(seen)):\n+ dupes = sorted({i for i in seen if seen.count(i) > 1})\n+ return refuse(\"duplicate_id\", f\"repeated ids: {', '.join(dupes[:5])}\")\n+ unknown = [i for i in seen if i not in expected]\n+ if unknown:\n+ return refuse(\"unknown_id\", f\"ids not in the request: {', '.join(sorted(unknown)[:5])}\")\n+ missing = [i for i in expected if i not in set(seen)]\n+ if missing:\n+ return refuse(\"missing_id\", f\"no verdict for: {', '.join(sorted(missing)[:5])}\")\n \n out: list[dict[str, Any]] = []\n- for i, article in enumerate(articles):\n- if i < len(results_list):\n- entry = results_list[i] if isinstance(results_list[i], dict) else {}\n- try:\n- score = max(0.0, min(1.0, float(entry.get(\"score\", 0.5))))\n- except Exception:\n- score = 0.5\n- relevant = bool(entry.get(\"relevant\", score >= 0.5))\n- out.append(verdict(article[\"id\"], relevant, score, str(entry.get(\"reason\", \"\"))))\n- else:\n- out.append(verdict(article[\"id\"], False, 0.0, \"scoring incomplete\"))\n- return ok(out)\n+ for entry in entries:\n+ article_id = entry.get(\"article_id\", entry.get(\"id\"))\n+ score = finite_unit_score(entry.get(\"score\"))\n+ if score is None:\n+ raw = entry.get(\"score\")\n+ if isinstance(raw, (int, float)) and not isinstance(raw, bool):\n+ # Distinguishes 9e9 (out of range) from NaN (not finite); both\n+ # are clamped silently by production.\n+ return refuse(\n+ \"score_not_finite\" if raw != raw or raw in (float(\"inf\"), float(\"-inf\"))\n+ else \"score_out_of_range\",\n+ f\"score={raw!r} for {article_id}\",\n+ )\n+ return refuse(\"invalid_type\", f\"score is {type(raw).__name__} for {article_id}\")\n+ relevant = entry.get(\"relevant\", score >= 0.5)\n+ if not isinstance(relevant, bool):\n+ return refuse(\"invalid_type\", f\"relevant is {type(relevant).__name__}\")\n+ out.append(verdict(article_id, relevant, score, str(entry.get(\"reason\", \"\"))))\n+\n+ # Emit in request order so downstream consumers see a stable sequence. The\n+ # association itself is by id and does not depend on this ordering — the\n+ # permutation property test asserts exactly that.\n+ by_id = {v[\"article_id\"]: v for v in out}\n+ return ok([by_id[a[\"id\"]] for a in articles])\n", "patch_base": "backend/lab/contract/versions/positional_v0.py" }, "provenance": { "executed_at_revision": "4e8bee7625820107e84184f8e24c6bd7125f2e86+dirty", "inputs_sha256": "02da1ab3e0e18c82", "executed_at": "2026-09-22T19:47:57.873Z", "evaluator_sha256": "277ec81521ba14d8", "spec_hash": "f027762ab4d08b35", "spec_version": 2, "execution_mode": "offline-replay", "execution_mode_basis": "The harness reads committed responses from disk and makes no network call. The candidate imports nothing beyond the standard library.", "python": "3.12.13", "runner": "local-known", "sandbox": null, "investigation": null, "case_suites": [ { "group": "observed", "path": "backend/lab/cases/observed.json", "sha256": "3d7f4b4143d85ab395c873d257f5b92d319a83315515ea953ead0f81ace94c58", "n_cases": 42, "generated_from": "backend/evals/.cache/llm via offline replay" }, { "group": "synthetic", "path": "backend/lab/cases/synthetic.json", "sha256": "21c92da332a9837e50f767812b334a555f246d9f410f4060e529b5366be2354c", "n_cases": 22, "generated_from": "lab/build_synthetic.py — fault injection, ground truth by construction" } ], "notes": [ { "severity": "info", "message": "A fault was deliberately injected into this run: the orchestrator was SIGKILLed with an attempt in flight. The recovery that follows is real.", "source": "backend/lab/orchestrate.py --kill-after" }, { "severity": "info", "message": "Every case ran offline against responses already committed to this repository. No inference call was made and no provider was charged.", "source": "backend/evals/.cache/llm" }, { "severity": "caution", "message": "The case suite is public. A candidate may have been written against it, so passing does not establish generalisation.", "source": "backend/lab/cases/" }, { "severity": "caution", "message": "Association correctness is measured; relevance quality under keyed-v2 is not. Sending article ids changes the request, which invalidates every recorded response for this runner — new budgeted recordings would be required and none exist.", "source": "backend/evals/llm_cache.py:133" } ] }, "verdict": "accepted-for-review", "verdict_reason": "all 6 criteria satisfied over 60 applicable cases", "verdict_scope": "Accepted means eligible for human review under this spec hash, against a public case suite. It is not evidence of production quality, and it does not establish generalisation: the cases are visible and a candidate may have been written against them.", "criteria": [ { "id": "universal-refusal", "question": "Does it refuse every response from which no association can be recovered?", "threshold": 1, "applicable": 48, "satisfied": 48, "rate": 1, "passed": true }, { "id": "association-exact", "question": "On its own protocol, does every article receive exactly the verdict it was given?", "threshold": 1, "applicable": 3, "satisfied": 3, "rate": 1, "passed": true }, { "id": "protocol-violation-refusal", "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?", "threshold": 1, "applicable": 9, "satisfied": 9, "rate": 1, "passed": true }, { "id": "no-crash", "question": "Does it terminate on every case without crashing or hanging?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "complete-evidence", "question": "Is there a prediction record for every applicable case?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "protocol-exclusivity", "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?", "threshold": 1, "applicable": 4, "satisfied": 4, "rate": 1, "passed": true } ], "gradings": [ { "spec_version": 1, "spec_hash": "008af8266204438a", "verdict": "accepted-for-review", "verdict_reason": "all 5 criteria satisfied over 60 applicable cases", "criteria": [ { "id": "universal-refusal", "question": "Does it refuse every response from which no association can be recovered?", "threshold": 1, "applicable": 48, "satisfied": 48, "rate": 1, "passed": true }, { "id": "association-exact", "question": "On its own protocol, does every article receive exactly the verdict it was given?", "threshold": 1, "applicable": 3, "satisfied": 3, "rate": 1, "passed": true }, { "id": "protocol-violation-refusal", "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?", "threshold": 1, "applicable": 9, "satisfied": 9, "rate": 1, "passed": true }, { "id": "no-crash", "question": "Does it terminate on every case without crashing or hanging?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "complete-evidence", "question": "Is there a prediction record for every applicable case?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true } ] }, { "spec_version": 2, "spec_hash": "f027762ab4d08b35", "verdict": "accepted-for-review", "verdict_reason": "all 6 criteria satisfied over 60 applicable cases", "criteria": [ { "id": "universal-refusal", "question": "Does it refuse every response from which no association can be recovered?", "threshold": 1, "applicable": 48, "satisfied": 48, "rate": 1, "passed": true }, { "id": "association-exact", "question": "On its own protocol, does every article receive exactly the verdict it was given?", "threshold": 1, "applicable": 3, "satisfied": 3, "rate": 1, "passed": true }, { "id": "protocol-violation-refusal", "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?", "threshold": 1, "applicable": 9, "satisfied": 9, "rate": 1, "passed": true }, { "id": "no-crash", "question": "Does it terminate on every case without crashing or hanging?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "complete-evidence", "question": "Is there a prediction record for every applicable case?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "protocol-exclusivity", "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?", "threshold": 1, "applicable": 4, "satisfied": 4, "rate": 1, "passed": true } ] } ], "outcomes": [ { "case_id": "observed-2026-09-02-000", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.02 }, { "case_id": "observed-2026-09-02-001", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.004 }, { "case_id": "observed-2026-09-02-002", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.009 }, { "case_id": "observed-2026-09-02-003", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.002 }, { "case_id": "observed-2026-09-02-004", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-005", "group": "observed", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [], "ms": 0.008 }, { "case_id": "observed-2026-09-02-006", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.053 }, { "case_id": "observed-2026-09-02-007", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.012 }, { "case_id": "observed-2026-09-02-008", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.012 }, { "case_id": "observed-2026-09-02-009", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-010", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.007 }, { "case_id": "observed-2026-09-02-011", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-012", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-013", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.009 }, { "case_id": "observed-2026-09-02-014", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-015", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.007 }, { "case_id": "observed-2026-09-02-016", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-017", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-018", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.015 }, { "case_id": "observed-2026-09-02-019", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-020", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.007 }, { "case_id": "observed-2026-09-02-021", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-022", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-023", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.012 }, { "case_id": "observed-2026-09-02-024", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.007 }, { "case_id": "observed-2026-09-02-025", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.01 }, { "case_id": "observed-2026-09-02-026", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-027", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.007 }, { "case_id": "observed-2026-09-02-028", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.011 }, { "case_id": "observed-2026-09-02-029", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-030", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-031", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.01 }, { "case_id": "observed-2026-09-02-032", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.014 }, { "case_id": "observed-2026-09-02-033", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.007 }, { "case_id": "observed-2026-09-02-034", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.011 }, { "case_id": "observed-2026-09-02-035", "group": "observed", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [], "ms": 0.006 }, { "case_id": "observed-2026-09-02-036", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-037", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-038", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.001 }, { "case_id": "observed-2026-09-02-039", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.009 }, { "case_id": "observed-2026-09-02-040", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.06 }, { "case_id": "observed-2026-09-02-041", "group": "observed", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [], "ms": 0.007 }, { "case_id": "syn-keyed-in-order", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "every article received its own verdict", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.018 }, { "case_id": "syn-keyed-reversed", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "every article received its own verdict", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.01 }, { "case_id": "syn-keyed-rotated", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "every article received its own verdict", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.009 }, { "case_id": "syn-duplicate-id", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (duplicate_id)", "counterexample": null, "observed_refusal_kind": "duplicate_id", "expected_refusal_kinds": [ "duplicate_id" ], "ms": 0.007 }, { "case_id": "syn-unknown-id", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (unknown_id)", "counterexample": null, "observed_refusal_kind": "unknown_id", "expected_refusal_kinds": [ "unknown_id" ], "ms": 0.005 }, { "case_id": "syn-missing-id", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (missing_id)", "counterexample": null, "observed_refusal_kind": "missing_id", "expected_refusal_kinds": [ "missing_id", "count_mismatch" ], "ms": 0.005 }, { "case_id": "syn-score-nan", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (score_not_finite)", "counterexample": null, "observed_refusal_kind": "score_not_finite", "expected_refusal_kinds": [ "score_not_finite", "invalid_type", "malformed_json" ], "ms": 0.007 }, { "case_id": "syn-score-out-of-range", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (score_out_of_range)", "counterexample": null, "observed_refusal_kind": "score_out_of_range", "expected_refusal_kinds": [ "score_out_of_range", "invalid_type" ], "ms": 0.006 }, { "case_id": "syn-score-string", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "invalid_type" ], "ms": 0.005 }, { "case_id": "syn-score-null", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "invalid_type" ], "ms": 0.005 }, { "case_id": "syn-relevant-not-bool", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "invalid_type" ], "ms": 0.005 }, { "case_id": "syn-malformed-json", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "malformed_json" ], "ms": 0.006 }, { "case_id": "syn-truncated-finish-reason", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (truncated_response)", "counterexample": null, "observed_refusal_kind": "truncated_response", "expected_refusal_kinds": [ "truncated_response", "count_mismatch", "missing_id" ], "ms": 0.001 }, { "case_id": "syn-not-an-object", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (unexpected_shape)", "counterexample": null, "observed_refusal_kind": "unexpected_shape", "expected_refusal_kinds": [ "unexpected_shape" ], "ms": 0.003 }, { "case_id": "syn-empty-results", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (missing_id)", "counterexample": null, "observed_refusal_kind": "missing_id", "expected_refusal_kinds": [ "missing_id", "count_mismatch" ], "ms": 0.004 }, { "case_id": "syn-exec-no-recording", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (no_recording)", "counterexample": null, "observed_refusal_kind": "no_recording", "expected_refusal_kinds": [ "no_recording" ], "ms": 0.001 }, { "case_id": "syn-exec-timeout", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (timeout)", "counterexample": null, "observed_refusal_kind": "timeout", "expected_refusal_kinds": [ "timeout" ], "ms": 0.001 }, { "case_id": "syn-exec-cancelled", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (cancelled)", "counterexample": null, "observed_refusal_kind": "cancelled", "expected_refusal_kinds": [ "cancelled" ], "ms": 0.001 }, { "case_id": "syn-exec-budget", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (retries_exhausted)", "counterexample": null, "observed_refusal_kind": "retries_exhausted", "expected_refusal_kinds": [ "retries_exhausted", "no_recording" ], "ms": 0.001 }, { "case_id": "syn-positional-short", "group": "synthetic", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch", "missing_id" ], "ms": 0.003 }, { "case_id": "syn-positional-long", "group": "synthetic", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch", "duplicate_id" ], "ms": 0.003 }, { "case_id": "syn-positional-reordered", "group": "synthetic", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "count_mismatch", "unexpected_shape", "unsupported_protocol", "invalid_type", "missing_id" ], "ms": 0.003 } ], "counts": { "correct": 60, "wrong-association": 0, "should-have-refused": 0, "should-have-parsed": 0, "crashed": 0, "timeout": 0, "missing-record": 0, "not-applicable": 4 }, "smallest_counterexample": null, "diagnostics": [ { "id": "out-of-protocol-association", "question": "On cases outside its declared protocol, did it refuse — or associate anyway?", "value": 0, "of": 4, "detail": "It produced no association on any case outside its declared protocol.", "case_ids": [] } ], "usage": { "model_calls": 0, "replay_spend_usd": 0, "recording_cost_usd": "unknown", "provider_reported": "unknown", "basis": "Offline replay of committed recordings: no inference call was made, so provider spend for this run is $0. What the original recordings cost is not attributed per batch anywhere in this repository, so it is left unknown rather than estimated." }, "attempts": [ { "attempt_id": "article-to-verdict-association__keyed-v2__bcde0d5383b5__008af826#01", "started_at": "2026-09-22T19:47:57.889Z", "ended_at": "2026-09-22T19:47:57.913Z", "status": "unknown-outcome", "runner": "local-known", "note": "orchestrator died with this attempt in flight; records file absent on recovery" }, { "attempt_id": "article-to-verdict-association__keyed-v2__bcde0d5383b5__008af826#02", "started_at": "2026-09-22T19:47:57.913Z", "ended_at": "2026-09-22T19:47:57.939Z", "status": "succeeded", "runner": "local-known", "note": "completed 64 cases in 25.5ms" } ] }