{
 "lab_artifact_version": 1,
 "run_id": "article-to-verdict-association__control-lenient-keyed__6a3f35522825__008af826__clean",
 "experiment_id": "article-to-verdict-association",
 "candidate": {
  "candidate_id": "control-lenient-keyed",
  "kind": "seeded-control",
  "description": "SEEDED DEFECT: keeps the last verdict when an article is judged twice, drops unknown ids silently, and coerces unusable scores to 0.0.",
  "declared_protocol": "keyed-v2",
  "source_path": "backend/lab/contract/controls/lenient_keyed.py",
  "source_sha256": "6a3f355228251f71a7e50e6101a3661ceed69636094366e4cec36f471d860d29",
  "source_bytes": 3176,
  "transcribed_from": "unknown",
  "patch": "diff --git a/backend/lab/contract/versions/positional_v0.py b/backend/lab/contract/controls/lenient_keyed.py\n--- a/backend/lab/contract/versions/positional_v0.py\n+++ b/backend/lab/contract/controls/lenient_keyed.py\n@@ -1,18 +1,11 @@\n-\"\"\"Historical behaviour, transcribed from `origin/main`.\n+\"\"\"CONTROL — keyed parsing with last-write-wins on duplicate ids.\n \n-Source: backend/app/services/openai_service.py, score_articles_batch, the\n-`normalized` loop. Verbatim semantics:\n+Seeded defect: accepts a response containing the same article twice, keeping\n+the later verdict. Plausible-looking and wrong: the model has contradicted\n+itself and the parser has silently picked a winner.\n \n-    if len(results_list) != len(articles):\n-        logger.warning(\"... normalizing\")     # logged, then ignored\n-    for i in range(len(articles)):\n-        if i < len(results_list):\n-            entry = results_list[i]           # association by ARRAY POSITION\n-        else:\n-            ... {\"relevant\": False, \"score\": 0.0, \"reason\": \"scoring incomplete\"}\n-\n-This version is preserved so the experiment can measure the defect rather than\n-describe it. It is not a control: it is what production does today.\n+Expected outcome: REJECTED on the duplicate-id cases. If the evaluator ever\n+accepts this, the evaluator is broken.\n \"\"\"\n \n from __future__ import annotations\n@@ -54,45 +47,39 @@ import json\n from typing import Any\n \n \n-VERSION_ID = \"positional-v0\"\n-PROTOCOL = \"positional-v0\"\n+VERSION_ID = \"control:lenient-keyed\"\n+PROTOCOL = \"keyed-v2\"\n \n \n def parse(articles: list[dict[str, Any]], response: dict[str, Any]) -> dict[str, Any]:\n     if response.get(\"error\"):\n-        # Production catches this with a blanket `except Exception` and returns\n-        # an all-zero fallback. Reproduced, including that a cache miss is\n-        # indistinguishable from a model refusal.\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n-\n+        return refuse(\"retries_exhausted\", str(response[\"error\"]))\n     content = response.get(\"content\")\n     if content is None:\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n-\n+        return refuse(\"retries_exhausted\", \"no content\")\n     try:\n         result = json.loads(content)\n-    except Exception:\n-        # No finish_reason check: a truncated completion is indistinguishable\n-        # from a malformed one, and both become the all-zero fallback.\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n-\n-    results_list = result.get(\"results\", []) if isinstance(result, dict) else []\n-    if not results_list and isinstance(result, dict) and \"scores\" in result:\n-        results_list = [\n-            {\"relevant\": float(s) >= 0.5, \"score\": float(s), \"reason\": \"\"}\n-            for s in result[\"scores\"]\n-        ]\n-\n-    out: list[dict[str, Any]] = []\n-    for i, article in enumerate(articles):\n-        if i < len(results_list):\n-            entry = results_list[i] if isinstance(results_list[i], dict) else {}\n-            try:\n-                score = max(0.0, min(1.0, float(entry.get(\"score\", 0.5))))\n-            except Exception:\n-                score = 0.5\n-            relevant = bool(entry.get(\"relevant\", score >= 0.5))\n-            out.append(verdict(article[\"id\"], relevant, score, str(entry.get(\"reason\", \"\"))))\n-        else:\n-            out.append(verdict(article[\"id\"], False, 0.0, \"scoring incomplete\"))\n-    return ok(out)\n+    except Exception as exc:\n+        return refuse(\"malformed_json\", str(exc))\n+    if not isinstance(result, dict) or not isinstance(result.get(\"results\"), list):\n+        return refuse(\"unexpected_shape\", \"no results array\")\n+\n+    expected = {a[\"id\"] for a in articles}\n+    by_id: dict[str, dict[str, Any]] = {}\n+    for entry in result[\"results\"]:\n+        if not isinstance(entry, dict):\n+            continue\n+        article_id = entry.get(\"article_id\", entry.get(\"id\"))\n+        if article_id not in expected:\n+            continue  # DEFECT: silently drops unknown ids instead of refusing\n+        score = finite_unit_score(entry.get(\"score\"))\n+        if score is None:\n+            score = 0.0  # DEFECT: coerces NaN / out-of-range to a real verdict\n+        by_id[article_id] = verdict(\n+            article_id, bool(entry.get(\"relevant\", score >= 0.5)), score,\n+            str(entry.get(\"reason\", \"\")),\n+        )  # DEFECT: last write wins on a duplicate id\n+\n+    if len(by_id) != len(expected):\n+        return refuse(\"missing_id\", f\"{len(by_id)} of {len(expected)}\")\n+    return ok([by_id[a[\"id\"]] for a in articles])\n",
  "patch_base": "backend/lab/contract/versions/positional_v0.py"
 },
 "provenance": {
  "executed_at_revision": "4e8bee7625820107e84184f8e24c6bd7125f2e86+dirty",
  "inputs_sha256": "90b370052dd759a0",
  "executed_at": "2026-09-22T19:47:57.650Z",
  "evaluator_sha256": "277ec81521ba14d8",
  "spec_hash": "f027762ab4d08b35",
  "spec_version": 2,
  "execution_mode": "offline-replay",
  "execution_mode_basis": "The harness reads committed responses from disk and makes no network call. The candidate imports nothing beyond the standard library.",
  "python": "3.12.13",
  "runner": "local-known",
  "sandbox": null,
  "investigation": null,
  "case_suites": [
   {
    "group": "observed",
    "path": "backend/lab/cases/observed.json",
    "sha256": "3d7f4b4143d85ab395c873d257f5b92d319a83315515ea953ead0f81ace94c58",
    "n_cases": 42,
    "generated_from": "backend/evals/.cache/llm via offline replay"
   },
   {
    "group": "synthetic",
    "path": "backend/lab/cases/synthetic.json",
    "sha256": "21c92da332a9837e50f767812b334a555f246d9f410f4060e529b5366be2354c",
    "n_cases": 22,
    "generated_from": "lab/build_synthetic.py — fault injection, ground truth by construction"
   }
  ],
  "notes": [
   {
    "severity": "warning",
    "message": "This is a seeded control with a deliberate defect. SEEDED DEFECT: keeps the last verdict when an article is judged twice, drops unknown ids silently, and coerces unusable scores to 0.0.",
    "source": "backend/lab/contract/controls/lenient_keyed.py"
   },
   {
    "severity": "info",
    "message": "Every case ran offline against responses already committed to this repository. No inference call was made and no provider was charged.",
    "source": "backend/evals/.cache/llm"
   },
   {
    "severity": "caution",
    "message": "The case suite is public. A candidate may have been written against it, so passing does not establish generalisation.",
    "source": "backend/lab/cases/"
   }
  ]
 },
 "verdict": "rejected",
 "verdict_reason": "protocol-violation-refusal: 3/9 cases satisfied, threshold 1",
 "verdict_scope": "Accepted means eligible for human review under this spec hash, against a public case suite. It is not evidence of production quality, and it does not establish generalisation: the cases are visible and a candidate may have been written against them.",
 "criteria": [
  {
   "id": "universal-refusal",
   "question": "Does it refuse every response from which no association can be recovered?",
   "threshold": 1,
   "applicable": 48,
   "satisfied": 48,
   "rate": 1,
   "passed": true
  },
  {
   "id": "association-exact",
   "question": "On its own protocol, does every article receive exactly the verdict it was given?",
   "threshold": 1,
   "applicable": 3,
   "satisfied": 3,
   "rate": 1,
   "passed": true
  },
  {
   "id": "protocol-violation-refusal",
   "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
   "threshold": 1,
   "applicable": 9,
   "satisfied": 3,
   "rate": 0.3333333333333333,
   "passed": false
  },
  {
   "id": "no-crash",
   "question": "Does it terminate on every case without crashing or hanging?",
   "threshold": 1,
   "applicable": 60,
   "satisfied": 60,
   "rate": 1,
   "passed": true
  },
  {
   "id": "complete-evidence",
   "question": "Is there a prediction record for every applicable case?",
   "threshold": 1,
   "applicable": 60,
   "satisfied": 60,
   "rate": 1,
   "passed": true
  },
  {
   "id": "protocol-exclusivity",
   "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?",
   "threshold": 1,
   "applicable": 4,
   "satisfied": 4,
   "rate": 1,
   "passed": true
  }
 ],
 "gradings": [
  {
   "spec_version": 1,
   "spec_hash": "008af8266204438a",
   "verdict": "rejected",
   "verdict_reason": "protocol-violation-refusal: 3/9 cases satisfied, threshold 1",
   "criteria": [
    {
     "id": "universal-refusal",
     "question": "Does it refuse every response from which no association can be recovered?",
     "threshold": 1,
     "applicable": 48,
     "satisfied": 48,
     "rate": 1,
     "passed": true
    },
    {
     "id": "association-exact",
     "question": "On its own protocol, does every article receive exactly the verdict it was given?",
     "threshold": 1,
     "applicable": 3,
     "satisfied": 3,
     "rate": 1,
     "passed": true
    },
    {
     "id": "protocol-violation-refusal",
     "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
     "threshold": 1,
     "applicable": 9,
     "satisfied": 3,
     "rate": 0.3333333333333333,
     "passed": false
    },
    {
     "id": "no-crash",
     "question": "Does it terminate on every case without crashing or hanging?",
     "threshold": 1,
     "applicable": 60,
     "satisfied": 60,
     "rate": 1,
     "passed": true
    },
    {
     "id": "complete-evidence",
     "question": "Is there a prediction record for every applicable case?",
     "threshold": 1,
     "applicable": 60,
     "satisfied": 60,
     "rate": 1,
     "passed": true
    }
   ]
  },
  {
   "spec_version": 2,
   "spec_hash": "f027762ab4d08b35",
   "verdict": "rejected",
   "verdict_reason": "protocol-violation-refusal: 3/9 cases satisfied, threshold 1",
   "criteria": [
    {
     "id": "universal-refusal",
     "question": "Does it refuse every response from which no association can be recovered?",
     "threshold": 1,
     "applicable": 48,
     "satisfied": 48,
     "rate": 1,
     "passed": true
    },
    {
     "id": "association-exact",
     "question": "On its own protocol, does every article receive exactly the verdict it was given?",
     "threshold": 1,
     "applicable": 3,
     "satisfied": 3,
     "rate": 1,
     "passed": true
    },
    {
     "id": "protocol-violation-refusal",
     "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
     "threshold": 1,
     "applicable": 9,
     "satisfied": 3,
     "rate": 0.3333333333333333,
     "passed": false
    },
    {
     "id": "no-crash",
     "question": "Does it terminate on every case without crashing or hanging?",
     "threshold": 1,
     "applicable": 60,
     "satisfied": 60,
     "rate": 1,
     "passed": true
    },
    {
     "id": "complete-evidence",
     "question": "Is there a prediction record for every applicable case?",
     "threshold": 1,
     "applicable": 60,
     "satisfied": 60,
     "rate": 1,
     "passed": true
    },
    {
     "id": "protocol-exclusivity",
     "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?",
     "threshold": 1,
     "applicable": 4,
     "satisfied": 4,
     "rate": 1,
     "passed": true
    }
   ]
  }
 ],
 "outcomes": [
  {
   "case_id": "observed-2026-09-02-000",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.024
  },
  {
   "case_id": "observed-2026-09-02-001",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.195
  },
  {
   "case_id": "observed-2026-09-02-002",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.011
  },
  {
   "case_id": "observed-2026-09-02-003",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.188
  },
  {
   "case_id": "observed-2026-09-02-004",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.175
  },
  {
   "case_id": "observed-2026-09-02-005",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is positional-v0; candidate declares keyed-v2",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [],
   "ms": 0.009
  },
  {
   "case_id": "observed-2026-09-02-006",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.055
  },
  {
   "case_id": "observed-2026-09-02-007",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-008",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-009",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.131
  },
  {
   "case_id": "observed-2026-09-02-010",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.008
  },
  {
   "case_id": "observed-2026-09-02-011",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.132
  },
  {
   "case_id": "observed-2026-09-02-012",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.134
  },
  {
   "case_id": "observed-2026-09-02-013",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.01
  },
  {
   "case_id": "observed-2026-09-02-014",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.188
  },
  {
   "case_id": "observed-2026-09-02-015",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.009
  },
  {
   "case_id": "observed-2026-09-02-016",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.345
  },
  {
   "case_id": "observed-2026-09-02-017",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.182
  },
  {
   "case_id": "observed-2026-09-02-018",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.018
  },
  {
   "case_id": "observed-2026-09-02-019",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.16
  },
  {
   "case_id": "observed-2026-09-02-020",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.008
  },
  {
   "case_id": "observed-2026-09-02-021",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.158
  },
  {
   "case_id": "observed-2026-09-02-022",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.176
  },
  {
   "case_id": "observed-2026-09-02-023",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.014
  },
  {
   "case_id": "observed-2026-09-02-024",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.008
  },
  {
   "case_id": "observed-2026-09-02-025",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-026",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.111
  },
  {
   "case_id": "observed-2026-09-02-027",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.008
  },
  {
   "case_id": "observed-2026-09-02-028",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-029",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.119
  },
  {
   "case_id": "observed-2026-09-02-030",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.119
  },
  {
   "case_id": "observed-2026-09-02-031",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-032",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.016
  },
  {
   "case_id": "observed-2026-09-02-033",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.008
  },
  {
   "case_id": "observed-2026-09-02-034",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-035",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is positional-v0; candidate declares keyed-v2",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [],
   "ms": 0.007
  },
  {
   "case_id": "observed-2026-09-02-036",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.145
  },
  {
   "case_id": "observed-2026-09-02-037",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.14
  },
  {
   "case_id": "observed-2026-09-02-038",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.141
  },
  {
   "case_id": "observed-2026-09-02-039",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.01
  },
  {
   "case_id": "observed-2026-09-02-040",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.066
  },
  {
   "case_id": "observed-2026-09-02-041",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is positional-v0; candidate declares keyed-v2",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [],
   "ms": 0.008
  },
  {
   "case_id": "syn-keyed-in-order",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "every article received its own verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.014
  },
  {
   "case_id": "syn-keyed-reversed",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "every article received its own verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.008
  },
  {
   "case_id": "syn-keyed-rotated",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "every article received its own verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.007
  },
  {
   "case_id": "syn-duplicate-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "the same article is judged twice; no winner may be picked",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "duplicate_id"
   ],
   "ms": 0.009
  },
  {
   "case_id": "syn-unknown-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "unknown_id"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-missing-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "missing_id",
    "count_mismatch"
   ],
   "ms": 0.005
  },
  {
   "case_id": "syn-score-nan",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "score is nan; production clamps it into a real verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "score_not_finite",
    "invalid_type",
    "malformed_json"
   ],
   "ms": 0.007
  },
  {
   "case_id": "syn-score-out-of-range",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "score is out-of-range; production clamps it into a real verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "score_out_of_range",
    "invalid_type"
   ],
   "ms": 0.007
  },
  {
   "case_id": "syn-score-string",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "score is string; production clamps it into a real verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-score-null",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "score is null; production clamps it into a real verdict",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-relevant-not-bool",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "relevant is a string",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-malformed-json",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "malformed_json"
   ],
   "ms": 0.004
  },
  {
   "case_id": "syn-truncated-finish-reason",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "truncated_response",
    "count_mismatch",
    "missing_id"
   ],
   "ms": 0.004
  },
  {
   "case_id": "syn-not-an-object",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (unexpected_shape)",
   "counterexample": null,
   "observed_refusal_kind": "unexpected_shape",
   "expected_refusal_kinds": [
    "unexpected_shape"
   ],
   "ms": 0.004
  },
  {
   "case_id": "syn-empty-results",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "missing_id",
    "count_mismatch"
   ],
   "ms": 0.002
  },
  {
   "case_id": "syn-exec-no-recording",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (retries_exhausted)",
   "counterexample": null,
   "observed_refusal_kind": "retries_exhausted",
   "expected_refusal_kinds": [
    "no_recording"
   ],
   "ms": 0.001
  },
  {
   "case_id": "syn-exec-timeout",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (retries_exhausted)",
   "counterexample": null,
   "observed_refusal_kind": "retries_exhausted",
   "expected_refusal_kinds": [
    "timeout"
   ],
   "ms": 0.001
  },
  {
   "case_id": "syn-exec-cancelled",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (retries_exhausted)",
   "counterexample": null,
   "observed_refusal_kind": "retries_exhausted",
   "expected_refusal_kinds": [
    "cancelled"
   ],
   "ms": 0.001
  },
  {
   "case_id": "syn-exec-budget",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (retries_exhausted)",
   "counterexample": null,
   "observed_refusal_kind": "retries_exhausted",
   "expected_refusal_kinds": [
    "retries_exhausted",
    "no_recording"
   ],
   "ms": 0.001
  },
  {
   "case_id": "syn-positional-short",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch",
    "missing_id"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-positional-long",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (missing_id)",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch",
    "duplicate_id"
   ],
   "ms": 0.004
  },
  {
   "case_id": "syn-positional-reordered",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is positional-v0; candidate declares keyed-v2",
   "counterexample": null,
   "observed_refusal_kind": "missing_id",
   "expected_refusal_kinds": [
    "count_mismatch",
    "unexpected_shape",
    "unsupported_protocol",
    "invalid_type",
    "missing_id"
   ],
   "ms": 0.003
  }
 ],
 "counts": {
  "correct": 54,
  "wrong-association": 0,
  "should-have-refused": 6,
  "should-have-parsed": 0,
  "crashed": 0,
  "timeout": 0,
  "missing-record": 0,
  "not-applicable": 4
 },
 "smallest_counterexample": null,
 "diagnostics": [
  {
   "id": "out-of-protocol-association",
   "question": "On cases outside its declared protocol, did it refuse — or associate anyway?",
   "value": 0,
   "of": 4,
   "detail": "It produced no association on any case outside its declared protocol.",
   "case_ids": []
  }
 ],
 "usage": {
  "model_calls": 0,
  "replay_spend_usd": 0,
  "recording_cost_usd": "unknown",
  "provider_reported": "unknown",
  "basis": "Offline replay of committed recordings: no inference call was made, so provider spend for this run is $0. What the original recordings cost is not attributed per batch anywhere in this repository, so it is left unknown rather than estimated."
 },
 "attempts": [
  {
   "attempt_id": "article-to-verdict-association__control-lenient-keyed__6a3f35522825__008af826#01",
   "started_at": "2026-09-22T19:47:57.667Z",
   "ended_at": "2026-09-22T19:47:57.697Z",
   "status": "succeeded",
   "runner": "local-known",
   "note": "completed 64 cases in 30.3ms"
  }
 ]
}