{
 "lab_artifact_version": 1,
 "run_id": "article-to-verdict-association__count-guard-v1__2c2fb884ebdf__008af826__clean",
 "experiment_id": "article-to-verdict-association",
 "candidate": {
  "candidate_id": "count-guard-v1",
  "kind": "preserved-version",
  "description": "PR #59: discards the batch when the verdict count differs from the article count, and lets CacheMiss and BudgetExceeded propagate.",
  "declared_protocol": "positional-v0",
  "source_path": "backend/lab/contract/versions/count_guard_v1.py",
  "source_sha256": "2c2fb884ebdf38e21bbb3fbb8738473b467a26dcc38baf03e4fccd560a68309f",
  "source_bytes": 4140,
  "transcribed_from": "81b20198:backend/app/services/openai_service.py",
  "patch": "diff --git a/backend/lab/contract/versions/positional_v0.py b/backend/lab/contract/versions/count_guard_v1.py\n--- a/backend/lab/contract/versions/positional_v0.py\n+++ b/backend/lab/contract/versions/count_guard_v1.py\n@@ -1,18 +1,23 @@\n-\"\"\"Historical behaviour, transcribed from `origin/main`.\n+\"\"\"The count-mismatch guard, transcribed from PR #59 (head 81b2019).\n \n-Source: backend/app/services/openai_service.py, score_articles_batch, the\n-`normalized` loop. Verbatim semantics:\n+Source: backend/app/services/openai_service.py on\n+`mmarufov/batch-scoring-alignment`:\n \n     if len(results_list) != len(articles):\n-        logger.warning(\"... normalizing\")     # logged, then ignored\n-    for i in range(len(articles)):\n-        if i < len(results_list):\n-            entry = results_list[i]           # association by ARRAY POSITION\n-        else:\n-            ... {\"relevant\": False, \"score\": 0.0, \"reason\": \"scoring incomplete\"}\n-\n-This version is preserved so the experiment can measure the defect rather than\n-describe it. It is not a control: it is what production does today.\n+        logger.warning(\"... discarding the batch rather than assigning \"\n+                       \"verdicts by position\")\n+        continue                              # retry, then report unscored\n+\n+That commit also stops swallowing the harness's stop signals:\n+\n+    except Exception as exc:\n+        if type(exc).__name__ in {\"CacheMiss\", \"BudgetExceeded\"}:\n+            raise\n+\n+Both are reproduced. The guard is a real improvement over positional-v0 and\n+this experiment is expected to show that — and also to show its ceiling: it\n+compares *lengths*, so an equal-length reordered response still passes through\n+and is still associated by position.\n \"\"\"\n \n from __future__ import annotations\n@@ -54,45 +59,55 @@ import json\n from typing import Any\n \n \n-VERSION_ID = \"positional-v0\"\n+VERSION_ID = \"count-guard-v1\"\n PROTOCOL = \"positional-v0\"\n \n \n def parse(articles: list[dict[str, Any]], response: dict[str, Any]) -> dict[str, Any]:\n-    if response.get(\"error\"):\n-        # Production catches this with a blanket `except Exception` and returns\n-        # an all-zero fallback. Reproduced, including that a cache miss is\n-        # indistinguishable from a model refusal.\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n+    error = response.get(\"error\")\n+    if error:\n+        # CacheMiss / BudgetExceeded propagate instead of degrading to zeros.\n+        if error in {\"no_recording\", \"budget_exceeded\"}:\n+            return refuse(\"no_recording\" if error == \"no_recording\" else \"retries_exhausted\", error)\n+        if error in {\"timeout\", \"cancelled\"}:\n+            return refuse(error, error)\n+        return refuse(\"retries_exhausted\", str(error))\n \n     content = response.get(\"content\")\n     if content is None:\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n+        return refuse(\"retries_exhausted\", \"no content returned\")\n \n     try:\n         result = json.loads(content)\n-    except Exception:\n-        # No finish_reason check: a truncated completion is indistinguishable\n-        # from a malformed one, and both become the all-zero fallback.\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n+    except Exception as exc:\n+        return refuse(\"malformed_json\", str(exc))\n \n-    results_list = result.get(\"results\", []) if isinstance(result, dict) else []\n-    if not results_list and isinstance(result, dict) and \"scores\" in result:\n+    if not isinstance(result, dict):\n+        return refuse(\"unexpected_shape\", f\"top level is {type(result).__name__}\")\n+\n+    results_list = result.get(\"results\", [])\n+    if not results_list and \"scores\" in result:\n         results_list = [\n             {\"relevant\": float(s) >= 0.5, \"score\": float(s), \"reason\": \"\"}\n             for s in result[\"scores\"]\n         ]\n+    if not isinstance(results_list, list):\n+        return refuse(\"unexpected_shape\", \"results is not a list\")\n+\n+    if len(results_list) != len(articles):\n+        return refuse(\n+            \"count_mismatch\",\n+            f\"{len(results_list)} results for {len(articles)} articles\",\n+        )\n \n     out: list[dict[str, Any]] = []\n-    for i, article in enumerate(articles):\n-        if i < len(results_list):\n-            entry = results_list[i] if isinstance(results_list[i], dict) else {}\n-            try:\n-                score = max(0.0, min(1.0, float(entry.get(\"score\", 0.5))))\n-            except Exception:\n-                score = 0.5\n-            relevant = bool(entry.get(\"relevant\", score >= 0.5))\n-            out.append(verdict(article[\"id\"], relevant, score, str(entry.get(\"reason\", \"\"))))\n-        else:\n-            out.append(verdict(article[\"id\"], False, 0.0, \"scoring incomplete\"))\n+    for article, entry in zip(articles, results_list):\n+        if not isinstance(entry, dict):\n+            return refuse(\"invalid_type\", f\"entry is {type(entry).__name__}\")\n+        try:\n+            score = max(0.0, min(1.0, float(entry.get(\"score\", 0.5))))\n+        except Exception:\n+            return refuse(\"invalid_type\", \"score is not numeric\")\n+        relevant = bool(entry.get(\"relevant\", score >= 0.5))\n+        out.append(verdict(article[\"id\"], relevant, score, str(entry.get(\"reason\", \"\"))))\n     return ok(out)\n",
  "patch_base": "backend/lab/contract/versions/positional_v0.py"
 },
 "provenance": {
  "executed_at_revision": "4e8bee7625820107e84184f8e24c6bd7125f2e86+dirty",
  "inputs_sha256": "ab8320e5dd1220c1",
  "executed_at": "2026-09-22T19:47:57.506Z",
  "evaluator_sha256": "277ec81521ba14d8",
  "spec_hash": "f027762ab4d08b35",
  "spec_version": 2,
  "execution_mode": "offline-replay",
  "execution_mode_basis": "The harness reads committed responses from disk and makes no network call. The candidate imports nothing beyond the standard library.",
  "python": "3.12.13",
  "runner": "local-known",
  "sandbox": null,
  "investigation": null,
  "case_suites": [
   {
    "group": "observed",
    "path": "backend/lab/cases/observed.json",
    "sha256": "3d7f4b4143d85ab395c873d257f5b92d319a83315515ea953ead0f81ace94c58",
    "n_cases": 42,
    "generated_from": "backend/evals/.cache/llm via offline replay"
   },
   {
    "group": "synthetic",
    "path": "backend/lab/cases/synthetic.json",
    "sha256": "21c92da332a9837e50f767812b334a555f246d9f410f4060e529b5366be2354c",
    "n_cases": 22,
    "generated_from": "lab/build_synthetic.py — fault injection, ground truth by construction"
   }
  ],
  "notes": [
   {
    "severity": "info",
    "message": "Every case ran offline against responses already committed to this repository. No inference call was made and no provider was charged.",
    "source": "backend/evals/.cache/llm"
   },
   {
    "severity": "caution",
    "message": "The case suite is public. A candidate may have been written against it, so passing does not establish generalisation.",
    "source": "backend/lab/cases/"
   }
  ]
 },
 "verdict": "rejected",
 "verdict_reason": "association-exact: 0/3 cases satisfied, threshold 1",
 "verdict_scope": "Accepted means eligible for human review under this spec hash, against a public case suite. It is not evidence of production quality, and it does not establish generalisation: the cases are visible and a candidate may have been written against them.",
 "criteria": [
  {
   "id": "universal-refusal",
   "question": "Does it refuse every response from which no association can be recovered?",
   "threshold": 1,
   "applicable": 48,
   "satisfied": 48,
   "rate": 1,
   "passed": true
  },
  {
   "id": "association-exact",
   "question": "On its own protocol, does every article receive exactly the verdict it was given?",
   "threshold": 1,
   "applicable": 3,
   "satisfied": 0,
   "rate": 0,
   "passed": false
  },
  {
   "id": "protocol-violation-refusal",
   "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
   "threshold": 1,
   "applicable": 1,
   "satisfied": 0,
   "rate": 0,
   "passed": false
  },
  {
   "id": "no-crash",
   "question": "Does it terminate on every case without crashing or hanging?",
   "threshold": 1,
   "applicable": 52,
   "satisfied": 52,
   "rate": 1,
   "passed": true
  },
  {
   "id": "complete-evidence",
   "question": "Is there a prediction record for every applicable case?",
   "threshold": 1,
   "applicable": 52,
   "satisfied": 52,
   "rate": 1,
   "passed": true
  },
  {
   "id": "protocol-exclusivity",
   "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?",
   "threshold": 1,
   "applicable": 12,
   "satisfied": 5,
   "rate": 0.4166666666666667,
   "passed": false
  }
 ],
 "gradings": [
  {
   "spec_version": 1,
   "spec_hash": "008af8266204438a",
   "verdict": "rejected",
   "verdict_reason": "association-exact: 0/3 cases satisfied, threshold 1",
   "criteria": [
    {
     "id": "universal-refusal",
     "question": "Does it refuse every response from which no association can be recovered?",
     "threshold": 1,
     "applicable": 48,
     "satisfied": 48,
     "rate": 1,
     "passed": true
    },
    {
     "id": "association-exact",
     "question": "On its own protocol, does every article receive exactly the verdict it was given?",
     "threshold": 1,
     "applicable": 3,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "protocol-violation-refusal",
     "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
     "threshold": 1,
     "applicable": 1,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "no-crash",
     "question": "Does it terminate on every case without crashing or hanging?",
     "threshold": 1,
     "applicable": 52,
     "satisfied": 52,
     "rate": 1,
     "passed": true
    },
    {
     "id": "complete-evidence",
     "question": "Is there a prediction record for every applicable case?",
     "threshold": 1,
     "applicable": 52,
     "satisfied": 52,
     "rate": 1,
     "passed": true
    }
   ]
  },
  {
   "spec_version": 2,
   "spec_hash": "f027762ab4d08b35",
   "verdict": "rejected",
   "verdict_reason": "association-exact: 0/3 cases satisfied, threshold 1",
   "criteria": [
    {
     "id": "universal-refusal",
     "question": "Does it refuse every response from which no association can be recovered?",
     "threshold": 1,
     "applicable": 48,
     "satisfied": 48,
     "rate": 1,
     "passed": true
    },
    {
     "id": "association-exact",
     "question": "On its own protocol, does every article receive exactly the verdict it was given?",
     "threshold": 1,
     "applicable": 3,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "protocol-violation-refusal",
     "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
     "threshold": 1,
     "applicable": 1,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "no-crash",
     "question": "Does it terminate on every case without crashing or hanging?",
     "threshold": 1,
     "applicable": 52,
     "satisfied": 52,
     "rate": 1,
     "passed": true
    },
    {
     "id": "complete-evidence",
     "question": "Is there a prediction record for every applicable case?",
     "threshold": 1,
     "applicable": 52,
     "satisfied": 52,
     "rate": 1,
     "passed": true
    },
    {
     "id": "protocol-exclusivity",
     "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?",
     "threshold": 1,
     "applicable": 12,
     "satisfied": 5,
     "rate": 0.4166666666666667,
     "passed": false
    }
   ]
  }
 ],
 "outcomes": [
  {
   "case_id": "observed-2026-09-02-000",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.02
  },
  {
   "case_id": "observed-2026-09-02-001",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.195
  },
  {
   "case_id": "observed-2026-09-02-002",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.008
  },
  {
   "case_id": "observed-2026-09-02-003",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.172
  },
  {
   "case_id": "observed-2026-09-02-004",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.17
  },
  {
   "case_id": "observed-2026-09-02-005",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "wrong-association",
   "detail": "no ground truth recorded",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.022
  },
  {
   "case_id": "observed-2026-09-02-006",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.045
  },
  {
   "case_id": "observed-2026-09-02-007",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.009
  },
  {
   "case_id": "observed-2026-09-02-008",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.01
  },
  {
   "case_id": "observed-2026-09-02-009",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.134
  },
  {
   "case_id": "observed-2026-09-02-010",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.006
  },
  {
   "case_id": "observed-2026-09-02-011",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.133
  },
  {
   "case_id": "observed-2026-09-02-012",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.134
  },
  {
   "case_id": "observed-2026-09-02-013",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.008
  },
  {
   "case_id": "observed-2026-09-02-014",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.179
  },
  {
   "case_id": "observed-2026-09-02-015",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.006
  },
  {
   "case_id": "observed-2026-09-02-016",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.342
  },
  {
   "case_id": "observed-2026-09-02-017",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.177
  },
  {
   "case_id": "observed-2026-09-02-018",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.014
  },
  {
   "case_id": "observed-2026-09-02-019",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.156
  },
  {
   "case_id": "observed-2026-09-02-020",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.006
  },
  {
   "case_id": "observed-2026-09-02-021",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.155
  },
  {
   "case_id": "observed-2026-09-02-022",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.155
  },
  {
   "case_id": "observed-2026-09-02-023",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.01
  },
  {
   "case_id": "observed-2026-09-02-024",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.006
  },
  {
   "case_id": "observed-2026-09-02-025",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.008
  },
  {
   "case_id": "observed-2026-09-02-026",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.105
  },
  {
   "case_id": "observed-2026-09-02-027",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.006
  },
  {
   "case_id": "observed-2026-09-02-028",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.009
  },
  {
   "case_id": "observed-2026-09-02-029",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.103
  },
  {
   "case_id": "observed-2026-09-02-030",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.113
  },
  {
   "case_id": "observed-2026-09-02-031",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.011
  },
  {
   "case_id": "observed-2026-09-02-032",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.016
  },
  {
   "case_id": "observed-2026-09-02-033",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.006
  },
  {
   "case_id": "observed-2026-09-02-034",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.011
  },
  {
   "case_id": "observed-2026-09-02-035",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "wrong-association",
   "detail": "no ground truth recorded",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.016
  },
  {
   "case_id": "observed-2026-09-02-036",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.14
  },
  {
   "case_id": "observed-2026-09-02-037",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.142
  },
  {
   "case_id": "observed-2026-09-02-038",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.138
  },
  {
   "case_id": "observed-2026-09-02-039",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.007
  },
  {
   "case_id": "observed-2026-09-02-040",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.054
  },
  {
   "case_id": "observed-2026-09-02-041",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "wrong-association",
   "detail": "no ground truth recorded",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.015
  },
  {
   "case_id": "syn-keyed-in-order",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.007
  },
  {
   "case_id": "syn-keyed-reversed",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.006
  },
  {
   "case_id": "syn-keyed-rotated",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.006
  },
  {
   "case_id": "syn-duplicate-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "duplicate_id"
   ],
   "ms": 0.004
  },
  {
   "case_id": "syn-unknown-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "unknown_id"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-missing-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "missing_id",
    "count_mismatch"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-score-nan",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "score_not_finite",
    "invalid_type",
    "malformed_json"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-score-out-of-range",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "score_out_of_range",
    "invalid_type"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-score-string",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": "invalid_type",
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.004
  },
  {
   "case_id": "syn-score-null",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": "invalid_type",
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.004
  },
  {
   "case_id": "syn-relevant-not-bool",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-malformed-json",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (malformed_json)",
   "counterexample": null,
   "observed_refusal_kind": "malformed_json",
   "expected_refusal_kinds": [
    "malformed_json"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-truncated-finish-reason",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "truncated_response",
    "count_mismatch",
    "missing_id"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-not-an-object",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (unexpected_shape)",
   "counterexample": null,
   "observed_refusal_kind": "unexpected_shape",
   "expected_refusal_kinds": [
    "unexpected_shape"
   ],
   "ms": 0.004
  },
  {
   "case_id": "syn-empty-results",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "missing_id",
    "count_mismatch"
   ],
   "ms": 0.002
  },
  {
   "case_id": "syn-exec-no-recording",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (no_recording)",
   "counterexample": null,
   "observed_refusal_kind": "no_recording",
   "expected_refusal_kinds": [
    "no_recording"
   ],
   "ms": 0.001
  },
  {
   "case_id": "syn-exec-timeout",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (timeout)",
   "counterexample": null,
   "observed_refusal_kind": "timeout",
   "expected_refusal_kinds": [
    "timeout"
   ],
   "ms": 0.001
  },
  {
   "case_id": "syn-exec-cancelled",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (cancelled)",
   "counterexample": null,
   "observed_refusal_kind": "cancelled",
   "expected_refusal_kinds": [
    "cancelled"
   ],
   "ms": 0.001
  },
  {
   "case_id": "syn-exec-budget",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (retries_exhausted)",
   "counterexample": null,
   "observed_refusal_kind": "retries_exhausted",
   "expected_refusal_kinds": [
    "retries_exhausted",
    "no_recording"
   ],
   "ms": 0.001
  },
  {
   "case_id": "syn-positional-short",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch",
    "missing_id"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-positional-long",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "correct",
   "detail": "refused (count_mismatch)",
   "counterexample": null,
   "observed_refusal_kind": "count_mismatch",
   "expected_refusal_kinds": [
    "count_mismatch",
    "duplicate_id"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-positional-reordered",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "equal length, internally reordered — a length check cannot detect this",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch",
    "unexpected_shape",
    "unsupported_protocol",
    "invalid_type",
    "missing_id"
   ],
   "ms": 0.007
  }
 ],
 "counts": {
  "correct": 48,
  "wrong-association": 3,
  "should-have-refused": 1,
  "should-have-parsed": 0,
  "crashed": 0,
  "timeout": 0,
  "missing-record": 0,
  "not-applicable": 12
 },
 "smallest_counterexample": null,
 "diagnostics": [
  {
   "id": "out-of-protocol-association",
   "question": "On cases outside its declared protocol, did it refuse — or associate anyway?",
   "value": 7,
   "of": 12,
   "detail": "It produced a complete association on 7 case(s) outside its declared protocol. Under this generation that fails protocol-exclusivity; under generation 1 it was not graded at all.",
   "case_ids": [
    "syn-keyed-in-order",
    "syn-keyed-reversed",
    "syn-keyed-rotated",
    "syn-unknown-id",
    "syn-score-nan",
    "syn-score-out-of-range",
    "syn-relevant-not-bool"
   ]
  }
 ],
 "usage": {
  "model_calls": 0,
  "replay_spend_usd": 0,
  "recording_cost_usd": "unknown",
  "provider_reported": "unknown",
  "basis": "Offline replay of committed recordings: no inference call was made, so provider spend for this run is $0. What the original recordings cost is not attributed per batch anywhere in this repository, so it is left unknown rather than estimated."
 },
 "attempts": [
  {
   "attempt_id": "article-to-verdict-association__count-guard-v1__2c2fb884ebdf__008af826#01",
   "started_at": "2026-09-22T19:47:57.523Z",
   "ended_at": "2026-09-22T19:47:57.552Z",
   "status": "succeeded",
   "runner": "local-known",
   "note": "completed 64 cases in 28.9ms"
  }
 ]
}