{
 "lab_artifact_version": 1,
 "run_id": "article-to-verdict-association__control-zero-filling__32afb5fd6da8__008af826__clean",
 "experiment_id": "article-to-verdict-association",
 "candidate": {
  "candidate_id": "control-zero-filling",
  "kind": "seeded-control",
  "description": "SEEDED DEFECT: turns every failure, including a missing recording, into a confident all-zero answer. This is the blanket except Exception on the production path.",
  "declared_protocol": "positional-v0",
  "source_path": "backend/lab/contract/controls/zero_filling.py",
  "source_sha256": "32afb5fd6da82f9089e27dee284a5220dcc82a56ad4dcd7ca9dc8f03d3e0b8ae",
  "source_bytes": 2449,
  "transcribed_from": "unknown",
  "patch": "diff --git a/backend/lab/contract/versions/positional_v0.py b/backend/lab/contract/controls/zero_filling.py\n--- a/backend/lab/contract/versions/positional_v0.py\n+++ b/backend/lab/contract/controls/zero_filling.py\n@@ -1,18 +1,12 @@\n-\"\"\"Historical behaviour, transcribed from `origin/main`.\n+\"\"\"CONTROL — turns every failure into a confident all-zero answer.\n \n-Source: backend/app/services/openai_service.py, score_articles_batch, the\n-`normalized` loop. Verbatim semantics:\n+Seeded defect: reproduces the blanket `except Exception` on the production\n+path, where `CacheMiss` (a RuntimeError) becomes a full set of 0.0 verdicts and\n+the run reports zero cache misses. An offline replay looks clean while having\n+scored nothing.\n \n-    if len(results_list) != len(articles):\n-        logger.warning(\"... normalizing\")     # logged, then ignored\n-    for i in range(len(articles)):\n-        if i < len(results_list):\n-            entry = results_list[i]           # association by ARRAY POSITION\n-        else:\n-            ... {\"relevant\": False, \"score\": 0.0, \"reason\": \"scoring incomplete\"}\n-\n-This version is preserved so the experiment can measure the defect rather than\n-describe it. It is not a control: it is what production does today.\n+Expected outcome: REJECTED on the no-recording and truncation cases, because\n+an unavailable answer must surface as incomplete and never as a verdict.\n \"\"\"\n \n from __future__ import annotations\n@@ -54,45 +48,23 @@ import json\n from typing import Any\n \n \n-VERSION_ID = \"positional-v0\"\n+VERSION_ID = \"control:zero-filling\"\n PROTOCOL = \"positional-v0\"\n \n \n-def parse(articles: list[dict[str, Any]], response: dict[str, Any]) -> dict[str, Any]:\n-    if response.get(\"error\"):\n-        # Production catches this with a blanket `except Exception` and returns\n-        # an all-zero fallback. Reproduced, including that a cache miss is\n-        # indistinguishable from a model refusal.\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n+def _zeros(articles: list[dict[str, Any]]) -> dict[str, Any]:\n+    return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n \n-    content = response.get(\"content\")\n-    if content is None:\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n \n+def parse(articles: list[dict[str, Any]], response: dict[str, Any]) -> dict[str, Any]:\n     try:\n-        result = json.loads(content)\n+        if response.get(\"error\"):\n+            return _zeros(articles)\n+        result = json.loads(response.get(\"content\") or \"\")\n+        entries = result[\"results\"]\n+        return ok([\n+            verdict(a[\"id\"], bool(e.get(\"relevant\")), float(e.get(\"score\", 0.0)), str(e.get(\"reason\", \"\")))\n+            for a, e in zip(articles, entries)\n+        ])\n     except Exception:\n-        # No finish_reason check: a truncated completion is indistinguishable\n-        # from a malformed one, and both become the all-zero fallback.\n-        return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n-\n-    results_list = result.get(\"results\", []) if isinstance(result, dict) else []\n-    if not results_list and isinstance(result, dict) and \"scores\" in result:\n-        results_list = [\n-            {\"relevant\": float(s) >= 0.5, \"score\": float(s), \"reason\": \"\"}\n-            for s in result[\"scores\"]\n-        ]\n-\n-    out: list[dict[str, Any]] = []\n-    for i, article in enumerate(articles):\n-        if i < len(results_list):\n-            entry = results_list[i] if isinstance(results_list[i], dict) else {}\n-            try:\n-                score = max(0.0, min(1.0, float(entry.get(\"score\", 0.5))))\n-            except Exception:\n-                score = 0.5\n-            relevant = bool(entry.get(\"relevant\", score >= 0.5))\n-            out.append(verdict(article[\"id\"], relevant, score, str(entry.get(\"reason\", \"\"))))\n-        else:\n-            out.append(verdict(article[\"id\"], False, 0.0, \"scoring incomplete\"))\n-    return ok(out)\n+        return _zeros(articles)\n",
  "patch_base": "backend/lab/contract/versions/positional_v0.py"
 },
 "provenance": {
  "executed_at_revision": "4e8bee7625820107e84184f8e24c6bd7125f2e86+dirty",
  "inputs_sha256": "01df3df6f1fa00e3",
  "executed_at": "2026-09-22T19:47:57.796Z",
  "evaluator_sha256": "277ec81521ba14d8",
  "spec_hash": "f027762ab4d08b35",
  "spec_version": 2,
  "execution_mode": "offline-replay",
  "execution_mode_basis": "The harness reads committed responses from disk and makes no network call. The candidate imports nothing beyond the standard library.",
  "python": "3.12.13",
  "runner": "local-known",
  "sandbox": null,
  "investigation": null,
  "case_suites": [
   {
    "group": "observed",
    "path": "backend/lab/cases/observed.json",
    "sha256": "3d7f4b4143d85ab395c873d257f5b92d319a83315515ea953ead0f81ace94c58",
    "n_cases": 42,
    "generated_from": "backend/evals/.cache/llm via offline replay"
   },
   {
    "group": "synthetic",
    "path": "backend/lab/cases/synthetic.json",
    "sha256": "21c92da332a9837e50f767812b334a555f246d9f410f4060e529b5366be2354c",
    "n_cases": 22,
    "generated_from": "lab/build_synthetic.py — fault injection, ground truth by construction"
   }
  ],
  "notes": [
   {
    "severity": "warning",
    "message": "This is a seeded control with a deliberate defect. SEEDED DEFECT: turns every failure, including a missing recording, into a confident all-zero answer. This is the blanket except Exception on the production path.",
    "source": "backend/lab/contract/controls/zero_filling.py"
   },
   {
    "severity": "info",
    "message": "Every case ran offline against responses already committed to this repository. No inference call was made and no provider was charged.",
    "source": "backend/evals/.cache/llm"
   },
   {
    "severity": "caution",
    "message": "The case suite is public. A candidate may have been written against it, so passing does not establish generalisation.",
    "source": "backend/lab/cases/"
   }
  ]
 },
 "verdict": "rejected",
 "verdict_reason": "universal-refusal: 0/48 cases satisfied, threshold 1",
 "verdict_scope": "Accepted means eligible for human review under this spec hash, against a public case suite. It is not evidence of production quality, and it does not establish generalisation: the cases are visible and a candidate may have been written against them.",
 "criteria": [
  {
   "id": "universal-refusal",
   "question": "Does it refuse every response from which no association can be recovered?",
   "threshold": 1,
   "applicable": 48,
   "satisfied": 0,
   "rate": 0,
   "passed": false
  },
  {
   "id": "association-exact",
   "question": "On its own protocol, does every article receive exactly the verdict it was given?",
   "threshold": 1,
   "applicable": 3,
   "satisfied": 0,
   "rate": 0,
   "passed": false
  },
  {
   "id": "protocol-violation-refusal",
   "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
   "threshold": 1,
   "applicable": 1,
   "satisfied": 0,
   "rate": 0,
   "passed": false
  },
  {
   "id": "no-crash",
   "question": "Does it terminate on every case without crashing or hanging?",
   "threshold": 1,
   "applicable": 52,
   "satisfied": 52,
   "rate": 1,
   "passed": true
  },
  {
   "id": "complete-evidence",
   "question": "Is there a prediction record for every applicable case?",
   "threshold": 1,
   "applicable": 52,
   "satisfied": 52,
   "rate": 1,
   "passed": true
  },
  {
   "id": "protocol-exclusivity",
   "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?",
   "threshold": 1,
   "applicable": 12,
   "satisfied": 0,
   "rate": 0,
   "passed": false
  }
 ],
 "gradings": [
  {
   "spec_version": 1,
   "spec_hash": "008af8266204438a",
   "verdict": "rejected",
   "verdict_reason": "universal-refusal: 0/48 cases satisfied, threshold 1",
   "criteria": [
    {
     "id": "universal-refusal",
     "question": "Does it refuse every response from which no association can be recovered?",
     "threshold": 1,
     "applicable": 48,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "association-exact",
     "question": "On its own protocol, does every article receive exactly the verdict it was given?",
     "threshold": 1,
     "applicable": 3,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "protocol-violation-refusal",
     "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
     "threshold": 1,
     "applicable": 1,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "no-crash",
     "question": "Does it terminate on every case without crashing or hanging?",
     "threshold": 1,
     "applicable": 52,
     "satisfied": 52,
     "rate": 1,
     "passed": true
    },
    {
     "id": "complete-evidence",
     "question": "Is there a prediction record for every applicable case?",
     "threshold": 1,
     "applicable": 52,
     "satisfied": 52,
     "rate": 1,
     "passed": true
    }
   ]
  },
  {
   "spec_version": 2,
   "spec_hash": "f027762ab4d08b35",
   "verdict": "rejected",
   "verdict_reason": "universal-refusal: 0/48 cases satisfied, threshold 1",
   "criteria": [
    {
     "id": "universal-refusal",
     "question": "Does it refuse every response from which no association can be recovered?",
     "threshold": 1,
     "applicable": 48,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "association-exact",
     "question": "On its own protocol, does every article receive exactly the verdict it was given?",
     "threshold": 1,
     "applicable": 3,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "protocol-violation-refusal",
     "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?",
     "threshold": 1,
     "applicable": 1,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    },
    {
     "id": "no-crash",
     "question": "Does it terminate on every case without crashing or hanging?",
     "threshold": 1,
     "applicable": 52,
     "satisfied": 52,
     "rate": 1,
     "passed": true
    },
    {
     "id": "complete-evidence",
     "question": "Is there a prediction record for every applicable case?",
     "threshold": 1,
     "applicable": 52,
     "satisfied": 52,
     "rate": 1,
     "passed": true
    },
    {
     "id": "protocol-exclusivity",
     "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?",
     "threshold": 1,
     "applicable": 12,
     "satisfied": 0,
     "rate": 0,
     "passed": false
    }
   ]
  }
 ],
 "outcomes": [
  {
   "case_id": "observed-2026-09-02-000",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "27 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.039
  },
  {
   "case_id": "observed-2026-09-02-001",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.231
  },
  {
   "case_id": "observed-2026-09-02-002",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "18 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.015
  },
  {
   "case_id": "observed-2026-09-02-003",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.192
  },
  {
   "case_id": "observed-2026-09-02-004",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.191
  },
  {
   "case_id": "observed-2026-09-02-005",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "wrong-association",
   "detail": "no ground truth recorded",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.014
  },
  {
   "case_id": "observed-2026-09-02-006",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "201 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.065
  },
  {
   "case_id": "observed-2026-09-02-007",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "33 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.021
  },
  {
   "case_id": "observed-2026-09-02-008",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "37 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.023
  },
  {
   "case_id": "observed-2026-09-02-009",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.166
  },
  {
   "case_id": "observed-2026-09-02-010",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "19 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-011",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.16
  },
  {
   "case_id": "observed-2026-09-02-012",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.161
  },
  {
   "case_id": "observed-2026-09-02-013",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "25 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.017
  },
  {
   "case_id": "observed-2026-09-02-014",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.196
  },
  {
   "case_id": "observed-2026-09-02-015",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "19 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-016",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.398
  },
  {
   "case_id": "observed-2026-09-02-017",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.189
  },
  {
   "case_id": "observed-2026-09-02-018",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "59 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.028
  },
  {
   "case_id": "observed-2026-09-02-019",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.167
  },
  {
   "case_id": "observed-2026-09-02-020",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "19 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-021",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.17
  },
  {
   "case_id": "observed-2026-09-02-022",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.166
  },
  {
   "case_id": "observed-2026-09-02-023",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "36 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.022
  },
  {
   "case_id": "observed-2026-09-02-024",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "21 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.013
  },
  {
   "case_id": "observed-2026-09-02-025",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "31 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.019
  },
  {
   "case_id": "observed-2026-09-02-026",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.12
  },
  {
   "case_id": "observed-2026-09-02-027",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "19 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-028",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "38 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.022
  },
  {
   "case_id": "observed-2026-09-02-029",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.114
  },
  {
   "case_id": "observed-2026-09-02-030",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.113
  },
  {
   "case_id": "observed-2026-09-02-031",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "31 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.019
  },
  {
   "case_id": "observed-2026-09-02-032",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "53 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.026
  },
  {
   "case_id": "observed-2026-09-02-033",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "19 verdicts recorded for 20 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-034",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "34 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.022
  },
  {
   "case_id": "observed-2026-09-02-035",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "wrong-association",
   "detail": "no ground truth recorded",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.012
  },
  {
   "case_id": "observed-2026-09-02-036",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.158
  },
  {
   "case_id": "observed-2026-09-02-037",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.153
  },
  {
   "case_id": "observed-2026-09-02-038",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "completion stopped for 'length'",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "malformed_json"
   ],
   "ms": 0.149
  },
  {
   "case_id": "observed-2026-09-02-039",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "25 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.016
  },
  {
   "case_id": "observed-2026-09-02-040",
   "group": "observed",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "254 verdicts recorded for 40 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch"
   ],
   "ms": 0.068
  },
  {
   "case_id": "observed-2026-09-02-041",
   "group": "observed",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "wrong-association",
   "detail": "no ground truth recorded",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.013
  },
  {
   "case_id": "syn-keyed-in-order",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.006
  },
  {
   "case_id": "syn-keyed-reversed",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.006
  },
  {
   "case_id": "syn-keyed-rotated",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [],
   "ms": 0.005
  },
  {
   "case_id": "syn-duplicate-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "duplicate_id"
   ],
   "ms": 0.005
  },
  {
   "case_id": "syn-unknown-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "unknown_id"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-missing-id",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "missing_id",
    "count_mismatch"
   ],
   "ms": 0.005
  },
  {
   "case_id": "syn-score-nan",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "score_not_finite",
    "invalid_type",
    "malformed_json"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-score-out-of-range",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "score_out_of_range",
    "invalid_type"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-score-string",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.007
  },
  {
   "case_id": "syn-score-null",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.006
  },
  {
   "case_id": "syn-relevant-not-bool",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "invalid_type"
   ],
   "ms": 0.005
  },
  {
   "case_id": "syn-malformed-json",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "the response is cut mid-object",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "malformed_json"
   ],
   "ms": 0.005
  },
  {
   "case_id": "syn-truncated-finish-reason",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "the completion stopped at the output ceiling",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "truncated_response",
    "count_mismatch",
    "missing_id"
   ],
   "ms": 0.004
  },
  {
   "case_id": "syn-not-an-object",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "top level is an array, not an object",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "unexpected_shape"
   ],
   "ms": 0.005
  },
  {
   "case_id": "syn-empty-results",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "keyed-v2",
   "applicability": "not-applicable",
   "status": "not-applicable",
   "detail": "case is keyed-v2; candidate declares positional-v0",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "missing_id",
    "count_mismatch"
   ],
   "ms": 0.002
  },
  {
   "case_id": "syn-exec-no-recording",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "execution ended as no_recording; an unavailable answer is not an answer",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "no_recording"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-exec-timeout",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "execution ended as timeout; an unavailable answer is not an answer",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "timeout"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-exec-cancelled",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "execution ended as cancelled; an unavailable answer is not an answer",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "cancelled"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-exec-budget",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "keyed-v2",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "execution ended as budget_exceeded; an unavailable answer is not an answer",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "retries_exhausted",
    "no_recording"
   ],
   "ms": 0.003
  },
  {
   "case_id": "syn-positional-short",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "4 verdicts for 6 articles: which belongs to which is unrecorded",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch",
    "missing_id"
   ],
   "ms": 0.004
  },
  {
   "case_id": "syn-positional-long",
   "group": "synthetic",
   "family": "universal-refusal",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "7 verdicts for 6 articles",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch",
    "duplicate_id"
   ],
   "ms": 0.005
  },
  {
   "case_id": "syn-positional-reordered",
   "group": "synthetic",
   "family": "protocol-association",
   "protocol": "positional-v0",
   "applicability": "scored",
   "status": "should-have-refused",
   "detail": "equal length, internally reordered — a length check cannot detect this",
   "counterexample": null,
   "observed_refusal_kind": null,
   "expected_refusal_kinds": [
    "count_mismatch",
    "unexpected_shape",
    "unsupported_protocol",
    "invalid_type",
    "missing_id"
   ],
   "ms": 0.005
  }
 ],
 "counts": {
  "correct": 0,
  "wrong-association": 3,
  "should-have-refused": 49,
  "should-have-parsed": 0,
  "crashed": 0,
  "timeout": 0,
  "missing-record": 0,
  "not-applicable": 12
 },
 "smallest_counterexample": null,
 "diagnostics": [
  {
   "id": "out-of-protocol-association",
   "question": "On cases outside its declared protocol, did it refuse — or associate anyway?",
   "value": 12,
   "of": 12,
   "detail": "It produced a complete association on 12 case(s) outside its declared protocol. Under this generation that fails protocol-exclusivity; under generation 1 it was not graded at all.",
   "case_ids": [
    "syn-keyed-in-order",
    "syn-keyed-reversed",
    "syn-keyed-rotated",
    "syn-duplicate-id",
    "syn-unknown-id",
    "syn-missing-id",
    "syn-score-nan",
    "syn-score-out-of-range",
    "syn-score-string",
    "syn-score-null",
    "syn-relevant-not-bool",
    "syn-empty-results"
   ]
  }
 ],
 "usage": {
  "model_calls": 0,
  "replay_spend_usd": 0,
  "recording_cost_usd": "unknown",
  "provider_reported": "unknown",
  "basis": "Offline replay of committed recordings: no inference call was made, so provider spend for this run is $0. What the original recordings cost is not attributed per batch anywhere in this repository, so it is left unknown rather than estimated."
 },
 "attempts": [
  {
   "attempt_id": "article-to-verdict-association__control-zero-filling__32afb5fd6da8__008af826#01",
   "started_at": "2026-09-22T19:47:57.813Z",
   "ended_at": "2026-09-22T19:47:57.845Z",
   "status": "succeeded",
   "runner": "local-known",
   "note": "completed 64 cases in 30.9ms"
  }
 ]
}