{ "lab_artifact_version": 1, "run_id": "article-to-verdict-association__agent-k2-generous-05__sandbox", "experiment_id": "article-to-verdict-association", "candidate": { "candidate_id": "agent-k2-generous-05", "kind": "agent-authored", "description": "The defect is positional association: results[i] maps to articles[i], which fails on reordering or count mismatch. The keyed-v2 protocol fixes this by using article_id fields for unambiguous association. This parser: (1) Uses keyed association when article_id is present, validating for duplicate_id, unknown_id, and missing_id; (2) Falls back to positional only when no article_id fields exist, with strict count validation; (3) Refuses when association is ambiguous.", "declared_protocol": "keyed-v2", "source_path": "backend/lab/contract/candidates/agent_k2_generous_05.py", "source_sha256": "57a914e27676dab3c5c0977f5e9532128c86203736c70ee91f73e49935614fdc", "source_bytes": 6620, "transcribed_from": "unknown", "patch": "diff --git a/backend/lab/contract/versions/positional_v0.py b/backend/lab/contract/candidates/agent_k2_generous_05.py\n--- a/backend/lab/contract/versions/positional_v0.py\n+++ b/backend/lab/contract/candidates/agent_k2_generous_05.py\n@@ -1,30 +1,16 @@\n-\"\"\"Historical behaviour, transcribed from `origin/main`.\n+\"\"\"Keyed association parser - fixes the positional association defect.\n \n-Source: backend/app/services/openai_service.py, score_articles_batch, the\n-`normalized` loop. Verbatim semantics:\n+The defect: positional-v0 and count_guard_v1 both associate results[i] with articles[i].\n+When the model returns results in a different order, or returns a different number of them,\n+articles receive wrong verdicts silently.\n \n- if len(results_list) != len(articles):\n- logger.warning(\"... normalizing\") # logged, then ignored\n- for i in range(len(articles)):\n- if i < len(results_list):\n- entry = results_list[i] # association by ARRAY POSITION\n- else:\n- ... {\"relevant\": False, \"score\": 0.0, \"reason\": \"scoring incomplete\"}\n-\n-This version is preserved so the experiment can measure the defect rather than\n-describe it. It is not a control: it is what production does today.\n+The fix: Use keyed association where each result includes an article_id field.\n+This enables unambiguous association and detects duplicates, unknown IDs, and missing IDs.\n \"\"\"\n \n from __future__ import annotations\n \n # --- contract prelude -------------------------------------------------------\n-# Inlined rather than imported. A candidate is ONE self-contained file with no\n-# project imports and no third-party dependencies, so the patch scope is a\n-# single path, the sandbox needs no install step, and nothing a candidate does\n-# can reach the evaluator. The canonical definitions live in\n-# lab/contract/types.py, and tests/test_contract_prelude.py asserts that every\n-# copy still agrees with it.\n-\n import math as _math\n \n \n@@ -54,45 +40,143 @@ import json\n from typing import Any\n \n \n-VERSION_ID = \"positional-v0\"\n-PROTOCOL = \"positional-v0\"\n+VERSION_ID = \"keyed-v2\"\n+PROTOCOL = \"keyed-v2\"\n \n \n def parse(articles: list[dict[str, Any]], response: dict[str, Any]) -> dict[str, Any]:\n- if response.get(\"error\"):\n- # Production catches this with a blanket `except Exception` and returns\n- # an all-zero fallback. Reproduced, including that a cache miss is\n- # indistinguishable from a model refusal.\n- return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n+ \"\"\"Parse response using keyed association when article_id is present.\n+ \n+ Keyed association (preferred): Each result has an article_id field.\n+ - Validates exact match of article IDs\n+ - Detects duplicates, unknown IDs, missing IDs\n+ - Unambiguous even if results are reordered\n+ \n+ Positional fallback: Only when no article_id fields are present.\n+ - Requires exact count match\n+ - Associates by position (results[i] -> articles[i])\n+ \"\"\"\n+ error = response.get(\"error\")\n+ if error:\n+ if error in {\"no_recording\", \"budget_exceeded\"}:\n+ return refuse(\"no_recording\" if error == \"no_recording\" else \"retries_exhausted\", error)\n+ if error in {\"timeout\", \"cancelled\"}:\n+ return refuse(error, error)\n+ return refuse(\"retries_exhausted\", str(error))\n \n content = response.get(\"content\")\n if content is None:\n- return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n+ return refuse(\"retries_exhausted\", \"no content returned\")\n \n try:\n result = json.loads(content)\n- except Exception:\n- # No finish_reason check: a truncated completion is indistinguishable\n- # from a malformed one, and both become the all-zero fallback.\n- return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n+ except Exception as exc:\n+ return refuse(\"malformed_json\", str(exc))\n \n- results_list = result.get(\"results\", []) if isinstance(result, dict) else []\n- if not results_list and isinstance(result, dict) and \"scores\" in result:\n+ if not isinstance(result, dict):\n+ return refuse(\"unexpected_shape\", f\"top level is {type(result).__name__}\")\n+\n+ # Extract results list\n+ results_list = result.get(\"results\", [])\n+ if not results_list and \"scores\" in result:\n+ # Legacy scores format - convert to results dicts\n results_list = [\n {\"relevant\": float(s) >= 0.5, \"score\": float(s), \"reason\": \"\"}\n for s in result[\"scores\"]\n ]\n+ \n+ if not isinstance(results_list, list):\n+ return refuse(\"unexpected_shape\", \"results is not a list\")\n+\n+ # Build set of expected article IDs\n+ expected_ids = {a[\"id\"] for a in articles}\n+\n+ # Check if any result has article_id field\n+ has_keyed_results = any(\n+ isinstance(r, dict) and r.get(\"article_id\") is not None \n+ for r in results_list\n+ )\n+\n+ if has_keyed_results:\n+ # Keyed association: use article_id for unambiguous matching\n+ return _keyed_association(articles, expected_ids, results_list)\n+ else:\n+ # Positional fallback: only when no article_id present\n+ return _positional_association(articles, results_list)\n+\n+\n+def _keyed_association(articles: list[dict[str, Any]], expected_ids: set[str], results_list: list) -> dict[str, Any]:\n+ \"\"\"Associate results using article_id fields.\"\"\"\n+ result_by_id: dict[str, dict] = {}\n+ \n+ for entry in results_list:\n+ if not isinstance(entry, dict):\n+ return refuse(\"invalid_type\", f\"entry is {type(entry).__name__}\")\n+ \n+ article_id = entry.get(\"article_id\")\n+ if article_id is None:\n+ # Mixed keyed/non-keyed is ambiguous - refuse\n+ return refuse(\"unexpected_shape\", \"mixed entries with and without article_id\")\n+ \n+ if article_id in result_by_id:\n+ return refuse(\"duplicate_id\", f\"duplicate article_id: {article_id}\")\n+ \n+ if article_id not in expected_ids:\n+ return refuse(\"unknown_id\", f\"unknown article_id: {article_id}\")\n+ \n+ result_by_id[article_id] = entry\n+ \n+ # Check for missing IDs\n+ missing_ids = expected_ids - set(result_by_id.keys())\n+ if missing_ids:\n+ return refuse(\"missing_id\", f\"missing article_ids: {sorted(missing_ids)}\")\n+ \n+ # Build verdicts in article order\n+ out: list[dict[str, Any]] = []\n+ for article in articles:\n+ article_id = article[\"id\"]\n+ entry = result_by_id[article_id]\n+ \n+ # Validate and extract score\n+ score_raw = entry.get(\"score\")\n+ score = finite_unit_score(score_raw)\n+ if score is None:\n+ return refuse(\"score_out_of_range\", f\"invalid score for {article_id}: {score_raw}\")\n+ \n+ relevant = bool(entry.get(\"relevant\", score >= 0.5))\n+ reason = str(entry.get(\"reason\", \"\"))\n+ \n+ out.append(verdict(article_id, relevant, score, reason))\n+ \n+ return ok(out)\n+\n \n+def _positional_association(articles: list[dict[str, Any]], results_list: list) -> dict[str, Any]:\n+ \"\"\"Associate results by position - only used when no article_id present.\"\"\"\n+ if len(results_list) != len(articles):\n+ return refuse(\n+ \"count_mismatch\",\n+ f\"{len(results_list)} results for {len(articles)} articles\",\n+ )\n+ \n out: list[dict[str, Any]] = []\n- for i, article in enumerate(articles):\n- if i < len(results_list):\n- entry = results_list[i] if isinstance(results_list[i], dict) else {}\n+ for article, entry in zip(articles, results_list):\n+ if not isinstance(entry, dict):\n+ return refuse(\"invalid_type\", f\"entry is {type(entry).__name__}\")\n+ \n+ # Validate and extract score\n+ score_raw = entry.get(\"score\")\n+ # Use finite_unit_score if available, else clamp\n+ score = finite_unit_score(score_raw)\n+ if score is None:\n try:\n score = max(0.0, min(1.0, float(entry.get(\"score\", 0.5))))\n- except Exception:\n- score = 0.5\n- relevant = bool(entry.get(\"relevant\", score >= 0.5))\n- out.append(verdict(article[\"id\"], relevant, score, str(entry.get(\"reason\", \"\"))))\n- else:\n- out.append(verdict(article[\"id\"], False, 0.0, \"scoring incomplete\"))\n+ except (TypeError, ValueError):\n+ return refuse(\"invalid_type\", \"score is not numeric\")\n+ \n+ relevant = bool(entry.get(\"relevant\", score >= 0.5))\n+ reason = str(entry.get(\"reason\", \"\"))\n+ \n+ out.append(verdict(article[\"id\"], relevant, score, reason))\n+ \n return ok(out)\n", "patch_base": "backend/lab/contract/versions/positional_v0.py" }, "provenance": { "executed_at_revision": "1c97bc0483f43673fc9f31820d8187d53659a0ff+dirty", "inputs_sha256": "201d390524ff79f4", "executed_at": "2026-09-30T03:52:40.599Z", "evaluator_sha256": "277ec81521ba14d8", "spec_hash": "f027762ab4d08b35", "spec_version": 2, "execution_mode": "offline-replay", "execution_mode_basis": "The harness reads committed responses from disk and makes no network call. The candidate imports nothing beyond the standard library.", "python": "3.13.1", "runner": "vercel-sandbox", "sandbox": { "sandbox_id": "moccasin-endless-felidae-TKFPiP", "runtime": "python3.13", "region": "iad1", "network_policy": "deny-all", "egress_bytes": 34495, "active_cpu_ms": 1857, "boot_ms": 280, "wall_clock_ms": 376, "exit_code": 0, "uploaded": [ { "path": "lab/__init__.py", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes": 0 }, { "path": "lab/harness.py", "sha256": "5a459209f6a0c84e4dd3e9d1cc7e478ba90a0b12911239183a5849fe4b63c7cd", "bytes": 9807 }, { "path": "lab/cases/observed.json", "sha256": "8f23376a7e53f3bf55764945208c2ce37e2a8716a314902ed6175bec823a8753", "bytes": 1749847 }, { "path": "lab/cases/synthetic.json", "sha256": "72e8c341c8a9fbbe9b062539ebb4245734ec7ef255ccd9a37c6780e2145bcbf9", "bytes": 26035 }, { "path": "lab/candidate.py", "sha256": "57a914e27676dab3c5c0977f5e9532128c86203736c70ee91f73e49935614fdc", "bytes": 6620 } ], "isolation": [ { "name": "egress-dns", "command": "python3 -c import socket;socket.setdefaulttimeout(8);socket.gethostbyname(\"api.openai.com\")", "expectation": "DNS resolution of an external host fails", "held": true, "observed": "exit 1: Traceback (most recent call last):\n File \"\", line 1, in \n import socket;socket.setdefaulttimeout(8);socket.gethostbyname(\"api.openai.com\")\n ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^\nsocket.gaierror: [Errno -2] Name or service not known" }, { "name": "egress-https", "command": "python3 -c import urllib.request;urllib.request.urlopen(\"https://api.github.com/meta\",timeout=8).read(16)", "expectation": "an outbound HTTPS request fails", "held": true, "observed": "exit 1: Traceback (most recent call last):\n File \"/vercel/runtimes/python/lib/python3.13/urllib/request.py\", line 1319, in do_open\n h.request(req.get_method(), req.selector, req.data, headers,\n ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n encode_chunked=req.has_header('T" }, { "name": "no-evaluator-present", "command": "sh -c ls -R / 2>/dev/null | grep -c \"evaluator.ts\" || true", "expectation": "the code that grades the candidate is nowhere on the filesystem", "held": true, "observed": "exit 0: 0" }, { "name": "no-secrets-in-env", "command": "sh -c env | grep -ciE \"^(VERCEL_TOKEN|AI_GATEWAY|OPENAI|BLOB_READ_WRITE|DATABASE|ANTHROPIC)\" || true", "expectation": "no production credential is present in the environment", "held": true, "observed": "exit 0: 0" } ] }, "investigation": { "model": "moonshotai/kimi-k2", "gateway": "vercel-ai-gateway", "gateway_auth": "oidc", "started_at": "2026-09-30T03:52:40.601Z", "wall_clock_ms": 228713, "finish_reason": "stop", "stop_cause": "completed", "tool_calls_made": 19, "max_tool_calls": 24, "model_calls_made": 12, "budget_label": "generous", "budget": { "max_proposals": 1, "max_tool_calls": 24, "max_model_calls": 12, "max_output_tokens": 4096, "max_total_tokens": 300000, "max_tool_result_chars": 8000, "max_usd": 1.5, "wall_clock_seconds": 360 }, "budget_ceiling_usd": 1, "usage": { "input_tokens": 294758, "output_tokens": 7789, "total_tokens": 302547 }, "tokens_used": 302547, "cost_usd": 0.09414308, "proposed": true, "evaluation_requested_by": "agent", "hypothesis": "The defect is positional association: results[i] maps to articles[i], which fails on reordering or count mismatch. The keyed-v2 protocol fixes this by using article_id fields for unambiguous association. This parser: (1) Uses keyed association when article_id is present, validating for duplicate_id, unknown_id, and missing_id; (2) Falls back to positional only when no article_id fields exist, with strict count validation; (3) Refuses when association is ambiguous.", "evidence": [ "backend/lab/contract/types.py defines PROTOCOL_KEYED = 'keyed-v2' as the alternative to PROTOCOL_POSITIONAL = 'positional-v0'", "backend/lab/contract/versions/positional_v0.py lines 86-97 show pure positional association with no validation", "backend/lab/contract/versions/count_guard_v1.py lines 97-101 add count validation but still use positional association", "REFUSAL_KINDS in types.py includes 'duplicate_id', 'unknown_id', 'missing_id' for keyed validation", "backend/app/services/ranking_contract.py ArticleJudgment has article_id field showing keyed support" ], "scope_accepted": true, "scope_reason": "within scope; queued for sandboxed evaluation", "trace_path": "backend/lab/runs/agent-k2-generous-05-sandbox.trace.json", "n_trace_steps": 47 }, "case_suites": [ { "group": "observed", "path": "backend/lab/cases/observed.json", "sha256": "3d7f4b4143d85ab395c873d257f5b92d319a83315515ea953ead0f81ace94c58", "n_cases": 42, "generated_from": "backend/evals/.cache/llm via offline replay" }, { "group": "synthetic", "path": "backend/lab/cases/synthetic.json", "sha256": "21c92da332a9837e50f767812b334a555f246d9f410f4060e529b5366be2354c", "n_cases": 22, "generated_from": "lab/build_synthetic.py — fault injection, ground truth by construction" } ], "notes": [ { "severity": "info", "message": "Every case ran offline against responses already committed to this repository, so grading made no inference call. The investigation that wrote the candidate did, and its spend is recorded under usage.", "source": "backend/lab/runs/agent-k2-generous-05-sandbox.trace.json" }, { "severity": "caution", "message": "The case suite is public. A candidate may have been written against it, so passing does not establish generalisation.", "source": "backend/lab/cases/" } ] }, "verdict": "rejected", "verdict_reason": "protocol-violation-refusal: 8/9 cases satisfied, threshold 1", "verdict_scope": "Accepted means eligible for human review under this spec hash, against a public case suite. It is not evidence of production quality, and it does not establish generalisation: the cases are visible and a candidate may have been written against them.", "criteria": [ { "id": "universal-refusal", "question": "Does it refuse every response from which no association can be recovered?", "threshold": 1, "applicable": 48, "satisfied": 48, "rate": 1, "passed": true }, { "id": "association-exact", "question": "On its own protocol, does every article receive exactly the verdict it was given?", "threshold": 1, "applicable": 3, "satisfied": 3, "rate": 1, "passed": true }, { "id": "protocol-violation-refusal", "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?", "threshold": 1, "applicable": 9, "satisfied": 8, "rate": 0.8888888888888888, "passed": false }, { "id": "no-crash", "question": "Does it terminate on every case without crashing or hanging?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "complete-evidence", "question": "Is there a prediction record for every applicable case?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "protocol-exclusivity", "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?", "threshold": 1, "applicable": 4, "satisfied": 0, "rate": 0, "passed": false } ], "gradings": [ { "spec_version": 1, "spec_hash": "008af8266204438a", "verdict": "rejected", "verdict_reason": "protocol-violation-refusal: 8/9 cases satisfied, threshold 1", "criteria": [ { "id": "universal-refusal", "question": "Does it refuse every response from which no association can be recovered?", "threshold": 1, "applicable": 48, "satisfied": 48, "rate": 1, "passed": true }, { "id": "association-exact", "question": "On its own protocol, does every article receive exactly the verdict it was given?", "threshold": 1, "applicable": 3, "satisfied": 3, "rate": 1, "passed": true }, { "id": "protocol-violation-refusal", "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?", "threshold": 1, "applicable": 9, "satisfied": 8, "rate": 0.8888888888888888, "passed": false }, { "id": "no-crash", "question": "Does it terminate on every case without crashing or hanging?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "complete-evidence", "question": "Is there a prediction record for every applicable case?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true } ] }, { "spec_version": 2, "spec_hash": "f027762ab4d08b35", "verdict": "rejected", "verdict_reason": "protocol-violation-refusal: 8/9 cases satisfied, threshold 1", "criteria": [ { "id": "universal-refusal", "question": "Does it refuse every response from which no association can be recovered?", "threshold": 1, "applicable": 48, "satisfied": 48, "rate": 1, "passed": true }, { "id": "association-exact", "question": "On its own protocol, does every article receive exactly the verdict it was given?", "threshold": 1, "applicable": 3, "satisfied": 3, "rate": 1, "passed": true }, { "id": "protocol-violation-refusal", "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?", "threshold": 1, "applicable": 9, "satisfied": 8, "rate": 0.8888888888888888, "passed": false }, { "id": "no-crash", "question": "Does it terminate on every case without crashing or hanging?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "complete-evidence", "question": "Is there a prediction record for every applicable case?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "protocol-exclusivity", "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?", "threshold": 1, "applicable": 4, "satisfied": 0, "rate": 0, "passed": false } ] } ], "outcomes": [ { "case_id": "observed-2026-09-02-000", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.465 }, { "case_id": "observed-2026-09-02-001", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.403 }, { "case_id": "observed-2026-09-02-002", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.026 }, { "case_id": "observed-2026-09-02-003", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.373 }, { "case_id": "observed-2026-09-02-004", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.368 }, { "case_id": "observed-2026-09-02-005", "group": "observed", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.06 }, { "case_id": "observed-2026-09-02-006", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.134 }, { "case_id": "observed-2026-09-02-007", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.029 }, { "case_id": "observed-2026-09-02-008", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.031 }, { "case_id": "observed-2026-09-02-009", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.279 }, { "case_id": "observed-2026-09-02-010", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.036 }, { "case_id": "observed-2026-09-02-011", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.276 }, { "case_id": "observed-2026-09-02-012", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.268 }, { "case_id": "observed-2026-09-02-013", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.025 }, { "case_id": "observed-2026-09-02-014", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.388 }, { "case_id": "observed-2026-09-02-015", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.019 }, { "case_id": "observed-2026-09-02-016", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.381 }, { "case_id": "observed-2026-09-02-017", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.376 }, { "case_id": "observed-2026-09-02-018", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.043 }, { "case_id": "observed-2026-09-02-019", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.323 }, { "case_id": "observed-2026-09-02-020", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.018 }, { "case_id": "observed-2026-09-02-021", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.323 }, { "case_id": "observed-2026-09-02-022", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.325 }, { "case_id": "observed-2026-09-02-023", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.032 }, { "case_id": "observed-2026-09-02-024", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.018 }, { "case_id": "observed-2026-09-02-025", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.027 }, { "case_id": "observed-2026-09-02-026", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.263 }, { "case_id": "observed-2026-09-02-027", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.021 }, { "case_id": "observed-2026-09-02-028", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.03 }, { "case_id": "observed-2026-09-02-029", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.235 }, { "case_id": "observed-2026-09-02-030", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.231 }, { "case_id": "observed-2026-09-02-031", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.026 }, { "case_id": "observed-2026-09-02-032", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.037 }, { "case_id": "observed-2026-09-02-033", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.018 }, { "case_id": "observed-2026-09-02-034", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.03 }, { "case_id": "observed-2026-09-02-035", "group": "observed", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.044 }, { "case_id": "observed-2026-09-02-036", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.307 }, { "case_id": "observed-2026-09-02-037", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.281 }, { "case_id": "observed-2026-09-02-038", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.284 }, { "case_id": "observed-2026-09-02-039", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.025 }, { "case_id": "observed-2026-09-02-040", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.154 }, { "case_id": "observed-2026-09-02-041", "group": "observed", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.042 }, { "case_id": "syn-keyed-in-order", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "every article received its own verdict", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.029 }, { "case_id": "syn-keyed-reversed", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "every article received its own verdict", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.021 }, { "case_id": "syn-keyed-rotated", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "every article received its own verdict", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.018 }, { "case_id": "syn-duplicate-id", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (duplicate_id)", "counterexample": null, "observed_refusal_kind": "duplicate_id", "expected_refusal_kinds": [ "duplicate_id" ], "ms": 0.012 }, { "case_id": "syn-unknown-id", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (unknown_id)", "counterexample": null, "observed_refusal_kind": "unknown_id", "expected_refusal_kinds": [ "unknown_id" ], "ms": 0.011 }, { "case_id": "syn-missing-id", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (missing_id)", "counterexample": null, "observed_refusal_kind": "missing_id", "expected_refusal_kinds": [ "missing_id", "count_mismatch" ], "ms": 0.014 }, { "case_id": "syn-score-nan", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (score_out_of_range)", "counterexample": null, "observed_refusal_kind": "score_out_of_range", "expected_refusal_kinds": [ "score_not_finite", "invalid_type", "malformed_json" ], "ms": 0.015 }, { "case_id": "syn-score-out-of-range", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (score_out_of_range)", "counterexample": null, "observed_refusal_kind": "score_out_of_range", "expected_refusal_kinds": [ "score_out_of_range", "invalid_type" ], "ms": 0.012 }, { "case_id": "syn-score-string", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (score_out_of_range)", "counterexample": null, "observed_refusal_kind": "score_out_of_range", "expected_refusal_kinds": [ "invalid_type" ], "ms": 0.011 }, { "case_id": "syn-score-null", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (score_out_of_range)", "counterexample": null, "observed_refusal_kind": "score_out_of_range", "expected_refusal_kinds": [ "invalid_type" ], "ms": 0.011 }, { "case_id": "syn-relevant-not-bool", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "should-have-refused", "detail": "relevant is a string", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [ "invalid_type" ], "ms": 0.018 }, { "case_id": "syn-malformed-json", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "malformed_json" ], "ms": 0.01 }, { "case_id": "syn-truncated-finish-reason", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (missing_id)", "counterexample": null, "observed_refusal_kind": "missing_id", "expected_refusal_kinds": [ "truncated_response", "count_mismatch", "missing_id" ], "ms": 0.012 }, { "case_id": "syn-not-an-object", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (unexpected_shape)", "counterexample": null, "observed_refusal_kind": "unexpected_shape", "expected_refusal_kinds": [ "unexpected_shape" ], "ms": 0.009 }, { "case_id": "syn-empty-results", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "missing_id", "count_mismatch" ], "ms": 0.007 }, { "case_id": "syn-exec-no-recording", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (no_recording)", "counterexample": null, "observed_refusal_kind": "no_recording", "expected_refusal_kinds": [ "no_recording" ], "ms": 0.003 }, { "case_id": "syn-exec-timeout", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (timeout)", "counterexample": null, "observed_refusal_kind": "timeout", "expected_refusal_kinds": [ "timeout" ], "ms": 0.003 }, { "case_id": "syn-exec-cancelled", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (cancelled)", "counterexample": null, "observed_refusal_kind": "cancelled", "expected_refusal_kinds": [ "cancelled" ], "ms": 0.003 }, { "case_id": "syn-exec-budget", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (retries_exhausted)", "counterexample": null, "observed_refusal_kind": "retries_exhausted", "expected_refusal_kinds": [ "retries_exhausted", "no_recording" ], "ms": 0.003 }, { "case_id": "syn-positional-short", "group": "synthetic", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch", "missing_id" ], "ms": 0.009 }, { "case_id": "syn-positional-long", "group": "synthetic", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch", "duplicate_id" ], "ms": 0.01 }, { "case_id": "syn-positional-reordered", "group": "synthetic", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [ "count_mismatch", "unexpected_shape", "unsupported_protocol", "invalid_type", "missing_id" ], "ms": 0.018 } ], "counts": { "correct": 59, "wrong-association": 0, "should-have-refused": 1, "should-have-parsed": 0, "crashed": 0, "timeout": 0, "missing-record": 0, "not-applicable": 4 }, "smallest_counterexample": null, "diagnostics": [ { "id": "out-of-protocol-association", "question": "On cases outside its declared protocol, did it refuse — or associate anyway?", "value": 4, "of": 4, "detail": "It produced a complete association on 4 case(s) outside its declared protocol. Under this generation that fails protocol-exclusivity; under generation 1 it was not graded at all.", "case_ids": [ "observed-2026-09-02-005", "observed-2026-09-02-035", "observed-2026-09-02-041", "syn-positional-reordered" ] } ], "usage": { "model_calls": 12, "replay_spend_usd": 0, "recording_cost_usd": "unknown", "provider_reported": { "input_tokens": 294758, "output_tokens": 7789, "total_tokens": 302547, "tokens_used": 302547, "cost_usd": 0.09414308 }, "basis": "Grading replayed committed recordings and made no inference call; `replay_spend_usd` is that, and is $0. The investigation that authored the candidate did call a model: 12 call(s) to moonshotai/kimi-k2 through the AI Gateway, 302547 tokens, $0.09414308 as the gateway reported it per call. That is in `provider_reported`. The microVM is metered compute billed at a rate this repository does not record: 1857ms of active CPU across 656ms wall clock." }, "attempts": [ { "attempt_id": "article-to-verdict-association__agent-k2-generous-05#01", "started_at": "2026-09-30T03:56:10.786Z", "ended_at": "2026-09-30T03:56:16.956Z", "status": "succeeded", "runner": "vercel-sandbox", "note": "sandbox moccasin-endless-felidae-TKFPiP in iad1, exit 0 in 376ms" } ] }