{ "lab_artifact_version": 1, "run_id": "article-to-verdict-association__agent-k2-generous-out-02__sandbox", "experiment_id": "article-to-verdict-association", "candidate": { "candidate_id": "agent-k2-generous-out-02", "kind": "agent-authored", "description": "The defect is positional association - when model returns results in different order or wrong count, articles get wrong verdicts. The fix is keyed-v2 protocol: each result must contain article_id for unambiguous association. Parser validates exact ID set matching, refuses with duplicate_id/unknown_id/missing_id/count_mismatch when association is ambiguous.", "declared_protocol": "keyed-v2", "source_path": "backend/lab/contract/candidates/agent_k2_generous_out_02.py", "source_sha256": "b0227a086bb086b34152e5d813493810dd3c6bb86ce6cdd6245e0a32df352ac8", "source_bytes": 5476, "transcribed_from": "unknown", "patch": "diff --git a/backend/lab/contract/versions/positional_v0.py b/backend/lab/contract/candidates/agent_k2_generous_out_02.py\n--- a/backend/lab/contract/versions/positional_v0.py\n+++ b/backend/lab/contract/candidates/agent_k2_generous_out_02.py\n@@ -1,30 +1,13 @@\n-\"\"\"Historical behaviour, transcribed from `origin/main`.\n+\"\"\"Keyed protocol parser (keyed-v2) that associates verdicts by article_id.\n \n-Source: backend/app/services/openai_service.py, score_articles_batch, the\n-`normalized` loop. Verbatim semantics:\n-\n- if len(results_list) != len(articles):\n- logger.warning(\"... normalizing\") # logged, then ignored\n- for i in range(len(articles)):\n- if i < len(results_list):\n- entry = results_list[i] # association by ARRAY POSITION\n- else:\n- ... {\"relevant\": False, \"score\": 0.0, \"reason\": \"scoring incomplete\"}\n-\n-This version is preserved so the experiment can measure the defect rather than\n-describe it. It is not a control: it is what production does today.\n+This parser fixes the positional association defect by requiring each result\n+to explicitly identify its article via article_id, enabling unambiguous\n+association regardless of result order.\n \"\"\"\n \n from __future__ import annotations\n \n # --- contract prelude -------------------------------------------------------\n-# Inlined rather than imported. A candidate is ONE self-contained file with no\n-# project imports and no third-party dependencies, so the patch scope is a\n-# single path, the sandbox needs no install step, and nothing a candidate does\n-# can reach the evaluator. The canonical definitions live in\n-# lab/contract/types.py, and tests/test_contract_prelude.py asserts that every\n-# copy still agrees with it.\n-\n import math as _math\n \n \n@@ -54,45 +37,116 @@ import json\n from typing import Any\n \n \n-VERSION_ID = \"positional-v0\"\n-PROTOCOL = \"positional-v0\"\n+VERSION_ID = \"keyed-v2\"\n+PROTOCOL = \"keyed-v2\"\n \n \n def parse(articles: list[dict[str, Any]], response: dict[str, Any]) -> dict[str, Any]:\n- if response.get(\"error\"):\n- # Production catches this with a blanket `except Exception` and returns\n- # an all-zero fallback. Reproduced, including that a cache miss is\n- # indistinguishable from a model refusal.\n- return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n-\n+ \"\"\"Parse response using keyed association by article_id.\n+ \n+ Each result entry must contain an 'article_id' field that matches one of the\n+ input articles. This enables unambiguous association regardless of result order.\n+ \"\"\"\n+ # Handle error responses\n+ error = response.get(\"error\")\n+ if error:\n+ if error in {\"no_recording\", \"budget_exceeded\"}:\n+ return refuse(\"no_recording\" if error == \"no_recording\" else \"retries_exhausted\", error)\n+ if error in {\"timeout\", \"cancelled\"}:\n+ return refuse(error, error)\n+ return refuse(\"retries_exhausted\", str(error))\n+\n+ # Get content\n content = response.get(\"content\")\n if content is None:\n- return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n+ return refuse(\"truncated_response\", \"no content returned\")\n \n+ # Parse JSON\n try:\n result = json.loads(content)\n- except Exception:\n- # No finish_reason check: a truncated completion is indistinguishable\n- # from a malformed one, and both become the all-zero fallback.\n- return ok([verdict(a[\"id\"], False, 0.0, \"scoring unavailable\") for a in articles])\n-\n- results_list = result.get(\"results\", []) if isinstance(result, dict) else []\n- if not results_list and isinstance(result, dict) and \"scores\" in result:\n+ except Exception as exc:\n+ return refuse(\"malformed_json\", str(exc))\n+\n+ # Validate top-level structure\n+ if not isinstance(result, dict):\n+ return refuse(\"unexpected_shape\", f\"top level is {type(result).__name__}\")\n+\n+ # Get results list\n+ results_list = result.get(\"results\", [])\n+ if not isinstance(results_list, list):\n+ return refuse(\"unexpected_shape\", \"results is not a list\")\n+\n+ # Handle legacy \"scores\" format by converting to keyed format\n+ if not results_list and \"scores\" in result:\n+ scores = result[\"scores\"]\n+ if not isinstance(scores, list):\n+ return refuse(\"unexpected_shape\", \"scores is not a list\")\n+ if len(scores) != len(articles):\n+ return refuse(\"count_mismatch\", f\"{len(scores)} scores for {len(articles)} articles\")\n+ # Convert legacy scores to keyed results using positional mapping\n results_list = [\n- {\"relevant\": float(s) >= 0.5, \"score\": float(s), \"reason\": \"\"}\n- for s in result[\"scores\"]\n+ {\"article_id\": article[\"id\"], \"relevant\": float(s) >= 0.5, \"score\": float(s), \"reason\": \"\"}\n+ for article, s in zip(articles, scores)\n ]\n \n- out: list[dict[str, Any]] = []\n- for i, article in enumerate(articles):\n- if i < len(results_list):\n- entry = results_list[i] if isinstance(results_list[i], dict) else {}\n- try:\n- score = max(0.0, min(1.0, float(entry.get(\"score\", 0.5))))\n- except Exception:\n- score = 0.5\n- relevant = bool(entry.get(\"relevant\", score >= 0.5))\n- out.append(verdict(article[\"id\"], relevant, score, str(entry.get(\"reason\", \"\"))))\n+ # Check count match\n+ if len(results_list) != len(articles):\n+ return refuse(\"count_mismatch\", f\"{len(results_list)} results for {len(articles)} articles\")\n+\n+ # Build expected ID set from input articles\n+ expected_ids = {article[\"id\"] for article in articles}\n+ \n+ # Process results and validate keyed association\n+ verdicts = []\n+ seen_ids = set()\n+ \n+ for entry in results_list:\n+ if not isinstance(entry, dict):\n+ return refuse(\"invalid_type\", f\"entry is {type(entry).__name__}\")\n+ \n+ # Extract article_id\n+ article_id = entry.get(\"article_id\")\n+ if article_id is None:\n+ return refuse(\"missing_id\", \"result entry missing article_id field\")\n+ \n+ # Validate article_id is a string\n+ if not isinstance(article_id, str):\n+ return refuse(\"invalid_type\", \"article_id is not a string\")\n+ \n+ # Check for duplicates\n+ if article_id in seen_ids:\n+ return refuse(\"duplicate_id\", f\"duplicate article_id: {article_id}\")\n+ seen_ids.add(article_id)\n+ \n+ # Check article_id is in expected set\n+ if article_id not in expected_ids:\n+ return refuse(\"unknown_id\", f\"unknown article_id: {article_id}\")\n+ \n+ # Extract and validate score\n+ score_raw = entry.get(\"score\", 0.5)\n+ score = finite_unit_score(score_raw)\n+ if score is None:\n+ return refuse(\"score_out_of_range\", f\"invalid score: {score_raw}\")\n+ \n+ # Extract relevant (default based on score)\n+ relevant_raw = entry.get(\"relevant\")\n+ if relevant_raw is None:\n+ relevant = score >= 0.5\n+ elif isinstance(relevant_raw, bool):\n+ relevant = relevant_raw\n else:\n- out.append(verdict(article[\"id\"], False, 0.0, \"scoring incomplete\"))\n- return ok(out)\n+ return refuse(\"invalid_type\", f\"relevant is not boolean: {type(relevant_raw).__name__}\")\n+ \n+ # Extract reason\n+ reason = entry.get(\"reason\", \"\")\n+ if not isinstance(reason, str):\n+ reason = str(reason)\n+ \n+ verdicts.append(verdict(article_id, relevant, score, reason))\n+ \n+ # Check all expected IDs were present\n+ missing = expected_ids - seen_ids\n+ if missing:\n+ return refuse(\"missing_id\", f\"missing results for articles: {sorted(missing)}\")\n+ \n+ return ok(verdicts)\n", "patch_base": "backend/lab/contract/versions/positional_v0.py" }, "provenance": { "executed_at_revision": "1c97bc0483f43673fc9f31820d8187d53659a0ff+dirty", "inputs_sha256": "4dd632ee78ec3e56", "executed_at": "2026-09-30T03:57:19.779Z", "evaluator_sha256": "277ec81521ba14d8", "spec_hash": "f027762ab4d08b35", "spec_version": 2, "execution_mode": "offline-replay", "execution_mode_basis": "The harness reads committed responses from disk and makes no network call. The candidate imports nothing beyond the standard library.", "python": "3.13.1", "runner": "vercel-sandbox", "sandbox": { "sandbox_id": "blush-ethical-krill-415iwX", "runtime": "python3.13", "region": "iad1", "network_policy": "deny-all", "egress_bytes": 23267, "active_cpu_ms": 1922, "boot_ms": 297, "wall_clock_ms": 395, "exit_code": 0, "uploaded": [ { "path": "lab/__init__.py", "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes": 0 }, { "path": "lab/harness.py", "sha256": "5a459209f6a0c84e4dd3e9d1cc7e478ba90a0b12911239183a5849fe4b63c7cd", "bytes": 9807 }, { "path": "lab/cases/observed.json", "sha256": "8f23376a7e53f3bf55764945208c2ce37e2a8716a314902ed6175bec823a8753", "bytes": 1749847 }, { "path": "lab/cases/synthetic.json", "sha256": "72e8c341c8a9fbbe9b062539ebb4245734ec7ef255ccd9a37c6780e2145bcbf9", "bytes": 26035 }, { "path": "lab/candidate.py", "sha256": "b0227a086bb086b34152e5d813493810dd3c6bb86ce6cdd6245e0a32df352ac8", "bytes": 5476 } ], "isolation": [ { "name": "egress-dns", "command": "python3 -c import socket;socket.setdefaulttimeout(8);socket.gethostbyname(\"api.openai.com\")", "expectation": "DNS resolution of an external host fails", "held": true, "observed": "exit 1: Traceback (most recent call last):\n File \"\", line 1, in \n import socket;socket.setdefaulttimeout(8);socket.gethostbyname(\"api.openai.com\")\n ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^\nsocket.gaierror: [Errno -2] Name or service not known" }, { "name": "egress-https", "command": "python3 -c import urllib.request;urllib.request.urlopen(\"https://api.github.com/meta\",timeout=8).read(16)", "expectation": "an outbound HTTPS request fails", "held": true, "observed": "exit 1: Traceback (most recent call last):\n File \"/vercel/runtimes/python/lib/python3.13/urllib/request.py\", line 1319, in do_open\n h.request(req.get_method(), req.selector, req.data, headers,\n ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n encode_chunked=req.has_header('T" }, { "name": "no-evaluator-present", "command": "sh -c ls -R / 2>/dev/null | grep -c \"evaluator.ts\" || true", "expectation": "the code that grades the candidate is nowhere on the filesystem", "held": true, "observed": "exit 0: 0" }, { "name": "no-secrets-in-env", "command": "sh -c env | grep -ciE \"^(VERCEL_TOKEN|AI_GATEWAY|OPENAI|BLOB_READ_WRITE|DATABASE|ANTHROPIC)\" || true", "expectation": "no production credential is present in the environment", "held": true, "observed": "exit 0: 0" } ] }, "investigation": { "model": "moonshotai/kimi-k2", "gateway": "vercel-ai-gateway", "gateway_auth": "oidc", "started_at": "2026-09-30T03:57:19.781Z", "wall_clock_ms": 253448, "finish_reason": "tool-calls", "stop_cause": "step-limit", "tool_calls_made": 20, "max_tool_calls": 24, "model_calls_made": 12, "budget_label": "generous-out", "budget": { "max_proposals": 1, "max_tool_calls": 24, "max_model_calls": 12, "max_output_tokens": 8192, "max_total_tokens": 300000, "max_tool_result_chars": 8000, "max_usd": 1.5, "wall_clock_seconds": 360 }, "budget_ceiling_usd": 1, "usage": { "input_tokens": 259125, "output_tokens": 7000, "total_tokens": 266125 }, "tokens_used": 266125, "cost_usd": 0.04567901, "proposed": true, "evaluation_requested_by": "agent", "hypothesis": "The defect is positional association - when model returns results in different order or wrong count, articles get wrong verdicts. The fix is keyed-v2 protocol: each result must contain article_id for unambiguous association. Parser validates exact ID set matching, refuses with duplicate_id/unknown_id/missing_id/count_mismatch when association is ambiguous.", "evidence": [ "backend/lab/contract/types.py defines PROTOCOL_KEYED = 'keyed-v2' and refusal kinds duplicate_id, unknown_id, missing_id for keyed failures", "backend/app/services/ranking_contract.py ArticleJudgment has article_id and validate_judgments checks exact ID set matching", "observed-2026-09-02-002: 18 results for 20 articles should refuse count_mismatch", "observed-2026-09-02-007: truncated response should refuse", "count_guard_v1 only checks length, still associates by position, fails on reordering" ], "scope_accepted": true, "scope_reason": "within scope; queued for sandboxed evaluation", "trace_path": "backend/lab/runs/agent-k2-generous-out-02-sandbox.trace.json", "n_trace_steps": 49 }, "case_suites": [ { "group": "observed", "path": "backend/lab/cases/observed.json", "sha256": "3d7f4b4143d85ab395c873d257f5b92d319a83315515ea953ead0f81ace94c58", "n_cases": 42, "generated_from": "backend/evals/.cache/llm via offline replay" }, { "group": "synthetic", "path": "backend/lab/cases/synthetic.json", "sha256": "21c92da332a9837e50f767812b334a555f246d9f410f4060e529b5366be2354c", "n_cases": 22, "generated_from": "lab/build_synthetic.py — fault injection, ground truth by construction" } ], "notes": [ { "severity": "info", "message": "Every case ran offline against responses already committed to this repository, so grading made no inference call. The investigation that wrote the candidate did, and its spend is recorded under usage.", "source": "backend/lab/runs/agent-k2-generous-out-02-sandbox.trace.json" }, { "severity": "caution", "message": "The case suite is public. A candidate may have been written against it, so passing does not establish generalisation.", "source": "backend/lab/cases/" } ] }, "verdict": "accepted-for-review", "verdict_reason": "all 6 criteria satisfied over 60 applicable cases", "verdict_scope": "Accepted means eligible for human review under this spec hash, against a public case suite. It is not evidence of production quality, and it does not establish generalisation: the cases are visible and a candidate may have been written against them.", "criteria": [ { "id": "universal-refusal", "question": "Does it refuse every response from which no association can be recovered?", "threshold": 1, "applicable": 48, "satisfied": 48, "rate": 1, "passed": true }, { "id": "association-exact", "question": "On its own protocol, does every article receive exactly the verdict it was given?", "threshold": 1, "applicable": 3, "satisfied": 3, "rate": 1, "passed": true }, { "id": "protocol-violation-refusal", "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?", "threshold": 1, "applicable": 9, "satisfied": 9, "rate": 1, "passed": true }, { "id": "no-crash", "question": "Does it terminate on every case without crashing or hanging?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "complete-evidence", "question": "Is there a prediction record for every applicable case?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "protocol-exclusivity", "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?", "threshold": 1, "applicable": 4, "satisfied": 4, "rate": 1, "passed": true } ], "gradings": [ { "spec_version": 1, "spec_hash": "008af8266204438a", "verdict": "accepted-for-review", "verdict_reason": "all 5 criteria satisfied over 60 applicable cases", "criteria": [ { "id": "universal-refusal", "question": "Does it refuse every response from which no association can be recovered?", "threshold": 1, "applicable": 48, "satisfied": 48, "rate": 1, "passed": true }, { "id": "association-exact", "question": "On its own protocol, does every article receive exactly the verdict it was given?", "threshold": 1, "applicable": 3, "satisfied": 3, "rate": 1, "passed": true }, { "id": "protocol-violation-refusal", "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?", "threshold": 1, "applicable": 9, "satisfied": 9, "rate": 1, "passed": true }, { "id": "no-crash", "question": "Does it terminate on every case without crashing or hanging?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "complete-evidence", "question": "Is there a prediction record for every applicable case?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true } ] }, { "spec_version": 2, "spec_hash": "f027762ab4d08b35", "verdict": "accepted-for-review", "verdict_reason": "all 6 criteria satisfied over 60 applicable cases", "criteria": [ { "id": "universal-refusal", "question": "Does it refuse every response from which no association can be recovered?", "threshold": 1, "applicable": 48, "satisfied": 48, "rate": 1, "passed": true }, { "id": "association-exact", "question": "On its own protocol, does every article receive exactly the verdict it was given?", "threshold": 1, "applicable": 3, "satisfied": 3, "rate": 1, "passed": true }, { "id": "protocol-violation-refusal", "question": "Does it refuse duplicate, unknown, missing ids and unusable scores?", "threshold": 1, "applicable": 9, "satisfied": 9, "rate": 1, "passed": true }, { "id": "no-crash", "question": "Does it terminate on every case without crashing or hanging?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "complete-evidence", "question": "Is there a prediction record for every applicable case?", "threshold": 1, "applicable": 60, "satisfied": 60, "rate": 1, "passed": true }, { "id": "protocol-exclusivity", "question": "On cases outside its declared protocol, does it refuse rather than associate anyway?", "threshold": 1, "applicable": 4, "satisfied": 4, "rate": 1, "passed": true } ] } ], "outcomes": [ { "case_id": "observed-2026-09-02-000", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.306 }, { "case_id": "observed-2026-09-02-001", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.502 }, { "case_id": "observed-2026-09-02-002", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.021 }, { "case_id": "observed-2026-09-02-003", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.384 }, { "case_id": "observed-2026-09-02-004", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.396 }, { "case_id": "observed-2026-09-02-005", "group": "observed", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": "missing_id", "expected_refusal_kinds": [], "ms": 0.023 }, { "case_id": "observed-2026-09-02-006", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.122 }, { "case_id": "observed-2026-09-02-007", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.025 }, { "case_id": "observed-2026-09-02-008", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.025 }, { "case_id": "observed-2026-09-02-009", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.293 }, { "case_id": "observed-2026-09-02-010", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.016 }, { "case_id": "observed-2026-09-02-011", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.276 }, { "case_id": "observed-2026-09-02-012", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.272 }, { "case_id": "observed-2026-09-02-013", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.02 }, { "case_id": "observed-2026-09-02-014", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.408 }, { "case_id": "observed-2026-09-02-015", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.016 }, { "case_id": "observed-2026-09-02-016", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.388 }, { "case_id": "observed-2026-09-02-017", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.382 }, { "case_id": "observed-2026-09-02-018", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.037 }, { "case_id": "observed-2026-09-02-019", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.353 }, { "case_id": "observed-2026-09-02-020", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.017 }, { "case_id": "observed-2026-09-02-021", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.332 }, { "case_id": "observed-2026-09-02-022", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.326 }, { "case_id": "observed-2026-09-02-023", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.025 }, { "case_id": "observed-2026-09-02-024", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.015 }, { "case_id": "observed-2026-09-02-025", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.022 }, { "case_id": "observed-2026-09-02-026", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.248 }, { "case_id": "observed-2026-09-02-027", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.016 }, { "case_id": "observed-2026-09-02-028", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.023 }, { "case_id": "observed-2026-09-02-029", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.253 }, { "case_id": "observed-2026-09-02-030", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.236 }, { "case_id": "observed-2026-09-02-031", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.02 }, { "case_id": "observed-2026-09-02-032", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.03 }, { "case_id": "observed-2026-09-02-033", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.013 }, { "case_id": "observed-2026-09-02-034", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.023 }, { "case_id": "observed-2026-09-02-035", "group": "observed", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": "missing_id", "expected_refusal_kinds": [], "ms": 0.019 }, { "case_id": "observed-2026-09-02-036", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.294 }, { "case_id": "observed-2026-09-02-037", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.29 }, { "case_id": "observed-2026-09-02-038", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "truncated_response", "malformed_json" ], "ms": 0.29 }, { "case_id": "observed-2026-09-02-039", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.018 }, { "case_id": "observed-2026-09-02-040", "group": "observed", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch" ], "ms": 0.186 }, { "case_id": "observed-2026-09-02-041", "group": "observed", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": "missing_id", "expected_refusal_kinds": [], "ms": 0.025 }, { "case_id": "syn-keyed-in-order", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "every article received its own verdict", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.077 }, { "case_id": "syn-keyed-reversed", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "every article received its own verdict", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.025 }, { "case_id": "syn-keyed-rotated", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "every article received its own verdict", "counterexample": null, "observed_refusal_kind": null, "expected_refusal_kinds": [], "ms": 0.018 }, { "case_id": "syn-duplicate-id", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "duplicate_id" ], "ms": 0.01 }, { "case_id": "syn-unknown-id", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (unknown_id)", "counterexample": null, "observed_refusal_kind": "unknown_id", "expected_refusal_kinds": [ "unknown_id" ], "ms": 0.016 }, { "case_id": "syn-missing-id", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "missing_id", "count_mismatch" ], "ms": 0.008 }, { "case_id": "syn-score-nan", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (score_out_of_range)", "counterexample": null, "observed_refusal_kind": "score_out_of_range", "expected_refusal_kinds": [ "score_not_finite", "invalid_type", "malformed_json" ], "ms": 0.02 }, { "case_id": "syn-score-out-of-range", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (score_out_of_range)", "counterexample": null, "observed_refusal_kind": "score_out_of_range", "expected_refusal_kinds": [ "score_out_of_range", "invalid_type" ], "ms": 0.012 }, { "case_id": "syn-score-string", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (score_out_of_range)", "counterexample": null, "observed_refusal_kind": "score_out_of_range", "expected_refusal_kinds": [ "invalid_type" ], "ms": 0.011 }, { "case_id": "syn-score-null", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (score_out_of_range)", "counterexample": null, "observed_refusal_kind": "score_out_of_range", "expected_refusal_kinds": [ "invalid_type" ], "ms": 0.014 }, { "case_id": "syn-relevant-not-bool", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (invalid_type)", "counterexample": null, "observed_refusal_kind": "invalid_type", "expected_refusal_kinds": [ "invalid_type" ], "ms": 0.012 }, { "case_id": "syn-malformed-json", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (malformed_json)", "counterexample": null, "observed_refusal_kind": "malformed_json", "expected_refusal_kinds": [ "malformed_json" ], "ms": 0.012 }, { "case_id": "syn-truncated-finish-reason", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "truncated_response", "count_mismatch", "missing_id" ], "ms": 0.007 }, { "case_id": "syn-not-an-object", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (unexpected_shape)", "counterexample": null, "observed_refusal_kind": "unexpected_shape", "expected_refusal_kinds": [ "unexpected_shape" ], "ms": 0.009 }, { "case_id": "syn-empty-results", "group": "synthetic", "family": "protocol-association", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "missing_id", "count_mismatch" ], "ms": 0.005 }, { "case_id": "syn-exec-no-recording", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (no_recording)", "counterexample": null, "observed_refusal_kind": "no_recording", "expected_refusal_kinds": [ "no_recording" ], "ms": 0.003 }, { "case_id": "syn-exec-timeout", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (timeout)", "counterexample": null, "observed_refusal_kind": "timeout", "expected_refusal_kinds": [ "timeout" ], "ms": 0.003 }, { "case_id": "syn-exec-cancelled", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (cancelled)", "counterexample": null, "observed_refusal_kind": "cancelled", "expected_refusal_kinds": [ "cancelled" ], "ms": 0.003 }, { "case_id": "syn-exec-budget", "group": "synthetic", "family": "universal-refusal", "protocol": "keyed-v2", "applicability": "scored", "status": "correct", "detail": "refused (retries_exhausted)", "counterexample": null, "observed_refusal_kind": "retries_exhausted", "expected_refusal_kinds": [ "retries_exhausted", "no_recording" ], "ms": 0.003 }, { "case_id": "syn-positional-short", "group": "synthetic", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch", "missing_id" ], "ms": 0.007 }, { "case_id": "syn-positional-long", "group": "synthetic", "family": "universal-refusal", "protocol": "positional-v0", "applicability": "scored", "status": "correct", "detail": "refused (count_mismatch)", "counterexample": null, "observed_refusal_kind": "count_mismatch", "expected_refusal_kinds": [ "count_mismatch", "duplicate_id" ], "ms": 0.008 }, { "case_id": "syn-positional-reordered", "group": "synthetic", "family": "protocol-association", "protocol": "positional-v0", "applicability": "not-applicable", "status": "not-applicable", "detail": "case is positional-v0; candidate declares keyed-v2", "counterexample": null, "observed_refusal_kind": "missing_id", "expected_refusal_kinds": [ "count_mismatch", "unexpected_shape", "unsupported_protocol", "invalid_type", "missing_id" ], "ms": 0.009 } ], "counts": { "correct": 60, "wrong-association": 0, "should-have-refused": 0, "should-have-parsed": 0, "crashed": 0, "timeout": 0, "missing-record": 0, "not-applicable": 4 }, "smallest_counterexample": null, "diagnostics": [ { "id": "out-of-protocol-association", "question": "On cases outside its declared protocol, did it refuse — or associate anyway?", "value": 0, "of": 4, "detail": "It produced no association on any case outside its declared protocol.", "case_ids": [] } ], "usage": { "model_calls": 12, "replay_spend_usd": 0, "recording_cost_usd": "unknown", "provider_reported": { "input_tokens": 259125, "output_tokens": 7000, "total_tokens": 266125, "tokens_used": 266125, "cost_usd": 0.04567901 }, "basis": "Grading replayed committed recordings and made no inference call; `replay_spend_usd` is that, and is $0. The investigation that authored the candidate did call a model: 12 call(s) to moonshotai/kimi-k2 through the AI Gateway, 266125 tokens, $0.04567901 as the gateway reported it per call. That is in `provider_reported`. The microVM is metered compute billed at a rate this repository does not record: 1922ms of active CPU across 692ms wall clock." }, "attempts": [ { "attempt_id": "article-to-verdict-association__agent-k2-generous-out-02#01", "started_at": "2026-09-30T04:00:41.799Z", "ended_at": "2026-09-30T04:00:47.188Z", "status": "succeeded", "runner": "vercel-sandbox", "note": "sandbox blush-ethical-krill-415iwX in iad1, exit 0 in 395ms" } ] }