{
  "corpus_fingerprint": "180-0a2e675b6673",
  "revision": "2026-10-12",
  "published_at": "2026-10-12T01:25:02Z",
  "grader_version": "2026-10-10",
  "rows": [
    {
      "track": "closed",
      "model": "claude-opus-5.5",
      "provider": "claude-code",
      "total": 178,
      "correct": 99,
      "incorrect": 11,
      "abstained": 68,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 77,
      "score_per_100": 43.258426966292134,
      "runs": 1,
      "reasoning_correct": 43,
      "reasoning_total": 43,
      "grounded_correct": 56,
      "grounded_total": 135,
      "grounded_wrong": 11,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 41.48148148148148,
      "grounded_hallucination_pct": 16.417910447761194,
      "cost_usd": 2.579628
    },
    {
      "track": "closed",
      "model": "grok-4.7",
      "provider": "xai",
      "total": 178,
      "correct": 69,
      "incorrect": 2,
      "abstained": 107,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 65,
      "score_per_100": 36.51685393258427,
      "runs": 1,
      "reasoning_correct": 41,
      "reasoning_total": 43,
      "grounded_correct": 28,
      "grounded_total": 135,
      "grounded_wrong": 1,
      "reasoning_accuracy": 95.34883720930233,
      "grounded_accuracy": 20.74074074074074,
      "grounded_hallucination_pct": 3.4482758620689653,
      "cost_usd": 2.266119806451613
    },
    {
      "track": "closed",
      "model": "claude-fable-5.1",
      "provider": "claude-code",
      "total": 178,
      "correct": 97,
      "incorrect": 20,
      "abstained": 61,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 57,
      "score_per_100": 32.02247191011236,
      "runs": 1,
      "reasoning_correct": 43,
      "reasoning_total": 43,
      "grounded_correct": 54,
      "grounded_total": 135,
      "grounded_wrong": 20,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 40,
      "grounded_hallucination_pct": 27.027027027027028,
      "cost_usd": 7.046769999999999
    },
    {
      "track": "closed",
      "model": "claude-sonnet-5.5",
      "provider": "claude-code",
      "total": 178,
      "correct": 91,
      "incorrect": 25,
      "abstained": 62,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 41,
      "score_per_100": 23.03370786516854,
      "runs": 1,
      "reasoning_correct": 42,
      "reasoning_total": 43,
      "grounded_correct": 49,
      "grounded_total": 135,
      "grounded_wrong": 24,
      "reasoning_accuracy": 97.67441860465117,
      "grounded_accuracy": 36.2962962962963,
      "grounded_hallucination_pct": 32.87671232876713,
      "cost_usd": 1.2743420000000008
    },
    {
      "track": "closed",
      "model": "claude-haiku-5.5",
      "provider": "claude-code",
      "total": 178,
      "correct": 70,
      "incorrect": 15,
      "abstained": 93,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 40,
      "score_per_100": 22.471910112359552,
      "runs": 1,
      "reasoning_correct": 43,
      "reasoning_total": 43,
      "grounded_correct": 27,
      "grounded_total": 135,
      "grounded_wrong": 15,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 20,
      "grounded_hallucination_pct": 35.714285714285715,
      "cost_usd": 0.08053180000000003
    },
    {
      "track": "closed",
      "model": "gemma4-12b",
      "provider": "ollama",
      "total": 178,
      "correct": 35,
      "incorrect": 63,
      "abstained": 63,
      "no_answer": 0,
      "errors": 0,
      "truncated": 17,
      "score": -91,
      "score_per_100": -56.52173913043478,
      "runs": 1,
      "reasoning_correct": 23,
      "reasoning_total": 33,
      "grounded_correct": 12,
      "grounded_total": 128,
      "grounded_wrong": 53,
      "reasoning_accuracy": 69.6969696969697,
      "grounded_accuracy": 9.375,
      "grounded_hallucination_pct": 81.53846153846153,
      "cost_usd": 0
    },
    {
      "track": "closed",
      "model": "gpt-oss-20b",
      "provider": "ollama",
      "total": 178,
      "correct": 36,
      "incorrect": 88,
      "abstained": 52,
      "no_answer": 1,
      "errors": 0,
      "truncated": 1,
      "score": -140,
      "score_per_100": -79.09604519774011,
      "runs": 1,
      "reasoning_correct": 26,
      "reasoning_total": 41,
      "grounded_correct": 10,
      "grounded_total": 135,
      "grounded_wrong": 74,
      "reasoning_accuracy": 63.41463414634146,
      "grounded_accuracy": 7.407407407407407,
      "grounded_hallucination_pct": 88.0952380952381,
      "cost_usd": 0
    },
    {
      "track": "closed",
      "model": "deepseek-r1-14b",
      "provider": "ollama",
      "total": 178,
      "correct": 29,
      "incorrect": 87,
      "abstained": 59,
      "no_answer": 0,
      "errors": 0,
      "truncated": 3,
      "score": -145,
      "score_per_100": -82.85714285714286,
      "runs": 1,
      "reasoning_correct": 22,
      "reasoning_total": 40,
      "grounded_correct": 7,
      "grounded_total": 135,
      "grounded_wrong": 69,
      "reasoning_accuracy": 55,
      "grounded_accuracy": 5.185185185185185,
      "grounded_hallucination_pct": 90.78947368421052,
      "cost_usd": 0
    },
    {
      "track": "mcp",
      "model": "claude-fable-5.1",
      "provider": "claude-code",
      "total": 356,
      "correct": 322,
      "incorrect": 18,
      "abstained": 9,
      "no_answer": 7,
      "errors": 0,
      "truncated": 0,
      "score": 286,
      "score_per_100": 80.33707865168539,
      "runs": 2,
      "score_per_100_min": 79.7752808988764,
      "score_per_100_max": 80.89887640449439,
      "reasoning_correct": 85,
      "reasoning_total": 86,
      "grounded_correct": 237,
      "grounded_total": 263,
      "grounded_wrong": 17,
      "reasoning_accuracy": 98.83720930232558,
      "grounded_accuracy": 90.11406844106465,
      "grounded_hallucination_pct": 6.692913385826771,
      "cost_usd": 174.7945705000001
    },
    {
      "track": "mcp",
      "model": "claude-opus-5.5",
      "provider": "claude-code",
      "total": 534,
      "correct": 458,
      "incorrect": 16,
      "abstained": 16,
      "no_answer": 44,
      "errors": 0,
      "truncated": 0,
      "score": 426,
      "score_per_100": 79.7752808988764,
      "runs": 3,
      "score_per_100_min": 77.52808988764045,
      "score_per_100_max": 80.89887640449439,
      "reasoning_correct": 126,
      "reasoning_total": 129,
      "grounded_correct": 332,
      "grounded_total": 361,
      "grounded_wrong": 13,
      "reasoning_accuracy": 97.67441860465117,
      "grounded_accuracy": 91.96675900277009,
      "grounded_hallucination_pct": 3.7681159420289854,
      "cost_usd": 62.888695599999984
    },
    {
      "track": "mcp",
      "model": "claude-sonnet-5.5",
      "provider": "claude-code",
      "total": 534,
      "correct": 456,
      "incorrect": 43,
      "abstained": 14,
      "no_answer": 21,
      "errors": 0,
      "truncated": 0,
      "score": 370,
      "score_per_100": 69.28838951310861,
      "runs": 3,
      "score_per_100_min": 67.97752808988764,
      "score_per_100_max": 70.2247191011236,
      "reasoning_correct": 120,
      "reasoning_total": 129,
      "grounded_correct": 336,
      "grounded_total": 384,
      "grounded_wrong": 34,
      "reasoning_accuracy": 93.02325581395348,
      "grounded_accuracy": 87.5,
      "grounded_hallucination_pct": 9.18918918918919,
      "cost_usd": 25.488547099999995
    },
    {
      "track": "mcp",
      "model": "grok-4.7",
      "provider": "xai",
      "total": 356,
      "correct": 285,
      "incorrect": 20,
      "abstained": 51,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 245,
      "score_per_100": 68.82022471910112,
      "runs": 2,
      "score_per_100_min": 68.53932584269663,
      "score_per_100_max": 69.10112359550561,
      "reasoning_correct": 83,
      "reasoning_total": 86,
      "grounded_correct": 202,
      "grounded_total": 270,
      "grounded_wrong": 18,
      "reasoning_accuracy": 96.51162790697674,
      "grounded_accuracy": 74.81481481481481,
      "grounded_hallucination_pct": 8.181818181818182,
      "cost_usd": 30.73281135483871
    },
    {
      "track": "mcp",
      "model": "claude-haiku-5.5",
      "provider": "claude-code",
      "total": 534,
      "correct": 422,
      "incorrect": 40,
      "abstained": 50,
      "no_answer": 22,
      "errors": 0,
      "truncated": 0,
      "score": 342,
      "score_per_100": 64.04494382022472,
      "runs": 3,
      "score_per_100_min": 61.79775280898876,
      "score_per_100_max": 66.29213483146067,
      "reasoning_correct": 119,
      "reasoning_total": 129,
      "grounded_correct": 303,
      "grounded_total": 383,
      "grounded_wrong": 36,
      "reasoning_accuracy": 92.24806201550388,
      "grounded_accuracy": 79.11227154046998,
      "grounded_hallucination_pct": 10.619469026548673,
      "cost_usd": 2.5468347099999997
    },
    {
      "track": "mcp",
      "model": "gpt-oss-20b",
      "provider": "ollama",
      "total": 534,
      "correct": 277,
      "incorrect": 102,
      "abstained": 17,
      "no_answer": 138,
      "errors": 0,
      "truncated": 0,
      "score": 73,
      "score_per_100": 13.670411985018726,
      "runs": 3,
      "score_per_100_min": 12.92134831460674,
      "score_per_100_max": 15.168539325842696,
      "reasoning_correct": 80,
      "reasoning_total": 110,
      "grounded_correct": 197,
      "grounded_total": 286,
      "grounded_wrong": 72,
      "reasoning_accuracy": 72.72727272727273,
      "grounded_accuracy": 68.88111888111888,
      "grounded_hallucination_pct": 26.765799256505577,
      "cost_usd": 0
    },
    {
      "track": "mcp",
      "model": "gemma4-12b",
      "provider": "ollama",
      "total": 534,
      "correct": 256,
      "incorrect": 128,
      "abstained": 125,
      "no_answer": 1,
      "errors": 0,
      "truncated": 24,
      "score": 0,
      "score_per_100": 0,
      "runs": 3,
      "score_per_100_min": -11.904761904761905,
      "score_per_100_max": 5.9171597633136095,
      "reasoning_correct": 80,
      "reasoning_total": 113,
      "grounded_correct": 176,
      "grounded_total": 396,
      "grounded_wrong": 110,
      "reasoning_accuracy": 70.79646017699115,
      "grounded_accuracy": 44.44444444444444,
      "grounded_hallucination_pct": 38.46153846153846,
      "cost_usd": 0
    }
  ],
  "gaps": [
    {
      "model": "claude-fable-5.1",
      "score_gap_per_100": 48.31460674157303,
      "grounded_acc_closed": 40,
      "grounded_acc_mcp": 90.11406844106465,
      "halluc_closed": 27.027027027027028,
      "halluc_mcp": 6.692913385826771
    },
    {
      "model": "claude-haiku-5.5",
      "score_gap_per_100": 41.573033707865164,
      "grounded_acc_closed": 20,
      "grounded_acc_mcp": 79.11227154046998,
      "halluc_closed": 35.714285714285715,
      "halluc_mcp": 10.619469026548673
    },
    {
      "model": "claude-opus-5.5",
      "score_gap_per_100": 36.51685393258427,
      "grounded_acc_closed": 41.48148148148148,
      "grounded_acc_mcp": 91.96675900277009,
      "halluc_closed": 16.417910447761194,
      "halluc_mcp": 3.7681159420289854
    },
    {
      "model": "claude-sonnet-5.5",
      "score_gap_per_100": 46.254681647940075,
      "grounded_acc_closed": 36.2962962962963,
      "grounded_acc_mcp": 87.5,
      "halluc_closed": 32.87671232876713,
      "halluc_mcp": 9.18918918918919
    },
    {
      "model": "gemma4-12b",
      "score_gap_per_100": 56.52173913043478,
      "grounded_acc_closed": 9.375,
      "grounded_acc_mcp": 44.44444444444444,
      "halluc_closed": 81.53846153846153,
      "halluc_mcp": 38.46153846153846
    },
    {
      "model": "gpt-oss-20b",
      "score_gap_per_100": 92.76645718275884,
      "grounded_acc_closed": 7.407407407407407,
      "grounded_acc_mcp": 68.88111888111888,
      "halluc_closed": 88.0952380952381,
      "halluc_mcp": 26.765799256505577
    },
    {
      "model": "grok-4.7",
      "score_gap_per_100": 32.30337078651685,
      "grounded_acc_closed": 20.74074074074074,
      "grounded_acc_mcp": 74.81481481481481,
      "halluc_closed": 3.4482758620689653,
      "halluc_mcp": 8.181818181818182
    }
  ],
  "jobs": [
    "baseline",
    "catalog",
    "constellation",
    "debris",
    "ground",
    "market",
    "regulatory"
  ],
  "errata": [
    {
      "id": "con-galileo-launched",
      "action": "rekey",
      "value": "34",
      "reason": "The question counts IOV and FOC satellites; the ground-truth SQL matched only names starting \"GALILEO \" and missed the 4 IOV satellites, giving 30. 34 Galileo satellites had been launched (latest 2025-12-17).",
      "date": "2026-10-11"
    },
    {
      "id": "deb-atlas5-centaur-afspc11-debris",
      "action": "drop",
      "reason": "The question names the wrong launch. The Centaur that fragmented in September 2024 is 2018-079B from the AEHF-4 launch (2018-10-17, 214 debris cataloged). AFSPC-11 (2018-036) has 1 debris object, which is what models answered.",
      "date": "2026-10-11"
    },
    {
      "id": "reg-ngso-market-access-count",
      "action": "rekey",
      "value": "9",
      "reason": "The question asks for foreign-licensed NGSO systems with US market access; the ground-truth SQL counted every granted NGSO system, U.S.-licensed ones included, giving 52. Excluding a U.S. licensing administration leaves 9 (Kepler, Telesat Ka-band, ExactView-1, ExactView-7, Kineis, Arctic Satellite Broadband Mission, two OneWeb systems, O3B).",
      "date": "2026-10-11"
    },
    {
      "id": "reg-noaa-remote-sensing-licensees",
      "action": "drop",
      "reason": "The question asks how many companies currently hold a license, but the answer key comes from a NOAA CRSRA licensee snapshot last seen 2024-07-27; no current list was available to verify it. To be restated as an as-of question, or rekeyed from a current list, in the next corpus.",
      "date": "2026-10-11"
    }
  ],
  "verdicts": [
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 25,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 25,
      "gradeable": 50,
      "correct": 50,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 17,
      "incorrect": 1,
      "abstained": 4,
      "no_answer": 0,
      "correct_pct": 0.7727272727272727,
      "wrong_pct": 0.045454545454545456,
      "correct_lo": 0.5655964393704065,
      "correct_hi": 0.8987710920517964,
      "wrong_lo": 0.008069219301147859,
      "wrong_hi": 0.21798489499518053,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 22,
      "gradeable": 44,
      "correct": 44,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8513404742740388,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.14865952572596125,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 11,
      "incorrect": 3,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.44,
      "wrong_pct": 0.12,
      "correct_lo": 0.2666537494700256,
      "correct_hi": 0.6293298298043557,
      "wrong_lo": 0.041667413570680906,
      "wrong_hi": 0.299561921833735,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 25,
      "gradeable": 50,
      "correct": 49,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 1,
      "correct_pct": 0.98,
      "wrong_pct": 0,
      "correct_lo": 0.8342228568014084,
      "correct_hi": 0.9979085090035399,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 24,
      "gradeable": 24,
      "correct": 14,
      "incorrect": 4,
      "abstained": 6,
      "no_answer": 0,
      "correct_pct": 0.5833333333333334,
      "wrong_pct": 0.16666666666666666,
      "correct_lo": 0.3883433764938864,
      "correct_hi": 0.7553265275346177,
      "wrong_lo": 0.06678568000728921,
      "wrong_hi": 0.35853470387869435,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 24,
      "gradeable": 48,
      "correct": 45,
      "incorrect": 3,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9375,
      "wrong_pct": 0.0625,
      "correct_lo": 0.7688333876424633,
      "correct_hi": 0.9854336085071835,
      "wrong_lo": 0.014566391492816647,
      "wrong_hi": 0.23116661235753683,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 5,
      "incorrect": 6,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.22727272727272727,
      "wrong_pct": 0.2727272727272727,
      "correct_lo": 0.10122890794820358,
      "correct_hi": 0.4344035606295935,
      "wrong_lo": 0.13150582015814438,
      "wrong_hi": 0.4815212369900198,
      "verdict": "dont_use"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 22,
      "gradeable": 44,
      "correct": 27,
      "incorrect": 7,
      "abstained": 4,
      "no_answer": 6,
      "correct_pct": 0.6136363636363636,
      "wrong_pct": 0.1590909090909091,
      "correct_lo": 0.40824765068543845,
      "correct_hi": 0.7852388207404795,
      "wrong_lo": 0.05991630151660696,
      "wrong_hi": 0.3596242842056394,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 23,
      "gradeable": 23,
      "correct": 7,
      "incorrect": 3,
      "abstained": 13,
      "no_answer": 0,
      "correct_pct": 0.30434782608695654,
      "wrong_pct": 0.13043478260869565,
      "correct_lo": 0.15603828020972457,
      "correct_hi": 0.5086612906057262,
      "wrong_lo": 0.04537576514344702,
      "wrong_hi": 0.32127897973018194,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 23,
      "gradeable": 46,
      "correct": 44,
      "incorrect": 2,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9565217391304348,
      "wrong_pct": 0.043478260869565216,
      "correct_lo": 0.7900841350594088,
      "correct_hi": 0.9922835330378734,
      "wrong_lo": 0.0077164669621267445,
      "wrong_hi": 0.20991586494059142,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 21,
      "gradeable": 21,
      "correct": 13,
      "incorrect": 2,
      "abstained": 6,
      "no_answer": 0,
      "correct_pct": 0.6190476190476191,
      "wrong_pct": 0.09523809523809523,
      "correct_lo": 0.40878295815501514,
      "correct_hi": 0.7924923220201749,
      "wrong_lo": 0.026518208796947246,
      "wrong_hi": 0.2891458386074067,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 21,
      "gradeable": 42,
      "correct": 39,
      "incorrect": 0,
      "abstained": 3,
      "no_answer": 0,
      "correct_pct": 0.9285714285714286,
      "wrong_pct": 0,
      "correct_lo": 0.7412604415701762,
      "correct_hi": 0.9833305670605077,
      "wrong_lo": 0,
      "wrong_hi": 0.15464382326420198,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 11,
      "gradeable": 11,
      "correct": 0,
      "incorrect": 1,
      "abstained": 10,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.09090909090909091,
      "correct_lo": 0,
      "correct_hi": 0.25884001724881417,
      "wrong_lo": 0.01623175226282597,
      "wrong_hi": 0.3773646254862038,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 11,
      "gradeable": 22,
      "correct": 14,
      "incorrect": 6,
      "abstained": 2,
      "no_answer": 0,
      "correct_pct": 0.6363636363636364,
      "wrong_pct": 0.2727272727272727,
      "correct_lo": 0.35379677689163724,
      "correct_hi": 0.8483377638586862,
      "wrong_lo": 0.09745880573921467,
      "wrong_hi": 0.5656502930102463,
      "verdict": "dont_use"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 25,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 75,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 8,
      "incorrect": 4,
      "abstained": 10,
      "no_answer": 0,
      "correct_pct": 0.36363636363636365,
      "wrong_pct": 0.18181818181818182,
      "correct_lo": 0.19732972772607899,
      "correct_hi": 0.5704865065628195,
      "wrong_lo": 0.07306766024105121,
      "wrong_hi": 0.38517021976637866,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 22,
      "gradeable": 66,
      "correct": 61,
      "incorrect": 0,
      "abstained": 1,
      "no_answer": 4,
      "correct_pct": 0.9242424242424242,
      "wrong_pct": 0,
      "correct_lo": 0.7412313059959793,
      "correct_hi": 0.9811181873274474,
      "wrong_lo": 0,
      "wrong_hi": 0.14865952572596125,
      "verdict": "trust"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 9,
      "incorrect": 2,
      "abstained": 14,
      "no_answer": 0,
      "correct_pct": 0.36,
      "wrong_pct": 0.08,
      "correct_lo": 0.20247656431735636,
      "correct_hi": 0.5548184539895337,
      "wrong_lo": 0.02221993369109193,
      "wrong_hi": 0.24966512122957818,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 65,
      "incorrect": 9,
      "abstained": 0,
      "no_answer": 1,
      "correct_pct": 0.8666666666666667,
      "wrong_pct": 0.12,
      "correct_lo": 0.6844982297773028,
      "correct_hi": 0.9511576746570329,
      "wrong_lo": 0.041667413570680906,
      "wrong_hi": 0.299561921833735,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 24,
      "gradeable": 24,
      "correct": 6,
      "incorrect": 5,
      "abstained": 13,
      "no_answer": 0,
      "correct_pct": 0.25,
      "wrong_pct": 0.20833333333333334,
      "correct_lo": 0.1199920347934233,
      "correct_hi": 0.4489982531210644,
      "wrong_lo": 0.0924468910478809,
      "wrong_hi": 0.40470844485235474,
      "verdict": "dont_use"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 24,
      "gradeable": 72,
      "correct": 60,
      "incorrect": 4,
      "abstained": 8,
      "no_answer": 0,
      "correct_pct": 0.8333333333333334,
      "wrong_pct": 0.05555555555555555,
      "correct_lo": 0.6414652961213058,
      "correct_hi": 0.9332143199927109,
      "wrong_lo": 0.011997271381444935,
      "wrong_hi": 0.22176324046653315,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 1,
      "incorrect": 1,
      "abstained": 20,
      "no_answer": 0,
      "correct_pct": 0.045454545454545456,
      "wrong_pct": 0.045454545454545456,
      "correct_lo": 0.008069219301147859,
      "correct_hi": 0.21798489499518053,
      "wrong_lo": 0.008069219301147859,
      "wrong_hi": 0.21798489499518053,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 22,
      "gradeable": 66,
      "correct": 30,
      "incorrect": 6,
      "abstained": 17,
      "no_answer": 13,
      "correct_pct": 0.45454545454545453,
      "wrong_pct": 0.09090909090909091,
      "correct_lo": 0.26920014432004846,
      "correct_hi": 0.6534052671095845,
      "wrong_lo": 0.025294896282837764,
      "wrong_hi": 0.27815380658385785,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 23,
      "gradeable": 23,
      "correct": 6,
      "incorrect": 1,
      "abstained": 16,
      "no_answer": 0,
      "correct_pct": 0.2608695652173913,
      "wrong_pct": 0.043478260869565216,
      "correct_lo": 0.1254844588924897,
      "correct_hi": 0.4647039054375056,
      "wrong_lo": 0.0077164669621267445,
      "wrong_hi": 0.20991586494059142,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 23,
      "gradeable": 69,
      "correct": 61,
      "incorrect": 4,
      "abstained": 3,
      "no_answer": 1,
      "correct_pct": 0.8840579710144928,
      "wrong_pct": 0.057971014492753624,
      "correct_lo": 0.6960829280288614,
      "correct_hi": 0.9620993959259949,
      "wrong_lo": 0.012523190309352533,
      "wrong_hi": 0.2299462104218508,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 21,
      "gradeable": 21,
      "correct": 10,
      "incorrect": 2,
      "abstained": 9,
      "no_answer": 0,
      "correct_pct": 0.47619047619047616,
      "wrong_pct": 0.09523809523809523,
      "correct_lo": 0.28343712296762946,
      "correct_hi": 0.6763078209973324,
      "wrong_lo": 0.026518208796947246,
      "wrong_hi": 0.2891458386074067,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 21,
      "gradeable": 63,
      "correct": 41,
      "incorrect": 2,
      "abstained": 19,
      "no_answer": 1,
      "correct_pct": 0.6507936507936508,
      "wrong_pct": 0.031746031746031744,
      "correct_lo": 0.4385607593915072,
      "correct_hi": 0.8163879288304002,
      "wrong_lo": 0.004173324950389634,
      "wrong_hi": 0.20414390636052968,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 11,
      "gradeable": 11,
      "correct": 0,
      "incorrect": 0,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0,
      "correct_lo": 0,
      "correct_hi": 0.25884001724881417,
      "wrong_lo": 0,
      "wrong_hi": 0.25884001724881417,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 11,
      "gradeable": 33,
      "correct": 17,
      "incorrect": 12,
      "abstained": 2,
      "no_answer": 2,
      "correct_pct": 0.5151515151515151,
      "wrong_pct": 0.36363636363636365,
      "correct_lo": 0.25693466435313284,
      "correct_hi": 0.7655247290635697,
      "wrong_lo": 0.15166223614131383,
      "wrong_hi": 0.6462032231083628,
      "verdict": "dont_use"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 25,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 75,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 19,
      "incorrect": 1,
      "abstained": 2,
      "no_answer": 0,
      "correct_pct": 0.8636363636363636,
      "wrong_pct": 0.045454545454545456,
      "correct_lo": 0.6666458812053,
      "correct_hi": 0.9525108273576373,
      "wrong_lo": 0.008069219301147859,
      "wrong_hi": 0.21798489499518053,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 22,
      "gradeable": 66,
      "correct": 66,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8513404742740388,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.14865952572596125,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 11,
      "incorrect": 2,
      "abstained": 12,
      "no_answer": 0,
      "correct_pct": 0.44,
      "wrong_pct": 0.08,
      "correct_lo": 0.2666537494700256,
      "correct_hi": 0.6293298298043557,
      "wrong_lo": 0.02221993369109193,
      "wrong_hi": 0.24966512122957818,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 74,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 1,
      "correct_pct": 0.9866666666666667,
      "wrong_pct": 0,
      "correct_lo": 0.8446808865024286,
      "correct_hi": 0.9990078593831441,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 24,
      "gradeable": 24,
      "correct": 13,
      "incorrect": 1,
      "abstained": 10,
      "no_answer": 0,
      "correct_pct": 0.5416666666666666,
      "wrong_pct": 0.041666666666666664,
      "correct_lo": 0.35074553553106635,
      "correct_hi": 0.7210894164831857,
      "wrong_lo": 0.007393265354805112,
      "wrong_hi": 0.20242226248842232,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 24,
      "gradeable": 72,
      "correct": 68,
      "incorrect": 3,
      "abstained": 0,
      "no_answer": 1,
      "correct_pct": 0.9444444444444444,
      "wrong_pct": 0.041666666666666664,
      "correct_lo": 0.7782367595334669,
      "correct_hi": 0.9880027286185551,
      "wrong_lo": 0.007393265354805112,
      "wrong_hi": 0.20242226248842232,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 6,
      "incorrect": 4,
      "abstained": 12,
      "no_answer": 0,
      "correct_pct": 0.2727272727272727,
      "wrong_pct": 0.18181818181818182,
      "correct_lo": 0.13150582015814438,
      "correct_hi": 0.4815212369900198,
      "wrong_lo": 0.07306766024105121,
      "wrong_hi": 0.38517021976637866,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 22,
      "gradeable": 66,
      "correct": 34,
      "incorrect": 5,
      "abstained": 3,
      "no_answer": 24,
      "correct_pct": 0.5151515151515151,
      "wrong_pct": 0.07575757575757576,
      "correct_lo": 0.3201925147022562,
      "correct_hi": 0.7056056814878662,
      "wrong_lo": 0.01888181267255261,
      "wrong_hi": 0.2587686940040206,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 23,
      "gradeable": 23,
      "correct": 7,
      "incorrect": 2,
      "abstained": 14,
      "no_answer": 0,
      "correct_pct": 0.30434782608695654,
      "wrong_pct": 0.08695652173913043,
      "correct_lo": 0.15603828020972457,
      "correct_hi": 0.5086612906057262,
      "wrong_lo": 0.024179495407233423,
      "wrong_hi": 0.26796404298094023,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 23,
      "gradeable": 69,
      "correct": 63,
      "incorrect": 0,
      "abstained": 2,
      "no_answer": 4,
      "correct_pct": 0.9130434782608695,
      "wrong_pct": 0,
      "correct_lo": 0.7320359570190598,
      "correct_hi": 0.9758205045927666,
      "wrong_lo": 0,
      "wrong_hi": 0.14312112541726277,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 21,
      "gradeable": 21,
      "correct": 13,
      "incorrect": 1,
      "abstained": 7,
      "no_answer": 0,
      "correct_pct": 0.6190476190476191,
      "wrong_pct": 0.047619047619047616,
      "correct_lo": 0.40878295815501514,
      "correct_hi": 0.7924923220201749,
      "wrong_lo": 0.008455769471923322,
      "wrong_hi": 0.22669816586235464,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 21,
      "gradeable": 63,
      "correct": 43,
      "incorrect": 0,
      "abstained": 7,
      "no_answer": 13,
      "correct_pct": 0.6825396825396826,
      "wrong_pct": 0,
      "correct_lo": 0.4690950508404568,
      "correct_hi": 0.8395270454281678,
      "wrong_lo": 0,
      "wrong_hi": 0.15464382326420198,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 11,
      "gradeable": 11,
      "correct": 0,
      "incorrect": 0,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0,
      "correct_lo": 0,
      "correct_hi": 0.25884001724881417,
      "wrong_lo": 0,
      "wrong_hi": 0.25884001724881417,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 11,
      "gradeable": 33,
      "correct": 20,
      "incorrect": 8,
      "abstained": 4,
      "no_answer": 1,
      "correct_pct": 0.6060606060606061,
      "wrong_pct": 0.24242424242424243,
      "correct_lo": 0.32850389146288966,
      "correct_hi": 0.8287118624540286,
      "wrong_lo": 0.08109909467209159,
      "wrong_hi": 0.5370912172439641,
      "verdict": "dont_use"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 25,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 74,
      "incorrect": 1,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9866666666666667,
      "wrong_pct": 0.013333333333333334,
      "correct_lo": 0.8446808865024286,
      "correct_hi": 0.9990078593831441,
      "wrong_lo": 0.0009921406168558883,
      "wrong_hi": 0.15531911349757144,
      "verdict": "trust"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 15,
      "incorrect": 3,
      "abstained": 4,
      "no_answer": 0,
      "correct_pct": 0.6818181818181818,
      "wrong_pct": 0.13636363636363635,
      "correct_lo": 0.47318222782839336,
      "correct_hi": 0.8363961264530752,
      "wrong_lo": 0.04748917264236269,
      "wrong_hi": 0.3333541187947,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 22,
      "gradeable": 66,
      "correct": 65,
      "incorrect": 1,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9848484848484849,
      "wrong_pct": 0.015151515151515152,
      "correct_lo": 0.8266698422816354,
      "correct_hi": 0.9988724358022811,
      "wrong_lo": 0.0011275641977189083,
      "wrong_hi": 0.17333015771836474,
      "verdict": "trust"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 12,
      "incorrect": 2,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.48,
      "wrong_pct": 0.08,
      "correct_lo": 0.30031003928960015,
      "correct_hi": 0.6650178204685269,
      "wrong_lo": 0.02221993369109193,
      "wrong_hi": 0.24966512122957818,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 71,
      "incorrect": 4,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9466666666666667,
      "wrong_pct": 0.05333333333333334,
      "correct_lo": 0.7858582126617453,
      "correct_hi": 0.9884862527400815,
      "wrong_lo": 0.011513747259918342,
      "wrong_hi": 0.21414178733825467,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 24,
      "gradeable": 24,
      "correct": 13,
      "incorrect": 6,
      "abstained": 5,
      "no_answer": 0,
      "correct_pct": 0.5416666666666666,
      "wrong_pct": 0.25,
      "correct_lo": 0.35074553553106635,
      "correct_hi": 0.7210894164831857,
      "wrong_lo": 0.1199920347934233,
      "wrong_hi": 0.4489982531210644,
      "verdict": "dont_use"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 24,
      "gradeable": 72,
      "correct": 67,
      "incorrect": 5,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9305555555555556,
      "wrong_pct": 0.06944444444444445,
      "correct_lo": 0.7595859892196655,
      "correct_hi": 0.982708514927606,
      "wrong_lo": 0.0172914850723941,
      "wrong_hi": 0.2404140107803347,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 5,
      "incorrect": 4,
      "abstained": 13,
      "no_answer": 0,
      "correct_pct": 0.22727272727272727,
      "wrong_pct": 0.18181818181818182,
      "correct_lo": 0.10122890794820358,
      "correct_hi": 0.4344035606295935,
      "wrong_lo": 0.07306766024105121,
      "wrong_hi": 0.38517021976637866,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 22,
      "gradeable": 66,
      "correct": 34,
      "incorrect": 11,
      "abstained": 6,
      "no_answer": 15,
      "correct_pct": 0.5151515151515151,
      "wrong_pct": 0.16666666666666666,
      "correct_lo": 0.3201925147022562,
      "correct_hi": 0.7056056814878662,
      "wrong_lo": 0.06422420536802462,
      "wrong_hi": 0.36821547844928293,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 23,
      "gradeable": 23,
      "correct": 6,
      "incorrect": 4,
      "abstained": 13,
      "no_answer": 0,
      "correct_pct": 0.2608695652173913,
      "wrong_pct": 0.17391304347826086,
      "correct_lo": 0.1254844588924897,
      "correct_hi": 0.4647039054375056,
      "wrong_lo": 0.06978540350612775,
      "wrong_hi": 0.3713805478529567,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 23,
      "gradeable": 69,
      "correct": 63,
      "incorrect": 4,
      "abstained": 1,
      "no_answer": 1,
      "correct_pct": 0.9130434782608695,
      "wrong_pct": 0.057971014492753624,
      "correct_lo": 0.7320359570190598,
      "correct_hi": 0.9758205045927666,
      "wrong_lo": 0.012523190309352533,
      "wrong_hi": 0.2299462104218508,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 21,
      "gradeable": 21,
      "correct": 10,
      "incorrect": 4,
      "abstained": 7,
      "no_answer": 0,
      "correct_pct": 0.47619047619047616,
      "wrong_pct": 0.19047619047619047,
      "correct_lo": 0.28343712296762946,
      "correct_hi": 0.6763078209973324,
      "wrong_lo": 0.07667440615177307,
      "wrong_hi": 0.4000098653927329,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 21,
      "gradeable": 63,
      "correct": 51,
      "incorrect": 2,
      "abstained": 7,
      "no_answer": 3,
      "correct_pct": 0.8095238095238095,
      "wrong_pct": 0.031746031746031744,
      "correct_lo": 0.599990134607267,
      "correct_hi": 0.9233255938482269,
      "wrong_lo": 0.004173324950389634,
      "wrong_hi": 0.20414390636052968,
      "verdict": "trust"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 11,
      "gradeable": 11,
      "correct": 0,
      "incorrect": 2,
      "abstained": 9,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.18181818181818182,
      "correct_lo": 0,
      "correct_hi": 0.25884001724881417,
      "wrong_lo": 0.05136660067100063,
      "wrong_hi": 0.4769861375782448,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 11,
      "gradeable": 33,
      "correct": 19,
      "incorrect": 12,
      "abstained": 0,
      "no_answer": 2,
      "correct_pct": 0.5757575757575758,
      "wrong_pct": 0.36363636363636365,
      "correct_lo": 0.30394026892404263,
      "correct_hi": 0.8083566981594703,
      "wrong_lo": 0.15166223614131383,
      "wrong_hi": 0.6462032231083628,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 15,
      "incorrect": 7,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.6818181818181818,
      "wrong_pct": 0.3181818181818182,
      "correct_lo": 0.47318222782839336,
      "correct_hi": 0.8363961264530752,
      "wrong_lo": 0.1636038735469248,
      "wrong_hi": 0.5268177721716065,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 0,
      "incorrect": 17,
      "abstained": 5,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.7727272727272727,
      "correct_lo": 0,
      "correct_hi": 0.14865952572596125,
      "wrong_lo": 0.5655964393704065,
      "wrong_hi": 0.8987710920517964,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 1,
      "incorrect": 15,
      "abstained": 9,
      "no_answer": 0,
      "correct_pct": 0.04,
      "wrong_pct": 0.6,
      "correct_lo": 0.007096050386699165,
      "correct_hi": 0.19544472405022525,
      "wrong_lo": 0.40739127785127344,
      "wrong_hi": 0.7659694233580909,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 24,
      "gradeable": 24,
      "correct": 2,
      "incorrect": 18,
      "abstained": 4,
      "no_answer": 0,
      "correct_pct": 0.08333333333333333,
      "wrong_pct": 0.75,
      "correct_lo": 0.023158327842973503,
      "correct_hi": 0.258492152014506,
      "wrong_lo": 0.5510017468789357,
      "wrong_hi": 0.8800079652065766,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 3,
      "incorrect": 8,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.13636363636363635,
      "wrong_pct": 0.36363636363636365,
      "correct_lo": 0.04748917264236269,
      "correct_hi": 0.3333541187947,
      "wrong_lo": 0.19732972772607899,
      "wrong_hi": 0.5704865065628195,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 23,
      "gradeable": 23,
      "correct": 1,
      "incorrect": 6,
      "abstained": 16,
      "no_answer": 0,
      "correct_pct": 0.043478260869565216,
      "wrong_pct": 0.2608695652173913,
      "correct_lo": 0.0077164669621267445,
      "correct_hi": 0.20991586494059142,
      "wrong_lo": 0.1254844588924897,
      "wrong_hi": 0.4647039054375056,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 21,
      "gradeable": 21,
      "correct": 3,
      "incorrect": 12,
      "abstained": 6,
      "no_answer": 0,
      "correct_pct": 0.14285714285714285,
      "wrong_pct": 0.5714285714285714,
      "correct_lo": 0.049809217982134946,
      "correct_hi": 0.346364941492295,
      "wrong_lo": 0.3654621629703244,
      "wrong_hi": 0.7553030051347894,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "kind": "os-derived",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 11,
      "gradeable": 11,
      "correct": 0,
      "incorrect": 3,
      "abstained": 8,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.2727272727272727,
      "correct_lo": 0,
      "correct_hi": 0.25884001724881417,
      "wrong_lo": 0.09745880573921467,
      "wrong_hi": 0.5656502930102463,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 16,
      "gradeable": 16,
      "correct": 14,
      "incorrect": 2,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.875,
      "wrong_pct": 0.125,
      "correct_lo": 0.6397666856325173,
      "correct_hi": 0.9650232506629428,
      "wrong_lo": 0.034976749337057134,
      "wrong_hi": 0.3602333143674828,
      "verdict": "verify"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 22,
      "gradeable": 60,
      "correct": 51,
      "incorrect": 3,
      "abstained": 6,
      "no_answer": 0,
      "correct_pct": 0.85,
      "wrong_pct": 0.05,
      "correct_lo": 0.6507913918001443,
      "correct_hi": 0.9451469401916829,
      "wrong_lo": 0.009488651287639496,
      "wrong_hi": 0.22430492186572565,
      "verdict": "trust"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 20,
      "gradeable": 20,
      "correct": 3,
      "incorrect": 13,
      "abstained": 4,
      "no_answer": 0,
      "correct_pct": 0.15,
      "wrong_pct": 0.65,
      "correct_lo": 0.05236779195949584,
      "correct_hi": 0.36042329588695743,
      "wrong_lo": 0.43285051021946463,
      "wrong_hi": 0.8188104521320554,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 22,
      "gradeable": 65,
      "correct": 54,
      "incorrect": 5,
      "abstained": 5,
      "no_answer": 1,
      "correct_pct": 0.8307692307692308,
      "wrong_pct": 0.07692307692307693,
      "correct_lo": 0.6288943094512687,
      "correct_hi": 0.9343001581454031,
      "wrong_lo": 0.01935416373577313,
      "wrong_hi": 0.26028081957080945,
      "verdict": "verify"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 20,
      "gradeable": 20,
      "correct": 2,
      "incorrect": 8,
      "abstained": 10,
      "no_answer": 0,
      "correct_pct": 0.1,
      "wrong_pct": 0.4,
      "correct_lo": 0.027865893984153206,
      "correct_hi": 0.30103820641179335,
      "wrong_lo": 0.21880396741419272,
      "wrong_hi": 0.613422057684794,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 32,
      "incorrect": 24,
      "abstained": 19,
      "no_answer": 0,
      "correct_pct": 0.4266666666666667,
      "wrong_pct": 0.32,
      "correct_lo": 0.2556631974009841,
      "correct_hi": 0.6172056217121488,
      "wrong_lo": 0.17204987516891143,
      "wrong_hi": 0.5159008626542329,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 23,
      "gradeable": 23,
      "correct": 3,
      "incorrect": 8,
      "abstained": 12,
      "no_answer": 0,
      "correct_pct": 0.13043478260869565,
      "wrong_pct": 0.34782608695652173,
      "correct_lo": 0.04537576514344702,
      "correct_hi": 0.32127897973018194,
      "wrong_lo": 0.18811054636982244,
      "wrong_hi": 0.5511002309310837,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 24,
      "gradeable": 72,
      "correct": 33,
      "incorrect": 18,
      "abstained": 21,
      "no_answer": 0,
      "correct_pct": 0.4583333333333333,
      "wrong_pct": 0.25,
      "correct_lo": 0.2789105835168143,
      "correct_hi": 0.6492544644689336,
      "wrong_lo": 0.1199920347934233,
      "wrong_hi": 0.4489982531210644,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 1,
      "incorrect": 10,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.045454545454545456,
      "wrong_pct": 0.45454545454545453,
      "correct_lo": 0.008069219301147859,
      "correct_hi": 0.21798489499518053,
      "wrong_lo": 0.26920014432004846,
      "wrong_hi": 0.6534052671095845,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 22,
      "gradeable": 62,
      "correct": 12,
      "incorrect": 29,
      "abstained": 21,
      "no_answer": 0,
      "correct_pct": 0.1935483870967742,
      "wrong_pct": 0.46774193548387094,
      "correct_lo": 0.08011063418332148,
      "correct_hi": 0.3981000428745257,
      "wrong_lo": 0.2800973213916451,
      "wrong_hi": 0.6649774867197072,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 23,
      "gradeable": 23,
      "correct": 0,
      "incorrect": 8,
      "abstained": 15,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.34782608695652173,
      "correct_lo": 0,
      "correct_hi": 0.14312112541726277,
      "wrong_lo": 0.18811054636982244,
      "wrong_hi": 0.5511002309310837,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 23,
      "gradeable": 67,
      "correct": 33,
      "incorrect": 22,
      "abstained": 12,
      "no_answer": 0,
      "correct_pct": 0.4925373134328358,
      "wrong_pct": 0.3283582089552239,
      "correct_lo": 0.304466671940458,
      "correct_hi": 0.6827440911254712,
      "wrong_lo": 0.17357141089174985,
      "wrong_hi": 0.532276139624624,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 21,
      "gradeable": 21,
      "correct": 7,
      "incorrect": 9,
      "abstained": 5,
      "no_answer": 0,
      "correct_pct": 0.3333333333333333,
      "wrong_pct": 0.42857142857142855,
      "correct_lo": 0.17194536432683688,
      "correct_hi": 0.5462692434278971,
      "wrong_lo": 0.24469699486521043,
      "wrong_hi": 0.6345378370296754,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 21,
      "gradeable": 63,
      "correct": 16,
      "incorrect": 9,
      "abstained": 38,
      "no_answer": 0,
      "correct_pct": 0.25396825396825395,
      "wrong_pct": 0.14285714285714285,
      "correct_lo": 0.11666529167404971,
      "correct_hi": 0.46736579596389094,
      "wrong_lo": 0.049809217982134946,
      "wrong_hi": 0.346364941492295,
      "verdict": "verify"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "kind": "os-derived",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 11,
      "gradeable": 11,
      "correct": 0,
      "incorrect": 5,
      "abstained": 6,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.45454545454545453,
      "correct_lo": 0,
      "correct_hi": 0.25884001724881417,
      "wrong_lo": 0.21270957543988045,
      "wrong_hi": 0.7199122443100118,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 11,
      "gradeable": 31,
      "correct": 13,
      "incorrect": 15,
      "abstained": 3,
      "no_answer": 0,
      "correct_pct": 0.41935483870967744,
      "wrong_pct": 0.4838709677419355,
      "correct_lo": 0.1883117188236234,
      "correct_hi": 0.6921463484745726,
      "wrong_lo": 0.23376230848412044,
      "wrong_hi": 0.7423293049755189,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 24,
      "gradeable": 24,
      "correct": 20,
      "incorrect": 3,
      "abstained": 0,
      "no_answer": 1,
      "correct_pct": 0.8333333333333334,
      "wrong_pct": 0.125,
      "correct_lo": 0.6414652961213058,
      "correct_hi": 0.9332143199927109,
      "wrong_lo": 0.04344255039315581,
      "wrong_hi": 0.3100428814785757,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 65,
      "incorrect": 6,
      "abstained": 0,
      "no_answer": 4,
      "correct_pct": 0.8666666666666667,
      "wrong_pct": 0.08,
      "correct_lo": 0.6844982297773028,
      "correct_hi": 0.9511576746570329,
      "wrong_lo": 0.02221993369109193,
      "wrong_hi": 0.24966512122957818,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 3,
      "incorrect": 12,
      "abstained": 7,
      "no_answer": 0,
      "correct_pct": 0.13636363636363635,
      "wrong_pct": 0.5454545454545454,
      "correct_lo": 0.04748917264236269,
      "correct_hi": 0.3333541187947,
      "wrong_lo": 0.34659473289041554,
      "wrong_hi": 0.7307998556799515,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 22,
      "gradeable": 66,
      "correct": 54,
      "incorrect": 6,
      "abstained": 0,
      "no_answer": 6,
      "correct_pct": 0.8181818181818182,
      "wrong_pct": 0.09090909090909091,
      "correct_lo": 0.6148297802336213,
      "correct_hi": 0.9269323397589488,
      "wrong_lo": 0.025294896282837764,
      "wrong_hi": 0.27815380658385785,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 1,
      "incorrect": 13,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.04,
      "wrong_pct": 0.52,
      "correct_lo": 0.007096050386699165,
      "correct_hi": 0.19544472405022525,
      "wrong_lo": 0.334982179531473,
      "wrong_hi": 0.6996899607103998,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 35,
      "incorrect": 15,
      "abstained": 2,
      "no_answer": 23,
      "correct_pct": 0.4666666666666667,
      "wrong_pct": 0.2,
      "correct_lo": 0.28897791408695633,
      "correct_hi": 0.6532351855099223,
      "wrong_lo": 0.08860454100652487,
      "wrong_hi": 0.3913133553653824,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 24,
      "gradeable": 24,
      "correct": 1,
      "incorrect": 17,
      "abstained": 6,
      "no_answer": 0,
      "correct_pct": 0.041666666666666664,
      "wrong_pct": 0.7083333333333334,
      "correct_lo": 0.007393265354805112,
      "correct_hi": 0.20242226248842232,
      "wrong_lo": 0.508319363412903,
      "wrong_hi": 0.8508553966583573,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 24,
      "gradeable": 72,
      "correct": 33,
      "incorrect": 18,
      "abstained": 5,
      "no_answer": 16,
      "correct_pct": 0.4583333333333333,
      "wrong_pct": 0.25,
      "correct_lo": 0.2789105835168143,
      "correct_hi": 0.6492544644689336,
      "wrong_lo": 0.1199920347934233,
      "wrong_hi": 0.4489982531210644,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 3,
      "incorrect": 10,
      "abstained": 9,
      "no_answer": 0,
      "correct_pct": 0.13636363636363635,
      "wrong_pct": 0.45454545454545453,
      "correct_lo": 0.04748917264236269,
      "correct_hi": 0.3333541187947,
      "wrong_lo": 0.26920014432004846,
      "wrong_hi": 0.6534052671095845,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 22,
      "gradeable": 66,
      "correct": 19,
      "incorrect": 13,
      "abstained": 8,
      "no_answer": 26,
      "correct_pct": 0.2878787878787879,
      "wrong_pct": 0.19696969696969696,
      "correct_lo": 0.14201401248825027,
      "correct_hi": 0.49681124085003636,
      "wrong_lo": 0.08219556781810852,
      "wrong_hi": 0.40184050837944374,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 23,
      "gradeable": 23,
      "correct": 2,
      "incorrect": 14,
      "abstained": 7,
      "no_answer": 0,
      "correct_pct": 0.08695652173913043,
      "wrong_pct": 0.6086956521739131,
      "correct_lo": 0.024179495407233423,
      "correct_hi": 0.26796404298094023,
      "wrong_lo": 0.4078517960374599,
      "wrong_hi": 0.7784262201761787,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 23,
      "gradeable": 69,
      "correct": 36,
      "incorrect": 15,
      "abstained": 0,
      "no_answer": 18,
      "correct_pct": 0.5217391304347826,
      "wrong_pct": 0.21739130434782608,
      "correct_lo": 0.3296242986421849,
      "correct_hi": 0.7076313046005427,
      "wrong_lo": 0.09663835601719065,
      "wrong_hi": 0.4190388018273492,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 21,
      "gradeable": 21,
      "correct": 2,
      "incorrect": 15,
      "abstained": 4,
      "no_answer": 0,
      "correct_pct": 0.09523809523809523,
      "wrong_pct": 0.7142857142857143,
      "correct_lo": 0.026518208796947246,
      "correct_hi": 0.2891458386074067,
      "wrong_lo": 0.500432261771624,
      "wrong_hi": 0.8618632425437179,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 21,
      "gradeable": 63,
      "correct": 10,
      "incorrect": 11,
      "abstained": 0,
      "no_answer": 42,
      "correct_pct": 0.15873015873015872,
      "wrong_pct": 0.1746031746031746,
      "correct_lo": 0.05841860601860449,
      "correct_hi": 0.36459225747918417,
      "wrong_lo": 0.06738346177864377,
      "wrong_hi": 0.38246410574250345,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "kind": "os-derived",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 11,
      "gradeable": 11,
      "correct": 0,
      "incorrect": 3,
      "abstained": 8,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.2727272727272727,
      "correct_lo": 0,
      "correct_hi": 0.25884001724881417,
      "wrong_lo": 0.09745880573921467,
      "wrong_hi": 0.5656502930102463,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 11,
      "gradeable": 33,
      "correct": 17,
      "incorrect": 13,
      "abstained": 2,
      "no_answer": 1,
      "correct_pct": 0.5151515151515151,
      "wrong_pct": 0.3939393939393939,
      "correct_lo": 0.25693466435313284,
      "correct_hi": 0.7655247290635697,
      "wrong_lo": 0.1712881375459714,
      "wrong_hi": 0.6714961085371104,
      "verdict": "dont_use"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 24,
      "incorrect": 0,
      "abstained": 1,
      "no_answer": 0,
      "correct_pct": 0.96,
      "wrong_pct": 0,
      "correct_lo": 0.8045552759497747,
      "correct_hi": 0.9929039496133008,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 25,
      "gradeable": 50,
      "correct": 49,
      "incorrect": 1,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.98,
      "wrong_pct": 0.02,
      "correct_lo": 0.8342228568014084,
      "correct_hi": 0.9979085090035399,
      "wrong_lo": 0.0020914909964600825,
      "wrong_hi": 0.16577714319859152,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 12,
      "incorrect": 0,
      "abstained": 10,
      "no_answer": 0,
      "correct_pct": 0.5454545454545454,
      "wrong_pct": 0,
      "correct_lo": 0.34659473289041554,
      "correct_hi": 0.7307998556799515,
      "wrong_lo": 0,
      "wrong_hi": 0.14865952572596125,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 22,
      "gradeable": 44,
      "correct": 40,
      "incorrect": 0,
      "abstained": 4,
      "no_answer": 0,
      "correct_pct": 0.9090909090909091,
      "wrong_pct": 0,
      "correct_lo": 0.7218461934161421,
      "correct_hi": 0.9747051037171622,
      "wrong_lo": 0,
      "wrong_hi": 0.14865952572596125,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 5,
      "incorrect": 0,
      "abstained": 20,
      "no_answer": 0,
      "correct_pct": 0.2,
      "wrong_pct": 0,
      "correct_lo": 0.08860454100652487,
      "correct_hi": 0.3913133553653824,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 25,
      "gradeable": 50,
      "correct": 45,
      "incorrect": 3,
      "abstained": 2,
      "no_answer": 0,
      "correct_pct": 0.9,
      "wrong_pct": 0.06,
      "correct_lo": 0.7249581344927519,
      "correct_hi": 0.9684846703447051,
      "wrong_lo": 0.013978549490517175,
      "wrong_hi": 0.2232343651882801,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 24,
      "gradeable": 24,
      "correct": 6,
      "incorrect": 0,
      "abstained": 18,
      "no_answer": 0,
      "correct_pct": 0.25,
      "wrong_pct": 0,
      "correct_lo": 0.1199920347934233,
      "correct_hi": 0.4489982531210644,
      "wrong_lo": 0,
      "wrong_hi": 0.13798057582897535,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 24,
      "gradeable": 48,
      "correct": 44,
      "incorrect": 1,
      "abstained": 3,
      "no_answer": 0,
      "correct_pct": 0.9166666666666666,
      "wrong_pct": 0.020833333333333332,
      "correct_lo": 0.741507847985494,
      "correct_hi": 0.9768416721570266,
      "wrong_lo": 0.002178790970731939,
      "wrong_hi": 0.17171926086536943,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 22,
      "gradeable": 22,
      "correct": 3,
      "incorrect": 1,
      "abstained": 18,
      "no_answer": 0,
      "correct_pct": 0.13636363636363635,
      "wrong_pct": 0.045454545454545456,
      "correct_lo": 0.04748917264236269,
      "correct_hi": 0.3333541187947,
      "wrong_lo": 0.008069219301147859,
      "wrong_hi": 0.21798489499518053,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 22,
      "gradeable": 44,
      "correct": 20,
      "incorrect": 5,
      "abstained": 19,
      "no_answer": 0,
      "correct_pct": 0.45454545454545453,
      "wrong_pct": 0.11363636363636363,
      "correct_lo": 0.26920014432004846,
      "correct_hi": 0.6534052671095845,
      "wrong_lo": 0.03589747812594399,
      "wrong_hi": 0.3062485190259352,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 23,
      "gradeable": 23,
      "correct": 4,
      "incorrect": 1,
      "abstained": 18,
      "no_answer": 0,
      "correct_pct": 0.17391304347826086,
      "wrong_pct": 0.043478260869565216,
      "correct_lo": 0.06978540350612775,
      "correct_hi": 0.3713805478529567,
      "wrong_lo": 0.0077164669621267445,
      "wrong_hi": 0.20991586494059142,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 23,
      "gradeable": 46,
      "correct": 37,
      "incorrect": 2,
      "abstained": 7,
      "no_answer": 0,
      "correct_pct": 0.8043478260869565,
      "wrong_pct": 0.043478260869565216,
      "correct_lo": 0.6045180896164389,
      "correct_hi": 0.917060355781749,
      "wrong_lo": 0.0077164669621267445,
      "wrong_hi": 0.20991586494059142,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 21,
      "gradeable": 21,
      "correct": 10,
      "incorrect": 0,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.47619047619047616,
      "wrong_pct": 0,
      "correct_lo": 0.28343712296762946,
      "correct_hi": 0.6763078209973324,
      "wrong_lo": 0,
      "wrong_hi": 0.15464382326420198,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 21,
      "gradeable": 42,
      "correct": 28,
      "incorrect": 1,
      "abstained": 13,
      "no_answer": 0,
      "correct_pct": 0.6666666666666666,
      "wrong_pct": 0.023809523809523808,
      "correct_lo": 0.45373075657210293,
      "correct_hi": 0.8280546356731631,
      "wrong_lo": 0.0024906779134592077,
      "wrong_hi": 0.19240820138578074,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "kind": "os-derived",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 11,
      "gradeable": 11,
      "correct": 0,
      "incorrect": 0,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0,
      "correct_lo": 0,
      "correct_hi": 0.25884001724881417,
      "wrong_lo": 0,
      "wrong_hi": 0.25884001724881417,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 11,
      "gradeable": 22,
      "correct": 12,
      "incorrect": 7,
      "abstained": 3,
      "no_answer": 0,
      "correct_pct": 0.5454545454545454,
      "wrong_pct": 0.3181818181818182,
      "correct_lo": 0.28008775568998817,
      "correct_hi": 0.7872904245601196,
      "wrong_lo": 0.12364867979887728,
      "wrong_hi": 0.6068385992006916,
      "verdict": "dont_use"
    }
  ]
}