{
  "corpus_fingerprint": "299-21b69d52efdc",
  "revision": "2026-10-28",
  "published_at": "2026-10-28T13:52:26Z",
  "grader_version": "2026-10-10",
  "rows": [
    {
      "track": "closed",
      "model": "claude-opus-5.5",
      "provider": "claude-code",
      "total": 295,
      "correct": 136,
      "incorrect": 28,
      "abstained": 131,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 80,
      "score_per_100": 27.11864406779661,
      "runs": 1,
      "reasoning_correct": 42,
      "reasoning_total": 43,
      "grounded_correct": 94,
      "grounded_total": 252,
      "grounded_wrong": 27,
      "reasoning_accuracy": 97.67441860465117,
      "grounded_accuracy": 37.301587301587304,
      "grounded_hallucination_pct": 22.31404958677686,
      "cost_usd": 4.7546440000000025
    },
    {
      "track": "closed",
      "model": "grok-4.7",
      "provider": "xai",
      "total": 295,
      "correct": 86,
      "incorrect": 6,
      "abstained": 203,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 74,
      "score_per_100": 25.084745762711865,
      "runs": 1,
      "reasoning_correct": 39,
      "reasoning_total": 43,
      "grounded_correct": 47,
      "grounded_total": 252,
      "grounded_wrong": 3,
      "reasoning_accuracy": 90.69767441860465,
      "grounded_accuracy": 18.650793650793652,
      "grounded_hallucination_pct": 6,
      "cost_usd": 3.1057719999999973
    },
    {
      "track": "closed",
      "model": "claude-fable-5.1",
      "provider": "claude-code",
      "total": 295,
      "correct": 143,
      "incorrect": 36,
      "abstained": 116,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 71,
      "score_per_100": 24.06779661016949,
      "runs": 1,
      "reasoning_correct": 43,
      "reasoning_total": 43,
      "grounded_correct": 100,
      "grounded_total": 252,
      "grounded_wrong": 36,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 39.682539682539684,
      "grounded_hallucination_pct": 26.470588235294116,
      "cost_usd": 14.285832499999987
    },
    {
      "track": "closed",
      "model": "claude-sonnet-5.5",
      "provider": "claude-code",
      "total": 295,
      "correct": 131,
      "incorrect": 50,
      "abstained": 114,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 31,
      "score_per_100": 10.508474576271187,
      "runs": 1,
      "reasoning_correct": 43,
      "reasoning_total": 43,
      "grounded_correct": 88,
      "grounded_total": 252,
      "grounded_wrong": 50,
      "reasoning_accuracy": 100,
      "grounded_accuracy": 34.92063492063492,
      "grounded_hallucination_pct": 36.231884057971016,
      "cost_usd": 2.349849999999998
    },
    {
      "track": "closed",
      "model": "claude-haiku-5.5",
      "provider": "claude-code",
      "total": 295,
      "correct": 80,
      "incorrect": 33,
      "abstained": 182,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 14,
      "score_per_100": 4.745762711864407,
      "runs": 1,
      "reasoning_correct": 41,
      "reasoning_total": 43,
      "grounded_correct": 39,
      "grounded_total": 252,
      "grounded_wrong": 31,
      "reasoning_accuracy": 95.34883720930233,
      "grounded_accuracy": 15.476190476190476,
      "grounded_hallucination_pct": 44.285714285714285,
      "cost_usd": 0.13366040000000015
    },
    {
      "track": "closed",
      "model": "gemma4-12b",
      "provider": "ollama",
      "total": 295,
      "correct": 33,
      "incorrect": 110,
      "abstained": 113,
      "no_answer": 0,
      "errors": 0,
      "truncated": 39,
      "score": -187,
      "score_per_100": -73.046875,
      "runs": 1,
      "reasoning_correct": 22,
      "reasoning_total": 30,
      "grounded_correct": 11,
      "grounded_total": 226,
      "grounded_wrong": 102,
      "reasoning_accuracy": 73.33333333333333,
      "grounded_accuracy": 4.867256637168142,
      "grounded_hallucination_pct": 90.26548672566372,
      "cost_usd": 0
    },
    {
      "track": "closed",
      "model": "gpt-oss-20b",
      "provider": "ollama",
      "total": 295,
      "correct": 41,
      "incorrect": 158,
      "abstained": 93,
      "no_answer": 2,
      "errors": 0,
      "truncated": 1,
      "score": -275,
      "score_per_100": -93.5374149659864,
      "runs": 1,
      "reasoning_correct": 27,
      "reasoning_total": 40,
      "grounded_correct": 14,
      "grounded_total": 252,
      "grounded_wrong": 145,
      "reasoning_accuracy": 67.5,
      "grounded_accuracy": 5.555555555555555,
      "grounded_hallucination_pct": 91.19496855345912,
      "cost_usd": 0
    },
    {
      "track": "closed",
      "model": "deepseek-r1-14b",
      "provider": "ollama",
      "total": 295,
      "correct": 36,
      "incorrect": 158,
      "abstained": 100,
      "no_answer": 0,
      "errors": 0,
      "truncated": 1,
      "score": -280,
      "score_per_100": -95.23809523809524,
      "runs": 1,
      "reasoning_correct": 22,
      "reasoning_total": 42,
      "grounded_correct": 14,
      "grounded_total": 252,
      "grounded_wrong": 138,
      "reasoning_accuracy": 52.38095238095238,
      "grounded_accuracy": 5.555555555555555,
      "grounded_hallucination_pct": 90.78947368421052,
      "cost_usd": 0
    },
    {
      "track": "mcp",
      "model": "claude-fable-5.1",
      "provider": "claude-code",
      "total": 590,
      "correct": 543,
      "incorrect": 12,
      "abstained": 34,
      "no_answer": 1,
      "errors": 0,
      "truncated": 0,
      "score": 519,
      "score_per_100": 87.96610169491525,
      "runs": 2,
      "score_per_100_min": 87.45762711864407,
      "score_per_100_max": 88.47457627118644,
      "reasoning_correct": 84,
      "reasoning_total": 86,
      "grounded_correct": 459,
      "grounded_total": 503,
      "grounded_wrong": 10,
      "reasoning_accuracy": 97.67441860465117,
      "grounded_accuracy": 91.25248508946322,
      "grounded_hallucination_pct": 2.1321961620469083,
      "cost_usd": 203.36271050000002
    },
    {
      "track": "mcp",
      "model": "claude-opus-5.5",
      "provider": "claude-code",
      "total": 885,
      "correct": 808,
      "incorrect": 17,
      "abstained": 27,
      "no_answer": 33,
      "errors": 0,
      "truncated": 0,
      "score": 774,
      "score_per_100": 87.45762711864407,
      "runs": 3,
      "score_per_100_min": 85.42372881355932,
      "score_per_100_max": 90.84745762711864,
      "reasoning_correct": 126,
      "reasoning_total": 129,
      "grounded_correct": 682,
      "grounded_total": 723,
      "grounded_wrong": 14,
      "reasoning_accuracy": 97.67441860465117,
      "grounded_accuracy": 94.32918395573998,
      "grounded_hallucination_pct": 2.0114942528735633,
      "cost_usd": 91.39165739999999
    },
    {
      "track": "mcp",
      "model": "claude-sonnet-5.5",
      "provider": "claude-code",
      "total": 885,
      "correct": 807,
      "incorrect": 26,
      "abstained": 40,
      "no_answer": 12,
      "errors": 0,
      "truncated": 0,
      "score": 755,
      "score_per_100": 85.31073446327683,
      "runs": 3,
      "score_per_100_min": 83.38983050847457,
      "score_per_100_max": 86.77966101694915,
      "reasoning_correct": 122,
      "reasoning_total": 129,
      "grounded_correct": 685,
      "grounded_total": 744,
      "grounded_wrong": 19,
      "reasoning_accuracy": 94.57364341085271,
      "grounded_accuracy": 92.06989247311827,
      "grounded_hallucination_pct": 2.6988636363636362,
      "cost_usd": 34.973462800000014
    },
    {
      "track": "mcp",
      "model": "grok-4.7",
      "provider": "xai",
      "total": 590,
      "correct": 501,
      "incorrect": 16,
      "abstained": 73,
      "no_answer": 0,
      "errors": 0,
      "truncated": 0,
      "score": 469,
      "score_per_100": 79.49152542372882,
      "runs": 2,
      "score_per_100_min": 77.62711864406779,
      "score_per_100_max": 81.35593220338983,
      "reasoning_correct": 81,
      "reasoning_total": 86,
      "grounded_correct": 420,
      "grounded_total": 504,
      "grounded_wrong": 13,
      "reasoning_accuracy": 94.18604651162791,
      "grounded_accuracy": 83.33333333333333,
      "grounded_hallucination_pct": 3.0023094688221708,
      "cost_usd": 39.31951000000001
    },
    {
      "track": "mcp",
      "model": "claude-haiku-5.5",
      "provider": "claude-code",
      "total": 885,
      "correct": 728,
      "incorrect": 52,
      "abstained": 91,
      "no_answer": 14,
      "errors": 0,
      "truncated": 0,
      "score": 624,
      "score_per_100": 70.50847457627118,
      "runs": 3,
      "score_per_100_min": 66.77966101694915,
      "score_per_100_max": 75.2542372881356,
      "reasoning_correct": 118,
      "reasoning_total": 129,
      "grounded_correct": 610,
      "grounded_total": 742,
      "grounded_wrong": 47,
      "reasoning_accuracy": 91.47286821705427,
      "grounded_accuracy": 82.21024258760107,
      "grounded_hallucination_pct": 7.153729071537291,
      "cost_usd": 3.616504530000003
    },
    {
      "track": "mcp",
      "model": "gemma4-12b",
      "provider": "ollama",
      "total": 885,
      "correct": 494,
      "incorrect": 169,
      "abstained": 195,
      "no_answer": 1,
      "errors": 0,
      "truncated": 26,
      "score": 156,
      "score_per_100": 18.160651920838184,
      "runs": 3,
      "score_per_100_min": 13.541666666666666,
      "score_per_100_max": 22.027972027972027,
      "reasoning_correct": 76,
      "reasoning_total": 115,
      "grounded_correct": 418,
      "grounded_total": 743,
      "grounded_wrong": 149,
      "reasoning_accuracy": 66.08695652173913,
      "grounded_accuracy": 56.258411843876175,
      "grounded_hallucination_pct": 26.278659611992946,
      "cost_usd": 0
    },
    {
      "track": "mcp",
      "model": "gpt-oss-20b",
      "provider": "ollama",
      "total": 885,
      "correct": 431,
      "incorrect": 170,
      "abstained": 26,
      "no_answer": 258,
      "errors": 0,
      "truncated": 0,
      "score": 91,
      "score_per_100": 10.282485875706215,
      "runs": 3,
      "score_per_100_min": 8.474576271186441,
      "score_per_100_max": 13.220338983050848,
      "reasoning_correct": 79,
      "reasoning_total": 111,
      "grounded_correct": 352,
      "grounded_total": 516,
      "grounded_wrong": 139,
      "reasoning_accuracy": 71.17117117117117,
      "grounded_accuracy": 68.21705426356588,
      "grounded_hallucination_pct": 28.30957230142566,
      "cost_usd": 0
    }
  ],
  "gaps": [
    {
      "model": "claude-fable-5.1",
      "score_gap_per_100": 63.89830508474576,
      "grounded_acc_closed": 39.682539682539684,
      "grounded_acc_mcp": 91.25248508946322,
      "halluc_closed": 26.470588235294116,
      "halluc_mcp": 2.1321961620469083
    },
    {
      "model": "claude-haiku-5.5",
      "score_gap_per_100": 65.76271186440678,
      "grounded_acc_closed": 15.476190476190476,
      "grounded_acc_mcp": 82.21024258760107,
      "halluc_closed": 44.285714285714285,
      "halluc_mcp": 7.153729071537291
    },
    {
      "model": "claude-opus-5.5",
      "score_gap_per_100": 60.33898305084746,
      "grounded_acc_closed": 37.301587301587304,
      "grounded_acc_mcp": 94.32918395573998,
      "halluc_closed": 22.31404958677686,
      "halluc_mcp": 2.0114942528735633
    },
    {
      "model": "claude-sonnet-5.5",
      "score_gap_per_100": 74.80225988700565,
      "grounded_acc_closed": 34.92063492063492,
      "grounded_acc_mcp": 92.06989247311827,
      "halluc_closed": 36.231884057971016,
      "halluc_mcp": 2.6988636363636362
    },
    {
      "model": "gemma4-12b",
      "score_gap_per_100": 91.20752692083818,
      "grounded_acc_closed": 4.867256637168142,
      "grounded_acc_mcp": 56.258411843876175,
      "halluc_closed": 90.26548672566372,
      "halluc_mcp": 26.278659611992946
    },
    {
      "model": "gpt-oss-20b",
      "score_gap_per_100": 103.81990084169261,
      "grounded_acc_closed": 5.555555555555555,
      "grounded_acc_mcp": 68.21705426356588,
      "halluc_closed": 91.19496855345912,
      "halluc_mcp": 28.30957230142566
    },
    {
      "model": "grok-4.7",
      "score_gap_per_100": 54.406779661016955,
      "grounded_acc_closed": 18.650793650793652,
      "grounded_acc_mcp": 83.33333333333333,
      "halluc_closed": 6,
      "halluc_mcp": 3.0023094688221708
    }
  ],
  "jobs": [
    "baseline",
    "catalog",
    "constellation",
    "debris",
    "ground",
    "market",
    "regulatory"
  ],
  "errata": [
    {
      "id": "con-galileo-launched",
      "action": "rekey",
      "value": "34",
      "reason": "The question counts IOV and FOC satellites; the ground-truth SQL matched only names starting \"GALILEO \" and missed the 4 IOV satellites, giving 30. 34 Galileo satellites had been launched (latest 2025-12-17).",
      "date": "2026-10-11"
    },
    {
      "id": "deb-atlas5-centaur-afspc11-debris",
      "action": "drop",
      "reason": "The question names the wrong launch. The Centaur that fragmented in September 2024 is 2018-079B from the AEHF-4 launch (2018-10-17, 214 debris cataloged). AFSPC-11 (2018-036) has 1 debris object, which is what models answered.",
      "date": "2026-10-11"
    },
    {
      "id": "reg-ngso-market-access-count",
      "action": "rekey",
      "value": "9",
      "reason": "The question asks for foreign-licensed NGSO systems with US market access; the ground-truth SQL counted every granted NGSO system, U.S.-licensed ones included, giving 52. Excluding a U.S. licensing administration leaves 9 (Kepler, Telesat Ka-band, ExactView-1, ExactView-7, Kineis, Arctic Satellite Broadband Mission, two OneWeb systems, O3B).",
      "date": "2026-10-11"
    },
    {
      "id": "reg-noaa-remote-sensing-licensees",
      "action": "drop",
      "reason": "The question asks how many companies currently hold a license, but the answer key comes from a NOAA CRSRA licensee snapshot last seen 2024-07-27; no current list was available to verify it. To be restated as an as-of question, or rekeyed from a current list, in the next corpus.",
      "date": "2026-10-11"
    },
    {
      "id": "gnd-ofcom-top-licensee",
      "action": "rekey",
      "value": "Intelsat Global Sales \u0026 Marketing Ltd",
      "accept": [
        "Intelsat"
      ],
      "reason": "The question asks which licensee holds the most licences; the ground-truth SQL counted register rows, which are one per licensed frequency. Counting distinct licence numbers, Intelsat Global Sales \u0026 Marketing Ltd holds 13, ahead of Ground Control Technologies (11); Goonhilly Earth Station Limited holds 6.",
      "date": "2026-10-28"
    },
    {
      "id": "gnd-acma-capricorn-locations",
      "action": "drop",
      "reason": "\"Distinct sites\" is ambiguous in the ACMA register: Capricorn Space's registrations sit at 8 distinct recorded coordinates (ACMA sites), which cluster into 4 geographic locations. The answer key used the 4; the register's own count is 8. Not scored; to be restated in the next corpus.",
      "date": "2026-10-28"
    },
    {
      "id": "gnd-acma-optus-locations",
      "action": "drop",
      "reason": "Same ambiguity as gnd-acma-capricorn-locations: Optus's registrations sit at 12 distinct recorded coordinates, clustering into 6 geographic locations. Not scored; to be restated in the next corpus.",
      "date": "2026-10-28"
    }
  ],
  "verdicts": [
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 25,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 25,
      "gradeable": 50,
      "correct": 50,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 45,
      "gradeable": 45,
      "correct": 29,
      "incorrect": 3,
      "abstained": 13,
      "no_answer": 0,
      "correct_pct": 0.6444444444444445,
      "wrong_pct": 0.06666666666666667,
      "correct_lo": 0.49835546011952203,
      "correct_hi": 0.7678110864309596,
      "wrong_lo": 0.0229316164621471,
      "wrong_hi": 0.17856874388640825,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 45,
      "gradeable": 90,
      "correct": 90,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.9213457380593594,
      "correct_hi": 1,
      "wrong_lo": 6.938893903907228e-18,
      "wrong_hi": 0.07865426194064076,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 43,
      "gradeable": 43,
      "correct": 23,
      "incorrect": 9,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.5348837209302325,
      "wrong_pct": 0.20930232558139536,
      "correct_lo": 0.38915389728872773,
      "correct_hi": 0.6748917373608144,
      "wrong_lo": 0.11422705997811457,
      "wrong_hi": 0.35205931794236645,
      "verdict": "dont_use"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 43,
      "gradeable": 86,
      "correct": 84,
      "incorrect": 1,
      "abstained": 1,
      "no_answer": 0,
      "correct_pct": 0.9767441860465116,
      "wrong_pct": 0.011627906976744186,
      "correct_lo": 0.8794073213878312,
      "correct_hi": 0.9958830188225801,
      "wrong_lo": 0.0012151180745690188,
      "wrong_hi": 0.10214599683183896,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 42,
      "gradeable": 42,
      "correct": 19,
      "incorrect": 4,
      "abstained": 19,
      "no_answer": 0,
      "correct_pct": 0.4523809523809524,
      "wrong_pct": 0.09523809523809523,
      "correct_lo": 0.3122316381863852,
      "correct_hi": 0.6005113725287732,
      "wrong_lo": 0.03766163882831211,
      "wrong_hi": 0.22065395225053333,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 42,
      "gradeable": 84,
      "correct": 77,
      "incorrect": 3,
      "abstained": 4,
      "no_answer": 0,
      "correct_pct": 0.9166666666666666,
      "wrong_pct": 0.03571428571428571,
      "correct_lo": 0.7944525206321794,
      "correct_hi": 0.9690461356101855,
      "wrong_lo": 0.008290820071836139,
      "wrong_hi": 0.14095353440095715,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 38,
      "gradeable": 38,
      "correct": 10,
      "incorrect": 10,
      "abstained": 18,
      "no_answer": 0,
      "correct_pct": 0.2631578947368421,
      "wrong_pct": 0.2631578947368421,
      "correct_lo": 0.14971462389701296,
      "correct_hi": 0.42009150206375323,
      "wrong_lo": 0.14971462389701296,
      "wrong_hi": 0.42009150206375323,
      "verdict": "dont_use"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 38,
      "gradeable": 76,
      "correct": 72,
      "incorrect": 3,
      "abstained": 1,
      "no_answer": 0,
      "correct_pct": 0.9473684210526315,
      "wrong_pct": 0.039473684210526314,
      "correct_lo": 0.8271423311893115,
      "correct_hi": 0.9854460975514631,
      "wrong_lo": 0.009168595461421056,
      "wrong_hi": 0.1543433161289579,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 41,
      "gradeable": 41,
      "correct": 18,
      "incorrect": 7,
      "abstained": 16,
      "no_answer": 0,
      "correct_pct": 0.43902439024390244,
      "wrong_pct": 0.17073170731707318,
      "correct_lo": 0.2988990477210983,
      "correct_hi": 0.5895973484824271,
      "wrong_lo": 0.08525143995273901,
      "wrong_hi": 0.3126290995462977,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 41,
      "gradeable": 82,
      "correct": 77,
      "incorrect": 1,
      "abstained": 4,
      "no_answer": 0,
      "correct_pct": 0.9390243902439024,
      "wrong_pct": 0.012195121951219513,
      "correct_lo": 0.8219151646070804,
      "correct_hi": 0.9809107827275374,
      "wrong_lo": 0.0012744535125377085,
      "wrong_hi": 0.1066967161156647,
      "verdict": "trust"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 40,
      "gradeable": 40,
      "correct": 14,
      "incorrect": 2,
      "abstained": 24,
      "no_answer": 0,
      "correct_pct": 0.35,
      "wrong_pct": 0.05,
      "correct_lo": 0.2213433317104915,
      "correct_hi": 0.5049440300646263,
      "wrong_lo": 0.013820377146101098,
      "wrong_hi": 0.165041708179252,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 40,
      "gradeable": 80,
      "correct": 55,
      "incorrect": 2,
      "abstained": 22,
      "no_answer": 1,
      "correct_pct": 0.6875,
      "wrong_pct": 0.025,
      "correct_lo": 0.532883871337421,
      "correct_hi": 0.8092569264436819,
      "wrong_lo": 0.0044267174077949195,
      "wrong_hi": 0.1288165948800778,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "closed",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 16,
      "gradeable": 16,
      "correct": 0,
      "incorrect": 1,
      "abstained": 15,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.0625,
      "correct_lo": 0,
      "correct_hi": 0.19361341827271994,
      "wrong_lo": 0.01111905730833121,
      "wrong_hi": 0.2832926836802987,
      "verdict": "verify"
    },
    {
      "model": "claude-fable-5.1",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 2,
      "questions": 16,
      "gradeable": 32,
      "correct": 28,
      "incorrect": 2,
      "abstained": 2,
      "no_answer": 0,
      "correct_pct": 0.875,
      "wrong_pct": 0.0625,
      "correct_lo": 0.6397666856325173,
      "correct_hi": 0.9650232506629428,
      "wrong_lo": 0.01111905730833121,
      "wrong_hi": 0.2832926836802987,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 25,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 74,
      "incorrect": 1,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9866666666666667,
      "wrong_pct": 0.013333333333333334,
      "correct_lo": 0.8446808865024286,
      "correct_hi": 0.9990078593831441,
      "wrong_lo": 0.0009921406168558883,
      "wrong_hi": 0.15531911349757144,
      "verdict": "trust"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 45,
      "gradeable": 45,
      "correct": 13,
      "incorrect": 8,
      "abstained": 24,
      "no_answer": 0,
      "correct_pct": 0.28888888888888886,
      "wrong_pct": 0.17777777777777778,
      "correct_lo": 0.17729918031900166,
      "correct_hi": 0.4336881747226023,
      "wrong_lo": 0.0929427892585383,
      "wrong_hi": 0.3133010684365414,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 45,
      "gradeable": 135,
      "correct": 128,
      "incorrect": 1,
      "abstained": 2,
      "no_answer": 4,
      "correct_pct": 0.9481481481481482,
      "wrong_pct": 0.007407407407407408,
      "correct_lo": 0.8414195055372256,
      "correct_hi": 0.9843792670937559,
      "wrong_lo": 0.0005509796220901919,
      "wrong_hi": 0.09175284880831888,
      "verdict": "trust"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 43,
      "gradeable": 43,
      "correct": 10,
      "incorrect": 6,
      "abstained": 27,
      "no_answer": 0,
      "correct_pct": 0.23255813953488372,
      "wrong_pct": 0.13953488372093023,
      "correct_lo": 0.1315353846924483,
      "correct_hi": 0.37744808299439414,
      "wrong_lo": 0.06555634267713947,
      "wrong_hi": 0.27263876594425684,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 43,
      "gradeable": 129,
      "correct": 107,
      "incorrect": 15,
      "abstained": 2,
      "no_answer": 5,
      "correct_pct": 0.8294573643410853,
      "wrong_pct": 0.11627906976744186,
      "correct_lo": 0.6913909603342344,
      "correct_hi": 0.9134844780225538,
      "wrong_lo": 0.050703529948620846,
      "wrong_hi": 0.24479448890641395,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 42,
      "gradeable": 42,
      "correct": 6,
      "incorrect": 6,
      "abstained": 30,
      "no_answer": 0,
      "correct_pct": 0.14285714285714285,
      "wrong_pct": 0.14285714285714285,
      "correct_lo": 0.06715875684189936,
      "correct_hi": 0.2784138235217878,
      "wrong_lo": 0.06715875684189936,
      "wrong_hi": 0.2784138235217878,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 42,
      "gradeable": 126,
      "correct": 108,
      "incorrect": 2,
      "abstained": 15,
      "no_answer": 1,
      "correct_pct": 0.8571428571428571,
      "wrong_pct": 0.015873015873015872,
      "correct_lo": 0.7215861764782122,
      "correct_hi": 0.9328412431581007,
      "wrong_lo": 0.0020833061958375337,
      "wrong_hi": 0.1108039694082717,
      "verdict": "trust"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 38,
      "gradeable": 38,
      "correct": 4,
      "incorrect": 3,
      "abstained": 31,
      "no_answer": 0,
      "correct_pct": 0.10526315789473684,
      "wrong_pct": 0.07894736842105263,
      "correct_lo": 0.041701897932665793,
      "correct_hi": 0.24130831200194466,
      "wrong_lo": 0.02721410139426496,
      "wrong_hi": 0.20799678920265297,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 38,
      "gradeable": 114,
      "correct": 89,
      "incorrect": 15,
      "abstained": 9,
      "no_answer": 1,
      "correct_pct": 0.7807017543859649,
      "wrong_pct": 0.13157894736842105,
      "correct_lo": 0.6269329384439006,
      "correct_hi": 0.8829264678248208,
      "wrong_lo": 0.05753782875731439,
      "wrong_hi": 0.2732717005149888,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 41,
      "gradeable": 41,
      "correct": 9,
      "incorrect": 5,
      "abstained": 27,
      "no_answer": 0,
      "correct_pct": 0.21951219512195122,
      "wrong_pct": 0.12195121951219512,
      "correct_lo": 0.12003040155386346,
      "correct_hi": 0.36705302098235293,
      "wrong_lo": 0.053232579973408245,
      "wrong_hi": 0.25544507648844866,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 41,
      "gradeable": 123,
      "correct": 102,
      "incorrect": 2,
      "abstained": 18,
      "no_answer": 1,
      "correct_pct": 0.8292682926829268,
      "wrong_pct": 0.016260162601626018,
      "correct_lo": 0.6873709004537024,
      "correct_hi": 0.914748560047261,
      "wrong_lo": 0.0021342021336350214,
      "wrong_hi": 0.11327054108099902,
      "verdict": "trust"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 40,
      "gradeable": 40,
      "correct": 8,
      "incorrect": 5,
      "abstained": 27,
      "no_answer": 0,
      "correct_pct": 0.2,
      "wrong_pct": 0.125,
      "correct_lo": 0.10499864973166874,
      "correct_hi": 0.34757607381856676,
      "wrong_lo": 0.05459419773541284,
      "wrong_hi": 0.26112420670238146,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 40,
      "gradeable": 120,
      "correct": 68,
      "incorrect": 7,
      "abstained": 43,
      "no_answer": 2,
      "correct_pct": 0.5666666666666667,
      "wrong_pct": 0.058333333333333334,
      "correct_lo": 0.4140230650400791,
      "correct_hi": 0.7076269963932018,
      "wrong_lo": 0.017592246196403025,
      "wrong_hi": 0.17647609680811022,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "closed",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 16,
      "gradeable": 16,
      "correct": 0,
      "incorrect": 0,
      "abstained": 16,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0,
      "correct_lo": 0,
      "correct_hi": 0.19361341827271994,
      "wrong_lo": 0,
      "wrong_hi": 0.19361341827271994,
      "verdict": "verify"
    },
    {
      "model": "claude-haiku-5.5",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 16,
      "gradeable": 48,
      "correct": 40,
      "incorrect": 6,
      "abstained": 2,
      "no_answer": 0,
      "correct_pct": 0.8333333333333334,
      "wrong_pct": 0.125,
      "correct_lo": 0.592568738833555,
      "correct_hi": 0.9450223156512985,
      "wrong_lo": 0.034976749337057134,
      "wrong_hi": 0.3602333143674828,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 25,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 75,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 45,
      "gradeable": 45,
      "correct": 30,
      "incorrect": 5,
      "abstained": 10,
      "no_answer": 0,
      "correct_pct": 0.6666666666666666,
      "wrong_pct": 0.1111111111111111,
      "correct_lo": 0.520702168092554,
      "correct_hi": 0.7864130779272324,
      "wrong_lo": 0.04840400437825558,
      "wrong_hi": 0.23499375490890945,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 45,
      "gradeable": 135,
      "correct": 133,
      "incorrect": 2,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9851851851851852,
      "wrong_pct": 0.014814814814814815,
      "correct_lo": 0.8959908160834458,
      "correct_hi": 0.998055788996377,
      "wrong_lo": 0.0019442110036230614,
      "wrong_hi": 0.10400918391655428,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 43,
      "gradeable": 43,
      "correct": 22,
      "incorrect": 5,
      "abstained": 16,
      "no_answer": 0,
      "correct_pct": 0.5116279069767442,
      "wrong_pct": 0.11627906976744186,
      "correct_lo": 0.36752063766074083,
      "correct_hi": 0.6538279072224399,
      "wrong_lo": 0.050703529948620846,
      "wrong_hi": 0.24479448890641395,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 43,
      "gradeable": 129,
      "correct": 127,
      "incorrect": 2,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9844961240310077,
      "wrong_pct": 0.015503875968992248,
      "correct_lo": 0.8915574846957087,
      "correct_hi": 0.9979652187701565,
      "wrong_lo": 0.002034781229843595,
      "wrong_hi": 0.1084425153042913,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 42,
      "gradeable": 42,
      "correct": 17,
      "incorrect": 3,
      "abstained": 22,
      "no_answer": 0,
      "correct_pct": 0.40476190476190477,
      "wrong_pct": 0.07142857142857142,
      "correct_lo": 0.27042647671209086,
      "correct_hi": 0.5550595447182258,
      "wrong_lo": 0.024589963551172878,
      "wrong_hi": 0.19009713288525168,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 42,
      "gradeable": 126,
      "correct": 117,
      "incorrect": 4,
      "abstained": 5,
      "no_answer": 0,
      "correct_pct": 0.9285714285714286,
      "wrong_pct": 0.031746031746031744,
      "correct_lo": 0.8099028671147483,
      "correct_hi": 0.9754100364488272,
      "wrong_lo": 0.006832557179303311,
      "wrong_hi": 0.13514038151975316,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 38,
      "gradeable": 38,
      "correct": 8,
      "incorrect": 8,
      "abstained": 22,
      "no_answer": 0,
      "correct_pct": 0.21052631578947367,
      "wrong_pct": 0.21052631578947367,
      "correct_lo": 0.1107466035209593,
      "correct_hi": 0.36346088376442165,
      "wrong_lo": 0.1107466035209593,
      "wrong_hi": 0.36346088376442165,
      "verdict": "dont_use"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 38,
      "gradeable": 114,
      "correct": 108,
      "incorrect": 4,
      "abstained": 0,
      "no_answer": 2,
      "correct_pct": 0.9473684210526315,
      "wrong_pct": 0.03508771929824561,
      "correct_lo": 0.8271423311893115,
      "correct_hi": 0.9854460975514631,
      "wrong_lo": 0.007555324531409738,
      "wrong_hi": 0.14799003383602044,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 41,
      "gradeable": 41,
      "correct": 15,
      "incorrect": 6,
      "abstained": 20,
      "no_answer": 0,
      "correct_pct": 0.36585365853658536,
      "wrong_pct": 0.14634146341463414,
      "correct_lo": 0.23589630416936816,
      "correct_hi": 0.5187957674783876,
      "wrong_lo": 0.06884154529455412,
      "wrong_hi": 0.28443755268589266,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 41,
      "gradeable": 123,
      "correct": 115,
      "incorrect": 1,
      "abstained": 4,
      "no_answer": 3,
      "correct_pct": 0.9349593495934959,
      "wrong_pct": 0.008130081300813009,
      "correct_lo": 0.8164669157161448,
      "correct_hi": 0.9789254580320415,
      "wrong_lo": 0.0006047617687712711,
      "wrong_hi": 0.09993283427299948,
      "verdict": "trust"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 40,
      "gradeable": 40,
      "correct": 14,
      "incorrect": 1,
      "abstained": 25,
      "no_answer": 0,
      "correct_pct": 0.35,
      "wrong_pct": 0.025,
      "correct_lo": 0.2213433317104915,
      "correct_hi": 0.5049440300646263,
      "wrong_lo": 0.0044267174077949195,
      "wrong_hi": 0.1288165948800778,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 40,
      "gradeable": 120,
      "correct": 78,
      "incorrect": 1,
      "abstained": 18,
      "no_answer": 23,
      "correct_pct": 0.65,
      "wrong_pct": 0.008333333333333333,
      "correct_lo": 0.4950559699353738,
      "correct_hi": 0.7786566682895086,
      "wrong_lo": 0.0006198888965553984,
      "wrong_hi": 0.10221090803299705,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "closed",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 16,
      "gradeable": 16,
      "correct": 0,
      "incorrect": 0,
      "abstained": 16,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0,
      "correct_lo": 0,
      "correct_hi": 0.19361341827271994,
      "wrong_lo": 0,
      "wrong_hi": 0.19361341827271994,
      "verdict": "verify"
    },
    {
      "model": "claude-opus-5.5",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 16,
      "gradeable": 48,
      "correct": 40,
      "incorrect": 3,
      "abstained": 0,
      "no_answer": 5,
      "correct_pct": 0.8333333333333334,
      "wrong_pct": 0.0625,
      "correct_lo": 0.592568738833555,
      "correct_hi": 0.9450223156512985,
      "wrong_lo": 0.01111905730833121,
      "wrong_hi": 0.2832926836802987,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 25,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 74,
      "incorrect": 1,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9866666666666667,
      "wrong_pct": 0.013333333333333334,
      "correct_lo": 0.8446808865024286,
      "correct_hi": 0.9990078593831441,
      "wrong_lo": 0.0009921406168558883,
      "wrong_hi": 0.15531911349757144,
      "verdict": "trust"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 45,
      "gradeable": 45,
      "correct": 26,
      "incorrect": 11,
      "abstained": 8,
      "no_answer": 0,
      "correct_pct": 0.5777777777777777,
      "wrong_pct": 0.24444444444444444,
      "correct_lo": 0.433005327959445,
      "correct_hi": 0.7103151201831219,
      "wrong_lo": 0.14235335025509283,
      "wrong_hi": 0.3867366058479013,
      "verdict": "dont_use"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 45,
      "gradeable": 135,
      "correct": 134,
      "incorrect": 1,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9925925925925926,
      "wrong_pct": 0.007407407407407408,
      "correct_lo": 0.9082471511916812,
      "correct_hi": 0.9994490203779098,
      "wrong_lo": 0.0005509796220901919,
      "wrong_hi": 0.09175284880831888,
      "verdict": "trust"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 43,
      "gradeable": 43,
      "correct": 21,
      "incorrect": 5,
      "abstained": 17,
      "no_answer": 0,
      "correct_pct": 0.4883720930232558,
      "wrong_pct": 0.11627906976744186,
      "correct_lo": 0.34617209277756,
      "correct_hi": 0.6324793623392592,
      "wrong_lo": 0.050703529948620846,
      "wrong_hi": 0.24479448890641395,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 43,
      "gradeable": 129,
      "correct": 122,
      "incorrect": 4,
      "abstained": 1,
      "no_answer": 2,
      "correct_pct": 0.9457364341085271,
      "wrong_pct": 0.031007751937984496,
      "correct_lo": 0.8347147058945641,
      "correct_hi": 0.9836461812940318,
      "wrong_lo": 0.0066729680551528975,
      "wrong_hi": 0.13226905498988967,
      "verdict": "trust"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 42,
      "gradeable": 42,
      "correct": 19,
      "incorrect": 7,
      "abstained": 16,
      "no_answer": 0,
      "correct_pct": 0.4523809523809524,
      "wrong_pct": 0.16666666666666666,
      "correct_lo": 0.3122316381863852,
      "correct_hi": 0.6005113725287732,
      "wrong_lo": 0.08315811295836502,
      "wrong_hi": 0.30604296204774295,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 42,
      "gradeable": 126,
      "correct": 119,
      "incorrect": 3,
      "abstained": 4,
      "no_answer": 0,
      "correct_pct": 0.9444444444444444,
      "wrong_pct": 0.023809523809523808,
      "correct_lo": 0.8311453536891364,
      "correct_hi": 0.9832532129693863,
      "wrong_lo": 0.004215295499100304,
      "wrong_hi": 0.12321481165248255,
      "verdict": "trust"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 38,
      "gradeable": 38,
      "correct": 7,
      "incorrect": 11,
      "abstained": 20,
      "no_answer": 0,
      "correct_pct": 0.18421052631578946,
      "wrong_pct": 0.2894736842105263,
      "correct_lo": 0.09221648453257492,
      "correct_hi": 0.33419168341511346,
      "wrong_lo": 0.17003106263656131,
      "wrong_hi": 0.44757438266189764,
      "verdict": "dont_use"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 38,
      "gradeable": 114,
      "correct": 105,
      "incorrect": 8,
      "abstained": 1,
      "no_answer": 0,
      "correct_pct": 0.9210526315789473,
      "wrong_pct": 0.07017543859649122,
      "correct_lo": 0.792003210797347,
      "correct_hi": 0.972785898605735,
      "wrong_lo": 0.022758308954978573,
      "wrong_hi": 0.19651947519604177,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 41,
      "gradeable": 41,
      "correct": 17,
      "incorrect": 11,
      "abstained": 13,
      "no_answer": 0,
      "correct_pct": 0.4146341463414634,
      "wrong_pct": 0.2682926829268293,
      "correct_lo": 0.2775633082533015,
      "correct_hi": 0.5663316464316339,
      "wrong_lo": 0.15694805281336685,
      "wrong_hi": 0.41933825276002934,
      "verdict": "dont_use"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 41,
      "gradeable": 123,
      "correct": 119,
      "incorrect": 0,
      "abstained": 4,
      "no_answer": 0,
      "correct_pct": 0.967479674796748,
      "wrong_pct": 0,
      "correct_lo": 0.8618609293365356,
      "correct_hi": 0.9930000331031039,
      "wrong_lo": 0,
      "wrong_hi": 0.08567044886890746,
      "verdict": "trust"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 40,
      "gradeable": 40,
      "correct": 11,
      "incorrect": 4,
      "abstained": 25,
      "no_answer": 0,
      "correct_pct": 0.275,
      "wrong_pct": 0.1,
      "correct_lo": 0.1610785343314309,
      "correct_hi": 0.4283525083312457,
      "wrong_lo": 0.03957888759557489,
      "wrong_hi": 0.23052074380473903,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 40,
      "gradeable": 120,
      "correct": 80,
      "incorrect": 3,
      "abstained": 30,
      "no_answer": 7,
      "correct_pct": 0.6666666666666666,
      "wrong_pct": 0.025,
      "correct_lo": 0.5117579108873644,
      "correct_hi": 0.7923672426958381,
      "wrong_lo": 0.0044267174077949195,
      "wrong_hi": 0.1288165948800778,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "closed",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 1,
      "questions": 16,
      "gradeable": 16,
      "correct": 0,
      "incorrect": 1,
      "abstained": 15,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.0625,
      "correct_lo": 0,
      "correct_hi": 0.19361341827271994,
      "wrong_lo": 0.01111905730833121,
      "wrong_hi": 0.2832926836802987,
      "verdict": "verify"
    },
    {
      "model": "claude-sonnet-5.5",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "claude-code",
      "harness": "2.1.296 (Claude Code)",
      "runs": 3,
      "questions": 16,
      "gradeable": 48,
      "correct": 42,
      "incorrect": 3,
      "abstained": 0,
      "no_answer": 3,
      "correct_pct": 0.875,
      "wrong_pct": 0.0625,
      "correct_lo": 0.6397666856325173,
      "correct_hi": 0.9650232506629428,
      "wrong_lo": 0.01111905730833121,
      "wrong_hi": 0.2832926836802987,
      "verdict": "verify"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 24,
      "gradeable": 24,
      "correct": 14,
      "incorrect": 10,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.5833333333333334,
      "wrong_pct": 0.4166666666666667,
      "correct_lo": 0.3883433764938864,
      "correct_hi": 0.7553265275346177,
      "wrong_lo": 0.24467347246538232,
      "wrong_hi": 0.6116566235061137,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 45,
      "gradeable": 45,
      "correct": 1,
      "incorrect": 34,
      "abstained": 10,
      "no_answer": 0,
      "correct_pct": 0.022222222222222223,
      "wrong_pct": 0.7555555555555555,
      "correct_lo": 0.003933497522757505,
      "correct_hi": 0.11566946388718813,
      "wrong_lo": 0.6132633941520986,
      "wrong_hi": 0.8576466497449071,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 43,
      "gradeable": 43,
      "correct": 3,
      "incorrect": 29,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.06976744186046512,
      "wrong_pct": 0.6744186046511628,
      "correct_lo": 0.024011153791962503,
      "correct_hi": 0.18609268553034927,
      "wrong_lo": 0.5251593250655096,
      "wrong_hi": 0.7950688481822019,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 42,
      "gradeable": 42,
      "correct": 5,
      "incorrect": 21,
      "abstained": 16,
      "no_answer": 0,
      "correct_pct": 0.11904761904761904,
      "wrong_pct": 0.5,
      "correct_lo": 0.05193726765019084,
      "correct_hi": 0.2500068180710754,
      "wrong_lo": 0.3552574591652803,
      "wrong_hi": 0.6447425408347197,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 38,
      "gradeable": 38,
      "correct": 5,
      "incorrect": 18,
      "abstained": 15,
      "no_answer": 0,
      "correct_pct": 0.13157894736842105,
      "wrong_pct": 0.47368421052631576,
      "correct_lo": 0.05753782875731439,
      "correct_hi": 0.2732717005149888,
      "wrong_lo": 0.3247877880894159,
      "wrong_hi": 0.6274128925728913,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 41,
      "gradeable": 41,
      "correct": 1,
      "incorrect": 18,
      "abstained": 22,
      "no_answer": 0,
      "correct_pct": 0.024390243902439025,
      "wrong_pct": 0.43902439024390244,
      "correct_lo": 0.00431842027071648,
      "correct_hi": 0.12595347011678087,
      "wrong_lo": 0.2988990477210983,
      "wrong_hi": 0.5895973484824271,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 40,
      "gradeable": 40,
      "correct": 3,
      "incorrect": 20,
      "abstained": 17,
      "no_answer": 0,
      "correct_pct": 0.075,
      "wrong_pct": 0.5,
      "correct_lo": 0.025835556771858226,
      "correct_hi": 0.19864530159097524,
      "wrong_lo": 0.35199278797099753,
      "wrong_hi": 0.6480072120290025,
      "verdict": "dont_use"
    },
    {
      "model": "deepseek-r1-14b",
      "track": "closed",
      "kind": "os-derived",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 16,
      "gradeable": 16,
      "correct": 0,
      "incorrect": 7,
      "abstained": 9,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.4375,
      "correct_lo": 0,
      "correct_hi": 0.19361341827271994,
      "wrong_lo": 0.2309836333518552,
      "wrong_hi": 0.6682180439322347,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 13,
      "gradeable": 13,
      "correct": 12,
      "incorrect": 1,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 0.9230769230769231,
      "wrong_pct": 0.07692307692307693,
      "correct_lo": 0.6668546607175265,
      "correct_hi": 0.9862899336203037,
      "wrong_lo": 0.013710066379696295,
      "wrong_hi": 0.3331453392824736,
      "verdict": "verify"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 24,
      "gradeable": 61,
      "correct": 50,
      "incorrect": 2,
      "abstained": 9,
      "no_answer": 0,
      "correct_pct": 0.819672131147541,
      "wrong_pct": 0.03278688524590164,
      "correct_lo": 0.6260964155072175,
      "correct_hi": 0.9250307573234378,
      "wrong_lo": 0.0048869096226989706,
      "wrong_hi": 0.18961953008634358,
      "verdict": "trust"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 38,
      "gradeable": 38,
      "correct": 0,
      "incorrect": 26,
      "abstained": 12,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.6842105263157895,
      "correct_lo": 0,
      "correct_hi": 0.09181293258384,
      "wrong_lo": 0.5254394356005149,
      "wrong_hi": 0.8091557997633335,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 45,
      "gradeable": 134,
      "correct": 97,
      "incorrect": 16,
      "abstained": 21,
      "no_answer": 0,
      "correct_pct": 0.7238805970149254,
      "wrong_pct": 0.11940298507462686,
      "correct_lo": 0.5796566710954191,
      "correct_hi": 0.8328861966923536,
      "wrong_lo": 0.05359769905053112,
      "wrong_hi": 0.24507942571025515,
      "verdict": "verify"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 33,
      "gradeable": 33,
      "correct": 2,
      "incorrect": 19,
      "abstained": 12,
      "no_answer": 0,
      "correct_pct": 0.06060606060606061,
      "wrong_pct": 0.5757575757575758,
      "correct_lo": 0.01678053042907096,
      "correct_hi": 0.19606581717798194,
      "wrong_lo": 0.4080697369146003,
      "wrong_hi": 0.7276464100497011,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 43,
      "gradeable": 128,
      "correct": 58,
      "incorrect": 50,
      "abstained": 20,
      "no_answer": 0,
      "correct_pct": 0.453125,
      "wrong_pct": 0.390625,
      "correct_lo": 0.31435896775649097,
      "correct_hi": 0.5995797106831866,
      "wrong_lo": 0.2595862028430773,
      "wrong_hi": 0.5396040468495036,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 41,
      "gradeable": 41,
      "correct": 4,
      "incorrect": 14,
      "abstained": 23,
      "no_answer": 0,
      "correct_pct": 0.0975609756097561,
      "wrong_pct": 0.34146341463414637,
      "correct_lo": 0.03859645975296143,
      "correct_hi": 0.22547975519030555,
      "wrong_lo": 0.2155861736664346,
      "wrong_hi": 0.4945044564627312,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 42,
      "gradeable": 125,
      "correct": 56,
      "incorrect": 22,
      "abstained": 46,
      "no_answer": 1,
      "correct_pct": 0.448,
      "wrong_pct": 0.176,
      "correct_lo": 0.3083340990610177,
      "correct_hi": 0.5963812686399352,
      "wrong_lo": 0.08961555603586711,
      "wrong_hi": 0.3166878888700699,
      "verdict": "verify"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 38,
      "gradeable": 38,
      "correct": 3,
      "incorrect": 17,
      "abstained": 18,
      "no_answer": 0,
      "correct_pct": 0.07894736842105263,
      "wrong_pct": 0.4473684210526316,
      "correct_lo": 0.02721410139426496,
      "correct_hi": 0.20799678920265297,
      "wrong_lo": 0.3014616531155089,
      "wrong_hi": 0.6029397082091058,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 38,
      "gradeable": 111,
      "correct": 68,
      "incorrect": 23,
      "abstained": 20,
      "no_answer": 0,
      "correct_pct": 0.6126126126126126,
      "wrong_pct": 0.2072072072072072,
      "correct_lo": 0.45430102634379393,
      "correct_hi": 0.7502456104616475,
      "wrong_lo": 0.10837151428318953,
      "wrong_hi": 0.3598072300226626,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 36,
      "gradeable": 36,
      "correct": 3,
      "incorrect": 11,
      "abstained": 22,
      "no_answer": 0,
      "correct_pct": 0.08333333333333333,
      "wrong_pct": 0.3055555555555556,
      "correct_lo": 0.0287480892712918,
      "correct_hi": 0.21827010276918848,
      "wrong_lo": 0.18004270282805354,
      "wrong_hi": 0.46856578679083716,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 41,
      "gradeable": 119,
      "correct": 74,
      "incorrect": 24,
      "abstained": 21,
      "no_answer": 0,
      "correct_pct": 0.6218487394957983,
      "wrong_pct": 0.20168067226890757,
      "correct_lo": 0.46909116775160653,
      "correct_hi": 0.7537286388265589,
      "wrong_lo": 0.10704386244156801,
      "wrong_hi": 0.347431783522234,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 36,
      "gradeable": 36,
      "correct": 4,
      "incorrect": 16,
      "abstained": 16,
      "no_answer": 0,
      "correct_pct": 0.1111111111111111,
      "wrong_pct": 0.4444444444444444,
      "correct_lo": 0.04406568908741833,
      "correct_hi": 0.25315129015036325,
      "wrong_lo": 0.2954102987654314,
      "wrong_hi": 0.6041921268399659,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 40,
      "gradeable": 118,
      "correct": 44,
      "incorrect": 22,
      "abstained": 52,
      "no_answer": 0,
      "correct_pct": 0.3728813559322034,
      "wrong_pct": 0.1864406779661017,
      "correct_lo": 0.2404433723656139,
      "correct_hi": 0.5275967647319436,
      "wrong_lo": 0.09540101791837682,
      "wrong_hi": 0.3324313202555981,
      "verdict": "verify"
    },
    {
      "model": "gemma4-12b",
      "track": "closed",
      "kind": "os-derived",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 16,
      "gradeable": 16,
      "correct": 0,
      "incorrect": 6,
      "abstained": 10,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.375,
      "correct_lo": 0,
      "correct_hi": 0.19361341827271994,
      "wrong_lo": 0.18480982554752806,
      "wrong_hi": 0.613593529020652,
      "verdict": "dont_use"
    },
    {
      "model": "gemma4-12b",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 16,
      "gradeable": 48,
      "correct": 36,
      "incorrect": 6,
      "abstained": 6,
      "no_answer": 0,
      "correct_pct": 0.75,
      "wrong_pct": 0.125,
      "correct_lo": 0.5050123107060135,
      "correct_hi": 0.8981809801576265,
      "wrong_lo": 0.034976749337057134,
      "wrong_hi": 0.3602333143674828,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 24,
      "gradeable": 24,
      "correct": 18,
      "incorrect": 4,
      "abstained": 0,
      "no_answer": 2,
      "correct_pct": 0.75,
      "wrong_pct": 0.16666666666666666,
      "correct_lo": 0.5510017468789357,
      "correct_hi": 0.8800079652065766,
      "wrong_lo": 0.06678568000728921,
      "wrong_hi": 0.35853470387869435,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 25,
      "gradeable": 75,
      "correct": 61,
      "incorrect": 9,
      "abstained": 0,
      "no_answer": 5,
      "correct_pct": 0.8133333333333334,
      "wrong_pct": 0.12,
      "correct_lo": 0.6233959025563518,
      "correct_hi": 0.9198009612329896,
      "wrong_lo": 0.041667413570680906,
      "wrong_hi": 0.299561921833735,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 45,
      "gradeable": 45,
      "correct": 3,
      "incorrect": 28,
      "abstained": 14,
      "no_answer": 0,
      "correct_pct": 0.06666666666666667,
      "wrong_pct": 0.6222222222222222,
      "correct_lo": 0.0229316164621471,
      "correct_hi": 0.17856874388640825,
      "wrong_lo": 0.4762967000646754,
      "wrong_hi": 0.7489211470165014,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 45,
      "gradeable": 135,
      "correct": 95,
      "incorrect": 16,
      "abstained": 4,
      "no_answer": 20,
      "correct_pct": 0.7037037037037037,
      "wrong_pct": 0.11851851851851852,
      "correct_lo": 0.558621471840562,
      "correct_hi": 0.8167416066280659,
      "wrong_lo": 0.053038233384756134,
      "wrong_hi": 0.2440090923921772,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 43,
      "gradeable": 43,
      "correct": 2,
      "incorrect": 22,
      "abstained": 19,
      "no_answer": 0,
      "correct_pct": 0.046511627906976744,
      "wrong_pct": 0.5116279069767442,
      "correct_lo": 0.012848990317319689,
      "correct_hi": 0.15455775923863058,
      "wrong_lo": 0.36752063766074083,
      "wrong_hi": 0.6538279072224399,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 43,
      "gradeable": 129,
      "correct": 36,
      "incorrect": 43,
      "abstained": 0,
      "no_answer": 50,
      "correct_pct": 0.27906976744186046,
      "wrong_pct": 0.3333333333333333,
      "correct_lo": 0.16746451821827157,
      "correct_hi": 0.42691312900129397,
      "wrong_lo": 0.2113118204580397,
      "wrong_hi": 0.4826923695497027,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 42,
      "gradeable": 42,
      "correct": 5,
      "incorrect": 27,
      "abstained": 10,
      "no_answer": 0,
      "correct_pct": 0.11904761904761904,
      "wrong_pct": 0.6428571428571429,
      "correct_lo": 0.05193726765019084,
      "correct_hi": 0.2500068180710754,
      "wrong_lo": 0.4916608928707938,
      "wrong_hi": 0.7701100749837315,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 42,
      "gradeable": 126,
      "correct": 58,
      "incorrect": 19,
      "abstained": 10,
      "no_answer": 39,
      "correct_pct": 0.4603174603174603,
      "wrong_pct": 0.15079365079365079,
      "correct_lo": 0.31931863666675875,
      "correct_hi": 0.6079672055958733,
      "wrong_lo": 0.07241201776631612,
      "wrong_hi": 0.28770339414484464,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 38,
      "gradeable": 38,
      "correct": 3,
      "incorrect": 21,
      "abstained": 14,
      "no_answer": 0,
      "correct_pct": 0.07894736842105263,
      "wrong_pct": 0.5526315789473685,
      "correct_lo": 0.02721410139426496,
      "correct_hi": 0.20799678920265297,
      "wrong_lo": 0.3970602917908942,
      "wrong_hi": 0.6985383468844911,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 38,
      "gradeable": 114,
      "correct": 46,
      "incorrect": 21,
      "abstained": 4,
      "no_answer": 43,
      "correct_pct": 0.40350877192982454,
      "wrong_pct": 0.18421052631578946,
      "correct_lo": 0.2634488492703231,
      "correct_hi": 0.5612869798248038,
      "wrong_lo": 0.09221648453257492,
      "wrong_hi": 0.33419168341511346,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 41,
      "gradeable": 41,
      "correct": 3,
      "incorrect": 24,
      "abstained": 14,
      "no_answer": 0,
      "correct_pct": 0.07317073170731707,
      "wrong_pct": 0.5853658536585366,
      "correct_lo": 0.02519737325256413,
      "correct_hi": 0.19427740017211295,
      "wrong_lo": 0.433668353568366,
      "wrong_hi": 0.7224366917466984,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 41,
      "gradeable": 123,
      "correct": 62,
      "incorrect": 26,
      "abstained": 2,
      "no_answer": 33,
      "correct_pct": 0.5040650406504065,
      "wrong_pct": 0.21138211382113822,
      "correct_lo": 0.3573736358366719,
      "correct_hi": 0.6500599377497596,
      "wrong_lo": 0.11407283240921819,
      "wrong_hi": 0.3581434429541349,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 40,
      "gradeable": 40,
      "correct": 3,
      "incorrect": 24,
      "abstained": 13,
      "no_answer": 0,
      "correct_pct": 0.075,
      "wrong_pct": 0.6,
      "correct_lo": 0.025835556771858226,
      "correct_hi": 0.19864530159097524,
      "wrong_lo": 0.445956198273935,
      "wrong_hi": 0.7365188938759863,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 40,
      "gradeable": 120,
      "correct": 40,
      "incorrect": 15,
      "abstained": 6,
      "no_answer": 59,
      "correct_pct": 0.3333333333333333,
      "wrong_pct": 0.125,
      "correct_lo": 0.20763275730416197,
      "correct_hi": 0.48824208911263556,
      "wrong_lo": 0.05459419773541284,
      "wrong_hi": 0.26112420670238146,
      "verdict": "verify"
    },
    {
      "model": "gpt-oss-20b",
      "track": "closed",
      "kind": "os-derived",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 16,
      "gradeable": 16,
      "correct": 0,
      "incorrect": 7,
      "abstained": 9,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0.4375,
      "correct_lo": 0,
      "correct_hi": 0.19361341827271994,
      "wrong_lo": 0.2309836333518552,
      "wrong_hi": 0.6682180439322347,
      "verdict": "dont_use"
    },
    {
      "model": "gpt-oss-20b",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "ollama",
      "harness": "spacebench-runner",
      "runs": 3,
      "questions": 16,
      "gradeable": 48,
      "correct": 26,
      "incorrect": 16,
      "abstained": 0,
      "no_answer": 6,
      "correct_pct": 0.5416666666666666,
      "wrong_pct": 0.3333333333333333,
      "correct_lo": 0.3142086929396215,
      "correct_hi": 0.7529901888709851,
      "wrong_lo": 0.15568204639135047,
      "wrong_hi": 0.5755224263662229,
      "verdict": "dont_use"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 25,
      "gradeable": 25,
      "correct": 24,
      "incorrect": 0,
      "abstained": 1,
      "no_answer": 0,
      "correct_pct": 0.96,
      "wrong_pct": 0,
      "correct_lo": 0.8045552759497747,
      "correct_hi": 0.9929039496133008,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "baseline",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 25,
      "gradeable": 50,
      "correct": 50,
      "incorrect": 0,
      "abstained": 0,
      "no_answer": 0,
      "correct_pct": 1,
      "wrong_pct": 0,
      "correct_lo": 0.8668035060468213,
      "correct_hi": 1,
      "wrong_lo": 0,
      "wrong_hi": 0.13319649395317873,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 45,
      "gradeable": 45,
      "correct": 21,
      "incorrect": 0,
      "abstained": 24,
      "no_answer": 0,
      "correct_pct": 0.4666666666666667,
      "wrong_pct": 0,
      "correct_lo": 0.3293490394201473,
      "correct_hi": 0.6092279113758954,
      "wrong_lo": 6.938893903907228e-18,
      "wrong_hi": 0.07865426194064076,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "catalog",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 45,
      "gradeable": 90,
      "correct": 78,
      "incorrect": 1,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.8666666666666667,
      "wrong_pct": 0.011111111111111112,
      "correct_lo": 0.738224177587027,
      "correct_hi": 0.9374293636565033,
      "wrong_lo": 0.0011610618786614471,
      "wrong_hi": 0.09796754979663178,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 43,
      "gradeable": 43,
      "correct": 8,
      "incorrect": 0,
      "abstained": 35,
      "no_answer": 0,
      "correct_pct": 0.18604651162790697,
      "wrong_pct": 0,
      "correct_lo": 0.0974163558010598,
      "correct_hi": 0.3261729323530596,
      "wrong_lo": 0,
      "wrong_hi": 0.08201257002322722,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "constellation",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 43,
      "gradeable": 86,
      "correct": 80,
      "incorrect": 3,
      "abstained": 3,
      "no_answer": 0,
      "correct_pct": 0.9302325581395349,
      "wrong_pct": 0.03488372093023256,
      "correct_lo": 0.8139073144696507,
      "correct_hi": 0.9759888462080375,
      "wrong_lo": 0.008097024246267959,
      "wrong_hi": 0.13796118042650155,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 42,
      "gradeable": 42,
      "correct": 5,
      "incorrect": 2,
      "abstained": 35,
      "no_answer": 0,
      "correct_pct": 0.11904761904761904,
      "wrong_pct": 0.047619047619047616,
      "correct_lo": 0.05193726765019084,
      "correct_hi": 0.2500068180710754,
      "wrong_lo": 0.013157248639206379,
      "wrong_hi": 0.1579013531547973,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "debris",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 42,
      "gradeable": 84,
      "correct": 70,
      "incorrect": 3,
      "abstained": 11,
      "no_answer": 0,
      "correct_pct": 0.8333333333333334,
      "wrong_pct": 0.03571428571428571,
      "correct_lo": 0.693957037952257,
      "correct_hi": 0.916841887041635,
      "wrong_lo": 0.008290820071836139,
      "wrong_hi": 0.14095353440095715,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 38,
      "gradeable": 38,
      "correct": 4,
      "incorrect": 2,
      "abstained": 32,
      "no_answer": 0,
      "correct_pct": 0.10526315789473684,
      "wrong_pct": 0.05263157894736842,
      "correct_lo": 0.041701897932665793,
      "correct_hi": 0.24130831200194466,
      "wrong_lo": 0.014553902448536812,
      "wrong_hi": 0.17285766881068843,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "ground",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 38,
      "gradeable": 76,
      "correct": 64,
      "incorrect": 5,
      "abstained": 7,
      "no_answer": 0,
      "correct_pct": 0.8421052631578947,
      "wrong_pct": 0.06578947368421052,
      "correct_lo": 0.6958287736272311,
      "correct_hi": 0.9255623777627731,
      "wrong_lo": 0.02061301436334438,
      "wrong_hi": 0.19069821656472719,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 41,
      "gradeable": 41,
      "correct": 8,
      "incorrect": 2,
      "abstained": 31,
      "no_answer": 0,
      "correct_pct": 0.1951219512195122,
      "wrong_pct": 0.04878048780487805,
      "correct_lo": 0.10234315749205929,
      "correct_hi": 0.34013882352556724,
      "wrong_lo": 0.013480662282573552,
      "wrong_hi": 0.16139266962351367,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "market",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 41,
      "gradeable": 82,
      "correct": 75,
      "incorrect": 0,
      "abstained": 7,
      "no_answer": 0,
      "correct_pct": 0.9146341463414634,
      "wrong_pct": 0,
      "correct_lo": 0.7899448419883656,
      "correct_hi": 0.9682796638276626,
      "wrong_lo": 0,
      "wrong_hi": 0.08567044886890746,
      "verdict": "trust"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 40,
      "gradeable": 40,
      "correct": 11,
      "incorrect": 0,
      "abstained": 29,
      "no_answer": 0,
      "correct_pct": 0.275,
      "wrong_pct": 0,
      "correct_lo": 0.1610785343314309,
      "correct_hi": 0.4283525083312457,
      "wrong_lo": 0,
      "wrong_hi": 0.08762453925039233,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "job": "regulatory",
      "kind": "public-verifiable",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 40,
      "gradeable": 80,
      "correct": 48,
      "incorrect": 1,
      "abstained": 31,
      "no_answer": 0,
      "correct_pct": 0.6,
      "wrong_pct": 0.0125,
      "correct_lo": 0.445956198273935,
      "correct_hi": 0.7365188938759863,
      "wrong_lo": 0.0013063486749930187,
      "wrong_hi": 0.10912757709413952,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "closed",
      "kind": "os-derived",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 1,
      "questions": 16,
      "gradeable": 16,
      "correct": 0,
      "incorrect": 0,
      "abstained": 16,
      "no_answer": 0,
      "correct_pct": 0,
      "wrong_pct": 0,
      "correct_lo": 0,
      "correct_hi": 0.19361341827271994,
      "wrong_lo": 0,
      "wrong_hi": 0.19361341827271994,
      "verdict": "verify"
    },
    {
      "model": "grok-4.7",
      "track": "mcp",
      "kind": "os-derived",
      "provider": "xai",
      "harness": "spacebench-runner",
      "runs": 2,
      "questions": 16,
      "gradeable": 32,
      "correct": 27,
      "incorrect": 2,
      "abstained": 3,
      "no_answer": 0,
      "correct_pct": 0.84375,
      "wrong_pct": 0.0625,
      "correct_lo": 0.6041210596215869,
      "correct_hi": 0.9502697153159181,
      "wrong_lo": 0.01111905730833121,
      "wrong_hi": 0.2832926836802987,
      "verdict": "verify"
    }
  ]
}