{
  "updated": "2026-10-10",
  "rows": [
    {
      "workload": "broader150",
      "id": "jev",
      "name": "JEV 1.13",
      "correct": 138,
      "total": 150,
      "packets": 50,
      "seconds": 0.25322409998625517,
      "cost": 0.05647152,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 1,
      "source": "data/exports/decision_new_three_150_v01/summary.json",
      "date": "2026-10-06 or earlier",
      "questions": 4,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "luna",
      "name": "Luna Decisions",
      "correct": 139,
      "total": 150,
      "packets": 50,
      "seconds": 0.3299103499739431,
      "cost": 0.129632,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 1,
      "source": "data/exports/decision_new_three_150_v01/summary.json",
      "date": "2026-10-06 or earlier",
      "questions": 4,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "perplexity",
      "name": "Perplexity Decider v1",
      "correct": 141,
      "total": 150,
      "packets": 50,
      "seconds": 0.2861112000537105,
      "cost": 0.1331888,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "source": "data/exports/decision_new_three_150_v01/summary.json",
      "date": "2026-10-06 or earlier",
      "questions": 4,
      "hidden_reason": "Earlier version; retained for comparison"
    },
    {
      "workload": "broader150",
      "id": "d1",
      "name": "Liquid d1",
      "correct": 133,
      "total": 150,
      "packets": 50,
      "seconds": 0.6861290500382893,
      "cost": 0.1329792,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 2,
      "source": "data/exports/decision_new_three_150_v01/summary.json",
      "date": "2026-10-06 or earlier",
      "questions": 4,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "kev",
      "name": "KEV 4B",
      "correct": 127,
      "total": 150,
      "packets": 50,
      "seconds": 0.6101652500219643,
      "cost": 0.03762612,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "source": "data/exports/decision_breadth_stress_clean_hard_compare_v01/ledger.jsonl",
      "date": "2026-10-06 or earlier",
      "questions": 4,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "mercury",
      "name": "Mercury Decide \u00b7 Free",
      "correct": 142,
      "total": 150,
      "packets": 50,
      "seconds": 0.48289380001369864,
      "cost": 0.0,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Promotional free route; availability and price can change",
      "unknown_charges": 0,
      "source": "data/exports/decision_breadth_stress_clean_hard_compare_v01/ledger.jsonl",
      "date": "2026-10-06 or earlier",
      "questions": 4,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "clef",
      "name": "Clef",
      "correct": 135,
      "total": 150,
      "packets": 50,
      "seconds": 0.6410424000350758,
      "cost": 0.2897616,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "source": "data/exports/decision_clef_full_v01/summary.json",
      "date": "2026-10-06 or earlier",
      "questions": 4,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "clef_flash",
      "name": "Clef Flash",
      "correct": 128,
      "total": 150,
      "packets": 50,
      "seconds": 0.4692182999999659,
      "cost": 0.10866060000000001,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "source": "data/exports/decision_accuracy_remaining_v01/summary.json",
      "date": "2026-10-06 or earlier",
      "questions": 4,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "tev",
      "name": "TEV 4B Experimental",
      "correct": 119,
      "total": 150,
      "packets": 50,
      "seconds": 0.389670150000029,
      "cost": 0.13384224,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "source": "data/exports/decision_accuracy_remaining_v01/summary.json",
      "date": "2026-10-06 or earlier",
      "questions": 4,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "solar",
      "name": "Solar Decide",
      "correct": 134,
      "total": 150,
      "packets": 50,
      "seconds": 17.912027900000112,
      "cost": 0.209546,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "source": "data/exports/decision_accuracy_remaining_v01/summary.json",
      "date": "2026-10-06 or earlier",
      "questions": 4,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "perplexity_v11",
      "name": "Perplexity Decider v1.1",
      "correct": 141,
      "total": 150,
      "packets": 50,
      "seconds": 0.41886819999854197,
      "cost": 0.0665944,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "source": "data/exports/decision_perplexity_v11_v01/summary.json",
      "date": "2026-10-06 or later",
      "questions": 4,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "microsoft",
      "name": "Microsoft Decision-1",
      "correct": 138,
      "total": 150,
      "packets": 50,
      "seconds": 0.49482685000111815,
      "cost": 0.037695,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "questions": 4,
      "date": "2026-10-09",
      "source": "current_run/summary.json",
      "model_id": "microsoft/microsoft-decision-1",
      "provider": "Azure",
      "diagnostics": {
        "correct": 47,
        "total": 50,
        "accuracy": 0.94
      },
      "confident_errors": 2,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "drex",
      "name": "Drex v1.5",
      "correct": 129,
      "total": 150,
      "packets": 50,
      "seconds": 0.5518420000007609,
      "cost": 0.035834399999999995,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "questions": 4,
      "date": "2026-10-09",
      "source": "current_run/summary.json",
      "model_id": "nace-ai/drex-v1.5",
      "provider": "DeepInfra",
      "diagnostics": {
        "correct": 48,
        "total": 50,
        "accuracy": 0.96
      },
      "confident_errors": 5,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "clef_omni",
      "name": "Clef Omni",
      "correct": 114,
      "total": 150,
      "packets": 50,
      "seconds": 0.3784010999952443,
      "cost": 0.180309,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "questions": 4,
      "date": "2026-10-09",
      "source": "current_run/summary.json",
      "model_id": "cloudflare/clef-omni",
      "provider": "Cloudflare",
      "diagnostics": {
        "correct": 45,
        "total": 50,
        "accuracy": 0.9
      },
      "confident_errors": 11,
      "hidden_reason": ""
    },
    {
      "workload": "broader150",
      "id": "mercury_paid",
      "name": "Mercury Decide \u00b7 Paid",
      "correct": 139,
      "total": 150,
      "packets": 50,
      "seconds": 0.5881346499809297,
      "cost": 0.015926799999999998,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "unknown_charges": 0,
      "questions": 4,
      "date": "2026-10-10",
      "source": "current_run/mercury_paid_accuracy/summary.json",
      "model_id": "inception/mercury-decide",
      "provider": "Inception",
      "diagnostics": {
        "correct": 46,
        "total": 50,
        "accuracy": 0.92
      },
      "confident_errors": 4,
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "jev",
      "name": "JEV 1.13",
      "correct": 28,
      "total": 30,
      "packets": 10,
      "seconds": 0.2393780000275001,
      "cost": 0.056943600000000004,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "Run date not recorded in aggregate",
      "provider": "TypeSafe",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "luna",
      "name": "Luna Decisions",
      "correct": 29,
      "total": 30,
      "packets": 10,
      "seconds": 0.2761070500127971,
      "cost": 0.13085000000000002,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "Run date not recorded in aggregate",
      "provider": "OpenAI",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "perplexity",
      "name": "Perplexity Decider 27B",
      "correct": 28,
      "total": 30,
      "packets": 10,
      "seconds": 0.2861112000537105,
      "cost": 0.134048,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "Run date not recorded in aggregate",
      "provider": "Perplexity",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "d1",
      "name": "Liquid d1",
      "correct": 26,
      "total": 30,
      "packets": 10,
      "seconds": 0.9872249499894679,
      "cost": 0.134344,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "Run date not recorded in aggregate",
      "provider": "Liquid",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "kev",
      "name": "KEV 4B",
      "correct": 24,
      "total": 30,
      "packets": 10,
      "seconds": 0.6330053500132635,
      "cost": 0.0382326,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "Run date not recorded in aggregate",
      "provider": "SiliconFlow",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "mercury",
      "name": "Mercury Decide \u00b7 Free",
      "correct": 30,
      "total": 30,
      "packets": 10,
      "seconds": 0.483160300005693,
      "cost": 0.0,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Promotional free route; availability and price can change",
      "questions": 4,
      "date": "Run date not recorded in aggregate",
      "provider": "Inception",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "clef",
      "name": "Clef",
      "correct": 26,
      "total": 30,
      "packets": 10,
      "seconds": 0.6014733999618329,
      "cost": 0.293856,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "2026-10-06",
      "provider": "Cloudflare",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "clef_flash",
      "name": "Clef Flash",
      "correct": 24,
      "total": 30,
      "packets": 10,
      "seconds": 0.44847525004297495,
      "cost": 0.11019599999999999,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "2026-10-06",
      "provider": "Cloudflare",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "tev",
      "name": "TEV 4B",
      "correct": 22,
      "total": 30,
      "packets": 10,
      "seconds": 0.31460370001150295,
      "cost": 0.13479480000000002,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "2026-10-06",
      "provider": "Together",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "solar",
      "name": "Solar Decide",
      "correct": 25,
      "total": 30,
      "packets": 10,
      "seconds": 16.856599649996497,
      "cost": 0.21036000000000002,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "2026-10-06",
      "provider": "Upstage",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "qwen9",
      "name": "Qwen 3.5 9B + schema (FP4)",
      "correct": 22,
      "total": 30,
      "packets": 10,
      "seconds": 2.3380500999628566,
      "cost": 0.10469699999999998,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "2026-10-06",
      "provider": "Darkbloom",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "qwen27",
      "name": "Qwen 3.8 27B + schema (FP4)",
      "correct": 27,
      "total": 30,
      "packets": 10,
      "seconds": 2.858384000021033,
      "cost": 0.16126000000000001,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "2026-10-06",
      "provider": "Darkbloom",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "qwen9bf16",
      "name": "Qwen 3.5 9B + schema (BF16)",
      "correct": 22,
      "total": 30,
      "packets": 10,
      "seconds": 4.031829849991482,
      "cost": 0.11388999999999999,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "2026-10-06",
      "provider": "DeepInfra",
      "hidden_reason": ""
    },
    {
      "workload": "quick",
      "id": "qwen27bf16",
      "name": "Qwen 3.8 27B + schema (BF16)",
      "correct": 26,
      "total": 30,
      "packets": 10,
      "seconds": 2.7452700000139885,
      "cost": 0.24706499999999998,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 4,
      "date": "2026-10-06",
      "provider": "DeepInfra",
      "hidden_reason": ""
    },
    {
      "workload": "general_core",
      "id": "jev",
      "name": "JEV 1.13",
      "correct": 1715,
      "total": 1800,
      "packets": 180,
      "seconds": 0.2683277000032831,
      "cost": 0.11011606666666669,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 10,
      "date": "2026-09-26",
      "gpu_gib": null,
      "fp": null,
      "fn": null,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "general_core",
      "id": "kev",
      "name": "KEV-4B",
      "correct": 1575,
      "total": 1800,
      "packets": 180,
      "seconds": 0.759520750012598,
      "cost": 0.08427860000000001,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Reported charges",
      "questions": 10,
      "date": "2026-09-26",
      "gpu_gib": null,
      "fp": null,
      "fn": null,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "general_smoke",
      "id": "jev",
      "name": "JEV 1.13",
      "correct": 78,
      "total": 80,
      "packets": 8,
      "seconds": 0.2701378499768907,
      "cost": 0.107688,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 10,
      "date": "2026-09-26",
      "gpu_gib": null,
      "fp": null,
      "fn": null,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "general_smoke",
      "id": "kev",
      "name": "KEV-4B",
      "correct": 72,
      "total": 80,
      "packets": 8,
      "seconds": 0.7503021499869647,
      "cost": 0.08186325,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Reported charges",
      "questions": 10,
      "date": "2026-09-26",
      "gpu_gib": null,
      "fp": null,
      "fn": null,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "general_smoke",
      "id": "kev-4b_\u00b7_bf16",
      "name": "KEV-4B \u00b7 BF16",
      "correct": 72,
      "total": 80,
      "packets": 8,
      "seconds": 5.865846500004409,
      "cost": null,
      "deployment": "Local GPU",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Local compute not measured",
      "questions": 10,
      "date": "2026-09-27",
      "gpu_gib": 10.198810577392578,
      "fp": null,
      "fn": null,
      "hardware": "AMD RX 6800 XT",
      "hidden_reason": ""
    },
    {
      "workload": "general_smoke",
      "id": "kev-0.8b_\u00b7_bf16",
      "name": "KEV-0.8B \u00b7 BF16",
      "correct": 41,
      "total": 80,
      "packets": 8,
      "seconds": 0.9796066999842878,
      "cost": null,
      "deployment": "Local GPU",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Local compute not measured",
      "questions": 10,
      "date": "2026-09-27",
      "gpu_gib": 2.4486145973205566,
      "fp": null,
      "fn": null,
      "hardware": "AMD RX 6800 XT",
      "hidden_reason": ""
    },
    {
      "workload": "general_smoke",
      "id": "kev-0.8b_\u00b7_fp32",
      "name": "KEV-0.8B \u00b7 FP32",
      "correct": 40,
      "total": 80,
      "packets": 8,
      "seconds": 0.6180222000111826,
      "cost": null,
      "deployment": "Local GPU",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Local compute not measured",
      "questions": 10,
      "date": "2026-09-27",
      "gpu_gib": 3.931704521179199,
      "fp": null,
      "fn": null,
      "hardware": "AMD RX 6800 XT",
      "hidden_reason": ""
    },
    {
      "workload": "general_smoke",
      "id": "julia-1",
      "name": "Julia-1",
      "correct": 33,
      "total": 80,
      "packets": 8,
      "seconds": 0.7073948500037659,
      "cost": null,
      "deployment": "Local GPU",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Local compute not measured",
      "questions": 10,
      "date": "2026-09-28",
      "gpu_gib": 3.038177490234375,
      "fp": null,
      "fn": null,
      "hardware": "AMD RX 6800 XT",
      "hidden_reason": ""
    },
    {
      "workload": "general_smoke",
      "id": "decider-2b_v11",
      "name": "Decider-2B v11",
      "correct": 61,
      "total": 80,
      "packets": 8,
      "seconds": 13.136242000007769,
      "cost": null,
      "deployment": "Local GPU",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Local compute not measured",
      "questions": 10,
      "date": "2026-09-28",
      "gpu_gib": 4.3595170974731445,
      "fp": null,
      "fn": null,
      "hardware": "AMD RX 6800 XT",
      "hidden_reason": ""
    },
    {
      "workload": "general_smoke",
      "id": "solar_decide",
      "name": "Solar Decide",
      "correct": 60,
      "total": 80,
      "packets": 8,
      "seconds": 1.3107373999955598,
      "cost": 0.80321875,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 10,
      "date": "2026-09-28",
      "gpu_gib": null,
      "fp": null,
      "fn": null,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "binary_core",
      "id": "jev",
      "name": "JEV 1.13",
      "correct": 513,
      "total": 540,
      "packets": 180,
      "seconds": 0.2683277000032831,
      "cost": 0.11011606666666669,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 10,
      "date": "2026-09-26",
      "gpu_gib": null,
      "fp": 27,
      "fn": 0,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "binary_core",
      "id": "kev",
      "name": "KEV-4B",
      "correct": 453,
      "total": 540,
      "packets": 180,
      "seconds": 0.759520750012598,
      "cost": 0.08427860000000001,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Reported charges",
      "questions": 10,
      "date": "2026-09-26",
      "gpu_gib": null,
      "fp": 87,
      "fn": 0,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "binary_core",
      "id": "span-01",
      "name": "Span-01",
      "correct": 497,
      "total": 540,
      "packets": 180,
      "seconds": 0.49353680000058375,
      "cost": 0.04076877777777778,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 3,
      "date": "2026-09-27",
      "gpu_gib": null,
      "fp": 43,
      "fn": 0,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "binary_core",
      "id": "span-01_lite",
      "name": "Span-01 Lite",
      "correct": 350,
      "total": 540,
      "packets": 180,
      "seconds": 0.507799699989846,
      "cost": 0.0,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 3,
      "date": "2026-09-27",
      "gpu_gib": null,
      "fp": 141,
      "fn": 49,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "binary_smoke",
      "id": "jev",
      "name": "JEV 1.13",
      "correct": 23,
      "total": 24,
      "packets": 8,
      "seconds": 0.2701378499768907,
      "cost": 0.107688,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 10,
      "date": "2026-09-26",
      "gpu_gib": null,
      "fp": 1,
      "fn": 0,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "binary_smoke",
      "id": "kev",
      "name": "KEV-4B",
      "correct": 22,
      "total": 24,
      "packets": 8,
      "seconds": 0.7503021499869647,
      "cost": 0.08186325,
      "deployment": "API",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Reported charges",
      "questions": 10,
      "date": "2026-09-26",
      "gpu_gib": null,
      "fp": 2,
      "fn": 0,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "binary_smoke",
      "id": "kev-4b_\u00b7_bf16",
      "name": "KEV-4B \u00b7 BF16",
      "correct": 22,
      "total": 24,
      "packets": 8,
      "seconds": 5.865846500004409,
      "cost": null,
      "deployment": "Local GPU",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Local compute not measured",
      "questions": 10,
      "date": "2026-09-27",
      "gpu_gib": 10.198810577392578,
      "fp": 2,
      "fn": 0,
      "hardware": "AMD RX 6800 XT",
      "hidden_reason": ""
    },
    {
      "workload": "binary_smoke",
      "id": "kev-0.8b_\u00b7_bf16",
      "name": "KEV-0.8B \u00b7 BF16",
      "correct": 17,
      "total": 24,
      "packets": 8,
      "seconds": 0.9796066999842878,
      "cost": null,
      "deployment": "Local GPU",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Local compute not measured",
      "questions": 10,
      "date": "2026-09-27",
      "gpu_gib": 2.4486145973205566,
      "fp": 7,
      "fn": 0,
      "hardware": "AMD RX 6800 XT",
      "hidden_reason": ""
    },
    {
      "workload": "binary_smoke",
      "id": "kev-0.8b_\u00b7_fp32",
      "name": "KEV-0.8B \u00b7 FP32",
      "correct": 16,
      "total": 24,
      "packets": 8,
      "seconds": 0.6180222000111826,
      "cost": null,
      "deployment": "Local GPU",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Local compute not measured",
      "questions": 10,
      "date": "2026-09-27",
      "gpu_gib": 3.931704521179199,
      "fp": 8,
      "fn": 0,
      "hardware": "AMD RX 6800 XT",
      "hidden_reason": ""
    },
    {
      "workload": "binary_smoke",
      "id": "julia-1",
      "name": "Julia-1",
      "correct": 13,
      "total": 24,
      "packets": 8,
      "seconds": 0.7073948500037659,
      "cost": null,
      "deployment": "Local GPU",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Local compute not measured",
      "questions": 10,
      "date": "2026-09-28",
      "gpu_gib": 3.038177490234375,
      "fp": 9,
      "fn": 2,
      "hardware": "AMD RX 6800 XT",
      "hidden_reason": ""
    },
    {
      "workload": "binary_smoke",
      "id": "decider-2b_v11",
      "name": "Decider-2B v11",
      "correct": 23,
      "total": 24,
      "packets": 8,
      "seconds": 13.136242000007769,
      "cost": null,
      "deployment": "Local GPU",
      "open_weights": true,
      "local_deployable": true,
      "cost_basis": "Local compute not measured",
      "questions": 10,
      "date": "2026-09-28",
      "gpu_gib": 4.3595170974731445,
      "fp": 1,
      "fn": 0,
      "hardware": "AMD RX 6800 XT",
      "hidden_reason": ""
    },
    {
      "workload": "binary_smoke",
      "id": "solar_decide",
      "name": "Solar Decide",
      "correct": 24,
      "total": 24,
      "packets": 8,
      "seconds": 1.3107373999955598,
      "cost": 0.80321875,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported charges",
      "questions": 10,
      "date": "2026-09-28",
      "gpu_gib": null,
      "fp": 0,
      "fn": 0,
      "hardware": null,
      "hidden_reason": ""
    },
    {
      "workload": "image_focused",
      "id": "luna",
      "name": "Luna Decisions",
      "correct": 47,
      "total": 50,
      "packets": 25,
      "seconds": 0.42625889999908395,
      "cost": 0.25026000000000004,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Token-based estimate",
      "questions": 2,
      "date": "2026-10-07",
      "abstentions": 1,
      "incorrect": 2,
      "failures": 0,
      "hidden_reason": ""
    },
    {
      "workload": "image_focused",
      "id": "perplexity_v11",
      "name": "Perplexity Decider v1.1",
      "correct": 49,
      "total": 50,
      "packets": 25,
      "seconds": 0.7865024000057019,
      "cost": 0.082292,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported usage.cost",
      "questions": 2,
      "date": "2026-10-07",
      "abstentions": 0,
      "incorrect": 1,
      "failures": 0,
      "hidden_reason": ""
    },
    {
      "workload": "image_focused",
      "id": "clef",
      "name": "Clef",
      "correct": 50,
      "total": 50,
      "packets": 25,
      "seconds": 1.3122364000009838,
      "cost": 0.6429888,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported usage.cost",
      "questions": 2,
      "date": "2026-10-07",
      "abstentions": 0,
      "incorrect": 0,
      "failures": 0,
      "hidden_reason": ""
    },
    {
      "workload": "image_focused",
      "id": "clef_flash",
      "name": "Clef Flash",
      "correct": 50,
      "total": 50,
      "packets": 25,
      "seconds": 0.7657443000061903,
      "cost": 0.2411208,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported usage.cost",
      "questions": 2,
      "date": "2026-10-07",
      "abstentions": 0,
      "incorrect": 0,
      "failures": 2,
      "hidden_reason": ""
    },
    {
      "workload": "image_website",
      "id": "luna",
      "name": "Luna Decisions",
      "correct": 15,
      "total": 19,
      "packets": 20,
      "seconds": 0.5149252499977592,
      "cost": 1.37765,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Token-based estimate",
      "questions": null,
      "date": "2026-10-07",
      "abstentions": null,
      "incorrect": null,
      "failures": 0,
      "hidden_reason": ""
    },
    {
      "workload": "image_website",
      "id": "perplexity_v11",
      "name": "Perplexity Decider v1.1",
      "correct": 16,
      "total": 19,
      "packets": 20,
      "seconds": 2.3995673000026727,
      "cost": 2.5840799999999997,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported usage.cost",
      "questions": null,
      "date": "2026-10-07",
      "abstentions": null,
      "incorrect": null,
      "failures": 0,
      "hidden_reason": ""
    },
    {
      "workload": "image_website",
      "id": "clef",
      "name": "Clef",
      "correct": 14,
      "total": 19,
      "packets": 20,
      "seconds": 3.090460549999989,
      "cost": 3.582552,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported usage.cost",
      "questions": null,
      "date": "2026-10-07",
      "abstentions": null,
      "incorrect": null,
      "failures": 1,
      "hidden_reason": ""
    },
    {
      "workload": "image_website",
      "id": "clef_flash",
      "name": "Clef Flash",
      "correct": 13,
      "total": 19,
      "packets": 20,
      "seconds": 1.5884208500028763,
      "cost": 1.343457,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported usage.cost",
      "questions": null,
      "date": "2026-10-07",
      "abstentions": null,
      "incorrect": null,
      "failures": 0,
      "hidden_reason": ""
    },
    {
      "workload": "image_documents",
      "id": "luna",
      "name": "Luna Decisions",
      "correct": 279,
      "total": 288,
      "packets": 24,
      "seconds": 0.5765315499957069,
      "cost": 0.9840708333333335,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Token-based estimate",
      "questions": 12,
      "date": "2026-10-07",
      "abstentions": null,
      "incorrect": null,
      "failures": 0,
      "hidden_reason": ""
    },
    {
      "workload": "image_documents",
      "id": "perplexity_v11",
      "name": "Perplexity Decider v1.1",
      "correct": 284,
      "total": 288,
      "packets": 24,
      "seconds": 1.5708765000017593,
      "cost": 1.16083,
      "deployment": "API",
      "open_weights": false,
      "local_deployable": false,
      "cost_basis": "Reported usage.cost",
      "questions": 12,
      "date": "2026-10-07",
      "abstentions": null,
      "incorrect": null,
      "failures": 0,
      "hidden_reason": ""
    }
  ],
  "workloads": [
    {
      "id": "broader150",
      "view": "text",
      "label": "Broader comparison \u00b7 150 findings",
      "note": "50 evidence packets, 150 primary findings and 50 separate diagnostics. Author-defined development labels across 25 related scenarios and 15 insurance families. Results are not independent externally adjudicated cases.",
      "timing": "Median request-to-validated completion. Reported charges exclude unresolved failed-call charges. Runs occurred at different times."
    },
    {
      "id": "quick",
      "view": "text",
      "label": "Quick screen \u00b7 30 findings",
      "note": "10 packets, 30 primary findings and 10 separate diagnostics. Small screening workload, not the broader benchmark. FP4 and BF16 provider routes remain separate measurements.",
      "timing": "Saved median end-to-end times; reused rows may lack exact run dates. Cost per 1,000 four-question packets."
    },
    {
      "id": "general_core",
      "view": "text",
      "label": "Core \u00b7 180 packets",
      "note": "Frozen related submission scenarios. Author keys await independent review. Core and exploratory subsets have different denominators. Binary compares only identical boolean questions.",
      "timing": "Historical successful-request medians exclude retry waits. API timing includes network; local timing is in-process. Binary cost/time covers original requests: 10 mixed questions for general models versus 3 for Span. Do not interpret this as matched binary-only serving cost."
    },
    {
      "id": "general_smoke",
      "view": "text",
      "label": "Exploratory \u00b7 8 packets",
      "note": "Frozen related submission scenarios. Author keys await independent review. Core and exploratory subsets have different denominators. Binary compares only identical boolean questions.",
      "timing": "Historical successful-request medians exclude retry waits. API timing includes network; local timing is in-process. Binary cost/time covers original requests: 10 mixed questions for general models versus 3 for Span. Do not interpret this as matched binary-only serving cost."
    },
    {
      "id": "binary_core",
      "view": "binary",
      "label": "Core \u00b7 180 packets",
      "note": "Frozen related submission scenarios. Author keys await independent review. Core and exploratory subsets have different denominators. Binary compares only identical boolean questions.",
      "timing": "Historical successful-request medians exclude retry waits. API timing includes network; local timing is in-process. Binary cost/time covers original requests: 10 mixed questions for general models versus 3 for Span. Do not interpret this as matched binary-only serving cost."
    },
    {
      "id": "binary_smoke",
      "view": "binary",
      "label": "Exploratory \u00b7 8 packets",
      "note": "Frozen related submission scenarios. Author keys await independent review. Core and exploratory subsets have different denominators. Binary compares only identical boolean questions.",
      "timing": "Historical successful-request medians exclude retry waits. API timing includes network; local timing is in-process. Binary cost/time covers original requests: 10 mixed questions for general models versus 3 for Span. Do not interpret this as matched binary-only serving cost."
    },
    {
      "id": "image_focused",
      "view": "image",
      "label": "Focused photos \u00b7 50 questions",
      "note": "25 packets, two focused questions each. Incorrect answers and abstentions are reported separately. Saved pilot labels, not proof of production reliability.",
      "timing": "End-to-end model workflow; offline image preparation excluded. Luna cost uses token-based estimates; other routes use reported charges. Differences in billing basis are labelled."
    },
    {
      "id": "image_website",
      "view": "image",
      "label": "Website photos \u00b7 19 primary answers",
      "note": "20 cases, 19 scored primary answers. One mixed-scope case excluded for every model. Frozen visible-image review; some corrections remain provisional. Clef lanes split larger requests at 64 questions; cost/time includes all parts.",
      "timing": "End-to-end model workflow; offline image preparation excluded. Luna cost uses token-based estimates; other routes use reported charges. Differences in billing basis are labelled."
    },
    {
      "id": "image_documents",
      "view": "image",
      "label": "Document images \u00b7 288 checks",
      "note": "24 synthetic packets from six base entities, four variants each; 12 checks per packet. Native image model results only. OCR and verified-text controls remain separate diagnostics.",
      "timing": "End-to-end model workflow; offline image preparation excluded. Luna cost uses token-based estimates; other routes use reported charges. Differences in billing basis are labelled."
    }
  ],
  "scaling": [
    {
      "id": "jev",
      "size": 1,
      "seconds": 0.258819000038784,
      "cost": 0.09047807999999999,
      "requests": 50,
      "native": true,
      "correct": 46,
      "total": 50,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "jev",
      "size": 10,
      "seconds": 0.25508874998195097,
      "cost": 0.01253196,
      "requests": 10,
      "native": true,
      "correct": 92,
      "total": 100,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "jev",
      "size": 25,
      "seconds": 0.2921169499750249,
      "cost": 0.00732816,
      "requests": 10,
      "native": true,
      "correct": 231,
      "total": 250,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "jev",
      "size": 50,
      "seconds": 0.29428360000019893,
      "cost": 0.005593308,
      "requests": 10,
      "native": true,
      "correct": 468,
      "total": 500,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "jev",
      "size": 100,
      "seconds": 0.3613525499822572,
      "cost": 0.004727814,
      "requests": 10,
      "native": true,
      "correct": 931,
      "total": 1000,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "luna",
      "size": 1,
      "seconds": 0.2537832999951206,
      "cost": 0.147618,
      "requests": 50,
      "native": true,
      "correct": 47,
      "total": 50,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "luna",
      "size": 10,
      "seconds": 0.29048184998100623,
      "cost": 0.02762,
      "requests": 10,
      "native": true,
      "correct": 91,
      "total": 100,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "luna",
      "size": 25,
      "seconds": 0.3009248000453226,
      "cost": 0.0196048,
      "requests": 10,
      "native": true,
      "correct": 234,
      "total": 250,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "luna",
      "size": 50,
      "seconds": 0.35788510006386787,
      "cost": 0.0169294,
      "requests": 10,
      "native": true,
      "correct": 463,
      "total": 500,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "luna",
      "size": 100,
      "seconds": 0.3994669999810867,
      "cost": 0.015598000000000002,
      "requests": 10,
      "native": true,
      "correct": 927,
      "total": 1000,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "perplexity",
      "size": 1,
      "seconds": 0.3006173499743454,
      "cost": 0.06642559999999999,
      "requests": 50,
      "native": true,
      "correct": 49,
      "total": 50,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "perplexity",
      "size": 10,
      "seconds": 0.3918455499224365,
      "cost": 0.0664136,
      "requests": 10,
      "native": true,
      "correct": 97,
      "total": 100,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "perplexity",
      "size": 25,
      "seconds": 0.5415060500381514,
      "cost": 0.0664064,
      "requests": 10,
      "native": true,
      "correct": 244,
      "total": 250,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "perplexity",
      "size": 50,
      "seconds": 0.8205098999897018,
      "cost": 0.0664036,
      "requests": 10,
      "native": true,
      "correct": 491,
      "total": 500,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "perplexity",
      "size": 100,
      "seconds": 1.3101861500763334,
      "cost": 0.066404,
      "requests": 10,
      "native": true,
      "correct": 982,
      "total": 1000,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "d1",
      "size": 1,
      "seconds": 0.40135945001384243,
      "cost": 0.072436,
      "requests": 50,
      "native": true,
      "correct": 46,
      "total": 50,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "d1",
      "size": 10,
      "seconds": 0.7341571000288241,
      "cost": 0.0724252,
      "requests": 10,
      "native": true,
      "correct": 92,
      "total": 100,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "d1",
      "size": 25,
      "seconds": 0.70186949998606,
      "cost": 0.07241648,
      "requests": 10,
      "native": true,
      "correct": 234,
      "total": 250,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "d1",
      "size": 50,
      "seconds": 0.8494248000206426,
      "cost": 0.07241328,
      "requests": 10,
      "native": true,
      "correct": 471,
      "total": 500,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "d1",
      "size": 100,
      "seconds": 1.3477004999876954,
      "cost": 0.072414,
      "requests": 10,
      "native": true,
      "correct": 945,
      "total": 1000,
      "source": "data/exports/decision_scaling_four_v01/summary.json"
    },
    {
      "id": "clef",
      "size": 1,
      "seconds": 0.788659350015223,
      "cost": 0.3722496,
      "requests": 50,
      "native": true,
      "correct": 47,
      "total": 50,
      "source": "data/exports/decision_clef_full_v01/summary.json"
    },
    {
      "id": "clef",
      "size": 10,
      "seconds": 0.8939198000007309,
      "cost": 0.0616584,
      "requests": 10,
      "native": true,
      "correct": 95,
      "total": 100,
      "source": "data/exports/decision_clef_full_v01/summary.json"
    },
    {
      "id": "clef",
      "size": 25,
      "seconds": 1.2166253000032157,
      "cost": 0.04104192,
      "requests": 10,
      "native": true,
      "correct": 243,
      "total": 250,
      "source": "data/exports/decision_clef_full_v01/summary.json"
    },
    {
      "id": "clef",
      "size": 50,
      "seconds": 1.6204389500198886,
      "cost": 0.03416736,
      "requests": 10,
      "native": true,
      "correct": 479,
      "total": 500,
      "source": "data/exports/decision_clef_full_v01/summary.json"
    },
    {
      "id": "clef",
      "size": 100,
      "seconds": 1.770862000004854,
      "cost": 0.03416976,
      "requests": 10,
      "native": false,
      "correct": 957,
      "total": 1000,
      "source": "data/exports/decision_clef_full_v01/summary.json"
    },
    {
      "id": "perplexity_v11",
      "size": 1,
      "seconds": 0.4372911499995098,
      "cost": 0.033212799999999994,
      "requests": 50,
      "native": true,
      "correct": 49,
      "total": 50,
      "source": "data/exports/decision_perplexity_v11_v01/summary.json"
    },
    {
      "id": "perplexity_v11",
      "size": 10,
      "seconds": 0.5408523999976751,
      "cost": 0.0332068,
      "requests": 10,
      "native": true,
      "correct": 97,
      "total": 100,
      "source": "data/exports/decision_perplexity_v11_v01/summary.json"
    },
    {
      "id": "perplexity_v11",
      "size": 25,
      "seconds": 0.8175086499977624,
      "cost": 0.0332032,
      "requests": 10,
      "native": true,
      "correct": 244,
      "total": 250,
      "source": "data/exports/decision_perplexity_v11_v01/summary.json"
    },
    {
      "id": "perplexity_v11",
      "size": 50,
      "seconds": 1.2689003499981482,
      "cost": 0.0332018,
      "requests": 10,
      "native": true,
      "correct": 491,
      "total": 500,
      "source": "data/exports/decision_perplexity_v11_v01/summary.json"
    },
    {
      "id": "perplexity_v11",
      "size": 100,
      "seconds": 2.1163545999988855,
      "cost": 0.033202,
      "requests": 10,
      "native": true,
      "correct": 982,
      "total": 1000,
      "source": "data/exports/decision_perplexity_v11_v01/summary.json"
    },
    {
      "id": "microsoft",
      "size": 1,
      "seconds": 0.4045168000011472,
      "cost": 0.06070092,
      "requests": 50,
      "native": true,
      "correct": 48,
      "total": 50,
      "source": "current_run/microsoft_scaling/summary.json",
      "fixed_ten_quality": null,
      "failed_attempts": 2,
      "unknown_charge_attempts": 2,
      "accounted_usd": 0.003724434
    },
    {
      "id": "microsoft",
      "size": 10,
      "seconds": 0.5915132999944035,
      "cost": 0.00846174,
      "requests": 10,
      "native": true,
      "correct": 96,
      "total": 100,
      "source": "current_run/microsoft_scaling/summary.json",
      "fixed_ten_quality": {
        "size": 10,
        "correct": 96,
        "total": 100
      },
      "failed_attempts": 0,
      "unknown_charge_attempts": 0,
      "accounted_usd": 0.000846174
    },
    {
      "id": "microsoft",
      "size": 25,
      "seconds": 0.6330549499980407,
      "cost": 0.004972296,
      "requests": 10,
      "native": true,
      "correct": 226,
      "total": 250,
      "source": "current_run/microsoft_scaling/summary.json",
      "fixed_ten_quality": {
        "size": 25,
        "correct": 91,
        "total": 100
      },
      "failed_attempts": 0,
      "unknown_charge_attempts": 0,
      "accounted_usd": 0.001243074
    },
    {
      "id": "microsoft",
      "size": 50,
      "seconds": 0.7737085999979172,
      "cost": 0.003809568,
      "requests": 10,
      "native": true,
      "correct": 458,
      "total": 500,
      "source": "current_run/microsoft_scaling/summary.json",
      "fixed_ten_quality": {
        "size": 50,
        "correct": 90,
        "total": 100
      },
      "failed_attempts": 0,
      "unknown_charge_attempts": 0,
      "accounted_usd": 0.001904784
    },
    {
      "id": "microsoft",
      "size": 100,
      "seconds": 1.1744899500044994,
      "cost": 0.003230514,
      "requests": 10,
      "native": true,
      "correct": 915,
      "total": 1000,
      "source": "current_run/microsoft_scaling/summary.json",
      "fixed_ten_quality": {
        "size": 100,
        "correct": 91,
        "total": 100
      },
      "failed_attempts": 1,
      "unknown_charge_attempts": 1,
      "accounted_usd": 0.020672631
    },
    {
      "id": "mercury_paid",
      "size": 1,
      "seconds": 0.48301494999032,
      "cost": 0.0320968,
      "requests": 50,
      "native": true,
      "correct": 49,
      "total": 50,
      "source": "current_run/mercury_paid_scaling/summary.json",
      "fixed_ten_quality": null,
      "failed_attempts": 0,
      "unknown_charge_attempts": 0,
      "accounted_usd": 0.00160484
    },
    {
      "id": "mercury_paid",
      "size": 10,
      "seconds": 0.8374758999998448,
      "cost": 0.00323,
      "requests": 10,
      "native": true,
      "correct": 98,
      "total": 100,
      "source": "current_run/mercury_paid_scaling/summary.json",
      "fixed_ten_quality": {
        "size": 10,
        "correct": 98,
        "total": 100
      },
      "failed_attempts": 0,
      "unknown_charge_attempts": 0,
      "accounted_usd": 0.000323
    },
    {
      "id": "mercury_paid",
      "size": 25,
      "seconds": 1.1745656000130111,
      "cost": 0.00128864,
      "requests": 10,
      "native": true,
      "correct": 246,
      "total": 250,
      "source": "current_run/mercury_paid_scaling/summary.json",
      "fixed_ten_quality": {
        "size": 25,
        "correct": 98,
        "total": 100
      },
      "failed_attempts": 0,
      "unknown_charge_attempts": 0,
      "accounted_usd": 0.00032216
    },
    {
      "id": "mercury_paid",
      "size": 50,
      "seconds": 1.546954100005678,
      "cost": 0.0006382,
      "requests": 10,
      "native": true,
      "correct": 488,
      "total": 500,
      "source": "current_run/mercury_paid_scaling/summary.json",
      "fixed_ten_quality": {
        "size": 50,
        "correct": 98,
        "total": 100
      },
      "failed_attempts": 0,
      "unknown_charge_attempts": 0,
      "accounted_usd": 0.0003191
    },
    {
      "id": "mercury_paid",
      "size": 100,
      "seconds": 1.5228332999977283,
      "cost": 0.00032278,
      "requests": 10,
      "native": true,
      "correct": 980,
      "total": 1000,
      "source": "current_run/mercury_paid_scaling/summary.json",
      "fixed_ten_quality": {
        "size": 100,
        "correct": 98,
        "total": 100
      },
      "failed_attempts": 0,
      "unknown_charge_attempts": 0,
      "accounted_usd": 0.00032278
    }
  ],
  "models": {
    "jev": {
      "name": "JEV 1.13",
      "open_weights": false,
      "local_deployable": false,
      "icon": "assets/typesafe.png"
    },
    "luna": {
      "name": "Luna Decisions",
      "open_weights": false,
      "local_deployable": false,
      "icon": "assets/openai.svg"
    },
    "perplexity": {
      "name": "Perplexity Decider v1",
      "open_weights": false,
      "local_deployable": false,
      "icon": "assets/perplexity.png"
    },
    "d1": {
      "name": "Liquid d1",
      "open_weights": false,
      "local_deployable": false,
      "icon": "assets/liquid.png"
    },
    "kev": {
      "name": "KEV 4B",
      "open_weights": true,
      "local_deployable": true,
      "icon": "assets/jaredpalmer.png"
    },
    "mercury": {
      "name": "Mercury Decide \u00b7 Free",
      "open_weights": false,
      "local_deployable": false,
      "icon": "assets/inception.svg"
    },
    "clef": {
      "name": "Clef",
      "open_weights": false,
      "local_deployable": false,
      "icon": "assets/cloudflare.png"
    },
    "clef_flash": {
      "name": "Clef Flash",
      "open_weights": false,
      "local_deployable": false,
      "icon": "assets/cloudflare.png"
    },
    "tev": {
      "name": "TEV 4B Experimental",
      "open_weights": true,
      "local_deployable": false,
      "icon": "assets/togethercomputer.png"
    },
    "solar": {
      "name": "Solar Decide",
      "open_weights": false,
      "local_deployable": false,
      "icon": "assets/upstage.png"
    },
    "perplexity_v11": {
      "name": "Perplexity Decider v1.1",
      "open_weights": false,
      "local_deployable": false,
      "icon": "assets/perplexity.png"
    },
    "microsoft": {
      "name": "Microsoft Decision-1",
      "open_weights": false,
      "local_deployable": false
    },
    "drex": {
      "name": "Drex v1.5",
      "open_weights": true,
      "local_deployable": true
    },
    "clef_omni": {
      "name": "Clef Omni",
      "open_weights": true,
      "local_deployable": false,
      "icon": "assets/cloudflare.png"
    },
    "mercury_paid": {
      "name": "Mercury Decide \u00b7 Paid",
      "open_weights": false,
      "local_deployable": false,
      "icon": "assets/inception.svg"
    },
    "qwen9": {
      "name": "Qwen 3.5 9B + schema (FP4)",
      "open_weights": true,
      "local_deployable": false
    },
    "qwen27": {
      "name": "Qwen 3.8 27B + schema (FP4)",
      "open_weights": true,
      "local_deployable": false
    },
    "qwen9bf16": {
      "name": "Qwen 3.5 9B + schema (BF16)",
      "open_weights": true,
      "local_deployable": false
    },
    "qwen27bf16": {
      "name": "Qwen 3.8 27B + schema (BF16)",
      "open_weights": true,
      "local_deployable": false
    },
    "kev-4b_\u00b7_bf16": {
      "name": "KEV-4B \u00b7 BF16",
      "open_weights": true,
      "local_deployable": true
    },
    "kev-0.8b_\u00b7_bf16": {
      "name": "KEV-0.8B \u00b7 BF16",
      "open_weights": true,
      "local_deployable": true
    },
    "kev-0.8b_\u00b7_fp32": {
      "name": "KEV-0.8B \u00b7 FP32",
      "open_weights": true,
      "local_deployable": true
    },
    "julia-1": {
      "name": "Julia-1",
      "open_weights": true,
      "local_deployable": true
    },
    "decider-2b_v11": {
      "name": "Decider-2B v11",
      "open_weights": true,
      "local_deployable": true
    },
    "solar_decide": {
      "name": "Solar Decide",
      "open_weights": false,
      "local_deployable": false
    },
    "span-01": {
      "name": "Span-01",
      "open_weights": false,
      "local_deployable": false
    },
    "span-01_lite": {
      "name": "Span-01 Lite",
      "open_weights": false,
      "local_deployable": false
    }
  },
  "candidates": [
    {
      "requested_model_id": "microsoft/microsoft-decision-1",
      "display_name": "Microsoft Decision-1",
      "creator": "Microsoft",
      "status": "evaluated",
      "announced_on": "2026-10-09",
      "official_source": "https://commandline.microsoft.com/microsoft-decision-1-model-foundry/",
      "announced_outputs": [
        "yes_no",
        "multiple_choice",
        "rating"
      ],
      "openrouter_status": "verified_2026-10-09",
      "pricing": null,
      "benchmark_results": {
        "model": "microsoft/microsoft-decision-1",
        "correct": 138,
        "total": 150,
        "diagnostics": {
          "correct": 47,
          "total": 50,
          "accuracy": 0.94
        },
        "median_seconds": 0.49482685000111815,
        "reported_usd": 0.00188475,
        "cost_per_1000_packets": 0.037695,
        "by_type": {
          "choice": {
            "correct": 44,
            "total": 50
          },
          "noul": {
            "correct": 94,
            "total": 100
          }
        },
        "confident_errors_90": 2,
        "returned_model_ids": [
          "microsoft/microsoft-decision-1-20261009"
        ],
        "reported_providers": [
          "Azure"
        ],
        "reported_input_tokens": 44875,
        "missing_primary": 0,
        "reported_output_tokens": 200,
        "cached_tokens_status": "No cached-token field reported on this route"
      },
      "next_steps": [
        "verify_catalog_route_version_and_pricing",
        "freeze_adapter_and_request_manifest",
        "contract_check_with_new_budget",
        "quick_screen",
        "broader150",
        "selective_scaling"
      ],
      "provider": "Azure",
      "input_usd_per_million": 0.042
    },
    {
      "requested_model_id": "nace-ai/drex-v1.5",
      "display_name": "Drex v1.5",
      "status": "evaluated",
      "provider": "DeepInfra",
      "input_usd_per_million": 0.04,
      "open_weights": true,
      "locally_deployable": true,
      "official_source": "https://openrouter.ai/nace-ai/drex-v1.5",
      "benchmark_results": {
        "model": "nace-ai/drex-v1.5",
        "correct": 129,
        "total": 150,
        "diagnostics": {
          "correct": 48,
          "total": 50,
          "accuracy": 0.96
        },
        "median_seconds": 0.5518420000007609,
        "reported_usd": 0.00179172,
        "cost_per_1000_packets": 0.035834399999999995,
        "by_type": {
          "choice": {
            "correct": 42,
            "total": 50
          },
          "noul": {
            "correct": 87,
            "total": 100
          }
        },
        "confident_errors_90": 5,
        "returned_model_ids": [
          "nace-ai/drex-v1.5-20261009"
        ],
        "reported_providers": [
          "DeepInfra"
        ],
        "reported_input_tokens": 44793,
        "missing_primary": 0,
        "reported_output_tokens": 11068,
        "cached_tokens_status": "No cached-token field reported on this route"
      }
    },
    {
      "requested_model_id": "cloudflare/clef-omni",
      "display_name": "Clef Omni",
      "status": "evaluated",
      "provider": "Cloudflare",
      "input_usd_per_million": 0.15,
      "open_weights": true,
      "locally_deployable": null,
      "official_source": "https://openrouter.ai/cloudflare/clef-omni",
      "benchmark_results": {
        "model": "cloudflare/clef-omni",
        "correct": 114,
        "total": 150,
        "diagnostics": {
          "correct": 45,
          "total": 50,
          "accuracy": 0.9
        },
        "median_seconds": 0.3784010999952443,
        "reported_usd": 0.00901545,
        "cost_per_1000_packets": 0.180309,
        "by_type": {
          "choice": {
            "correct": 33,
            "total": 50
          },
          "noul": {
            "correct": 81,
            "total": 100
          }
        },
        "confident_errors_90": 11,
        "returned_model_ids": [
          "cloudflare/clef-omni-20261009"
        ],
        "reported_providers": [
          "Cloudflare"
        ],
        "reported_input_tokens": 60103,
        "missing_primary": 0,
        "reported_output_tokens": 0,
        "cached_tokens_status": "No cached-token field reported on this route"
      }
    },
    {
      "requested_model_id": "inception/mercury-decide",
      "display_name": "Mercury Decide \u00b7 Paid",
      "status": "evaluated",
      "provider": "Inception",
      "input_usd_per_million": 0.02,
      "open_weights": false,
      "locally_deployable": false,
      "official_source": "https://openrouter.ai/inception/mercury-decide"
    }
  ],
  "sources": {
    "docs/blog/assets/jev-new-competitors/model-list.json": "11d763351c4fb616c352ab10d30f980b758575be8bf6eed5b5ff9f7931aea495",
    "output/blog/jev-new-competitors/full150-performance.json": "3be9d94fa88a06256903ee426d5098d775fbb3c2d96fd18fb912f6b3737bed63",
    "data/exports/decision_perplexity_v11_v01/summary.json": "0b96c273fc20f106f0a13b008e5560e92c4709b21ba57e83a459b5acb1b5a48e",
    "current_run/summary.json": "610509d073cec8491d1910797796f5f83a4fbc533a4f80db40e36879a2a485b8",
    "current_run/mercury_paid_accuracy/summary.json": "50719f5fcf11d0b69d54fac0e1c119230591eaa3e2997e46de76170e1f0d94aa",
    "data/exports/decision_quick_frontier_v01/matched-results.json": "02f72868bdd94d66ed935a57eb475ebb97230aec24c16e58fea62cc263cb3340",
    "reproduced/base-general-binary.json": "2d106d7428ae5516e3bee27500f1722012312ee71aadca20a495263c97e84243",
    "data/exports/decision_image_inputs_v01_20261007/benchmark.json": "d7e76d71678f2aa942339e1ac276a593b672e44bed1cdf58afeb48c74a89a473",
    "data/exports/decision_scaling_four_v01/summary.json": "2a64a6f34ecc846cd38e50d035932f7c403d1e3a49230e170598b637a0123f7e",
    "data/exports/decision_clef_full_v01/summary.json": "537ed286bc3d3e613693436b44d7b9fe201a1ca6da10aff088810941aad8fa3e",
    "current_run/microsoft_scaling/summary.json": "59b672fd8d6fdf668fbdd5aced1a2cf2fc95b73972c04eee48cdf5849ea1143b",
    "current_run/mercury_paid_scaling/summary.json": "ddf692a7a3d1728aabc2ea93f4abe4613cc64f9115b1d9a53e38e2308dc8c44e"
  }
}