{ "benchmark": "JevBench", "revision": "v1.6.1", "protocol": "jevbench::v1.6", "provisional": false, "status": "published", "run_kind": "scheduled-refresh", "label": "JevBench v1.6.1", "source_sha256": "e15347783b8bedeeddd9017a831608a957f5bf0386f05902c50161ce0468f911", "source_note": "Hosted APIs re-scored on the full item set (v1.6.1, 5 Oct 2026): scorer score-v161-3/out-O1S (2026-10-06 00:39:36 UTC), method option B (O1S). Self-hosted rows are identical to v1.6.0.", "sample": { "open": 300, "sealed": 1200, "total": 1500, "published_open": 300 }, "types": { "choice": 0.3333333333333333, "noul": 0.3333333333333333, "score": 0.3333333333333333 }, "tier_weights": { "easy": 0.1, "hard": 0.4, "judge": 0.3, "standard": 0.2 }, "sealed_share_of_intelligence": 0.5, "G_med": 2.573387642438244, "G_med_api_basis_P_vs_A": -1.1922125858968342, "G_med_flag_gt10": false, "headline": "A", "options": { "A": { "intelligence_floor": 50.0, "weights": { "calibration": 25, "cost": 25, "intelligence": 25, "speed": 25 } }, "B": { "intelligence_floor": 50.0, "weights": { "calibration": 20, "cost": 20, "intelligence": 40, "speed": 20 } }, "C": { "intelligence_floor": 60.0, "weights": { "calibration": 25, "cost": 25, "intelligence": 25, "speed": 25 } } }, "views": [ "Balanced 33:33:33", "Emphasis on Accuracy 60:20:20", "Emphasis on Cost 20:20:60", "Emphasis on Speed 20:60:20", "Intelligence only" ], "bootstrap": { "B": 1000, "bootstrap_seed": 16, "g_med_fixed": true }, "n_ranked": 135, "board": { "A": { "order": [ "sage-1.3.0", "h2o-lightning-4b", "mercury-decide", "decisio-gemma-4-31b-v080", "jev-1.13.0", "quyet-1-0-large", "decider-12b", "wity-1", "decider-12b-v1", "torchcast-decision-12b", "winnow-12b", "deck31b", "cygnet", "decisio-gemma-4-12b-v080", "decisio-gemma-4-12b-v090", "jev-omni", "xor-26b-a4b", "bobcat-flash-1.2", "spx-cd-flash", "decider-chat-gemma4-31b", "hopper-12b-trained", "surogate-rune-26b-a4b-v3-v16", "gev-26b-decide", "spx-cd-omni", "diffusion-jev", "rene-1-31b-fp8", "aplomb-1", "bespoke-nimble-9b-v3", "apus-openjev-v1-9b", "plumb-4b", "spx-cd-pro", "messier-one-v0.2", "spark-s1-4b-v6", "swanone", "quyet-1-0-medium", "lev", "ninfer-qwen3.8-flash-next", "jev-local", "decider-4b-v2", "classone-qwen3.5-9b", "jevk5-v0.3-4b", "metask-jev-4b", "deck-4b-v1-0", "clef-flash", "evalengine-decision-4b", "janus-4b", "qwen35-9b-jev-data-mix-v2", "jevk5-v02", "jevone", "kahn1", "typecastlm-1.4.0", "hopper", "imajev-4b-rtx5090-a2", "malkuth-4b", "jqv", "jpt-35b-a3b", "decider-35b-a3b", "coco-decision-4b", "manchego-v21-v15", "decision-4b-v12", "raw-qwen3-4b-instruct-2507", "jev-27b-autotrust", "torchcast-decision-27b", "nimble-9b", "reflex-4b", "pplx-decider-v1-1-27b", "jade", "typecastlm", "jebadiah-27b", "decision-2-0-vega-27b", "decision-4b-v11", "autojev-27b", "eikos-27b", "ninfer-qwen3.8-27b", "openjev-thinking", "open-jev-zefan-9b", "standardone-8b", "clef", "jev-qwen3.5-9b-base-nvfp4", "semif-qwen3.5-4b", "bev-bonsai-27b", "litjev", "localjev-qwen3.5-4b", "system-one-sg", "kev-8b", "decision-2b", "opensourcejev-qwen35-4b-q4km", "open-alternative-jev", "raw-qwen3-8b", "decider-2b", "nemotron-diffusion-8b", "kev-4b", "kev-27b", "malkuth-2b", "qwen3-reranker-4b", "simplejev-qwen3.8-27b-selfhost", "zerank-2", "exaone-4.0-1.2b-jev-v0.3", "open-jev-zefan-2b", "open-jev-zefan-27b-v1.1", "raw-phi-4-mini", "simplejev-qwen3.5-0.8b", "smalljev", "kev-0.6b", "jul-fast", "opendecision", "classone-gemma4-e2b", "mghafiri-qwen3.5-0.8b-decision-model", "fastino-gliner-2-5-decide", "deem-0.8-v1", "decision-fast", "quyet-1-0-small-en", "laya-typed-decisions", "watersheep", "openjev-verdict-1.4", "raw-qwen3-0.6b", "lev-350m", "raw-qwen3-1.7b", "kev-0.5b", "openjev-verdict", "quyet-1-0-small", "watt-flash-0.1", "quyet-1-0-tiny", "open-jev-deberta-v3-large", "verdict-small", "tacet-sonata", "laya", "mxbai-rerank-base-v2", "laya-multilingual", "certo", "clm-8b", "bge-reranker-v2-m3", "gte-reranker-modernbert-base", "mirror", "open-jev-json-canvas-joshuasp" ], "leader_wording": null, "markers": [] }, "B": { "order": [ "sage-1.3.0", "quyet-1-0-large", "decisio-gemma-4-31b-v080", "mercury-decide", "wity-1", "jev-1.13.0", "h2o-lightning-4b", "deck31b", "decider-12b", "decider-12b-v1", "torchcast-decision-12b", "bobcat-flash-1.2", "winnow-12b", "xor-26b-a4b", "cygnet", "jev-omni", "spx-cd-flash", "decider-chat-gemma4-31b", "decisio-gemma-4-12b-v080", "decisio-gemma-4-12b-v090", "surogate-rune-26b-a4b-v3-v16", "hopper-12b-trained", "gev-26b-decide", "spx-cd-omni", "diffusion-jev", "rene-1-31b-fp8", "bespoke-nimble-9b-v3", "aplomb-1", "spx-cd-pro", "apus-openjev-v1-9b", "plumb-4b", "swanone", "spark-s1-4b-v6", "messier-one-v0.2", "ninfer-qwen3.8-flash-next", "quyet-1-0-medium", "lev", "jev-local", "decider-4b-v2", "classone-qwen3.5-9b", "clef-flash", "metask-jev-4b", "deck-4b-v1-0", "jevk5-v0.3-4b", "evalengine-decision-4b", "qwen35-9b-jev-data-mix-v2", "janus-4b", "jevone", "kahn1", "jevk5-v02", "typecastlm-1.4.0", "hopper", "imajev-4b-rtx5090-a2", "jpt-35b-a3b", "decider-35b-a3b", "malkuth-4b", "jqv", "torchcast-decision-27b", "pplx-decider-v1-1-27b", "jev-27b-autotrust", "coco-decision-4b", "manchego-v21-v15", "decision-4b-v12", "nimble-9b", "raw-qwen3-4b-instruct-2507", "jade", "jebadiah-27b", "decision-2-0-vega-27b", "eikos-27b", "autojev-27b", "openjev-thinking", "reflex-4b", "ninfer-qwen3.8-27b", "clef", "typecastlm", "open-jev-zefan-9b", "decision-4b-v11", "standardone-8b", "jev-qwen3.5-9b-base-nvfp4", "semif-qwen3.5-4b", "bev-bonsai-27b", "litjev", "system-one-sg", "localjev-qwen3.5-4b", "kev-8b", "opensourcejev-qwen35-4b-q4km", "raw-qwen3-8b", "decision-2b", "open-alternative-jev", "decider-2b", "kev-27b", "nemotron-diffusion-8b", "kev-4b", "simplejev-qwen3.8-27b-selfhost", "open-jev-zefan-27b-v1.1", "malkuth-2b", "qwen3-reranker-4b", "open-jev-zefan-2b", "zerank-2", "exaone-4.0-1.2b-jev-v0.3", "raw-phi-4-mini", "simplejev-qwen3.5-0.8b", "smalljev", "kev-0.6b", "classone-gemma4-e2b", "jul-fast", "opendecision", "mghafiri-qwen3.5-0.8b-decision-model", "fastino-gliner-2-5-decide", "deem-0.8-v1", "decision-fast", "quyet-1-0-small-en", "laya-typed-decisions", "watersheep", "raw-qwen3-0.6b", "openjev-verdict-1.4", "raw-qwen3-1.7b", "lev-350m", "kev-0.5b", "openjev-verdict", "quyet-1-0-small", "watt-flash-0.1", "quyet-1-0-tiny", "open-jev-deberta-v3-large", "verdict-small", "tacet-sonata", "laya", "mxbai-rerank-base-v2", "laya-multilingual", "certo", "clm-8b", "bge-reranker-v2-m3", "gte-reranker-modernbert-base", "mirror", "open-jev-json-canvas-joshuasp" ], "leader_wording": null, "markers": [] }, "C": { "order": [ "sage-1.3.0", "h2o-lightning-4b", "mercury-decide", "decisio-gemma-4-31b-v080", "jev-1.13.0", "quyet-1-0-large", "decider-12b", "wity-1", "decider-12b-v1", "torchcast-decision-12b", "deck31b", "winnow-12b", "bobcat-flash-1.2", "xor-26b-a4b", "spx-cd-flash", "decider-chat-gemma4-31b", "jev-omni", "cygnet", "surogate-rune-26b-a4b-v3-v16", "rene-1-31b-fp8", "decisio-gemma-4-12b-v080", "decisio-gemma-4-12b-v090", "diffusion-jev", "hopper-12b-trained", "spx-cd-pro", "bespoke-nimble-9b-v3", "gev-26b-decide", "spx-cd-omni", "swanone", "aplomb-1", "apus-openjev-v1-9b", "plumb-4b", "messier-one-v0.2", "spark-s1-4b-v6", "quyet-1-0-medium", "lev", "ninfer-qwen3.8-flash-next", "jev-local", "decider-4b-v2", "classone-qwen3.5-9b", "jevk5-v0.3-4b", "metask-jev-4b", "deck-4b-v1-0", "clef-flash", "evalengine-decision-4b", "janus-4b", "qwen35-9b-jev-data-mix-v2", "jevk5-v02", "jevone", "kahn1", "torchcast-decision-27b", "typecastlm-1.4.0", "pplx-decider-v1-1-27b", "hopper", "imajev-4b-rtx5090-a2", "jev-27b-autotrust", "decision-2-0-vega-27b", "jpt-35b-a3b", "malkuth-4b", "eikos-27b", "jebadiah-27b", "jqv", "openjev-thinking", "decider-35b-a3b", "coco-decision-4b", "manchego-v21-v15", "decision-4b-v12", "clef", "autojev-27b", "raw-qwen3-4b-instruct-2507", "nimble-9b", "reflex-4b", "jade", "typecastlm", "ninfer-qwen3.8-27b", "decision-4b-v11", "open-jev-zefan-9b", "standardone-8b", "jev-qwen3.5-9b-base-nvfp4", "semif-qwen3.5-4b", "bev-bonsai-27b", "litjev", "localjev-qwen3.5-4b", "system-one-sg", "kev-8b", "decision-2b", "opensourcejev-qwen35-4b-q4km", "open-alternative-jev", "raw-qwen3-8b", "kev-27b", "decider-2b", "nemotron-diffusion-8b", "kev-4b", "simplejev-qwen3.8-27b-selfhost", "malkuth-2b", "qwen3-reranker-4b", "zerank-2", "exaone-4.0-1.2b-jev-v0.3", "open-jev-zefan-2b", "open-jev-zefan-27b-v1.1", "raw-phi-4-mini", "simplejev-qwen3.5-0.8b", "smalljev", "kev-0.6b", "jul-fast", "opendecision", "classone-gemma4-e2b", "mghafiri-qwen3.5-0.8b-decision-model", "fastino-gliner-2-5-decide", "deem-0.8-v1", "decision-fast", "quyet-1-0-small-en", "laya-typed-decisions", "watersheep", "openjev-verdict-1.4", "raw-qwen3-0.6b", "lev-350m", "raw-qwen3-1.7b", "kev-0.5b", "openjev-verdict", "quyet-1-0-small", "watt-flash-0.1", "quyet-1-0-tiny", "open-jev-deberta-v3-large", "verdict-small", "tacet-sonata", "laya", "mxbai-rerank-base-v2", "laya-multilingual", "certo", "clm-8b", "bge-reranker-v2-m3", "gte-reranker-modernbert-base", "mirror", "open-jev-json-canvas-joshuasp" ], "leader_wording": null, "markers": [] } }, "v16": { "a2_supplement": { "counts": { "A": 300, "P": 300, "S": 300, "api_input": 600, "api_smoke_public": 5, "core": 0, "selfhosted_input": 600 }, "release": "v1.6.0-A2", "set_manifest_sha256": { "A": "380f76186ec6e09130154579c6ae1db0eebd1cae1b219339ad7429458c177f5d", "P": "6b2321f8359d87769a95c8e52135542d95ccdc353197e1e5c96abb9b616ce5ba", "S": "380f76186ec6e09130154579c6ae1db0eebd1cae1b219339ad7429458c177f5d" }, "supplement_of": "v1.6.0" }, "capability_formula": "mean(Intelligence, Calibration)", "counts": { "A": 300, "P": 300, "S": 1200, "api_input": 1500, "api_smoke_public": 5, "core": 900, "selfhosted_input": 1500 }, "equating": { "equating_flag": null, "min_pool": 5, "offsets": { "C": 2.3428887908654588, "I": -2.209917383979878 }, "rule": "median over complete full-coverage self-hosted systems of metric(S u P) - metric(A u P), per metric (Intelligence, Calibration); API value = clip(raw_A + offset)", "pool_note": "Historical v1.6.0 equating offsets, kept for the record. Hosted APIs that answered the full set (v1.6.1) are not equated." }, "equating_A2": { "G_med_A2_basis_P_vs_A2": 3.824932047119341, "diagnostic_main_A_offset_same_systems": { "C": 1.9486117388517954, "I": -4.381056886943014 }, "equating_flag": null, "offset_bootstrap_B": 200, "offsets": { "C": 1.5638258868659562, "I": -0.2887166717454388 }, "per_system": { "hopper": { "C_A2": 87.82068552843725, "C_A_main": 87.40904942545944, "C_S": 90.07902805935959, "I_A2": 34.77120638990772, "I_A_main": 41.88389851544817, "I_S": 34.73970870443837, "dC": 2.2583425309223344, "dI": -0.03149768546934695 }, "kev-0.6b": { "C_A2": 68.46161035185293, "C_A_main": 65.17717796738081, "C_S": 66.26655558235618, "I_A2": 5.866380901403119, "I_A_main": 9.100409843273326, "I_S": 7.598789169415934, "dC": -2.1950547694967497, "dI": 1.732408268012815 }, "kev-4b": { "C_A2": 68.65918994957973, "C_A_main": 69.97372951951687, "C_S": 68.45285936403559, "I_A2": 19.558821195537895, "I_A_main": 25.023085194237723, "I_S": 19.510707720813777, "dC": -0.20633058554413708, "dI": -0.04811347472411853 }, "kev-8b": { "C_A2": 61.3782880436933, "C_A_main": 67.24332350856244, "C_S": 70.7561729087212, "I_A2": 26.42642690493986, "I_A_main": 34.2874494947909, "I_S": 29.26217941280341, "dC": 9.377884865027895, "dI": 2.8357525078635497 }, "localjev-qwen3.5-4b": { "C_A2": 76.77849725935185, "C_A_main": 81.40355743693019, "C_S": 85.84286362718517, "I_A2": 22.674863142962646, "I_A_main": 28.26930138473988, "I_S": 24.145363218466134, "dC": 9.064366367833316, "dI": 1.4705000755034874 }, "malkuth-4b": { "C_A2": 83.46678991361165, "C_A_main": 85.35554104202156, "C_S": 88.43741591513763, "I_A2": 34.73305384360792, "I_A_main": 38.819192380023466, "I_S": 33.840703805987175, "dC": 4.970626001525986, "dI": -0.8923500376207443 }, "metask-jev-4b": { "C_A2": 79.1415556283946, "C_A_main": 83.18850868556241, "C_S": 84.41575352936586, "I_A2": 41.051921128504205, "I_A_main": 41.86894313160032, "I_S": 39.166454612935595, "dC": 5.274197900971259, "dI": -1.8854665155686092 }, "raw-phi-4-mini": { "C_A2": 56.153985406303406, "C_A_main": 53.16092441580774, "C_S": 57.015465878675315, "I_A2": 14.189557846849354, "I_A_main": 12.927281214813327, "I_S": 13.660237978082595, "dC": 0.861480472371909, "dI": -0.5293198687667591 }, "raw-qwen3-1.7b": { "C_A2": 11.425912172792259, "C_A_main": 11.45520847967844, "C_S": 11.516326812547591, "I_A2": 4.994829708920566, "I_A_main": 6.997731901083261, "I_S": 5.124286554530066, "dC": 0.09041463975533226, "dI": 0.1294568456095 }, "raw-qwen3-4b-instruct-2507": { "C_A2": 34.31244267053921, "C_A_main": 37.75992930947443, "C_S": 35.18175191334879, "I_A2": 35.69189443401588, "I_A_main": 38.58935302861775, "I_S": 33.95117742100547, "dC": 0.8693092428095781, "dI": -1.7407170130104106 }, "raw-qwen3-8b": { "C_A2": 34.345565335170505, "C_A_main": 38.60246474366818, "C_S": 33.721548452619835, "I_A2": 30.34325924789806, "I_A_main": 31.73129272246627, "I_S": 26.54912920590792, "dC": -0.62401688255067, "dI": -3.7941300419901403 }, "typecastlm": { "C_A2": 76.627802179825, "C_A_main": 79.3986887571586, "C_S": 83.29951051808085, "I_A2": 31.896472425382363, "I_A_main": 31.998923946026117, "I_S": 29.78900656204624, "dC": 6.671708338255854, "dI": -2.107465863336124 } }, "pool": [ "hopper", "kev-0.6b", "kev-4b", "kev-8b", "localjev-qwen3.5-4b", "malkuth-4b", "metask-jev-4b", "raw-phi-4-mini", "raw-qwen3-1.7b", "raw-qwen3-4b-instruct-2507", "raw-qwen3-8b", "typecastlm" ], "pool_n": 12, "rule": "median over the A2 pool of metric(S u P, main run) - metric(A2 u P, A2 run), per metric; API A2 value = clip(raw_A2 + offset_A2)", "runs_status": { "hopper": { "answered_ok": 600, "missing": 0, "rows": 600, "status": "complete" }, "kev-0.6b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "kev-4b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "kev-8b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "localjev-qwen3.5-4b": { "answered_ok": 600, "missing": 0, "rows": 600, "status": "complete" }, "malkuth-4b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "metask-jev-4b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "raw-phi-4-mini": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "raw-qwen3-1.7b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "raw-qwen3-4b-instruct-2507": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "raw-qwen3-8b": { "answered_ok": 594, "missing": 0, "rows": 600, "status": "complete" }, "typecastlm": { "answered_ok": 600, "missing": 0, "rows": 600, "status": "complete" } } }, "g_med_rule": "median gap over ranked self-hosted systems; API systems use the same systems' P-vs-A gap", "offset_bootstrap_B": 200, "release": "v1.6.0", "scorer_sources_sha256": { "noul_method_v16.py": "119e59486ab3300b972fbcbadb0fe1dceceb124287607375aa61c66063c607d0", "score_v15.py": "0931e05c6410b1b971201834397cb609cdb25f942723c50375632952f6dcc447", "score_v15_headlineA.py": "826f2cbcb57007ec8a161e6eb0111182b8a656c0638027c6c78022a858fd8e2c", "score_v16.py": "02132271117653e6302c509232a581d0094f0e78eac8b1bfc67596fd5d5097e9" }, "scoring_parameters": { "B": 1000, "g_med_fixed": null, "noul_method": "O1S", "only": null, "seed": 16, "test_allow_partial": false, "workers": 10 }, "set_manifest_sha256": { "A": "19b196a71d58da260cd83069ffcd64bb323e158c24a5fcabeffbb5bee7c1eabb", "P": "6b2321f8359d87769a95c8e52135542d95ccdc353197e1e5c96abb9b616ce5ba", "S": "f13557a141fba12efc6bf6d39a0dd8e3bf803d4cacd6eab516d4f4f87045539a" }, "long_items_note": "23 items of this draw are very long (about 77,000 to 81,000 input tokens). Systems with a shorter context window refuse them; Jev-Omni runs out of GPU memory on them on its listed RTX 6000 (48 GB) recipe; decider-4b v2's server truncates them to about 32,800 tokens and answers; Plumb-4B reads them in full. Each outcome is the system's own recipe and is scored as such.", "item_sets": [ { "set": "S", "name": "Sealed release draw", "items": 1200, "by_type": { "choice": 600, "noul": 300, "score": 300 }, "answered_by": "self-hosted systems (run offline on our own GPU pods or Sandy) and, from v1.6.1, externally hosted APIs" }, { "set": "A", "name": "API subset (part of S; used by the v1.6.0 hosted-API measurements)", "items": 300, "by_type": { "choice": 150, "noul": 75, "score": 75 }, "answered_by": "every system (inside S since v1.6.1); retired for future draws" }, { "set": "P", "name": "Public set", "items": 300, "by_type": { "choice": 150, "noul": 75, "score": 75 }, "answered_by": "every system" } ], "full_set_api_rows": [ "fastino-gliner-2-5-decide", "jev-1.13.0", "sage-1.3.0", "wity-1", "wity-1-always", "wity-1-off" ], "packaging_partial": false }, "noul_method": { "applied": "O1S", "decisive_source_sha256": "159554cb4ebc276bac6e24559b3f326a47398b97479d0048647411ed64322e49" }, "systems": [ { "addendum": null, "api_flag": true, "api_subset": false, "author": "Levanto Labs", "axes": { "calibration": 91.59817606608073, "cost": 58.231203787388296, "intelligence": 65.59290944525489, "speed": 93.61638181239104 }, "calibration": { "parts": { "choice": { "ece_hard": 0.034436035156249975, "ece_score": 93.11279296875, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 93.11279296875 }, "noul": { "abstention_rate": 0.16533333333333333, "brier": 0.08087898763020833, "ece": 0.049375, "n": 375, "score": 90.125 }, "score": { "interval90_coverage": 0.968, "interval90_width": 0.2808888888888889, "n": 375, "nrps": 0.06392779541015624, "score": 91.55673522949218, "top_ece": 0.05246875 } }, "score": 91.59817606608073 }, "capability": 78.59554275566781, "composite_ci95": { "A": [ 71.70514550946575, 75.17412681523324 ], "B": [ 68.95926995267655, 74.03508105576502 ], "C": [ 71.32573829825638, 75.17412681523324 ] }, "cost": { "basis": "Levanto list tariff, top tier: USD 0.05 per 1M input tokens, USD 10 per 1M output tokens (https://levanto.ai/pricing, read 5 Oct 2026)", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.06497606666666667 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.024677048070413 }, "display": "Sage 1.3.0 (Levanto Labs)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 65.67742817331194, "I_sealed": 65.50839071719783, "base": 65.59290944525489, "excess": -2.4043501863241326, "gap": 0.16903745611411125, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 85.05619838199217, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 83.01352626612143, "judge": 84.17930125247197, "standard": 84.93093344495604 } }, "open|noul": { "cc": 54.291235334713605, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 56.52173913043479, "judge": 39.99999999999999, "standard": 55.55555555555556 } }, "open|score": { "cc": 57.68485080323005, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 71.80397727272727, "hard": 52.14120370370371, "judge": 68.19385593220339, "standard": 45.94907407407408 } }, "sealed|choice": { "cc": 79.68397614698543, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 82.70637794693668, "hard": 73.02036199095022, "judge": 85.1024208566108, "standard": 83.37233649464221 } }, "sealed|noul": { "cc": 56.36040792532405, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 41.747572815533985, "judge": 57.89473684210527, "standard": 77.46478873239437 } }, "sealed|score": { "cc": 60.480788079284, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 76.96974734042553, "hard": 51.94546568627452, "judge": 64.89053143712576, "standard": 62.692338197969534 } } } }, "jevbench_score": 74.04992179175468, "key": "sage-1.3.0", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 1, "ranked": true, "ranks": { "A": 1, "B": 1, "C": 1, "capability": 8 }, "scores": { "A": 74.04992179175468, "B": 72.1884444725083, "C": 74.04992179175468 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.15245595201849937, "p50_s_raw": 0.15245595201849937, "p95_s_adjusted": 0.2852446019649505, "p95_s_raw": 0.2852446019649505 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 86.55241460870812, 92.09209592889854 ], "I": [ 60.000783824692206, 70.03012866456243 ], "capability": [ 74.15252653755351, 80.27098937533351 ], "composite": { "A": [ 71.70514550946575, 75.17412681523324 ], "B": [ 68.95926995267655, 74.03508105576502 ], "C": [ 71.32573829825638, 75.17412681523324 ] }, "seed": 114 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": true, "g_med_used": 2.573387642438244, "lane": "api", "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 91.59817606608073, "capability": 78.59554275566781, "intelligence": 65.59290944525489 }, "per_type": { "choice": { "accuracy": 0.8493333333333334, "calibration": 93.11279296875, "cc": 80.73188522493676, "cc_by_type": { "choice": 80.73188522493676 }, "cc_tiered": 82.3700872644888, "cc_tiered_open": 85.05619838199217, "cc_tiered_sealed": 79.68397614698543, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 80.73188522493676 }, "noul": { "accuracy": 0.792, "calibration": 90.125, "cc": 58.400000000000006, "cc_by_type": { "noul": 58.400000000000006 }, "cc_tiered": 55.32582163001882, "cc_tiered_open": 54.291235334713605, "cc_tiered_sealed": 56.36040792532405, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 58.400000000000006 }, "score": { "accuracy": 0.68, "calibration": 91.55673522949218, "cc": 60.08634868421052, "cc_by_type": { "score": 60.08634868421052 }, "cc_tiered": 59.08281944125702, "cc_tiered_open": 57.68485080323005, "cc_tiered_sealed": 60.480788079284, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 60.08634868421052 } }, "raw_on_A": null, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 69.60497929786021, "Emphasis on Accuracy 60:20:20": 67.94266119076805, "Emphasis on Cost 20:20:60": 64.56093721161619, "Emphasis on Speed 20:60:20": 77.56249365186707, "Intelligence only": 65.59290944525489 }, "repo": "https://levanto.ai", "class": "decision-api", "open": "no", "licence": "proprietary hosted API", "model_pin": "levanto-sage-v1.3 (Sage 1.3.0)", "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "endpoint_condition": "Levanto hosted API, text only, measured from Sandy (Helsinki) on 5 Oct 2026; full set 1,200 sealed + P300 (v1.6.1); not evaluated on ImageJevBench.", "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).", "provenance": { "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)", "carried_metadata_from": "v1.6.0 release row" }, "noul_decisive": { "supported": true, "decisive_rate": 0.8346666666666667, "acc_among_decisive": 0.9488817891373802, "valid": 375 } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "H2O.ai", "axes": { "calibration": 89.95399367673842, "cost": 60.29882443070202, "intelligence": 60.03839906196252, "speed": 92.61084474720658 }, "calibration": { "parts": { "choice": { "ece_hard": 0.03794010455551056, "ece_score": 92.4119790888979, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 92.4119790888979 }, "noul": { "abstention_rate": 0.0, "brier": 0.1156787859671944, "ece": 0.055451181905507255, "n": 375, "score": 88.90976361889855 }, "score": { "interval90_coverage": 0.952, "interval90_width": 0.45600000000000007, "n": 375, "nrps": 0.08555524714180213, "score": 88.54023832241884, "top_ece": 0.0718199932049105 } }, "score": 89.95399367673842 }, "capability": 74.99619636935047, "composite_ci95": { "A": [ 69.79499879039663, 73.78891952313766 ], "B": [ 66.0714448834085, 71.5461599654 ], "C": [ 57.59678849695695, 73.78891952313766 ] }, "cost": { "basis": "ESTIMATE (self-hosted open weights, no public tariff): cost per 1,000 decisions measured for H2O-Lightning-4B v1.0 on the v1.5 pool (same base, shim and token path) carried for v1.1, the practice of every Qwen3.5-4B row on this board; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off. Cross-check: v1.1's own v1.6 input tokens on the common cost basis give USD 0.0204 per 1,000 (within 3 %)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.021055837438423645 }, "display": "H2O-Lightning-4B v1.1 (H2O.ai, Qwen3.5-4B fine-tune, stock vLLM + open shim)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 61.05316077664259, "I_sealed": 59.02363734728246, "base": 60.03839906196252, "excess": -0.5438642130781113, "gap": 2.0295234293601325, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 75.18196275852294, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 69.80182447310474, "judge": 80.22412656558997, "standard": 69.8618668899121 } }, "open|noul": { "cc": 60.94202898550725, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 56.52173913043479, "judge": 50.0, "standard": 66.66666666666667 } }, "open|score": { "cc": 47.035490585897584, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 54.83317199906027, "hard": 55.09009420986275, "judge": 42.073705022183916, "standard": 34.47012097695641 } }, "sealed|choice": { "cc": 64.65667579231118, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 72.06414899120539, "hard": 60.80316742081447, "judge": 67.41154562383613, "standard": 64.52765118857003 } }, "sealed|noul": { "cc": 68.08195642942992, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 87.99999999999999, "hard": 55.33980582524271, "judge": 68.42105263157893, "standard": 83.09859154929578 } }, "sealed|score": { "cc": 44.33227982010628, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 63.681000195734974, "hard": 40.36020718747171, "judge": 45.957875906717405, "standard": 40.163670767644376 } } } }, "jevbench_score": 72.5232271521435, "key": "h2o-lightning-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 2, "ranked": true, "ranks": { "A": 2, "B": 7, "C": 2, "capability": 15 }, "scores": { "A": 72.5232271521435, "B": 69.62745760423118, "C": 72.5232271521435 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.20856922594830393, "p50_s_raw": 0.02928461297415197, "p95_s_adjusted": 0.2628241665661335, "p95_s_raw": 0.056412083283066745 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.64761887040858, 90.96540455486856 ], "I": [ 54.48601679476801, 63.869097672212845 ], "capability": [ 70.2236559039794, 76.81358157643643 ], "composite": { "A": [ 69.79499879039663, 73.78891952313766 ], "B": [ 66.0714448834085, 71.5461599654 ], "C": [ 57.59678849695695, 73.78891952313766 ] }, "seed": 44 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.95399367673842, "capability": 74.99619636935047, "intelligence": 60.03839906196252 }, "per_type": { "choice": { "accuracy": 0.744, "calibration": 92.4119790888979, "cc": 67.26125631139698, "cc_by_type": { "choice": 67.26125631139698 }, "cc_tiered": 69.91931927541705, "cc_tiered_open": 75.18196275852294, "cc_tiered_sealed": 64.65667579231118, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 67.26125631139698 }, "noul": { "accuracy": 0.848, "calibration": 88.90976361889855, "cc": 69.6, "cc_by_type": { "noul": 69.6 }, "cc_tiered": 64.51199270746858, "cc_tiered_open": 60.94202898550725, "cc_tiered_sealed": 68.08195642942992, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 69.6 }, "score": { "accuracy": 0.5893333333333334, "calibration": 88.54023832241884, "cc": 45.79426376408395, "cc_by_type": { "score": 45.79426376408395 }, "cc_tiered": 45.683885203001935, "cc_tiered_open": 47.035490585897584, "cc_tiered_sealed": 44.33227982010628, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 45.79426376408395 } }, "raw_on_A": { "calibration": 86.31592326935305, "capability": 74.50666727956846, "intelligence": 62.69741128978386 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 68.12306233469367, "Emphasis on Accuracy 60:20:20": 64.64127121779333, "Emphasis on Cost 20:20:60": 64.76172866409156, "Emphasis on Speed 20:60:20": 76.18038522166987, "Intelligence only": 60.038399061962515 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/h2oai/h2o-lightning-4b", "gpu": "RTX 5090", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX 5090, official vLLM 0.30.0 image + the author's open standard-library shim (h2o_lightning_shim.py) as published on the model card, offline (all credentials unset), driven over loopback only by the JevBench client; label-free inputs; pod destroyed afterwards; JevBench add-request (GitHub #181), measured 4 Oct 2026", "underlying": "Qwen/Qwen3.5-4B", "model_pin": "h2oai/h2o-lightning-4b@193ad740925b176a3b70a5a13a7cff2f2fadd01e", "last_measured_on": "2026-10-04", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 1.0, "acc_among_decisive": 0.848, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a8 (scorer output sha256 7a5b4d4c7b7c, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": true, "api_subset": false, "author": "Inception (API, free on OpenRouter) - Florian work order, 6 Oct 2026", "axes": { "calibration": 81.34341651168008, "cost": 62.08313746631354, "intelligence": 65.90701345473542, "speed": 85.47752366478576 }, "calibration": { "parts": { "choice": { "ece_hard": 0.11045316563089375, "ece_score": 77.90936687382126, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 77.90936687382126 }, "noul": { "abstention_rate": 0.10242587601078168, "brier": 0.10151182638394682, "ece": 0.08813961357701941, "n": 371, "score": 82.37207728459612 }, "score": { "interval90_coverage": 0.8753387533875339, "interval90_width": 0.1458897922312557, "n": 369, "nrps": 0.06239733540901056, "score": 83.74880537662287, "top_ece": 0.13131327852926614 } }, "score": 81.34341651168008 }, "capability": 73.62521498320774, "composite_ci95": { "A": [ 69.93954261051735, 73.87316393178861 ], "B": [ 67.79015712351674, 72.99152541270097 ], "C": [ 69.74081685972264, 73.87316393178861 ] }, "cost": { "basis": "LABELLED ESTIMATE, not charged - and nothing WAS charged: usage.cost is 0 on every one of the 1,500 calls. Inception publishes NO paid tariff for Mercury Decide: OpenRouter lists it free-only (prompt 0, completion 0) and the non-free slug inception/mercury-decide has no endpoints at all (404 'No endpoints found', read 7 Oct 2026), so there is nothing bookable. Because the system is API-only with undisclosed weights, the open rows' size-class estimate table cannot apply either. The row is therefore priced from THE SAME OPERATOR'S cheapest published per-token tariff on the SAME platform, inception/mercury-2.5 at USD 0.04 per 1M input and USD 0.15 per 1M output, which IS a key in the frozen 25 Sep snapshot - x this row's own measured tokens. Output tokens are 3 per decision on average (4,434 over 1,477 answers) because the probability is read from the model rather than written out, so the output rate is nearly inert and the Cost axis is set by the input rate alone. RELEASE-LANE LINE, recorded and not acted on, and it is RUN 57'S SHAPE EXACTLY - one mapping line, not a new price: the rate inception/mercury-2.5 (0.04, 0.15) is already IN the dated snapshot; what is missing is only a base-model decision line pointing at it, so pricing_v15.reference() raises on every inception/* string. receipts/COST-ALTERNATIVES-R58.json has the two alternatives' arithmetic already done.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.01807942666666667 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.018360961408259987 }, "display": "Mercury Decide (Inception; System One decisions API, served free on OpenRouter as inception/mercury-decide:free)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 67.12906498043914, "I_sealed": 64.68496192903172, "base": 65.90701345473542, "excess": -0.12928459103081957, "gap": 2.4441030514074242, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 79.71168908755723, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 81.12614029569046, "judge": 80.22412656558997, "standard": 69.8618668899121 } }, "open|noul": { "cc": 58.71014492753623, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 82.6086956521739, "judge": 30.000000000000004, "standard": 33.33333333333333 } }, "open|score": { "cc": 62.96536092622397, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 62.561695179434636, "hard": 63.165822850913635, "judge": 76.80814242854491, "standard": 42.0020976967579 } }, "sealed|choice": { "cc": 74.09139259954716, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 77.38526346907102, "hard": 68.43891402714931, "judge": 79.51582867783985, "standard": 75.61276019214189 } }, "sealed|noul": { "cc": 57.27197132719671, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 87.99999999999999, "hard": 35.92233009708738, "judge": 65.78947368421053, "standard": 71.83098591549295 } }, "sealed|score": { "cc": 62.691521860351266, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 69.70067055435705, "hard": 51.597448628789465, "judge": 70.50916076191972, "standard": 69.64863562411932 } } } }, "jevbench_score": 72.36824049637728, "key": "mercury-decide", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 3, "ranked": true, "ranks": { "A": 3, "B": 4, "C": 3, "capability": 19 }, "scores": { "A": 72.36824049637728, "B": 70.97659528968215, "C": 72.36824049637728 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.33213604614138603, "p50_s_raw": 0.33213604614138603, "p95_s_adjusted": 0.8529658075422044, "p95_s_raw": 0.8529658075422044 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 76.84031884046848, 84.00969333082946 ], "I": [ 60.19803361460591, 70.00098427505237 ], "capability": [ 69.08542722449891, 76.48291495498401 ], "composite": { "A": [ 69.93954261051735, 73.87316393178861 ], "B": [ 67.79015712351674, 72.99152541270097 ], "C": [ 69.74081685972264, 73.87316393178861 ] }, "seed": 91 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": true, "g_med_used": 2.573387642438244, "lane": "api", "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 81.34341651168008, "capability": 73.62521498320774, "intelligence": 65.90701345473542 }, "per_type": { "choice": { "accuracy": 0.808, "calibration": 77.90936687382126, "cc": 75.44594223354774, "cc_by_type": { "choice": 75.44594223354774 }, "cc_tiered": 76.9015408435522, "cc_tiered_open": 79.71168908755723, "cc_tiered_sealed": 74.09139259954716, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 75.44594223354774 }, "noul": { "accuracy": 0.8026666666666666, "calibration": 82.37207728459612, "cc": 60.53333333333333, "cc_by_type": { "noul": 60.53333333333333 }, "cc_tiered": 57.99105812736647, "cc_tiered_open": 58.71014492753623, "cc_tiered_sealed": 57.27197132719671, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 60.53333333333333 }, "score": { "accuracy": 0.712, "calibration": 83.74880537662287, "cc": 62.21714605598172, "cc_by_type": { "score": 62.21714605598172 }, "cc_tiered": 62.82844139328762, "cc_tiered_open": 62.96536092622397, "cc_tiered_sealed": 62.691521860351266, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 62.21714605598172 } }, "raw_on_A": null, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 69.80103192778668, "Emphasis on Accuracy 60:20:20": 68.18948202584015, "Emphasis on Cost 20:20:60": 66.49451324072818, "Emphasis on Speed 20:60:20": 75.32698952826067, "Intelligence only": 65.90701345473542 }, "class": "Hosted decision API", "open": "closed weights", "licence": null, "repo": "https://openrouter.ai/inception/mercury-decide:free", "gpu": null, "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), COMPLETE: 1,500 of 1,500 dispatched, 1,477 answered, 23 invalid. Measured through the release lane's reviewed run_api_full_v161.py UNCHANGED, on Florian's 5 Oct amendment (hosted systems answer the same full S u P as self-hosted ones); provider inception, proxy openrouter, every sealed item exposure-logged before the first request. Florian work order of 6 Oct 2026 ('put it on JevBench', under https://x.com/airesearch12/status/2107569862486659111). NO MAPPING DECISION OF OURS: Mercury Decide publishes the same /v1/systemone contract as Jev and all three primitives come back in the canonical shape (noul a float; choice a label plus probabilities keyed by the item's own option labels; score probabilities keyed by level index '0'..'n-1' - 0-based, unlike ClassOne's), so the vendored typesafe adapter's request body, answer mapping and validations are reused UNCHANGED. or_decisions_r58.OpenRouterDecisionsAdapter rewrites ONLY the URL, because OpenRouter serves the contract at POST /api/alpha/decisions rather than /v1/systemone - its own 400 from chat/completions names that path - and it does so by calling the parent run(), so none of the reviewed mapping is duplicated. The pinned harness/vendor tree was NOT edited; the module is reached by PYTHONPATH. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW REFUSES THEM: the context window is 32,768 tokens and all 23 came back as HTTP 422 with the provider's own 'upstream_error: Decision service could not complete the request' - a disclosed refusal, which the harness counts as the system's own rather than as an outage. The invalid set is EXACTLY those 23 and nothing else failed in 1,477 answers. RUN IN TWO PASSES BECAUSE OF A FREE-TIER BURST LIMIT, AND IT DOES NOT MOVE THE SPEED FIGURE: items 1-736 ran back to back at ~1.7 req/s until OpenRouter returned a header-less 429 ('rate_limit_exceeded: Request or spending limit exceeded'), which is run_v16's own stop rule; the row was resumed at 1 req/s and completed. The two passes' medians differ by 0.038 s (0.3624 s vs 0.3246 s over 1,477 answers; 0.3624 vs 0.3104 on the 124 items the Speed axis actually uses), and run_v16 sleeps the spacing BEFORE starting the timer, so the pacing is not in the measurement. Receipt LATENCY-SEGMENTS-mercury-R58.txt. Production API, so NO latency adjustment. Serial, one decision per request, no retry, no batching. add-requests run 58", "underlying": null, "model_pin": "provider-reported measurement model slug: inception/mercury-decide:free", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.8975741239892183, "acc_among_decisive": 0.9039039039039038, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Current-pool API measurement registry" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Amin Roudaki (@aminry)", "axes": { "calibration": 88.71627256952715, "cost": 51.66530928015878, "intelligence": 70.42321091828492, "speed": 91.18580810090454 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06150753988606317, "ece_score": 87.69849202278736, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 87.69849202278736 }, "noul": { "abstention_rate": 0.06469002695417791, "brier": 0.04879523639868886, "ece": 0.04694201082318837, "n": 371, "score": 90.61159783536232 }, "score": { "interval90_coverage": 0.9349593495934959, "interval90_width": 0.1980578139114724, "n": 369, "nrps": 0.057879253805657324, "score": 87.83872785043177, "top_ece": 0.09267309459285371 } }, "score": 88.71627256952715 }, "capability": 79.56974174390604, "composite_ci95": { "A": [ 69.85754247280484, 72.80874617920621 ], "B": [ 68.87932855683242, 73.03493454427631 ], "C": [ 69.85754247280484, 72.80874617920621 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff) and CLEAN: unlike run 54's FP8 sibling, the frozen pricing snapshot resolves this EXACT base - google/gemma-4-31B-it - to USD 0.09 per 1M input and USD 0.34 per 1M output. Read before the pod and asserted in make_registry_r55.py. x this row's own measured tokens, the deck31b method. Nothing is provisional and no release-lane line is open on it. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.04022032666666667 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04084664184157075 }, "display": "decisio v0.8.0 on gemma-4-31B-it (frozen, FP8 on load, one prefill per state, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 67.59136348711468, "I_sealed": 73.25505834945518, "base": 70.42321091828492, "excess": -8.23708250477875, "gap": -5.663694862340506, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 70.82494232345373, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 69.80182447310474, "judge": 64.40342781806196, "standard": 69.8618668899121 } }, "open|noul": { "cc": 73.1207729468599, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 73.91304347826086, "judge": 60.00000000000001, "standard": 77.77777777777777 } }, "open|score": { "cc": 58.8283751910304, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 71.01492747107596, "hard": 63.1937957087831, "judge": 57.45226814139626, "standard": 46.068418589953396 } }, "sealed|choice": { "cc": 83.09170328497459, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 90.68804966373514, "hard": 76.07466063348416, "judge": 86.03351955307262, "standard": 88.91489099642814 } }, "sealed|noul": { "cc": 76.04956278293162, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 87.99999999999999, "hard": 61.165048543689316, "judge": 81.57894736842107, "standard": 91.54929577464787 } }, "sealed|score": { "cc": 60.62390898045936, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 72.91119795662057, "hard": 56.16498434122372, "judge": 63.285909697701555, "standard": 59.4051126949867 } } } }, "jevbench_score": 71.6926576652985, "key": "decisio-gemma-4-31b-v080", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 4, "ranked": true, "ranks": { "A": 4, "B": 3, "C": 4, "capability": 4 }, "scores": { "A": 71.6926576652985, "B": 71.43512018650192, "C": 71.6926576652985 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.24419243037700653, "p50_s_raw": 0.047096215188503265, "p95_s_adjusted": 0.3116642528679222, "p95_s_raw": 0.0808321264339611 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.0517072217947, 90.77969936508954 ], "I": [ 65.27867139621, 74.3007710998681 ], "capability": [ 75.67457118842692, 81.98658905786246 ], "composite": { "A": [ 69.85754247280484, 72.80874617920621 ], "B": [ 68.87932855683242, 73.03493454427631 ], "C": [ 69.85754247280484, 72.80874617920621 ] }, "seed": 41 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.71627256952715, "capability": 79.56974174390604, "intelligence": 70.42321091828492 }, "per_type": { "choice": { "accuracy": 0.8546666666666667, "calibration": 87.69849202278736, "cc": 81.41394238511599, "cc_by_type": { "choice": 81.41394238511599 }, "cc_tiered": 76.95832280421416, "cc_tiered_open": 70.82494232345373, "cc_tiered_sealed": 83.09170328497459, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 81.41394238511599 }, "noul": { "accuracy": 0.888, "calibration": 90.61159783536232, "cc": 77.60000000000001, "cc_by_type": { "noul": 77.60000000000001 }, "cc_tiered": 74.58516786489577, "cc_tiered_open": 73.1207729468599, "cc_tiered_sealed": 76.04956278293162, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 77.60000000000001 }, "score": { "accuracy": 0.7093333333333334, "calibration": 87.83872785043177, "cc": 60.762544130610976, "cc_by_type": { "score": 60.762544130610976 }, "cc_tiered": 59.72614208574488, "cc_tiered_open": 58.8283751910304, "cc_tiered_sealed": 60.62390898045936, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 60.762544130610976 } }, "raw_on_A": { "calibration": 84.78496723140272, "capability": 77.99664929333233, "intelligence": 71.20833135526192 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 67.38267528356181, "Emphasis on Accuracy 60:20:20": 68.56682969127965, "Emphasis on Cost 20:20:60": 60.07267047173585, "Emphasis on Speed 20:60:20": 75.23880708945377, "Intelligence only": 70.42321091828492 }, "class": "Jev-compatible decision readout on frozen base", "open": "open weights", "licence": "Apache-2.0 software; Gemma model terms", "repo": "https://github.com/aminry/decisio", "gpu": "H100", "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. github.com/aminry/decisio tag v0.8.0 (5b42101a191d222062a795194b0aedb5003bf6ea) - the same tag whose other two entries run 54 measured - installed with their own `uv sync --extra serve --frozen` so their lockfile fixes every version, and served with their own `python -m decisio.serve.vllm_engine --base gemma-4-31b`. --base carries google/gemma-4-31B-it@842da379, the official weights, FROZEN: no fine-tuning, no adapter, no task registered; vLLM 0.30.0 quantizes it to FP8 when it loads (30.6 GiB). That profile's measured settings: temperature 4.672 for choice questions and 5.252 for the rest, the 12B entry's prompt unchanged, a system turn, the chat template's own answer position, every single-token form of each letter summed, no tokens generated. New in 0.8.0 on this base and the reason it is worth its own row: several questions of a state are scored in ONE pass after a single prefill. The author's own limit is a 32,768-token prompt, so the pool's 23 items at ~80,000 tokens are a documented capacity limit, not a defect. THE AUTHOR STATES ONE 96 GB CARD; this is one H100 80 GB (Lium) and no flag of theirs was changed to make it fit. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 55, GitHub #169", "underlying": "google/gemma-4-31B-it", "model_pin": "aminry/decisio@5b42101a191d222062a795194b0aedb5003bf6ea + google/gemma-4-31B-it@842da379", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.9353099730458221, "acc_among_decisive": 0.9596541786743515, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "r54/r55/r61 registry and pinned author package" } }, { "addendum": null, "api_flag": true, "api_subset": false, "author": "TypeSafe AI", "axes": { "calibration": 90.63220203152369, "cost": 54.733842199536014, "intelligence": 63.62075784024779, "speed": 91.49989299046987 }, "calibration": { "parts": { "choice": { "ece_hard": 0.03783700194123639, "ece_score": 92.43259961175272, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 92.43259961175272 }, "noul": { "abstention_rate": 0.21293800539083563, "brier": 0.06638921832884097, "ece": 0.06361185983827498, "n": 371, "score": 87.277628032345 }, "score": { "interval90_coverage": 0.9728997289972899, "interval90_width": 0.24954832881662145, "n": 369, "nrps": 0.05074999798842498, "score": 92.18637845047334, "top_ece": 0.05276121650105413 } }, "score": 90.63220203152369 }, "capability": 77.12647993588574, "composite_ci95": { "A": [ 69.1394876572671, 72.39570149403289 ], "B": [ 66.59085018153897, 71.36079037075339 ], "C": [ 64.35267606102977, 72.39570149403289 ] }, "cost": { "basis": "TypeSafe list tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.031780532 }, "kind": "tariff", "usage_estimated_rows": 0, "usd_per_1000": 0.03227542180094787 }, "display": "Jev 1.13.0 (TypeSafe AI)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 65.60411433522728, "I_sealed": 61.637401345268295, "base": 63.62075784024779, "excess": 1.3933253475207414, "gap": 3.9667129899589852, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 79.0195653974535, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 71.68921044353571, "judge": 84.17930125247197, "standard": 77.39640016743407 } }, "open|noul": { "cc": 54.46376811594203, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 47.82608695652173, "judge": 39.99999999999999, "standard": 66.66666666666667 } }, "open|score": { "cc": 63.32900949228633, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 76.45454545454547, "hard": 65.44837261503929, "judge": 68.91525423728812, "standard": 44.148148148148145 } }, "sealed|choice": { "cc": 75.78213638772388, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 76.05498484960461, "hard": 73.02036199095022, "judge": 75.79143389199255, "standard": 81.15531469392782 } }, "sealed|noul": { "cc": 46.14871857614774, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 80.0, "hard": 28.155339805825253, "judge": 47.36842105263157, "standard": 63.38028169014085 } }, "sealed|score": { "cc": 62.981349071933266, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 81.82978723404256, "hard": 55.9193404634581, "judge": 71.57122119397567, "standard": 54.796339024765395 } } } }, "jevbench_score": 71.49118639284514, "key": "jev-1.13.0", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 5, "ranked": true, "ranks": { "A": 5, "B": 6, "C": 5, "capability": 11 }, "scores": { "A": 71.49118639284514, "B": 69.76507971323585, "C": 71.49118639284514 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.23910215869545937, "p50_s_raw": 0.23910215869545937, "p95_s_adjusted": 0.29609236158430574, "p95_s_raw": 0.29609236158430574 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.83738964744846, 90.80253311928502 ], "I": [ 57.864503927061655, 67.65512541027901 ], "capability": [ 72.76645552638956, 78.36978036921771 ], "composite": { "A": [ 69.1394876572671, 72.39570149403289 ], "B": [ 66.59085018153897, 71.36079037075339 ], "C": [ 64.35267606102977, 72.39570149403289 ] }, "seed": 44 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": true, "g_med_used": 2.573387642438244, "lane": "api", "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.63220203152369, "capability": 77.12647993588574, "intelligence": 63.62075784024779 }, "per_type": { "choice": { "accuracy": 0.8173333333333334, "calibration": 92.43259961175272, "cc": 76.63954226386139, "cc_by_type": { "choice": 76.63954226386139 }, "cc_tiered": 77.40085089258869, "cc_tiered_open": 79.0195653974535, "cc_tiered_sealed": 75.78213638772388, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 76.63954226386139 }, "noul": { "accuracy": 0.76, "calibration": 87.277628032345, "cc": 52.0, "cc_by_type": { "noul": 52.0 }, "cc_tiered": 50.30624334604489, "cc_tiered_open": 54.46376811594203, "cc_tiered_sealed": 46.14871857614774, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 52.0 }, "score": { "accuracy": 0.752, "calibration": 92.18637845047334, "cc": 63.113849016480586, "cc_by_type": { "score": 63.113849016480586 }, "cc_tiered": 63.1551792821098, "cc_tiered_open": 63.32900949228633, "cc_tiered_sealed": 62.981349071933266, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 63.113849016480586 } }, "raw_on_A": null, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 66.78934094542959, "Emphasis on Accuracy 60:20:20": 65.48477335742311, "Emphasis on Cost 20:20:60": 61.38146766808138, "Emphasis on Speed 20:60:20": 74.8779907919869, "Intelligence only": 63.620757840247784 }, "class": "jev", "open": "no", "licence": "proprietary API", "repo": "https://docs.typesafe.ai", "gpu": null, "endpoint_condition": "the operator's hosted API", "underlying": "closed", "adapter_id": "harness/specs/jev-1.13.0.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).", "model_pin": "jev-1.13.0 (response model id on all 590 successful rows)", "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "noul_decisive": { "supported": true, "decisive_rate": 0.7870619946091644, "acc_among_decisive": 0.976027397260274, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)", "carried_metadata_from": "v1.6.0 release row" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Chinh Nguyen (ncchinh)", "axes": { "calibration": 89.97624968733336, "cost": 50.546097335562465, "intelligence": 73.40135940607429, "speed": 86.87440406834797 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05296499999999975, "ece_score": 89.40700000000005, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 89.40700000000005 }, "noul": { "abstention_rate": 0.08799999999999997, "brier": 0.07739600085333337, "ece": 0.046341333333333186, "n": 375, "score": 90.73173333333337 }, "score": { "interval90_coverage": 0.9413333333333334, "interval90_width": 0.21599999999999997, "n": 375, "nrps": 0.054102618759999964, "score": 89.79001572866669, "top_ece": 0.07504853333333314 } }, "score": 89.97624968733336 }, "capability": 81.68880454670384, "composite_ci95": { "A": [ 69.83728305350246, 72.20621881721509 ], "B": [ 69.73317863481549, 73.04885527628076 ], "C": [ 69.83728305350246, 72.20621881721509 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-31b-it list price $0.09/M input, $0.34/M output (the exact base model google/gemma-4-31B-it; listed since Apr 2026), read 2026-10-04 x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04451059729064039 }, "display": "Quyet-1.0-Large (Chinh Nguyen, Gemma-4-31B decoder, option-letter logits)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 75.23869318243646, "I_sealed": 71.56402562971212, "base": 73.40135940607429, "excess": 1.1012799102861002, "gap": 3.674667552724344, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 85.01378955328916, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 84.90091223655237, "judge": 80.22412656558997, "standard": 84.93093344495604 } }, "open|noul": { "cc": 71.3768115942029, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 82.6086956521739, "judge": 50.0, "standard": 66.66666666666667 } }, "open|score": { "cc": 69.32547839981737, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 57.654545454545456, "hard": 72.09425925925925, "judge": 68.68847457627118, "standard": 70.5788888888889 } }, "sealed|choice": { "cc": 82.59643939217574, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 80.65610859728505, "judge": 81.3780260707635, "standard": 88.91489099642814 } }, "sealed|noul": { "cc": 66.16688233643043, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 84.00000000000001, "hard": 45.63106796116505, "judge": 76.3157894736842, "standard": 83.09859154929578 } }, "sealed|score": { "cc": 65.9287551605302, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 73.00248226950355, "hard": 67.1347549019608, "judge": 65.63335329341317, "standard": 60.42299492385786 } } } }, "jevbench_score": 71.38706329141472, "key": "quyet-1-0-large", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 6, "ranked": true, "ranks": { "A": 6, "B": 2, "C": 6, "capability": 2 }, "scores": { "A": 71.38706329141472, "B": 71.78102942459306, "C": 71.38706329141472 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3815564611926675, "p50_s_raw": 0.11577823059633374, "p95_s_adjusted": 0.5382707504555583, "p95_s_raw": 0.19413537522777913 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.58390587934255, 91.370785899262 ], "I": [ 69.18945219901103, 76.93636317458929 ], "capability": [ 77.9843219931321, 83.55754626687737 ], "composite": { "A": [ 69.83728305350246, 72.20621881721509 ], "B": [ 69.73317863481549, 73.04885527628076 ], "C": [ 69.83728305350246, 72.20621881721509 ] }, "seed": 86 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.97624968733336, "capability": 81.68880454670384, "intelligence": 73.40135940607429 }, "per_type": { "choice": { "accuracy": 0.8706666666666667, "calibration": 89.40700000000005, "cc": 83.46011386565368, "cc_by_type": { "choice": 83.46011386565368 }, "cc_tiered": 83.80511447273244, "cc_tiered_open": 85.01378955328916, "cc_tiered_sealed": 82.59643939217574, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 83.46011386565368 }, "noul": { "accuracy": 0.848, "calibration": 90.73173333333337, "cc": 69.6, "cc_by_type": { "noul": 69.6 }, "cc_tiered": 68.77184696531667, "cc_tiered_open": 71.3768115942029, "cc_tiered_sealed": 66.16688233643043, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 69.6 }, "score": { "accuracy": 0.7493333333333333, "calibration": 89.79001572866669, "cc": 66.83812280701756, "cc_by_type": { "score": 66.83812280701756 }, "cc_tiered": 67.6271167801738, "cc_tiered_open": 69.32547839981737, "cc_tiered_sealed": 65.9287551605302, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 66.83812280701756 } }, "raw_on_A": { "calibration": 86.55147389375357, "capability": 81.17122174668583, "intelligence": 75.7909695996181 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 66.78760078772265, "Emphasis on Accuracy 60:20:20": 69.28473442746802, "Emphasis on Cost 20:20:60": 59.18115107083417, "Emphasis on Speed 20:60:20": 73.59406855476429, "Intelligence only": 73.40135940607429 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.912, "acc_among_decisive": 0.9298245614035088, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Large", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)", "underlying": "google/gemma-4-31B-it" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mark Marosi (Mapika)", "axes": { "calibration": 81.98845142422223, "cost": 57.755449421401664, "intelligence": 63.01250112161242, "speed": 90.47583056481872 }, "calibration": { "parts": { "choice": { "ece_hard": 0.048897499999999955, "ece_score": 90.22050000000002, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 90.22050000000002 }, "noul": { "abstention_rate": 0.013333333333333308, "brier": 0.16693694264000003, "ece": 0.1691528, "n": 375, "score": 66.16944000000001 }, "score": { "interval90_coverage": 0.952, "interval90_width": 0.33711111111111114, "n": 375, "nrps": 0.07347944788000003, "score": 89.57541427266668, "top_ece": 0.06750613333333315 } }, "score": 81.98845142422223 }, "capability": 72.50047627291733, "composite_ci95": { "A": [ 68.54138708333946, 72.20753634823356 ], "B": [ 66.06197783195658, 71.11572775091496 ], "C": [ 63.43447024505396, 72.20753634823356 ] }, "cost": { "basis": "add-requests run 46 (v1.5 basis, unpublished) documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.025594796798029562 }, "display": "decider-12b v2 (Mapika)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 62.42359203105606, "I_sealed": 63.601410212168766, "base": 63.01250112161242, "excess": -3.7512058235509493, "gap": -1.1778181811127055, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 75.47889301775673, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 71.68921044353571, "judge": 76.26895187870797, "standard": 77.39640016743407 } }, "open|noul": { "cc": 58.42028985507247, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 65.21739130434783, "judge": 30.000000000000004, "standard": 66.66666666666667 } }, "open|score": { "cc": 53.37159322033899, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 55.93666666666668, "hard": 56.19055555555556, "judge": 47.0086440677966, "standard": 55.995555555555555 } }, "sealed|choice": { "cc": 73.57708692549191, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 68.43891402714931, "judge": 77.6536312849162, "standard": 74.50424929178472 } }, "sealed|noul": { "cc": 61.47997869691321, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 87.99999999999999, "hard": 47.57281553398058, "judge": 60.52631578947369, "standard": 77.46478873239437 } }, "sealed|score": { "cc": 55.74716501410119, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 69.177304964539, "hard": 52.92137254901961, "judge": 55.79760479041915, "standard": 54.60802030456852 } } } }, "jevbench_score": 70.87955164338175, "key": "decider-12b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 7, "ranked": true, "ranks": { "A": 7, "B": 9, "C": 7, "capability": 22 }, "scores": { "A": 70.87955164338175, "B": 69.15281897628992, "C": 70.87955164338175 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.25826345253735783, "p50_s_raw": 0.054131726268678904, "p95_s_adjusted": 0.3470195889472961, "p95_s_raw": 0.09850979447364806 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 76.76796045055926, 83.41866329732498 ], "I": [ 57.71486047412692, 67.18786674520106 ], "capability": [ 68.19784274796856, 74.82646556669576 ], "composite": { "A": [ 68.54138708333946, 72.20753634823356 ], "B": [ 66.06197783195658, 71.11572775091496 ], "C": [ 63.43447024505396, 72.20753634823356 ] }, "seed": 16 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 81.98845142422223, "capability": 72.50047627291733, "intelligence": 63.01250112161242 }, "per_type": { "choice": { "accuracy": 0.7973333333333333, "calibration": 90.22050000000002, "cc": 74.08182791318927, "cc_by_type": { "choice": 74.08182791318927 }, "cc_tiered": 74.52798997162432, "cc_tiered_open": 75.47889301775673, "cc_tiered_sealed": 73.57708692549191, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 74.08182791318927 }, "noul": { "accuracy": 0.8213333333333334, "calibration": 66.16944000000001, "cc": 64.26666666666667, "cc_by_type": { "noul": 64.26666666666667 }, "cc_tiered": 59.95013427599284, "cc_tiered_open": 58.42028985507247, "cc_tiered_sealed": 61.47997869691321, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 64.26666666666667 }, "score": { "accuracy": 0.6506666666666666, "calibration": 89.57541427266668, "cc": 55.808578947368446, "cc_by_type": { "score": 55.808578947368446 }, "cc_tiered": 54.55937911722009, "cc_tiered_open": 53.37159322033899, "cc_tiered_sealed": 55.74716501410119, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 55.808578947368446 } }, "raw_on_A": { "calibration": 81.87902912425925, "capability": 73.03621970023784, "intelligence": 64.19341027621644 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 67.81664735288307, "Emphasis on Accuracy 60:20:20": 65.80968465261019, "Emphasis on Cost 20:20:60": 63.39892316578739, "Emphasis on Speed 20:60:20": 75.36672286262451, "Intelligence only": 63.01250112161242 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/Mapika/decider-12b", "gpu": "RTX PRO6000", "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P83 on one evaluator-owned Lium RTX PRO6000. Offline contributor image, serial loopback-only requests, no batching or retry; standard selfhosted x2+0.15 s latency adjustment. Current stage receipt verifies the same model pin used by the prior add-request run. Cost is the carried documented estimate recorded in current registry.", "underlying": "google/gemma-4-12B-it", "model_pin": "Mapika/decider-12b@8ac1efa708b71b86ae33b01d2a8d7a3ddcb48e66", "last_measured_on": "2026-10-04", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.9866666666666667, "acc_among_decisive": 0.8324324324324325, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "run 46 author metadata + current P83 staged pin/measurement receipt" } }, { "addendum": null, "api_flag": true, "api_subset": false, "author": "Wity (alphanimble)", "axes": { "calibration": 87.9497550095679, "cost": 58.8349720563948, "intelligence": 70.28143483604077, "speed": 71.79000307789984 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09748974871312502, "ece_score": 80.502050257375, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 80.502050257375 }, "noul": { "abstention_rate": 0.09333333333333338, "brier": 0.06247285161168309, "ece": 0.035695432538436114, "n": 375, "score": 92.86091349231278 }, "score": { "interval90_coverage": 0.9466666666666667, "interval90_width": 0.19822222222222222, "n": 375, "nrps": 0.052696939614440746, "score": 90.48630127901592, "top_ece": 0.06878851740262039 } }, "score": 87.9497550095679 }, "capability": 79.11559492280433, "composite_ci95": { "A": [ 68.86577835236345, 71.94201428005243 ], "B": [ 67.99651553112257, 72.43020923475851 ], "C": [ 68.86577835236345, 71.94201428005243 ] }, "cost": { "basis": "Wity's stated API tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.074026736 }, "kind": "tariff", "usage_estimated_rows": 0, "usd_per_1000": 0.023559582938388626 }, "display": "wity-1 (Wity, reasoning auto)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 68.30373826167562, "I_sealed": 72.25913141040593, "base": 70.28143483604077, "excess": -6.528780791168552, "gap": -3.955393148730309, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 82.78833309663467, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 73.57659641396666, "judge": 84.17930125247197, "standard": 92.46546672247803 } }, "open|noul": { "cc": 58.2415458937198, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 47.82608695652173, "judge": 60.00000000000001, "standard": 55.55555555555556 } }, "open|score": { "cc": 63.8813357946724, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 81.26135069496807, "hard": 66.56206622564854, "judge": 62.42171848967287, "standard": 52.01929344007156 } }, "sealed|choice": { "cc": 77.28423947454772, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 67.92986425339366, "judge": 81.3780260707635, "standard": 87.80638009607095 } }, "sealed|noul": { "cc": 69.78487192958465, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 87.99999999999999, "hard": 53.398058252427184, "judge": 71.05263157894737, "standard": 91.54929577464787 } }, "sealed|score": { "cc": 69.70828282708541, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 88.63984239469578, "hard": 66.09013140977517, "judge": 70.29595017077045, "standard": 66.59730486237312 } } } }, "jevbench_score": 70.76534010331063, "key": "wity-1", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 8, "ranked": true, "ranks": { "A": 8, "B": 5, "C": 8, "capability": 5 }, "scores": { "A": 70.76534010331063, "B": 70.66802669351945, "C": 70.76534010331063 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 1.5746641159057617, "p50_s_raw": 1.5746641159057617, "p95_s_adjusted": 4.205443101748823, "p95_s_raw": 4.205443101748823 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.84467593779125, 90.08714023528091 ], "I": [ 64.90155627554498, 74.72711946846094 ], "capability": [ 74.9632364228138, 81.81203862791727 ], "composite": { "A": [ 68.86577835236345, 71.94201428005243 ], "B": [ 67.99651553112257, 72.43020923475851 ], "C": [ 68.86577835236345, 71.94201428005243 ] }, "seed": 111 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": true, "g_med_used": 2.573387642438244, "lane": "api", "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 87.9497550095679, "capability": 79.11559492280433, "intelligence": 70.28143483604077 }, "per_type": { "choice": { "accuracy": 0.8293333333333334, "calibration": 80.502050257375, "cc": 78.17417087426465, "cc_by_type": { "choice": 78.17417087426465 }, "cc_tiered": 80.03628628559119, "cc_tiered_open": 82.78833309663467, "cc_tiered_sealed": 77.28423947454772, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 78.17417087426465 }, "noul": { "accuracy": 0.8533333333333334, "calibration": 92.86091349231278, "cc": 70.66666666666667, "cc_by_type": { "noul": 70.66666666666667 }, "cc_tiered": 64.01320891165223, "cc_tiered_open": 58.2415458937198, "cc_tiered_sealed": 69.78487192958465, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 70.66666666666667 }, "score": { "accuracy": 0.7973333333333333, "calibration": 90.48630127901592, "cc": 69.54369663887998, "cc_by_type": { "score": 69.54369663887998 }, "cc_tiered": 66.7948093108789, "cc_tiered_open": 63.8813357946724, "cc_tiered_sealed": 69.70828282708541, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 69.54369663887998 } }, "raw_on_A": null, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 66.4382396447495, "Emphasis on Accuracy 60:20:20": 67.92395209017961, "Emphasis on Cost 20:20:60": 63.172705766286356, "Emphasis on Speed 20:60:20": 68.48025157142519, "Intelligence only": 70.28143483604077 }, "model_pin": null, "last_measured_on": "2026-10-06", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "noul_decisive": { "supported": true, "decisive_rate": 0.9066666666666666, "acc_among_decisive": 0.9411764705882353, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)", "carried_metadata_from": "v1.6.0 release row" }, "class": "decision-api", "open": "no", "licence": "proprietary hosted API", "repo": "https://wity.alphanimble.com/", "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=auto as the operator asked for their main row.", "underlying": "Base model: undisclosed (author request)", "adapter_id": "run/wity-1-auto.spec.json:wity_reasoning_adapter.WityReasoningAdapter", "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).", "alt": { "label": "Base-model reference price (base undisclosed at the author's request)", "note": "base model undisclosed at the author's request; frozen base-model reference price applied to the same measured tokens", "usd_per_1000": 0.08721059113300492, "axes": { "calibration": 90.45687167393245, "cost": 41.78292309262718, "intelligence": 78.56795761907267, "speed": 75.05991709119236 } } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mark Marosi (Mapika)", "axes": { "calibration": 83.7629373406296, "cost": 57.755449421401664, "intelligence": 60.62893554403879, "speed": 90.47654290454523 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0649509375000001, "ece_score": 87.00981249999998, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 87.00981249999998 }, "noul": { "abstention_rate": 0.040000000000000036, "brier": 0.1306117199200001, "ece": 0.12608400000000003, "n": 375, "score": 74.7832 }, "score": { "interval90_coverage": 0.9893333333333333, "interval90_width": 0.493333333333333, "n": 375, "nrps": 0.07613894289555556, "score": 89.49579952188888, "top_ece": 0.06697253333333336 } }, "score": 83.7629373406296 }, "capability": 72.1959364423342, "composite_ci95": { "A": [ 68.08779334144351, 71.63200792040908 ], "B": [ 65.27927398150561, 70.07926650409995 ], "C": [ 58.91555277575465, 71.63200792040908 ] }, "cost": { "basis": "add-requests run 46 (v1.5 basis, unpublished) documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.025594796798029562 }, "display": "decider-12b v1, stock Gemma-4-12B-it (Mapika)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 61.78178227747658, "I_sealed": 59.476088810601, "base": 60.62893554403879, "excess": -0.2676941755626623, "gap": 2.3056934668755815, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 69.97472688448558, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 66.02705253224286, "judge": 60.44825313117996, "standard": 84.93093344495604 } }, "open|noul": { "cc": 73.89855072463769, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 73.91304347826086, "judge": 70.0, "standard": 66.66666666666667 } }, "open|score": { "cc": 41.4720692233065, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 53.67363636363637, "hard": 39.991944444444435, "judge": 43.4459322033898, "standard": 35.370740740740736 } }, "sealed|choice": { "cc": 66.65509953327584, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 62.839366515837106, "judge": 66.4804469273743, "standard": 67.8531838896416 } }, "sealed|noul": { "cc": 60.08887273564741, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 87.99999999999999, "hard": 35.92233009708738, "judge": 65.78947368421053, "standard": 85.91549295774648 } }, "sealed|score": { "cc": 51.684294162879745, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 65.59943262411348, "hard": 48.76892156862744, "judge": 54.09401197604791, "standard": 46.94289340101522 } } } }, "jevbench_score": 70.42352588714175, "key": "decider-12b-v1", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 9, "ranked": true, "ranks": { "A": 9, "B": 10, "C": 9, "capability": 25 }, "scores": { "A": 70.42352588714175, "B": 68.21936151093459, "C": 70.42352588714175 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.25808657668530943, "p50_s_raw": 0.054043288342654705, "p95_s_adjusted": 0.3472004604525864, "p95_s_raw": 0.0986002302262932 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.60323827570184, 85.08662764834355 ], "I": [ 55.85565047982421, 64.5542584209754 ], "capability": [ 67.75349807475966, 74.15899513794021 ], "composite": { "A": [ 68.08779334144351, 71.63200792040908 ], "B": [ 65.27927398150561, 70.07926650409995 ], "C": [ 58.91555277575465, 71.63200792040908 ] }, "seed": 17 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.7629373406296, "capability": 72.1959364423342, "intelligence": 60.62893554403879 }, "per_type": { "choice": { "accuracy": 0.7506666666666667, "calibration": 87.00981249999998, "cc": 68.11382776162101, "cc_by_type": { "choice": 68.11382776162101 }, "cc_tiered": 68.31491320888071, "cc_tiered_open": 69.97472688448558, "cc_tiered_sealed": 66.65509953327584, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 68.11382776162101 }, "noul": { "accuracy": 0.832, "calibration": 74.7832, "cc": 66.39999999999999, "cc_by_type": { "noul": 66.39999999999999 }, "cc_tiered": 66.99371173014255, "cc_tiered_open": 73.89855072463769, "cc_tiered_sealed": 60.08887273564741, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 66.39999999999999 }, "score": { "accuracy": 0.6453333333333333, "calibration": 89.49579952188888, "cc": 50.03278070175441, "cc_by_type": { "score": 50.03278070175441 }, "cc_tiered": 46.57818169309312, "cc_tiered_open": 41.4720692233065, "cc_tiered_sealed": 51.684294162879745, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 50.03278070175441 } }, "raw_on_A": { "calibration": 81.0648832518091, "capability": 71.2842608897069, "intelligence": 61.5036385276047 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 66.87360828773896, "Emphasis on Accuracy 60:20:20": 64.22748164585495, "Emphasis on Cost 20:20:60": 62.90137869162178, "Emphasis on Speed 20:60:20": 74.66484030090665, "Intelligence only": 60.62893554403879 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/Mapika/decider-12b", "gpu": "RTX PRO6000", "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P83 on one evaluator-owned Lium RTX PRO6000. Offline contributor image, serial loopback-only requests, no batching or retry; standard selfhosted x2+0.15 s latency adjustment. Current stage receipt verifies the same model pin used by the prior add-request run. Cost is the carried documented estimate recorded in current registry.", "underlying": "google/gemma-4-12B-it", "model_pin": "Mapika/decider-12b@09e9e3387fa42fed740233be2290385d18a6044d", "last_measured_on": "2026-10-04", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.96, "acc_among_decisive": 0.8666666666666667, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "run 46 author metadata + current P83 staged pin/measurement receipt" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Torchcast AI", "axes": { "calibration": 82.93469131589123, "cost": 56.36034169226667, "intelligence": 60.54320091058725, "speed": 91.74573681567192 }, "calibration": { "parts": { "choice": { "ece_hard": 0.07394265024295099, "ece_score": 85.2114699514098, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 85.2114699514098 }, "noul": { "abstention_rate": 0.035040431266846306, "brier": 0.1393319820198383, "ece": 0.13485522993675053, "n": 371, "score": 73.02895401264989 }, "score": { "interval90_coverage": 0.964769647696477, "interval90_width": 0.33220415537488723, "n": 369, "nrps": 0.06922052747100878, "score": 90.56364998361398, "top_ece": 0.05975323642835579 } }, "score": 82.93469131589123 }, "capability": 71.73894611323924, "composite_ci95": { "A": [ 67.22388654747238, 71.4745219700647 ], "B": [ 64.35805126511531, 70.01822593897899 ], "C": [ 56.59214410663297, 71.4745219700647 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; no public tariff, customer asked for the base-model reference): OpenRouter google/gemma-3-12b-it list price $0.05/M input, $0.15/M output (gemma-4-12b-it is not listed on OpenRouter, read 2026-10-04; same reference as the Cygnet and Jev-Omni Gemma-4-12B rows) x the shim's own measured usage.prompt_tokens/completion_tokens; self-hosted on 1x H100 80GB (Lium), vLLM 0.30.0 BF16; estimated, not charged", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.02848756157635468 }, "display": "torchcast-decision-12b (Torchcast AI, Gemma-4-12B fine-tune, option-letter logprob readout)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 59.97536524790964, "I_sealed": 61.11103657326486, "base": 60.54320091058725, "excess": -3.7090589677934624, "gap": -1.1356713253552186, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 71.98506861847196, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 67.9144385026738, "judge": 68.35860250494396, "standard": 77.39640016743407 } }, "open|noul": { "cc": 56.49758454106281, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 56.52173913043479, "judge": 50.0, "standard": 44.44444444444444 } }, "open|score": { "cc": 51.44344258419417, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.22709032709656, "hard": 57.48378737898936, "judge": 45.79568230097426, "standard": 44.44256954798248 } }, "sealed|choice": { "cc": 68.40340626918382, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 60.80316742081447, "judge": 72.06703910614526, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 62.37236788127848, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": 45.63106796116505, "judge": 65.78947368421053, "standard": 85.91549295774648 } }, "sealed|score": { "cc": 52.557335569332295, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 69.05633413996496, "hard": 44.66535408470005, "judge": 55.981937871697106, "standard": 54.95489579973323 } } } }, "jevbench_score": 69.9087143535159, "key": "torchcast-decision-12b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 10, "ranked": true, "ranks": { "A": 10, "B": 11, "C": 10, "capability": 26 }, "scores": { "A": 69.9087143535159, "B": 67.81076568849952, "C": 69.9087143535159 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2243030747398734, "p50_s_raw": 0.037151537369936705, "p95_s_adjusted": 0.29825730649754406, "p95_s_raw": 0.07412865324877203 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.04707649811247, 84.79039923447887 ], "I": [ 55.00644510399965, 65.05987282834745 ], "capability": [ 67.02511371513799, 74.5041047187016 ], "composite": { "A": [ 67.22388654747238, 71.4745219700647 ], "B": [ 64.35805126511531, 70.01822593897899 ], "C": [ 56.59214410663297, 71.4745219700647 ] }, "seed": 106 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.93469131589123, "capability": 71.73894611323924, "intelligence": 60.54320091058725 }, "per_type": { "choice": { "accuracy": 0.7626666666666667, "calibration": 85.2114699514098, "cc": 69.64845637202428, "cc_by_type": { "choice": 69.64845637202428 }, "cc_tiered": 70.19423744382789, "cc_tiered_open": 71.98506861847196, "cc_tiered_sealed": 68.40340626918382, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 69.64845637202428 }, "noul": { "accuracy": 0.8186666666666667, "calibration": 73.02895401264989, "cc": 63.733333333333334, "cc_by_type": { "noul": 63.733333333333334 }, "cc_tiered": 59.43497621117064, "cc_tiered_open": 56.49758454106281, "cc_tiered_sealed": 62.37236788127848, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 63.733333333333334 }, "score": { "accuracy": 0.6666666666666666, "calibration": 90.56364998361398, "cc": 52.778195902361034, "cc_by_type": { "score": 52.778195902361034 }, "cc_tiered": 52.000389076763234, "cc_tiered_open": 51.44344258419417, "cc_tiered_sealed": 52.557335569332295, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 52.778195902361034 } }, "raw_on_A": { "calibration": 79.9965213222889, "capability": 69.86117451490037, "intelligence": 59.725827707511854 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 66.43077398924878, "Emphasis on Accuracy 60:20:20": 63.94347960496841, "Emphasis on Cost 20:20:60": 61.999556455891316, "Emphasis on Speed 20:60:20": 74.67237011934479, "Intelligence only": 60.54320091058725 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9649595687331537, "acc_among_decisive": 0.8575418994413407, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "cc-by-nc-4.0", "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-12b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order b113eac1)", "underlying": "google/gemma-4-12B-it" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Eldan Ring", "axes": { "calibration": 82.95806123333315, "cost": 56.556023424877196, "intelligence": 59.53366229166759, "speed": 86.69753831262085 }, "calibration": { "parts": { "choice": { "ece_hard": 0.11234009752278087, "ece_score": 77.53198049544383, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 77.53198049544383 }, "noul": { "abstention_rate": 0.14016172506738545, "brier": 0.09737227728287662, "ece": 0.07944785238178745, "n": 371, "score": 84.11042952364252 }, "score": { "interval90_coverage": 0.9214092140921409, "interval90_width": 0.2443541102077688, "n": 369, "nrps": 0.06941649910523198, "score": 87.23177368091314, "top_ece": 0.09297401363825267 } }, "score": 82.95806123333315 }, "capability": 71.24586176250037, "composite_ci95": { "A": [ 66.45196246998344, 70.4618307059656 ], "B": [ 63.55759159512183, 69.03636277102011 ], "C": [ 53.88529001998626, 70.4618307059656 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.028062900246305422 }, "display": "Winnow-12B Q8", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 59.81914414395652, "I_sealed": 59.24818043937866, "base": 59.53366229166759, "excess": -2.0024239378603803, "gap": 0.5709637045778635, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 70.47816196296756, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 67.9144385026738, "judge": 68.35860250494396, "standard": 69.8618668899121 } }, "open|noul": { "cc": 50.49758454106281, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 56.52173913043479, "judge": 30.000000000000004, "standard": 44.44444444444444 } }, "open|score": { "cc": 58.48168592783921, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 61.310580174513184, "hard": 65.34102213505747, "judge": 48.44185240673705, "standard": 58.40831667171895 } }, "sealed|choice": { "cc": 69.74580750216904, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 62.33031674208144, "judge": 76.72253258845438, "standard": 68.96169478999877 } }, "sealed|noul": { "cc": 52.512988405651086, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 28.155339805825253, "judge": 60.52631578947369, "standard": 77.46478873239437 } }, "sealed|score": { "cc": 55.48574541031586, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 72.39586027967081, "hard": 51.82211180030672, "judge": 58.61411366152696, "standard": 49.665402818839944 } } } }, "jevbench_score": 68.88542163459404, "key": "winnow-12b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 11, "ranked": true, "ranks": { "A": 11, "B": 13, "C": 12, "capability": 30 }, "scores": { "A": 68.88542163459404, "B": 66.78718784915615, "C": 67.81878724910808 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.38378054890781643, "p50_s_raw": 0.1168902744539082, "p95_s_adjusted": 0.5573951802216469, "p95_s_raw": 0.20369759011082347 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.28501959511246, 85.05351182481425 ], "I": [ 54.0544167472914, 64.20476080861759 ], "capability": [ 66.87367358483415, 74.02873882700322 ], "composite": { "A": [ 66.45196246998344, 70.4618307059656 ], "B": [ 63.55759159512183, 69.03636277102011 ], "C": [ 53.88529001998626, 70.4618307059656 ] }, "seed": 109 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.95806123333315, "capability": 71.24586176250037, "intelligence": 59.53366229166759 }, "per_type": { "choice": { "accuracy": 0.768, "calibration": 77.53198049544383, "cc": 70.33051353220351, "cc_by_type": { "choice": 70.33051353220351 }, "cc_tiered": 70.1119847325683, "cc_tiered_open": 70.47816196296756, "cc_tiered_sealed": 69.74580750216904, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 70.33051353220351 }, "noul": { "accuracy": 0.7786666666666666, "calibration": 84.11042952364252, "cc": 55.73333333333332, "cc_by_type": { "noul": 55.73333333333332 }, "cc_tiered": 51.505286473356946, "cc_tiered_open": 50.49758454106281, "cc_tiered_sealed": 52.512988405651086, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 55.73333333333332 }, "score": { "accuracy": 0.704, "calibration": 87.23177368091314, "cc": 56.525484762477454, "cc_by_type": { "score": 56.525484762477454 }, "cc_tiered": 56.98371566907753, "cc_tiered_open": 58.48168592783921, "cc_tiered_sealed": 55.48574541031586, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 56.525484762477454 } }, "raw_on_A": { "calibration": 80.53670554364732, "capability": 70.27835744009714, "intelligence": 60.02000933654696 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 65.19874575043826, "Emphasis on Accuracy 60:20:20": 62.8080780809742, "Emphasis on Cost 20:20:60": 61.44293550575936, "Emphasis on Speed 20:60:20": 72.37789977811173, "Intelligence only": 59.53366229166759 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0, including the applicable Gemma 4 base/derivative licence terms", "repo": "https://huggingface.co/EldanRing/Winnow-12B", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "google/gemma-4-12B-it LoRA fine-tune, merged and exported as Q8_0 GGUF", "adapter_id": "images/winnow-12b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8598382749326146, "acc_among_decisive": 0.9153605015673981, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "krishna765", "axes": { "calibration": 82.15636027953371, "cost": 49.69190284016272, "intelligence": 73.02512554551848, "speed": 86.69970536196755 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10701823999728471, "ece_score": 78.59635200054306, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 78.59635200054306 }, "noul": { "abstention_rate": 0.037735849056603765, "brier": 0.059638464379526884, "ece": 0.052855966211929055, "n": 371, "score": 89.42880675761418 }, "score": { "interval90_coverage": 0.8482384823848238, "interval90_width": 0.08897922312556458, "n": 369, "nrps": 0.06373815398690165, "score": 78.44392208044394, "top_ece": 0.18369170220210976 } }, "score": 82.15636027953371 }, "capability": 77.5907429125261, "composite_ci95": { "A": [ 67.07103330487536, 69.73758073723451 ], "B": [ 67.10121070795465, 70.84803238839223 ], "C": [ 67.07103330487536, 69.73758073723451 ] }, "cost": { "basis": "ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-31b-it list price $0.09/M input, $0.34/M output (the exact frozen base model google/gemma-4-31B-it; same reference as Quyet-1.0-Large), read 2026-10-04 x the server's own measured usage.input_tokens; letter-mass readout, nothing generated; TorchAO FP8 dynamic on 1x GPU (Lium), deck31b 1.0.0; estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.04679784 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04752658090724441 }, "display": "deck-31B (krishna765, frozen Gemma-4-31B-it, TorchAO FP8 dynamic)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 70.23424531362603, "I_sealed": 75.81600577741091, "base": 73.02512554551848, "excess": -8.155148106223118, "gap": -5.581760463784875, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 72.33184897895812, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 69.80182447310474, "judge": 64.40342781806196, "standard": 77.39640016743407 } }, "open|noul": { "cc": 77.39268461007592, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 82.6086956521739, "judge": 60.00000000000001, "standard": 88.88888888888889 } }, "open|score": { "cc": 60.978202351844075, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 73.27912483548366, "hard": 65.74514134257689, "judge": 59.223722663075364, "standard": 47.92558266171173 } }, "sealed|choice": { "cc": 83.85398241235396, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 90.68804966373514, "hard": 76.5837104072398, "judge": 87.89571694599628, "standard": 88.91489099642814 } }, "sealed|noul": { "cc": 78.05289930692999, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 96.0, "hard": 66.99029126213591, "judge": 81.57894736842107, "standard": 85.91549295774648 } }, "sealed|score": { "cc": 65.54113561294878, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 75.42107738087628, "hard": 61.37893873388316, "judge": 71.30554200754482, "standard": 60.278948895222165 } } } }, "jevbench_score": 68.68200118286292, "key": "deck31b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 12, "ranked": true, "ranks": { "A": 12, "B": 8, "C": 11, "capability": 9 }, "scores": { "A": 68.68200118286292, "B": 69.34459478009123, "C": 68.68200118286292 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4049364184960723, "p50_s_raw": 0.12746820924803615, "p95_s_adjusted": 0.5280105819925666, "p95_s_raw": 0.18900529099628327 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.16081966059086, 84.92278583205757 ], "I": [ 68.03652261338961, 76.64977164564989 ], "capability": [ 73.77453269228936, 80.24074177369353 ], "composite": { "A": [ 67.70893786128632, 70.4047103972742 ], "B": [ 67.72073941714547, 71.50656471292106 ], "C": [ 67.70893786128632, 70.4047103972742 ] }, "seed": 34 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.15636027953371, "capability": 77.5907429125261, "intelligence": 73.02512554551848 }, "per_type": { "choice": { "accuracy": 0.86, "calibration": 78.59635200054306, "cc": 82.09599954529521, "cc_by_type": { "choice": 82.09599954529521 }, "cc_tiered": 78.09291569565605, "cc_tiered_open": 72.33184897895812, "cc_tiered_sealed": 83.85398241235396, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 82.09599954529521 }, "noul": { "accuracy": 0.8986666666666666, "calibration": 89.42880675761418, "cc": 79.73333333333332, "cc_by_type": { "noul": 79.73333333333332 }, "cc_tiered": 77.72279195850295, "cc_tiered_open": 77.39268461007592, "cc_tiered_sealed": 78.05289930692999, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 79.73333333333332 }, "score": { "accuracy": 0.72, "calibration": 78.44392208044394, "cc": 64.7077492239505, "cc_by_type": { "score": 64.7077492239505 }, "cc_tiered": 63.25966898239643, "cc_tiered_open": 60.978202351844075, "cc_tiered_sealed": 65.54113561294878, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 64.7077492239505 } }, "raw_on_A": { "calibration": 79.57362236322788, "capability": 77.19870727297797, "intelligence": 74.82379218272806 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 65.98518077730212, "Emphasis on Accuracy 60:20:20": 68.52137025063348, "Emphasis on Cost 20:20:60": 58.32800868887079, "Emphasis on Speed 20:60:20": 72.85260809439177, "Intelligence only": 72.71357044848186 }, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9622641509433962, "acc_among_decisive": 0.9439775910364145, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": null }, "repo": "https://github.com/krishna-gogineni-765/deck31b" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "blockbrain", "axes": { "calibration": 86.96753988280044, "cost": 56.42912569481113, "intelligence": 54.83928122751086, "speed": 91.77283664982562 }, "calibration": { "parts": { "choice": { "ece_hard": 0.08631856803856107, "ece_score": 82.73628639228778, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 82.73628639228778 }, "noul": { "abstention_rate": 0.18867924528301883, "brier": 0.1059190859395021, "ece": 0.061302788041812155, "n": 371, "score": 87.73944239163757 }, "score": { "interval90_coverage": 0.978319783197832, "interval90_width": 0.46589882565492324, "n": 369, "nrps": 0.07366857377747793, "score": 90.42689086447598, "top_ece": 0.05889680446650118 } }, "score": 86.96753988280044 }, "capability": 70.90341055515565, "composite_ci95": { "A": [ 64.42215454949937, 69.93947552798573 ], "B": [ 60.49166104427309, 67.44884553134193 ], "C": [ 44.73760732604123, 67.5344744697005 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.028337561576354687 }, "display": "Cygnet (blockbrain, frozen Gemma-4-12B-it)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 58.329951877214064, "I_sealed": 51.348610577807655, "base": 54.83928122751086, "excess": 4.407953656968166, "gap": 6.9813412994064095, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 72.62577711735155, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 67.9144385026738, "judge": 60.44825313117996, "standard": 92.46546672247803 } }, "open|noul": { "cc": 57.7536231884058, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 65.21739130434783, "judge": 50.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 44.61045532588486, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 51.78845800105967, "hard": 46.33324418399161, "judge": 48.54954649888648, "standard": 31.66723951258148 } }, "sealed|choice": { "cc": 65.49755292537436, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 84.03665656640308, "hard": 56.730769230769226, "judge": 66.4804469273743, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 39.63579350399792, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 12.621359223300965, "judge": 44.736842105263165, "standard": 71.83098591549295 } }, "sealed|score": { "cc": 48.9124853040507, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 68.44540243240824, "hard": 39.569328601993966, "judge": 55.630472156586585, "standard": 47.75535986518157 } } } }, "jevbench_score": 68.55012424151383, "key": "cygnet", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 13, "ranked": true, "ranks": { "A": 13, "B": 15, "C": 18, "capability": 31 }, "scores": { "A": 68.55012424151383, "B": 65.28560070195185, "C": 57.26499844882472 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.22363614114001393, "p50_s_raw": 0.03681807057000697, "p95_s_adjusted": 0.29728592303581536, "p95_s_raw": 0.07364296151790768 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.02033013748918, 88.42190039096631 ], "I": [ 49.43932563439576, 59.02934294021137 ], "capability": [ 66.3438265961016, 72.89464887942813 ], "composite": { "A": [ 64.42215454949937, 69.93947552798573 ], "B": [ 60.49166104427309, 67.44884553134193 ], "C": [ 44.73760732604123, 67.5344744697005 ] }, "seed": 23 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.96753988280044, "capability": 70.90341055515565, "intelligence": 54.83928122751086 }, "per_type": { "choice": { "accuracy": 0.748, "calibration": 82.73628639228778, "cc": 67.7727991815314, "cc_by_type": { "choice": 67.7727991815314 }, "cc_tiered": 69.06166502136296, "cc_tiered_open": 72.62577711735155, "cc_tiered_sealed": 65.49755292537436, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 67.7727991815314 }, "noul": { "accuracy": 0.736, "calibration": 87.73944239163757, "cc": 47.199999999999996, "cc_by_type": { "noul": 47.199999999999996 }, "cc_tiered": 48.69470834620186, "cc_tiered_open": 57.7536231884058, "cc_tiered_sealed": 39.63579350399792, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 47.199999999999996 }, "score": { "accuracy": 0.6133333333333333, "calibration": 90.42689086447598, "cc": 48.18037785294212, "cc_by_type": { "score": 48.18037785294212 }, "cc_tiered": 46.76147031496778, "cc_tiered_open": 44.61045532588486, "cc_tiered_sealed": 48.9124853040507, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 48.18037785294212 } }, "raw_on_A": { "calibration": 83.24254074674259, "capability": 69.54334852201868, "intelligence": 55.844156297294745 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 64.03016157894142, "Emphasis on Accuracy 60:20:20": 60.00734907512389, "Emphasis on Cost 20:20:60": 60.756582833038394, "Emphasis on Speed 20:60:20": 72.83760421908612, "Intelligence only": 54.83928122751087 }, "class": "system-one-open", "open": "yes", "licence": "shim MIT; weights Apache-2.0 with Google's Gemma Prohibited Use Policy", "repo": "https://github.com/blockbrain-ai/cygnet-recipe", "gpu": "RTXPRO6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "underlying": null, "adapter_id": "images/cygnet/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8113207547169812, "acc_among_decisive": 0.9169435215946844, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Amin Roudaki (@aminry)", "axes": { "calibration": 89.15794375860679, "cost": 59.34583145470655, "intelligence": 52.324988528776075, "speed": 86.9860428376769 }, "calibration": { "parts": { "choice": { "ece_hard": 0.07110019971983926, "ece_score": 85.77996005603214, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 85.77996005603214 }, "noul": { "abstention_rate": 0.18328840970350402, "brier": 0.10602428963864989, "ece": 0.056082845451344523, "n": 371, "score": 88.7834309097311 }, "score": { "interval90_coverage": 0.986449864498645, "interval90_width": 0.49186991869918684, "n": 369, "nrps": 0.073791213570342, "score": 92.91044031005717, "top_ece": 0.03399999011425735 } }, "score": 89.15794375860679 }, "capability": 70.74146614369144, "composite_ci95": { "A": [ 57.374888808052155, 69.64960281748938 ], "B": [ 53.18263176533343, 66.57834556173977 ], "C": [ 39.84367278336956, 61.696012278542455 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): the exact base google/gemma-4-12B-it is in the frozen pricing module's UNLISTED_BASE_MODELS set, so no exact base-model rate and no floor exist. The reference is the one the rows PUBLISHED on this same base use - cygnet, jev-omni and winnow-12b - OpenRouter google/gemma-3-12b-it, the nearest publicly hosted 12B Gemma sibling, at its FULL catalog pair USD 0.05 per 1M input and USD 0.15 per 1M output. PREREG-R54-AMENDMENT-1 uses the full pair rather than run 53's output=0, so a row that emits tokens pays for them; where a row generates nothing the output term is zero anyway. x this row's own measured input/output tokens, the deck31b method. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.022306333333333338 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.022653689911983754 }, "display": "decisio v0.8.0 on gemma-4-12B-it (frozen, one forward pass, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 53.076253329484096, "I_sealed": 51.57372372806806, "base": 52.324988528776075, "excess": -1.0708580410222077, "gap": 1.502529601416036, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 72.23658792816236, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 67.9144385026738, "judge": 60.44825313117996, "standard": 92.46546672247803 } }, "open|noul": { "cc": 43.54106280193237, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 39.13043478260869, "judge": 30.000000000000004, "standard": 44.44444444444444 } }, "open|score": { "cc": 43.45110925835756, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 52.14556472904645, "hard": 43.95984283175309, "judge": 46.337851474151414, "standard": 33.75630105253128 } }, "sealed|choice": { "cc": 65.8737458551212, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 85.3669351858695, "hard": 57.74886877828053, "judge": 67.41154562383613, "standard": 70.07020569035596 } }, "sealed|noul": { "cc": 40.79862969333631, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 8.737864077669899, "judge": 47.36842105263157, "standard": 77.46478873239437 } }, "sealed|score": { "cc": 48.04879563574668, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 67.31002438791882, "hard": 37.90772388734433, "judge": 56.21653541756339, "standard": 46.44871508374021 } } } }, "jevbench_score": 68.17346094077045, "key": "decisio-gemma-4-12b-v080", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 14, "ranked": true, "ranks": { "A": 14, "B": 19, "C": 21, "capability": 32 }, "scores": { "A": 68.17346094077045, "B": 64.2795916317687, "C": 51.847894540473604 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3548810470150784, "p50_s_raw": 0.10244052350753918, "p95_s_adjusted": 0.564043904747814, "p95_s_raw": 0.20702195237390697 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.58456964369535, 89.43375634811495 ], "I": [ 46.8581794615679, 56.47360933655171 ], "capability": [ 66.07776568391779, 72.37388803479945 ], "composite": { "A": [ 57.374888808052155, 69.64960281748938 ], "B": [ 53.18263176533343, 66.57834556173977 ], "C": [ 39.84367278336956, 61.696012278542455 ] }, "seed": 39 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.15794375860679, "capability": 70.74146614369144, "intelligence": 52.324988528776075 }, "per_type": { "choice": { "accuracy": 0.7493333333333333, "calibration": 85.77996005603214, "cc": 67.9433134715762, "cc_by_type": { "choice": 67.9433134715762 }, "cc_tiered": 69.05516689164179, "cc_tiered_open": 72.23658792816236, "cc_tiered_sealed": 65.8737458551212, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 67.9433134715762 }, "noul": { "accuracy": 0.7333333333333333, "calibration": 88.7834309097311, "cc": 46.66666666666666, "cc_by_type": { "noul": 46.66666666666666 }, "cc_tiered": 42.16984624763434, "cc_tiered_open": 43.54106280193237, "cc_tiered_sealed": 40.79862969333631, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 46.66666666666666 }, "score": { "accuracy": 0.616, "calibration": 92.91044031005717, "cc": 47.02582727916151, "cc_by_type": { "score": 47.02582727916151 }, "cc_tiered": 45.749952447052124, "cc_tiered_open": 43.45110925835756, "cc_tiered_sealed": 48.04879563574668, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 47.02582727916151 } }, "raw_on_A": { "calibration": 85.4557717128137, "capability": 69.50074934822587, "intelligence": 53.54572698363802 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 63.21404479691977, "Emphasis on Accuracy 60:20:20": 58.35636046255001, "Emphasis on Cost 20:20:60": 61.60778501333468, "Emphasis on Speed 20:60:20": 70.97231847224265, "Intelligence only": 52.324988528776075 }, "class": "Jev-compatible decision readout on frozen base", "open": "open weights", "licence": "Apache-2.0 software; Gemma model terms", "repo": "https://github.com/aminry/decisio", "gpu": "RTX 6000 Ada", "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. github.com/aminry/decisio tag v0.8.0 (5b42101a191d222062a795194b0aedb5003bf6ea, verified against the GitHub refs API), installed with their own `uv sync --extra serve --frozen` so their lockfile fixes every version, and served with their own `python -m decisio.serve.vllm_engine --base gemma-4-12b`. --base carries google/gemma-4-12B-it@707f0a3b (bf16, 22.8 GiB) and that profile's measured settings - temperature 3.592 for every question type, a system turn, the chat template's own answer position, every single-token form of each letter summed. Frozen base: no fine-tuning, no adapter, no task registered. One RTX 6000 Ada 48 GB (Lium, Toronto). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 54, GitHub #169", "underlying": "google/gemma-4-12B-it", "model_pin": "aminry/decisio@5b42101a191d222062a795194b0aedb5003bf6ea + google/gemma-4-12B-it@707f0a3b", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.816711590296496, "acc_among_decisive": 0.9075907590759076, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "r54/r55/r61 registry and pinned author package" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Amin Roudaky / @aminry (roudaky@gmail.com)", "axes": { "calibration": 87.60965425617285, "cost": 59.34583145470655, "intelligence": 52.299106200484864, "speed": 85.86778537116794 }, "calibration": { "parts": { "choice": { "ece_hard": 0.08060951899685599, "ece_score": 83.8780962006288, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 83.8780962006288 }, "noul": { "abstention_rate": 0.18867924528301883, "brier": 0.10747754831036967, "ece": 0.06459999777321335, "n": 371, "score": 87.08000044535733 }, "score": { "interval90_coverage": 0.991869918699187, "interval90_width": 0.4905149051490515, "n": 369, "nrps": 0.07376404038589723, "score": 91.87086612253242, "top_ece": 0.044409318581727114 } }, "score": 87.60965425617285 }, "capability": 69.95438022832886, "composite_ci95": { "A": [ 54.37922382108821, 69.5635857308408 ], "B": [ 50.29209403611676, 66.65944464115974 ], "C": [ 37.763349875755715, 62.76631856626907 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff), CARRIED AND NOT NEW: the exact base google/gemma-4-12B-it is in the frozen pricing module's UNLISTED_BASE_MODELS set, so no exact base-model rate and no floor exist, and (0.05, 0.15) is the reference the rows PUBLISHED on this same base already use - cygnet, jev-omni, winnow-12b and torchcast-decision-12b - OpenRouter google/gemma-3-12b-it, the nearest publicly hosted 12B Gemma sibling, at its FULL catalog pair USD 0.05 per 1M input and USD 0.15 per 1M output. It is also the rate THIS ROW'S OWN v0.8.0 SIBLING carries, so the two versions are priced identically and the comparison between them is clean. PREREG-R54-AMENDMENT-1's full pair rather than output=0, x this row's own measured input/output tokens, the deck31b method. The server reports its own usage.input_tokens/output_tokens on every answer (664,759 in and 1,477 out over the answered rows: one prefill, one option-label readout step per decision), so no token-accounting amendment was needed. Estimated, not charged.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.022306333333333338 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.022653689911983754 }, "display": "decisio v0.9.0 on gemma-4-12B-it (frozen, one prefill per question, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 53.066754472907135, "I_sealed": 51.53145792806259, "base": 52.299106200484864, "excess": -1.038091097593702, "gap": 1.5352965448445417, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 72.23658792816236, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 67.9144385026738, "judge": 60.44825313117996, "standard": 92.46546672247803 } }, "open|noul": { "cc": 43.54106280193237, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 39.13043478260869, "judge": 30.000000000000004, "standard": 44.44444444444444 } }, "open|score": { "cc": 43.422612688626685, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 52.00297563421439, "hard": 44.200521547084826, "judge": 45.87070740823223, "standard": 33.90447141950822 } }, "sealed|choice": { "cc": 65.83450996318003, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 84.03665656640308, "hard": 56.730769230769226, "judge": 68.34264432029796, "standard": 71.17871659071314 } }, "sealed|noul": { "cc": 40.79862969333631, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 8.737864077669899, "judge": 47.36842105263157, "standard": 77.46478873239437 } }, "sealed|score": { "cc": 47.96123412767146, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 66.87191488571192, "hard": 38.251545396524136, "judge": 55.512516311610604, "standard": 46.59834793503713 } } } }, "jevbench_score": 67.76072338665118, "key": "decisio-gemma-4-12b-v090", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 15, "ranked": true, "ranks": { "A": 15, "B": 20, "C": 22, "capability": 37 }, "scores": { "A": 67.76072338665118, "B": 63.97786249241387, "C": 51.483026133201925 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4135970369447023, "p50_s_raw": 0.13179851847235113, "p95_s_adjusted": 0.6261004906438755, "p95_s_raw": 0.23805024532193778 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.75352521796918, 88.88216270564548 ], "I": [ 45.89453540642751, 56.94412894091952 ], "capability": [ 65.24251668071038, 72.31935391386574 ], "composite": { "A": [ 54.37922382108821, 69.5635857308408 ], "B": [ 50.29209403611676, 66.65944464115974 ], "C": [ 37.763349875755715, 62.76631856626907 ] }, "seed": 40 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 87.60965425617285, "capability": 69.95438022832886, "intelligence": 52.299106200484864 }, "per_type": { "choice": { "accuracy": 0.748, "calibration": 83.8780962006288, "cc": 67.7727991815314, "cc_by_type": { "choice": 67.7727991815314 }, "cc_tiered": 69.0355489456712, "cc_tiered_open": 72.23658792816236, "cc_tiered_sealed": 65.83450996318003, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 67.7727991815314 }, "noul": { "accuracy": 0.7333333333333333, "calibration": 87.08000044535733, "cc": 46.66666666666666, "cc_by_type": { "noul": 46.66666666666666 }, "cc_tiered": 42.16984624763434, "cc_tiered_open": 43.54106280193237, "cc_tiered_sealed": 40.79862969333631, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 46.66666666666666 }, "score": { "accuracy": 0.6133333333333333, "calibration": 91.87086612253242, "cc": 47.01542304941684, "cc_by_type": { "score": 47.01542304941684 }, "cc_tiered": 45.69192340814907, "cc_tiered_open": 43.422612688626685, "cc_tiered_sealed": 47.96123412767146, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 47.01542304941684 } }, "raw_on_A": { "calibration": 83.89503312669699, "capability": 68.67777484474954, "intelligence": 53.460516562802084 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 63.00273578153404, "Emphasis on Accuracy 60:20:20": 58.2353176446816, "Emphasis on Cost 20:20:60": 61.48719355296207, "Emphasis on Speed 20:60:20": 70.51330683201019, "Intelligence only": 52.299106200484864 }, "class": "Jev-compatible decision readout on frozen base", "open": "open weights", "licence": "Apache-2.0 software; Gemma model terms", "repo": "https://github.com/aminry/decisio", "gpu": "RTX 6000 Ada Generation", "endpoint_condition": "measured 2026-10-07 21:11:36-21:15:32Z on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), 1,477 of 1,500 answered. github.com/aminry/decisio tag v0.9.0 = 333a0a849de7658969a8f8933b34a8097a8c3489 (verified against the GitHub refs API before the run and re-read as the pod's checked-out HEAD after it), Apache-2.0, installed with their own `uv sync --extra serve --frozen` so their lockfile fixes every version (decisio 0.9.0, torch 2.13.0+cu130, vllm 0.30.0, transformers 5.18.0) and served with their own command `python -m decisio.serve.vllm_engine --base gemma-4-12b`. --base carries google/gemma-4-12B-it@707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7 (bf16, 22.8 GiB) and that profile's own settings - temperature 3.592 for every question type, a system turn, the spaced layout read at the chat template's own answer position, every single-token form of each letter summed. Frozen base: no fine-tuning, no adapter, no task registered. UNCHANGED typesafe adapter and UNCHANGED run_v16.py; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. A FOLLOW-UP OF THIS JOB'S OWN decisio-gemma-4-12b-v080 ROW (run 54, same pool, same basis, same rate), measured as a CANDIDATE: which version is listed is the release lane's decision. WHAT v0.9.0 CHANGES is prefix-cache state-boundary registration, request ticketing and engine-health probing; the whole readout package, serve/systemone.py, serve/temperature.py, serve/abstention.py and the entire vllm_plugin are BYTE-IDENTICAL between the two tags, and no temperature or prompt line moved (r61/receipts/DECISIO-V090-SOURCE-DIFF.json). THE 23 LONG ITEMS FAIL WITH A BARE HTTP 500 from the author's own server - vLLM raises VLLMValidationError because the rendered prompt exceeds the 32,768-token context the profile declares, and the server does not translate that into the 422 the harness scores as a refusal. They are counted wrong either way and they are exactly the 23 items the v1.6.1 common cost basis excludes (the ones the Jev reference itself refuses), but it is the author's own code and A FINDING TO REPORT, NOT A THING TO PATCH AROUND; no setting of theirs was changed and no retry was made. One NVIDIA RTX 6000 Ada Generation 48 GB (Lium, Toronto, CANADA) - the same card model and city on which run 54 measured the v0.8.0 row, chosen over the 96 GB class the author names because the 12B profile is proven to fit it. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline after setup, credential-free. add-requests run 61, PREREG-R61-AMENDMENT-2, benchmarkheaven.com/submit ref 65B71BD3 (bh-submit f05c3298-702f-4314-bc8e-f823d9bce858); previously GitHub #169 for v0.8.0", "underlying": "google/gemma-4-12B-it", "model_pin": "aminry/decisio@333a0a849de7658969a8f8933b34a8097a8c3489 + google/gemma-4-12B-it@707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.8113207547169812, "acc_among_decisive": 0.9136212624584718, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "r54/r55/r61 registry and pinned author package" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "akhilaaa3", "axes": { "calibration": 87.04450672835743, "cost": 56.05113155939739, "intelligence": 55.530225828826175, "speed": 85.43173201875288 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10691343524556984, "ece_score": 78.61731295088603, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 78.61731295088603 }, "noul": { "abstention_rate": 0.25876010781671155, "brier": 0.09426879548573859, "ece": 0.04284927173453476, "n": 371, "score": 91.43014565309304 }, "score": { "interval90_coverage": 0.94579945799458, "interval90_width": 0.32836495031616986, "n": 369, "nrps": 0.06732883113404022, "score": 91.08606158109325, "top_ece": 0.0554749686220474 } }, "score": 87.04450672835743 }, "capability": 71.2873662785918, "composite_ci95": { "A": [ 65.25170208143348, 68.96455653744843 ], "B": [ 61.61186910919615, 66.73364708003527 ], "C": [ 45.97947208044005, 67.32404729169104 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.02917173645320197 }, "display": "Jev-Omni (akhilaaa3, Gemma-4-12B merged)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 59.14991720973798, "I_sealed": 51.91053444791438, "base": 55.530225828826175, "excess": 4.6659951193853555, "gap": 7.239382761823599, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 75.02831016136747, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 73.57659641396666, "judge": 68.35860250494396, "standard": 77.39640016743407 } }, "open|noul": { "cc": 45.118702553485164, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 65.21739130434783, "judge": 10.000000000000009, "standard": 44.44444444444444 } }, "open|score": { "cc": 57.30273891436131, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.591582585888155, "hard": 52.24401347123031, "judge": 64.7750828511346, "standard": 55.56725205969997 } }, "sealed|choice": { "cc": 68.8767390165768, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 74.72470623013821, "hard": 62.33031674208144, "judge": 74.86033519553072, "standard": 70.07020569035596 } }, "sealed|noul": { "cc": 32.45047104291564, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": 12.621359223300965, "judge": 36.8421052631579, "standard": 57.746478873239425 } }, "sealed|score": { "cc": 54.404393284250695, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 74.52093494591816, "hard": 48.39164613508641, "judge": 58.117448525933035, "standard": 50.80203388922203 } } } }, "jevbench_score": 67.74758858028281, "key": "jev-omni", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 16, "ranked": true, "ranks": { "A": 16, "B": 16, "C": 17, "capability": 29 }, "scores": { "A": 67.74758858028281, "B": 64.89216659285307, "C": 58.02968592150493 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.45938353668898346, "p50_s_raw": 0.15469176834449172, "p95_s_adjusted": 0.6232343347743153, "p95_s_raw": 0.23661716738715766 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.22517864881327, 88.34500746248735 ], "I": [ 50.30228764136711, 59.27245651053757 ], "capability": [ 67.20628773376218, 73.03534010513394 ], "composite": { "A": [ 65.25170208143348, 68.96455653744843 ], "B": [ 61.61186910919615, 66.73364708003527 ], "C": [ 45.97947208044005, 67.32404729169104 ] }, "seed": 46 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 87.04450672835743, "capability": 71.2873662785918, "intelligence": 55.530225828826175 }, "per_type": { "choice": { "accuracy": 0.7693333333333333, "calibration": 78.61731295088603, "cc": 70.5010278222483, "cc_by_type": { "choice": 70.5010278222483 }, "cc_tiered": 71.95252458897214, "cc_tiered_open": 75.02831016136747, "cc_tiered_sealed": 68.8767390165768, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 70.5010278222483 }, "noul": { "accuracy": 0.688, "calibration": 91.43014565309304, "cc": 37.59999999999999, "cc_by_type": { "noul": 37.59999999999999 }, "cc_tiered": 38.7845867982004, "cc_tiered_open": 45.118702553485164, "cc_tiered_sealed": 32.45047104291564, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 37.59999999999999 }, "score": { "accuracy": 0.6906666666666667, "calibration": 91.08606158109325, "cc": 55.14278895136254, "cc_by_type": { "score": 55.14278895136254 }, "cc_tiered": 55.85356609930601, "cc_tiered_open": 57.30273891436131, "cc_tiered_sealed": 54.404393284250695, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 55.14278895136254 } }, "raw_on_A": { "calibration": 85.06452164862384, "capability": 71.04516964846138, "intelligence": 57.025817648298926 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 63.085756293696, "Emphasis on Accuracy 60:20:20": 59.82955470935818, "Emphasis on Cost 20:20:60": 60.07014592127774, "Emphasis on Speed 20:60:20": 70.45744510674369, "Intelligence only": 55.53022582882618 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0, following Gemma 4; dataset rights stated separately by the author", "repo": "https://huggingface.co/akhilaaa3/Jev-Omni", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "google/gemma-4-12B-it fine-tuned and merged, with a trained 256-way decision head", "adapter_id": "images/jev-omni/spec.json:omni_v15.JevOmni", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7412398921832885, "acc_among_decisive": 0.9381818181818182, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Juspay", "axes": { "calibration": 86.13921620236111, "cost": 50.69588844260597, "intelligence": 58.673823517178164, "speed": 91.03179229063062 }, "calibration": { "parts": { "choice": { "ece_hard": 0.12367230463806947, "ece_score": 75.2655390723861, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 75.2655390723861 }, "noul": { "abstention_rate": 0.19733333333333336, "brier": 0.08803851800016456, "ece": 0.03517045741230692, "n": 375, "score": 92.96590851753862 }, "score": { "interval90_coverage": 0.9573333333333334, "interval90_width": 0.31933333333333314, "n": 375, "nrps": 0.07179731457956051, "score": 90.18620101715864, "top_ece": 0.0622393325386334 } }, "score": 86.13921620236111 }, "capability": 72.40651985976965, "composite_ci95": { "A": [ 64.63565801053839, 68.71970276219201 ], "B": [ 61.75049706211547, 67.5877402852976 ], "C": [ 49.28381430026326, 68.71970276219201 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-26b-a4b-it in the frozen 25 Sep snapshot, the M2 reference of the published surogate-rune-26b-a4b-v3 row (the frozen module has no explicit listing-decision line for this base: release-lane line), USD 0.09 per 1M input and USD 0.3 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.15323633999999997 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04400179417738659 }, "display": "Xor 26B-A4B (Juspay, Gemma-4-26B-A4B, bf16)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 58.16452850885176, "I_sealed": 59.18311852550457, "base": 58.673823517178164, "excess": -3.5919776590910524, "gap": -1.0185900166528086, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 76.28669962802184, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 77.35136835482857, "judge": 76.26895187870797, "standard": 62.32733361239012 } }, "open|noul": { "cc": 43.24154589371981, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 47.82608695652173, "judge": 10.000000000000009, "standard": 55.55555555555556 } }, "open|score": { "cc": 54.96534000481365, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 50.747368826312524, "hard": 60.18508130733947, "judge": 63.88049297699079, "standard": 33.26211353074683 } }, "sealed|choice": { "cc": 74.42776387415822, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 86.6972138053359, "hard": 69.45701357466062, "judge": 77.6536312849162, "standard": 73.39573839142753 } }, "sealed|noul": { "cc": 47.28825379461234, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 80.0, "hard": 26.21359223300972, "judge": 50.0, "standard": 69.01408450704226 } }, "sealed|score": { "cc": 55.83333790774316, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 66.57630209327414, "hard": 54.326017090168286, "judge": 62.2590525874285, "standard": 43.83792543059939 } } } }, "jevbench_score": 67.3818936950357, "key": "xor-26b-a4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 17, "ranked": true, "ranks": { "A": 17, "B": 14, "C": 14, "capability": 23 }, "scores": { "A": 67.3818936950357, "B": 65.43945552763522, "C": 64.43613635369312 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.26713270749896767, "p50_s_raw": 0.058566353749483824, "p95_s_adjusted": 0.2951846059411764, "p95_s_raw": 0.0725923029705882 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.23954361455905, 87.31484254463867 ], "I": [ 52.394156281135615, 63.433561181193575 ], "capability": [ 67.76461763096295, 74.44897661397972 ], "composite": { "A": [ 64.63565801053839, 68.71970276219201 ], "B": [ 61.75049706211547, 67.5877402852976 ], "C": [ 49.28381430026326, 68.71970276219201 ] }, "seed": 127 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.13921620236111, "capability": 72.40651985976965, "intelligence": 58.673823517178164 }, "per_type": { "choice": { "accuracy": 0.812, "calibration": 75.2655390723861, "cc": 75.95748510368216, "cc_by_type": { "choice": 75.95748510368216 }, "cc_tiered": 75.35723175109003, "cc_tiered_open": 76.28669962802184, "cc_tiered_sealed": 74.42776387415822, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 75.95748510368216 }, "noul": { "accuracy": 0.7546666666666667, "calibration": 92.96590851753862, "cc": 50.933333333333344, "cc_by_type": { "noul": 50.933333333333344 }, "cc_tiered": 45.264899844166074, "cc_tiered_open": 43.24154589371981, "cc_tiered_sealed": 47.28825379461234, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 50.933333333333344 }, "score": { "accuracy": 0.672, "calibration": 90.18620101715864, "cc": 55.63796909061629, "cc_by_type": { "score": 55.63796909061629 }, "cc_tiered": 55.3993389562784, "cc_tiered_open": 54.96534000481365, "cc_tiered_sealed": 55.83333790774316, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 55.63796909061629 } }, "raw_on_A": { "calibration": 82.5975161439357, "capability": 72.08940491420412, "intelligence": 61.58129368447253 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 62.82194404983342, "Emphasis on Accuracy 60:20:20": 61.0942481872419, "Emphasis on Cost 20:20:60": 57.336197993965754, "Emphasis on Speed 20:60:20": 71.71094828654465, "Intelligence only": 58.67382351717816 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/juspay/Xor-26B-A4B", "gpu": "RTX PRO 6000", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB (the author's card class), the author's own serving bundle (SGLang image pinned by digest + their compatibility server, single GPU via their documented switch), offline (all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "google/gemma-4-26B-A4B-it", "model_pin": "juspay/Xor-26B-A4B@8afd4396492084c7c91846ed3791161d54665d34", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.8026666666666666, "acc_among_decisive": 0.9401993355481728, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a7 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Foxl AI / The Bobcat Authors (admin@foxl.ai)", "axes": { "calibration": 81.17269878237248, "cost": 49.68577376542501, "intelligence": 65.81464576505633, "speed": 90.781726629608 }, "calibration": { "parts": { "choice": { "ece_hard": 0.114937134190201, "ece_score": 77.01257316195979, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 77.01257316195979 }, "noul": { "abstention_rate": 0.10666666666666669, "brier": 0.08802311851821586, "ece": 0.07573463232351335, "n": 375, "score": 84.85307353529733 }, "score": { "interval90_coverage": 0.9066666666666666, "interval90_width": 0.1808888888888889, "n": 375, "nrps": 0.06948590052038994, "score": 81.65244964986034, "top_ece": 0.14873255324120163 } }, "score": 81.17269878237248 }, "capability": 73.4936722737144, "composite_ci95": { "A": [ 65.41706710923236, 68.6674732651027 ], "B": [ 64.1802436512738, 68.73104866859511 ], "C": [ 65.32359947723424, 68.6674732651027 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-26b-a4b-it in the frozen 25 Sep snapshot, USD 0.09 per 1M input and USD 0.3 per 1M output x this row's own measured tokens. Estimated, not charged. THIS IS THE REFERENCE, THE RATE AND THE LABEL THE PUBLISHED PAGE ALREADY CARRIES FOR THIS EXACT BASE on three rows published within the last day - xor-26b-a4b, surogate-rune-26b-a4b-v3-v16 and gev-26b-decide - so pricing this row any other way would make it inconsistent with the field it is placed in. RELEASE-LANE LINE, recorded and not acted on: pricing_v15.reference() RAISES on the author's exact declared spelling google/gemma-4-26B-A4B-it, because that string is in none of the three decision tables, while the lowercase google/gemma-4-26b-a4b-it IS in BASE_REFERENCES and in the dated snapshot at (0.09, 0.30). That is a CAPITALISATION gap - one notch tighter than run 56's -Base/instruct gap on Qwen3.5-35B-A3B - and the three published rows say in their own basis string that the frozen module has no explicit listing-decision line for this base. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: their build_response returns usage.output_tokens == 0 and the readout takes candidate logits at the first answer position of one forward pass, so this row's cost depends on the input rate alone and any other decision is reversible by arithmetic from the raw.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.15782652 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04754894380501015 }, "display": "Bobcat Flash 1.2 (Gemma-4-26B-A4B-it + two merged rank-64 LoRA adapters, typed-decision readout at the first answer position, FP8, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 65.86976952005253, "I_sealed": 65.75952201006011, "base": 65.81464576505633, "excess": -2.4631401324458246, "gap": 0.11024750999241917, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 74.45043236055656, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 66.02705253224286, "judge": 80.22412656558997, "standard": 69.8618668899121 } }, "open|noul": { "cc": 66.6425120772947, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 65.21739130434783, "judge": 50.0, "standard": 77.77777777777777 } }, "open|score": { "cc": 56.516364122306314, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 62.947350572836605, "hard": 60.92245146041372, "judge": 57.42924259191816, "standard": 43.1193785164086 } }, "sealed|choice": { "cc": 76.1313750303687, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 78.71554208853745, "hard": 67.92986425339366, "judge": 81.3780260707635, "standard": 83.37233649464221 } }, "sealed|noul": { "cc": 59.790998006434116, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 84.00000000000001, "hard": 37.86407766990292, "judge": 71.05263157894737, "standard": 74.64788732394365 } }, "sealed|score": { "cc": 61.35619299337752, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 85.14251727623912, "hard": 56.30342200492587, "judge": 54.946113599979476, "standard": 69.18369191894709 } } } }, "jevbench_score": 67.34032787154446, "key": "bobcat-flash-1.2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 18, "ranked": true, "ranks": { "A": 18, "B": 12, "C": 13, "capability": 20 }, "scores": { "A": 67.34032787154446, "B": 66.85676394909423, "C": 67.34032787154446 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2719114740379155, "p50_s_raw": 0.060955737018957734, "p95_s_adjusted": 0.3071848569670692, "p95_s_raw": 0.0785924284835346 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.39295521484306, 83.94013345398247 ], "I": [ 60.18260161381281, 69.94166977988647 ], "capability": [ 69.50208486636623, 76.43509664894793 ], "composite": { "A": [ 65.41706710923236, 68.6674732651027 ], "B": [ 64.1802436512738, 68.73104866859511 ], "C": [ 65.32359947723424, 68.6674732651027 ] }, "seed": 24 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 81.17269878237248, "capability": 73.4936722737144, "intelligence": 65.81464576505633 }, "per_type": { "choice": { "accuracy": 0.812, "calibration": 77.01257316195979, "cc": 75.95748510368216, "cc_by_type": { "choice": 75.95748510368216 }, "cc_tiered": 75.29090369546263, "cc_tiered_open": 74.45043236055656, "cc_tiered_sealed": 76.1313750303687, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 75.95748510368216 }, "noul": { "accuracy": 0.8213333333333334, "calibration": 84.85307353529733, "cc": 64.26666666666667, "cc_by_type": { "noul": 64.26666666666667 }, "cc_tiered": 63.2167550418644, "cc_tiered_open": 66.6425120772947, "cc_tiered_sealed": 59.790998006434116, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 64.26666666666667 }, "score": { "accuracy": 0.696, "calibration": 81.65244964986034, "cc": 62.27325899358171, "cc_by_type": { "score": 62.27325899358171 }, "cc_tiered": 58.936278557841916, "cc_tiered_open": 56.516364122306314, "cc_tiered_sealed": 61.35619299337752, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 62.27325899358171 } }, "raw_on_A": { "calibration": 76.73132293335215, "capability": 72.99647962418823, "intelligence": 69.26163631502432 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 63.93311146268058, "Emphasis on Accuracy 60:20:20": 64.35172177912958, "Emphasis on Cost 20:20:60": 57.02049836458103, "Emphasis on Speed 20:60:20": 72.21838604704122, "Intelligence only": 64.99001759823823 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/sanghwa-na/bobcat-flash-1.2", "gpu": "H100 PCIe", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights sanghwa-na/bobcat-flash-1.2@58d15456937ce58625b13a5e3d982846554a3aef, Apache-2.0, 51.68 GB BF16 - a merge of two rank-64 LoRA adapters on google/gemma-4-26B-A4B-it@4d7ae4984b7db7de8f8457170b3f1a419ee76d52 (0.6*A + 0.4*B, B being the Flash 1.1 adapter), file-by-file verified against the Hub's own manifest before the engine was started. Code github.com/foxl-ai/bobcat@43387a09bbc0310edcf57e2b21fa59ea81ac4166, Apache-2.0. NO MAPPING DECISION OF OURS EXISTS FOR THIS ROW, which is why it is worth saying so explicitly. The author implements TypeSafe's published System One contract, and their own src/bobcat/protocol.py parse_request accepts the JevBench canonical record EXACTLY as the UNCHANGED jevbench.adapters.typesafe already emits it: choice -> a dict of 1-255 named criteria, so the item's own labels are the keys and its own criteria[label] strings travel verbatim as the descriptions (their protocol renders {name, description} per option); score -> the item's ordered array of levels, which their own parse_request labels '0'..'n-1' - the same keys JevBench uses - with the item's own level text as each description, so the ordinal order is carried; noul -> criteria restricted to {true,false}, and the item's OWN true/false description strings are delivered (unlike run 56's APUS rows, whose runtime required canonical criteria there), with the answer returned as noul = P(yes). Their answer() refuses to reply unless the probabilities sum to one. Nothing was rewritten, no option was dropped, no item type was recorded as a miss, and no shim of ours sits anywhere in the path. STACK: the author's own published install line, verbatim from their model card's Quickstart - a uv-managed Python 3.12 (their note: it ships the headers Triton compiles against) and vllm==0.30.0 LETTING IT RESOLVE ITS OWN STACK, which gave torch 2.13.0+cu130, transformers 5.19.0, tokenizers 0.23.2. Served by THEIR OWN SERVER, bobcat.flash_server, with THEIR OWN published command verbatim: --engine vllm --quantization fp8 (their online dynamic FP8 of these BF16 weights, the served configuration) --temperature choice=0.594604,noul=0.353553,score=0.529732 --max-num-seqs 64 --max-model-len 98368 --schedule all --engine-arg max_num_batched_tokens=16384 --engine-arg attention_backend=TRITON_ATTN, with VLLM_USE_FLASHINFER_SAMPLER=0 as their command sets it. --local only because the server is on loopback on our own pod. One H100 PCIe 80 GB (Lium) - the second of the two cards the author measured. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. No file of theirs was edited and no prompt of ours exists. CALIBRATION IS THEIR PUBLISHED DEFAULT AND WAS NOT CHOSEN BY US. Their card ships two calibrations: the per-type triple used here, which is the Quickstart's - i.e. the published serve command of this release - and a single product temperature 0.9232 fitted by log-loss on their six product tasks. The triple was bound in PREREG-R57 s3.3 before any item was sent and the alternative was deliberately NOT probed on our items, because the author has ALREADY PUBLISHED both configurations' numbers on JevBench's public 231 (per-type Capability 68.2 = 57.5/78.9; product 68.3 = 54.3/82.3), so there was nothing to learn from our pool and choosing between them from our own measurement would have been exactly the mistake run 56's probe was designed to avoid. The author states the triple was fitted on a held-out split of 8,906 JevBench-FORMAT decisions (generated and public) for the highest JevBench-style Capability, and separately that no JevBench item was used for training, calibration, selection or prompt design, with generated rows audited against JevBench's public items at 0 exact and 0 8-gram overlaps. THE 23 LONG ITEMS: the server's limit is --max-model-len 98368 (their compiler gets max_model_len - 64) against a pool whose longest item is 128,111 BYTES of canonical record, and their RequestLimitError answers 422 with 'Inputs are never truncated' rather than truncating. This row therefore reads every item in full. Serial, one decision per request, no retry, no batching, offline, credential-free. add-requests run 57, benchmarkheaven.com/submit ref 7A714784 (bh-submit e9d576d7-e1f1-49d8-8151-05cdbbdd7ed4)", "underlying": "google/gemma-4-26B-A4B-it", "model_pin": "sanghwa-na/bobcat-flash-1.2@58d15456937ce58625b13a5e3d982846554a3aef", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.8933333333333333, "acc_among_decisive": 0.9194029850746268, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": true, "api_subset": false, "author": "SPX AI team (SurdAI, formerly Simplex AI)", "axes": { "calibration": 90.20036269195566, "cost": 52.107175358413464, "intelligence": 58.43051843707937, "speed": 79.39117060033084 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05772702157730632, "ece_score": 88.45459568453875, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 88.45459568453875 }, "noul": { "abstention_rate": 0.18598382749326148, "brier": 0.08492257448407475, "ece": 0.04230529224114798, "n": 371, "score": 91.5389415517704 }, "score": { "interval90_coverage": 0.9322493224932249, "interval90_width": 0.30013550135501355, "n": 369, "nrps": 0.06769358396529002, "score": 90.60755083955786, "top_ece": 0.06007769962177635 } }, "score": 90.20036269195566 }, "capability": 74.31544056451752, "composite_ci95": { "A": [ 64.36187068361473, 67.93122351228547 ], "B": [ 61.80904827802077, 66.83209430434223 ], "C": [ 50.5869412571725, 67.93122351228547 ] }, "cost": { "basis": "Operator's own public-beta overage tariff, conservative reading USD 0.04 per 1M input tokens, output USD 0 (the provider's /v1/models payload reports output_price \"0\"): three dated primary readings of the operator's own published overage tariff: 24 Sep 2026 provider console (flash 0.025, pro 0.08 per 1M input), 1 Oct 2026 docs page (flash 0.04, pro 0.02), 6 Oct 2026 docs page AND /v1/models payload agreeing (flash 0.02, pro 0.04). Sources conflict, so the per-model MAXIMUM is used, which is the published fastino-gliner-2-5-decide precedent (\"published sources conflict ... conservative higher\"). Today's agreeing pair would be USD 0.02/M, i.e. half this row's cost. No base-model floor applies: the submitter states the base model stays sealed until a later open release, so no base model exists to reference. Priced on usage.input_tokens as the provider reports them. The provider also reports billable_input_tokens = input_tokens / 2 on every single item (297/297 of the public set, exactly 2.0): its default effort 2 renders the prompt twice and its docs bundle the second pass free during public beta (\"default effort 2 is included at the single-pass input price\"). That bundle is a public-beta promotion, and TASK.md forbids a free or promotional tier setting the price, so the two passes actually consumed are charged. The operator's own billed figure is exactly half of this row's cost", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.03887914666666666 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.039484576844955994 }, "display": "SPX-CD Flash (SurdAI, hosted /v1/systemone, Oct-4 checkpoint)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 59.99669320470862, "I_sealed": 56.86434366945013, "base": 58.43051843707937, "excess": 0.5589618928202476, "gap": 3.1323495352584914, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 75.41749935055667, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 73.57659641396666, "judge": 68.35860250494396, "standard": 77.39640016743407 } }, "open|noul": { "cc": 52.89855072463768, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 73.91304347826086, "judge": 0.0, "standard": 66.66666666666667 } }, "open|score": { "cc": 51.67402953893152, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 54.86207439440385, "hard": 52.93207079861204, "judge": 54.720404668190085, "standard": 42.994361897946476 } }, "sealed|choice": { "cc": 71.02357528674706, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 67.92986425339366, "judge": 68.34264432029796, "standard": 76.7212710924991 } }, "sealed|noul": { "cc": 45.34763039144423, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 20.388349514563096, "judge": 52.63157894736843, "standard": 69.01408450704226 } }, "sealed|score": { "cc": 54.221825330159106, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 80.5246722641586, "hard": 48.238601087091794, "judge": 55.5539283927005, "standard": 51.038695755481854 } } } }, "jevbench_score": 66.68017155386212, "key": "spx-cd-flash", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 19, "ranked": true, "ranks": { "A": 19, "B": 17, "C": 15, "capability": 17 }, "scores": { "A": 66.68017155386212, "B": 64.84899920515487, "C": 63.23735362072274 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.9804960675537586, "p50_s_raw": 0.9804960675537586, "p95_s_adjusted": 1.1733756832778452, "p95_s_raw": 1.1733756832778452 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.82360765229315, 91.10972764372464 ], "I": [ 53.013471055498634, 62.79232050259918 ], "capability": [ 69.82626986507542, 76.43255727254187 ], "composite": { "A": [ 64.36187068361473, 67.93122351228547 ], "B": [ 61.80904827802077, 66.83209430434223 ], "C": [ 50.5869412571725, 67.93122351228547 ] }, "seed": 136 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": true, "g_med_used": 2.573387642438244, "lane": "api", "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.20036269195566, "capability": 74.31544056451752, "intelligence": 58.43051843707937 }, "per_type": { "choice": { "accuracy": 0.7893333333333333, "calibration": 88.45459568453875, "cc": 73.05874217292042, "cc_by_type": { "choice": 73.05874217292042 }, "cc_tiered": 73.22053731865186, "cc_tiered_open": 75.41749935055667, "cc_tiered_sealed": 71.02357528674706, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 73.05874217292042 }, "noul": { "accuracy": 0.7546666666666667, "calibration": 91.5389415517704, "cc": 50.933333333333344, "cc_by_type": { "noul": 50.933333333333344 }, "cc_tiered": 49.12309055804096, "cc_tiered_open": 52.89855072463768, "cc_tiered_sealed": 45.34763039144423, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 50.933333333333344 }, "score": { "accuracy": 0.6746666666666666, "calibration": 90.60755083955786, "cc": 54.63501832116133, "cc_by_type": { "score": 54.63501832116133 }, "cc_tiered": 52.94792743454531, "cc_tiered_open": 51.67402953893152, "cc_tiered_sealed": 54.221825330159106, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 54.63501832116133 } }, "raw_on_A": null, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 61.34791411142523, "Emphasis on Accuracy 60:20:20": 60.146681111986254, "Emphasis on Cost 20:20:60": 57.28436724874348, "Emphasis on Speed 20:60:20": 67.48263394824296, "Intelligence only": 58.43051843707938 }, "class": "Hosted decision API", "open": "closed weights", "licence": null, "repo": "https://surd.ai", "gpu": null, "endpoint_condition": "measured 2026-10-06 on the live v1.6.0 S u P (1,500 rows) under Florian's 5 Oct 2026 full-set amendment, through the release lane's own full-set API lane (run_api_full_v161.py, rotation.py expose --full-set logged 1,200 sealed items to provider 'surd' before the first request); stock jevbench.adapters.typesafe with an endpoint override only, reasoning at the provider default, one request at a time, no retries; add-requests run 53, benchmarkheaven.com/submit ref 3FF00545", "underlying": null, "model_pin": "provider-reported measurement model slug: spx-cd-flash", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.8140161725067385, "acc_among_decisive": 0.9370860927152318, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Current-pool API measurement registry" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mapika", "axes": { "calibration": 82.62244964732962, "cost": 50.78502198104071, "intelligence": 58.54290392078144, "speed": 85.78163968405963 }, "calibration": { "parts": { "choice": { "ece_hard": 0.13017175324675329, "ece_score": 73.96564935064934, "mean_tvd": null, "n": 738, "n_ece_hard": 308, "n_tvd": 0, "score": 73.96564935064934 }, "noul": { "abstention_rate": 0.1967654986522911, "brier": 0.06013664803234495, "ece": 0.0693757412398923, "n": 371, "score": 86.12485175202154 }, "score": { "interval90_coverage": 0.986449864498645, "interval90_width": 0.640469738030713, "n": 369, "nrps": 0.06538499443315268, "score": 87.77684783931798, "top_ece": 0.0895390243902439 } }, "score": 82.62244964732962 }, "capability": 70.58267678405554, "composite_ci95": { "A": [ 63.831274751690664, 67.00563695142573 ], "B": [ 61.40092144952185, 66.08555760610543 ], "C": [ 50.02666080334015, 67.00563695142573 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-31b-it, the reference of the published deck31b and quyet-1-0-large rows, USD 0.09 per 1M input and USD 0.34 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.04500444 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04370179417738659 }, "display": "decider chat on Gemma-4-31B-it (Mapika, frozen base, inference technique)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 55.72975554572549, "I_sealed": 61.356052295837394, "base": 58.54290392078144, "excess": -8.199684392550148, "gap": -5.626296750111905, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 76.96981695560446, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 75.46398238439761, "judge": 72.31377719182596, "standard": 77.39640016743407 } }, "open|noul": { "cc": 43.846790890269155, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 56.52173913043479, "judge": 19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 46.372658791302854, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 53.266060606060606, "hard": 49.83425925925926, "judge": 51.426101694915246, "standard": 28.422592592592586 } }, "sealed|choice": { "cc": 82.26456829463379, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 90.68804966373514, "hard": 73.02036199095022, "judge": 88.8268156424581, "standard": 86.69786919571378 } }, "sealed|noul": { "cc": 54.93243610873211, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 37.86407766990292, "judge": 63.1578947368421, "standard": 66.19718309859155 } }, "sealed|score": { "cc": 46.871152484146286, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 60.35531914893617, "hard": 42.373504901960835, "judge": 52.318562874251484, "standard": 40.95324873096443 } } } }, "jevbench_score": 66.07956202755186, "key": "decider-chat-gemma4-31b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 20, "ranked": true, "ranks": { "A": 20, "B": 18, "C": 16, "capability": 34 }, "scores": { "A": 66.07956202755186, "B": 64.42088721187244, "C": 62.90905728971202 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4011345919920132, "p50_s_raw": 0.12556729599600658, "p95_s_adjusted": 0.658485045604175, "p95_s_raw": 0.2542425228020875 }, "status": { "answered_ok": 1478, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.7847423077956, 82.36441170960092 ], "I": [ 53.001527166194506, 62.9130738579859 ], "capability": [ 66.59700207989862, 71.74809270745902 ], "composite": { "A": [ 63.831274751690664, 67.00563695142573 ], "B": [ 61.40092144952185, 66.08555760610543 ], "C": [ 50.02666080334015, 67.00563695142573 ] }, "seed": 30 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.62244964732962, "capability": 70.58267678405554, "intelligence": 58.54290392078144 }, "per_type": { "choice": { "accuracy": 0.856, "calibration": 73.96564935064934, "cc": 81.5844566751608, "cc_by_type": { "choice": 81.5844566751608 }, "cc_tiered": 79.61719262511912, "cc_tiered_open": 76.96981695560446, "cc_tiered_sealed": 82.26456829463379, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 81.5844566751608 }, "noul": { "accuracy": 0.776, "calibration": 86.12485175202154, "cc": 55.2, "cc_by_type": { "noul": 55.2 }, "cc_tiered": 49.38961349950063, "cc_tiered_open": 43.846790890269155, "cc_tiered_sealed": 54.93243610873211, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 55.2 }, "score": { "accuracy": 0.6613333333333333, "calibration": 87.77684783931798, "cc": 46.969219298245605, "cc_by_type": { "score": 46.969219298245605 }, "cc_tiered": 46.621905637724566, "cc_tiered_open": 46.372658791302854, "cc_tiered_sealed": 46.871152484146286, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 46.969219298245605 } }, "raw_on_A": { "calibration": 80.81065486925488, "capability": 71.24728363904796, "intelligence": 61.68391240884105 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.016000000000000014, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 61.94527894229558, "Emphasis on Accuracy 60:20:20": 60.537949991324034, "Emphasis on Cost 20:20:60": 56.94013011167525, "Emphasis on Speed 20:60:20": 69.69141016272579, "Intelligence only": 58.54290392078144 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://github.com/Mapika/decider", "gpu": "RTX PRO 6000", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "google/gemma-4-31B-it", "model_pin": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475 + Mapika/decider-chat-gemma4-31b@22975cbd1716170264a1d2a7e1105787e08c5d95", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.8032345013477089, "acc_among_decisive": 0.9765100671140939, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "HopitAI (@ArchieIndian)", "axes": { "calibration": 84.63452107883388, "cost": 56.15749203571245, "intelligence": 51.6576814732076, "speed": 84.57301814857726 }, "calibration": { "parts": { "choice": { "ece_hard": 0.11381338060621848, "ece_score": 77.2373238787563, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 77.2373238787563 }, "noul": { "abstention_rate": 0.2075471698113207, "brier": 0.09205588963446744, "ece": 0.056376852598614385, "n": 371, "score": 88.72462948027713 }, "score": { "interval90_coverage": 0.964769647696477, "interval90_width": 0.40492321589882563, "n": 369, "nrps": 0.08202603571136496, "score": 87.94160987746824, "top_ece": 0.0795708833696351 } }, "score": 84.63452107883388 }, "capability": 68.14610127602074, "composite_ci95": { "A": [ 54.67384317203267, 67.43166618883266 ], "B": [ 51.03564669331923, 64.73449720897486 ], "C": [ 37.9679466472449, 58.605012542436775 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): the exact base google/gemma-4-12B-it is in the frozen pricing module's UNLISTED_BASE_MODELS set, so no exact base-model rate and no floor exist. The reference is the one the rows PUBLISHED on this same base use - cygnet, jev-omni and winnow-12b - OpenRouter google/gemma-3-12b-it, the nearest publicly hosted 12B Gemma sibling, at its FULL catalog pair USD 0.05 per 1M input and USD 0.15 per 1M output. PREREG-R54-AMENDMENT-1 uses the full pair rather than run 53's output=0, so a row that emits tokens pays for them; where a row generates nothing the output term is zero anyway. x this row's own measured input/output tokens, the deck31b method. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.028490900000000006 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.02893456330399459 }, "display": "Hopper 12B trained (gemma-4-12B-it frozen + LoRA r32 unmerged, one forward pass, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 52.49257010742919, "I_sealed": 50.822792838986025, "base": 51.6576814732076, "excess": -0.9036103739950812, "gap": 1.6697772684431627, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 68.64489681634264, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 60.36489462094999, "judge": 72.31377719182596, "standard": 69.8618668899121 } }, "open|noul": { "cc": 43.547273982056595, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 65.21739130434783, "judge": 0.0, "standard": 44.44444444444444 } }, "open|score": { "cc": 45.285539523888346, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 63.928801818275204, "hard": 41.02383720746134, "judge": 39.409476186258296, "standard": 53.30140801599398 } }, "sealed|choice": { "cc": 65.07976853625175, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 82.70637794693668, "hard": 58.2579185520362, "judge": 62.756052141527014, "standard": 73.39573839142753 } }, "sealed|noul": { "cc": 41.18694322295551, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 18.446601941747566, "judge": 39.47368421052631, "standard": 71.83098591549295 } }, "sealed|score": { "cc": 46.20166675775083, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 67.66271785418321, "hard": 37.07866936088858, "judge": 45.777923536915246, "standard": 54.3527508345125 } } } }, "jevbench_score": 65.78419365065008, "key": "hopper-12b-trained", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 21, "ranked": true, "ranks": { "A": 21, "B": 22, "C": 24, "capability": 40 }, "scores": { "A": 65.78419365065008, "B": 62.37284887464997, "C": 48.76282692619866 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4776301250443794, "p50_s_raw": 0.1638150625221897, "p95_s_adjusted": 0.7304768792411778, "p95_s_raw": 0.2902384396205889 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 79.81171527967304, 87.00046880053556 ], "I": [ 46.53255595156734, 55.86954644502632 ], "capability": [ 64.05753668009696, 70.56574900316099 ], "composite": { "A": [ 54.67384317203267, 67.43166618883266 ], "B": [ 51.03564669331923, 64.73449720897486 ], "C": [ 37.9679466472449, 58.605012542436775 ] }, "seed": 59 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 84.63452107883388, "capability": 68.14610127602074, "intelligence": 51.6576814732076 }, "per_type": { "choice": { "accuracy": 0.7413333333333333, "calibration": 77.2373238787563, "cc": 66.92022773130735, "cc_by_type": { "choice": 66.92022773130735 }, "cc_tiered": 66.8623326762972, "cc_tiered_open": 68.64489681634264, "cc_tiered_sealed": 65.07976853625175, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 66.92022773130735 }, "noul": { "accuracy": 0.7306666666666667, "calibration": 88.72462948027713, "cc": 46.13333333333334, "cc_by_type": { "noul": 46.13333333333334 }, "cc_tiered": 42.36710860250605, "cc_tiered_open": 43.547273982056595, "cc_tiered_sealed": 41.18694322295551, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 46.13333333333334 }, "score": { "accuracy": 0.5973333333333334, "calibration": 87.94160987746824, "cc": 46.77672583471599, "cc_by_type": { "score": 46.77672583471599 }, "cc_tiered": 45.74360314081959, "cc_tiered_open": 45.285539523888346, "cc_tiered_sealed": 46.20166675775083, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 46.77672583471599 } }, "raw_on_A": { "calibration": 82.16875295682286, "capability": 68.22397555466662, "intelligence": 54.279198152510396 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 61.23777421409552, "Emphasis on Accuracy 60:20:20": 57.0087849790455, "Emphasis on Cost 20:20:60": 59.09921236284679, "Emphasis on Speed 20:60:20": 68.83489755597157, "Intelligence only": 51.6576814732076 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/HopitAI/hopper", "gpu": "RTX 6000 Ada", "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. Serving code github.com/hopit-ai/hopper tag 12b-1.1.1, installed with the author's own one-line pip install; adapter HopitAI/hopper@4b605f25eb903c2cf4d121040ffb097240b3bc37 (branch hopper-12b), LoRA r32 alpha 64, loaded UNMERGED because the author states a bf16 merge changes answers; base google/gemma-4-12B-it@707f0a3b, frozen; readout map maps/hopper-12b-trained-readout.json as shipped in their package. 1.1.1 is their own fix for corrupted option scores at prompt lengths one above a multiple of 32. Their flags left at their defaults: --shortlist tournament (their measured default) for menus over 26 options, --cuda-graphs OFF. The author asked on 6 Oct 2026 that the TRAINED version replace the untrained Hopper 12B request in GitHub #164; this is that row. NOT the published `hopper` row (HopitAI, rank 25), a different system. One RTX 6000 Ada 48 GB (Lium, Toronto); the author ran theirs on an L40S. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 54, GitHub #164", "underlying": "google/gemma-4-12B-it", "model_pin": "HopitAI/hopper@4b605f25eb903c2cf4d121040ffb097240b3bc37", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7924528301886793, "acc_among_decisive": 0.9319727891156463, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Surogate", "axes": { "calibration": 90.88042351612289, "cost": 49.597326937281316, "intelligence": 56.055735926690254, "speed": 86.95393114544444 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05932792591743529, "ece_score": 88.13441481651294, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 88.13441481651294 }, "noul": { "abstention_rate": 0.2695417789757413, "brier": 0.08438373400978966, "ece": 0.039029372562851536, "n": 371, "score": 92.19412548742969 }, "score": { "interval90_coverage": 0.967479674796748, "interval90_width": 0.37466124661246614, "n": 369, "nrps": 0.0647012985012109, "score": 92.31273024442605, "top_ece": 0.04452204830513414 } }, "score": 90.88042351612289 }, "capability": 73.46807972140657, "composite_ci95": { "A": [ 62.82778837763545, 66.19023981114698 ], "B": [ 59.77310724201828, 64.70527953464652 ], "C": [ 45.16787327358027, 66.15052716337196 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-26b-a4b-it in the frozen 25 Sep snapshot, the M2 reference of the published surogate-rune-26b-a4b-v3 row (the frozen module has no explicit listing-decision line for this base: release-lane line), USD 0.09 per 1M input and USD 0.3 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.04713878 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.047872830060934325 }, "display": "Surogate Rune 26B-A4B v3 (v1.6 pool)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 58.17716696493766, "I_sealed": 53.934304888442846, "base": 56.055735926690254, "excess": 1.6694744340565726, "gap": 4.242862076494816, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 75.95733901606096, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 66.02705253224286, "judge": 80.22412656558997, "standard": 77.39640016743407 } }, "open|noul": { "cc": 42.59075224292616, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 19.999999999999996, "standard": 44.44444444444444 } }, "open|score": { "cc": 55.98340963582588, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 62.28084236876889, "hard": 59.46587194956875, "judge": 61.146761005919444, "standard": 38.12474158672826 } }, "sealed|choice": { "cc": 73.06247198507991, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 84.03665656640308, "hard": 68.94796380090497, "judge": 73.9292364990689, "standard": 74.50424929178472 } }, "sealed|noul": { "cc": 34.31372825609765, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 16.50485436893203, "judge": 36.8421052631579, "standard": 49.29577464788733 } }, "sealed|score": { "cc": 54.426714424150994, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 71.74170476496207, "hard": 48.52758483569058, "judge": 67.30550244555229, "standard": 38.24929639856435 } } } }, "jevbench_score": 65.0490738113794, "key": "surogate-rune-26b-a4b-v3-v16", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 22, "ranked": true, "ranks": { "A": 22, "B": 21, "C": 19, "capability": 21 }, "scores": { "A": 65.0490738113794, "B": 62.796501415524304, "C": 56.77782262023472 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3823961119982414, "p50_s_raw": 0.1161980559991207, "p95_s_adjusted": 0.5273432816902641, "p95_s_raw": 0.18867164084513205 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.71263475548702, 91.20016913118003 ], "I": [ 50.88578039677805, 60.44466553566163 ], "capability": [ 69.30942731571565, 74.85354775896664 ], "composite": { "A": [ 62.82778837763545, 66.19023981114698 ], "B": [ 59.77310724201828, 64.70527953464652 ], "C": [ 45.16787327358027, 66.15052716337196 ] }, "seed": 116 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.88042351612289, "capability": 73.46807972140657, "intelligence": 56.055735926690254 }, "per_type": { "choice": { "accuracy": 0.8, "calibration": 88.13441481651294, "cc": 74.4228564932789, "cc_by_type": { "choice": 74.4228564932789 }, "cc_tiered": 74.50990550057043, "cc_tiered_open": 75.95733901606096, "cc_tiered_sealed": 73.06247198507991, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 74.4228564932789 }, "noul": { "accuracy": 0.6986666666666667, "calibration": 92.19412548742969, "cc": 39.733333333333334, "cc_by_type": { "noul": 39.733333333333334 }, "cc_tiered": 38.4522402495119, "cc_tiered_open": 42.59075224292616, "cc_tiered_sealed": 34.31372825609765, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 39.733333333333334 }, "score": { "accuracy": 0.6666666666666666, "calibration": 92.31273024442605, "cc": 54.21453874970139, "cc_by_type": { "score": 54.21453874970139 }, "cc_tiered": 55.20506202998844, "cc_tiered_open": 55.98340963582588, "cc_tiered_sealed": 54.426714424150994, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 54.21453874970139 } }, "raw_on_A": { "calibration": 89.39331463026028, "capability": 74.65140006965763, "intelligence": 59.909485509054974 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 59.63127588509936, "Emphasis on Accuracy 60:20:20": 57.75697016597956, "Emphasis on Cost 20:20:60": 54.76969389057124, "Emphasis on Speed 20:60:20": 67.8565544623977, "Intelligence only": 55.15648620822009 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/surogate/rune-26b-a4b-GGUF", "gpu": "RTX PRO 6000", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "google/gemma-4-26B-A4B-it", "model_pin": "surogate/rune-26b-a4b-GGUF@bd4a7cbbed66af1a95dfbac12cabf52e6e677411", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7304582210242587, "acc_among_decisive": 0.966789667896679, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a7 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card metadata" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "AutoTrust AI Lab", "axes": { "calibration": 90.33483359631919, "cost": 51.12030271996105, "intelligence": 49.817428608862244, "speed": 89.87310530072351 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05524139109810591, "ece_score": 88.95172178037882, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 88.95172178037882 }, "noul": { "abstention_rate": 0.30188679245283023, "brier": 0.091757346419259, "ece": 0.03659159632329119, "n": 371, "score": 92.68168073534177 }, "score": { "interval90_coverage": 0.9728997289972899, "interval90_width": 0.4146341463414633, "n": 369, "nrps": 0.07505075554087659, "score": 89.37109827323698, "top_ece": 0.06876363949719191 } }, "score": 90.33483359631919 }, "capability": 70.07613110259072, "composite_ci95": { "A": [ 48.122003834420376, 66.28949313101391 ], "B": [ 44.53704625593728, 63.445630864100984 ], "C": [ 33.418058218347475, 54.44178965547935 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter google/gemma-4-26b-a4b-it in the frozen 25 Sep snapshot, the M2 reference of the published surogate-rune-26b-a4b-v3 row (the frozen module has no explicit listing-decision line for this base: release-lane line), USD 0.09 per 1M input and USD 0.3 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.04193846 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04259153012863913 }, "display": "GEV-26B-Decide (AutoTrust, Gemma-4-26B-A4B + LoRA + head)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 51.11414911073161, "I_sealed": 48.520708106992885, "base": 49.817428608862244, "excess": 0.02005336130048141, "gap": 2.593441003738725, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 70.91276210170015, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 69.80182447310474, "judge": 72.31377719182596, "standard": 62.32733361239012 } }, "open|noul": { "cc": 35.146307798481715, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 10.000000000000009, "standard": 22.222222222222232 } }, "open|score": { "cc": 47.28337743201297, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 62.41806192777556, "hard": 49.795958510853765, "judge": 44.087055938179034, "standard": 39.48535526720095 } }, "sealed|choice": { "cc": 69.65586334909833, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 64.36651583710406, "judge": 69.27374301675978, "standard": 75.61276019214189 } }, "sealed|noul": { "cc": 27.311750523580937, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 63.99999999999999, "hard": 8.737864077669899, "judge": 28.947368421052634, "standard": 43.66197183098593 } }, "sealed|score": { "cc": 48.59451044829939, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 67.07336665308927, "hard": 39.678983511563985, "judge": 58.83673780283293, "standard": 41.82279518757497 } } } }, "jevbench_score": 64.2201412904201, "key": "gev-26b-decide", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 23, "ranked": true, "ranks": { "A": 23, "B": 23, "C": 27, "capability": 36 }, "scores": { "A": 64.2201412904201, "B": 60.60132222808652, "C": 44.59732034056951 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2535779389785603, "p50_s_raw": 0.051788969489280134, "p95_s_adjusted": 0.40604858579754366, "p95_s_raw": 0.12802429289877182 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.06256442860612, 90.45126283775208 ], "I": [ 44.10305960817391, 54.39355915396382 ], "capability": [ 65.77131123789238, 71.56937054169023 ], "composite": { "A": [ 48.122003834420376, 66.28949313101391 ], "B": [ 44.53704625593728, 63.445630864100984 ], "C": [ 33.418058218347475, 54.44178965547935 ] }, "seed": 42 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.33483359631919, "capability": 70.07613110259072, "intelligence": 49.817428608862244 }, "per_type": { "choice": { "accuracy": 0.772, "calibration": 88.95172178037882, "cc": 70.84205640233793, "cc_by_type": { "choice": 70.84205640233793 }, "cc_tiered": 70.28431272539925, "cc_tiered_open": 70.91276210170015, "cc_tiered_sealed": 69.65586334909833, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 70.84205640233793 }, "noul": { "accuracy": 0.664, "calibration": 92.68168073534177, "cc": 32.800000000000004, "cc_by_type": { "noul": 32.800000000000004 }, "cc_tiered": 31.229029161031328, "cc_tiered_open": 35.146307798481715, "cc_tiered_sealed": 27.311750523580937, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 32.800000000000004 }, "score": { "accuracy": 0.656, "calibration": 89.37109827323698, "cc": 48.08438271156575, "cc_by_type": { "score": 48.08438271156575 }, "cc_tiered": 47.93894394015618, "cc_tiered_open": 47.28337743201297, "cc_tiered_sealed": 48.59451044829939, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 48.08438271156575 } }, "raw_on_A": { "calibration": 88.68291663446959, "capability": 72.10556374473956, "intelligence": 55.528210855009505 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 58.6687573959346, "Emphasis on Accuracy 60:20:20": 54.59949882434838, "Emphasis on Cost 20:20:60": 55.22102328110309, "Emphasis on Speed 20:60:20": 67.97953354893902, "Intelligence only": 49.45428333114776 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/autotrust/GEV-26B-Decide", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "google/gemma-4-26B-A4B-it", "model_pin": "autotrust/GEV-26B-Decide@7c89590ead085bf77630b4bf68264ea30b6ddc78", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.6981132075471698, "acc_among_decisive": 0.9613899613899614, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "SPX AI team (SurdAI)", "axes": { "calibration": 87.47801599949784, "cost": 58.27072766098771, "intelligence": 48.779308810848946, "speed": 89.44891259349035 }, "calibration": { "parts": { "choice": { "ece_hard": 0.04837927211703187, "ece_score": 90.32414557659362, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 90.32414557659362 }, "noul": { "abstention_rate": 0.3207547169811321, "brier": 0.10397941406905582, "ece": 0.08441105054816544, "n": 371, "score": 83.11778989036691 }, "score": { "interval90_coverage": 0.975609756097561, "interval90_width": 0.31187895212285444, "n": 369, "nrps": 0.06867686607554577, "score": 88.99211253153302, "top_ece": 0.07574044164689696 } }, "score": 87.47801599949784 }, "capability": 68.1286624051734, "composite_ci95": { "A": [ 49.00994204245296, 68.5754476769007 ], "B": [ 44.94378680804397, 65.03574713274071 ], "C": [ 34.03468197392567, 55.062399131442234 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): the exact base google/gemma-4-12B-it is in the frozen pricing module's UNLISTED_BASE_MODELS set, so no exact base-model rate and no floor exist. The reference is the one the three rows PUBLISHED on this same base all use - cygnet, jev-omni and winnow-12b - OpenRouter google/gemma-3-12b-it list price USD 0.05 per 1M input, USD 0.00 per 1M output (the nearest publicly hosted 12B Gemma sibling; output is 0 because the runner reads candidate logits in one forward pass and generates nothing), x this row's own measured usage.input_tokens, the deck31b method. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.024225066666666666 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.024602301963439405 }, "display": "SPX-CD-Omni (SurdAI, google/gemma-4-12B-it + LoRA r32, one forward pass, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 47.456697185695944, "I_sealed": 50.10192043600195, "base": 48.779308810848946, "excess": -5.2186108927442465, "gap": -2.645223250306003, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 69.72620969563555, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 69.80182447310474, "judge": 68.35860250494396, "standard": 62.32733361239012 } }, "open|noul": { "cc": 16.937888198757765, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 39.13043478260869, "judge": -9.999999999999998, "standard": 0.0 } }, "open|score": { "cc": 55.70599366269451, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 45.44648364886738, "hard": 61.913039440117416, "judge": 59.403373958799065, "standard": 42.875586670605394 } }, "sealed|choice": { "cc": 73.93119843479649, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 84.03665656640308, "hard": 69.45701357466062, "judge": 73.9292364990689, "standard": 77.82978199285628 } }, "sealed|noul": { "cc": 24.24849331039893, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 32.00000000000001, "hard": 4.854368932038833, "judge": 28.947368421052634, "standard": 52.112676056338024 } }, "sealed|score": { "cc": 52.12606956281044, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 64.31573995848055, "hard": 49.879926355330106, "judge": 59.15248322255222, "standard": 39.98390029032339 } } } }, "jevbench_score": 63.16406085537614, "key": "spx-cd-omni", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 24, "ranked": true, "ranks": { "A": 24, "B": 24, "C": 28, "capability": 41 }, "scores": { "A": 63.16406085537614, "B": 58.9160449775072, "C": 43.86393114956675 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2966954869683832, "p50_s_raw": 0.0733477434841916, "p95_s_adjusted": 0.38264654851518576, "p95_s_raw": 0.11632327425759287 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.85250104999506, 89.14284725267011 ], "I": [ 43.845760787268546, 53.75726553555249 ], "capability": [ 64.04521719808471, 70.65155471508619 ], "composite": { "A": [ 49.00994204245296, 68.5754476769007 ], "B": [ 44.94378680804397, 65.03574713274071 ], "C": [ 34.03468197392567, 55.062399131442234 ] }, "seed": 137 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 87.47801599949784, "capability": 68.1286624051734, "intelligence": 48.779308810848946 }, "per_type": { "choice": { "accuracy": 0.7973333333333333, "calibration": 90.32414557659362, "cc": 74.08182791318927, "cc_by_type": { "choice": 74.08182791318927 }, "cc_tiered": 71.82870406521602, "cc_tiered_open": 69.72620969563555, "cc_tiered_sealed": 73.93119843479649, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 74.08182791318927 }, "noul": { "accuracy": 0.624, "calibration": 83.11778989036691, "cc": 24.8, "cc_by_type": { "noul": 24.8 }, "cc_tiered": 20.593190754578348, "cc_tiered_open": 16.937888198757765, "cc_tiered_sealed": 24.24849331039893, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 24.8 }, "score": { "accuracy": 0.6613333333333333, "calibration": 88.99211253153302, "cc": 52.652284619387025, "cc_by_type": { "score": 52.652284619387025 }, "cc_tiered": 53.91603161275248, "cc_tiered_open": 55.70599366269451, "cc_tiered_sealed": 52.12606956281044, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 52.652284619387025 } }, "raw_on_A": { "calibration": 83.85218691107563, "capability": 66.88160846988468, "intelligence": 49.91103002869373 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 58.4608318148256, "Emphasis on Accuracy 60:20:20": 52.9688184573952, "Emphasis on Cost 20:20:60": 57.22245674010682, "Emphasis on Speed 20:60:20": 66.8372282634097, "Intelligence only": 46.42660407688444 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/SurdAI/SPX-CD-Omni", "gpu": "RTX PRO 6000 Blackwell", "endpoint_condition": "measured 2026-10-06 on the live v1.6.0 S u P (1,500 rows, 1,477 ok; the 23 refusals are exactly the 23 long-context items, through the author's own \"Prompt exceeds context limit; no automatic truncation\" guard at their shipped --max-context 16384, although the base model itself declares 262,144 positions). Author's own code unmodified (infer.py / decision.py, sha256 1c5db5de.../43389f44...) with their shipped --backend transformers on the unquantized base and the flags their own EVALUATION.md prescribes FOR JEVBENCH: --prompt-format open-format --effort 1 --temperature 1.0. Their requirements-transformers.txt is missing torchvision, which transformers 5.16.1 needs for the Gemma 4 unified processor; adding it was the only deviation and it changes no numerics. One RTX PRO 6000 Blackwell (Lium, Brasov, Romania), torch 2.12.0+cu130, transformers 5.16.1, peft 0.20.0. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment; raw p50/p95 are in the run 53 receipts; add-requests run 53, benchmarkheaven.com/submit ref 75D5EF0C", "underlying": "google/gemma-4-12B-it", "model_pin": "SurdAI/SPX-CD-Omni@06a786a13f2ce9b8d1b8f541cf030fae007008fb + google/gemma-4-12B-it@707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.6792452830188679, "acc_among_decisive": 0.9285714285714286, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "r53 PREREG-R53.md, dl_r53.sh and current registry" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Hangzhi Zhou (@Hangzhi)", "axes": { "calibration": 61.31489461738089, "cost": 49.559714426383636, "intelligence": 54.860121638521335, "speed": 86.79193858661813 }, "calibration": { "parts": { "choice": { "ece_hard": 0.2516031734669799, "ece_score": 49.67936530660402, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 49.67936530660402 }, "noul": { "abstention_rate": 0.07027027027027022, "brier": 0.14489436428485655, "ece": 0.14040055215319622, "n": 370, "score": 71.91988956936075 }, "score": { "interval90_coverage": 0.6811989100817438, "interval90_width": 0.0510899182561308, "n": 367, "nrps": 0.11223140401478648, "score": 62.3454289761779, "top_ece": 0.3204300082308278 } }, "score": 61.31489461738089 }, "capability": 58.08750812795111, "composite_ci95": { "A": [ 52.773263175899025, 61.260192333972526 ], "B": [ 50.91942517195592, 60.65419970930148 ], "C": [ 36.648099427707656, 60.66776170099464 ] }, "cost": { "basis": "ESTIMATE, and the caveat travels with the number rather than sitting in a footnote. The frozen pricing snapshot returns None for google/diffusiongemma-26B-A4B-it (it is unlisted: no exact rate and no floor), and a scan of all 129 registry entries finds NO published row on any DiffusionGemma base at all - the page's openjev-razorback16 is the same model but a different submitter and carries no v1.6 score - so there was no precedent to copy. The reference used is google/gemma-4-26b-a4b-it, USD 0.09 per 1M input and USD 0.30 per 1M output: the identically sized and identically shaped publicly hosted Gemma 4 26B-A4B sibling of the unlisted diffusion checkpoint, a closer sibling than the one run 54 had to use for gemma-4-12B-it. x this row's own measured tokens. THE CAVEAT: a token-priced cost axis charges this row as though it ran one forward pass, when the author's own documentation denoises a 256-position canvas for up to 48 adaptive steps per question. The Cost axis therefore UNDERSTATES this configuration's compute, by construction and not by accident; the measured per-question denoising_steps are kept in the raw so the understatement is quantifiable. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.047275059999999994 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04801123222748815 }, "display": "Diffusion Jev (DiffusionGemma 26B-A4B on patched SGLang, 48-step diffusion readout, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 53.207343059355594, "I_sealed": 56.51290021768708, "base": 54.860121638521335, "excess": -5.878944800769732, "gap": -3.3055571583314887, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 66.57316141515619, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 60.36489462094999, "judge": 56.49307844429796, "standard": 77.39640016743407 } }, "open|noul": { "cc": 51.94202898550725, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 56.52173913043479, "judge": 19.999999999999996, "standard": 66.66666666666667 } }, "open|score": { "cc": 41.10683877740336, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 42.890763672952836, "hard": 43.8594983288807, "judge": 51.92844064579445, "standard": 18.477154424087306 } }, "sealed|choice": { "cc": 67.58861736481802, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 59.27601809954751, "judge": 70.2048417132216, "standard": 73.39573839142753 } }, "sealed|noul": { "cc": 53.955192987254144, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 32.038834951456316, "judge": 65.78947368421053, "standard": 69.01408450704226 } }, "sealed|score": { "cc": 47.994890300989084, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 68.90991267549208, "hard": 40.176230341477435, "judge": 50.19410623577225, "standard": 49.875875130586145 } } } }, "jevbench_score": 59.330428224994314, "key": "diffusion-jev", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 25, "ranked": true, "ranks": { "A": 25, "B": 25, "C": 23, "capability": 72 }, "scores": { "A": 59.330428224994314, "B": 58.15811643913557, "C": 49.60078097158421 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4113455285318196, "p50_s_raw": 0.1306727642659098, "p95_s_adjusted": 0.5088612267747521, "p95_s_raw": 0.17943061338737604 }, "status": { "answered_ok": 1474, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 56.00277250017865, 64.96308698031199 ], "I": [ 48.25688155312269, 59.81154987802872 ], "capability": [ 52.99878343969597, 61.730417849443256 ], "composite": { "A": [ 52.773263175899025, 61.260192333972526 ], "B": [ 50.91942517195592, 60.65419970930148 ], "C": [ 36.648099427707656, 60.66776170099464 ] }, "seed": 50 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 61.31489461738089, "capability": 58.08750812795111, "intelligence": 54.860121638521335 }, "per_type": { "choice": { "accuracy": 0.7533333333333333, "calibration": 49.67936530660402, "cc": 68.45485634171062, "cc_by_type": { "choice": 68.45485634171062 }, "cc_tiered": 67.0808893899871, "cc_tiered_open": 66.57316141515619, "cc_tiered_sealed": 67.58861736481802, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 68.45485634171062 }, "noul": { "accuracy": 0.784, "calibration": 71.91988956936075, "cc": 56.800000000000004, "cc_by_type": { "noul": 56.800000000000004 }, "cc_tiered": 52.9486109863807, "cc_tiered_open": 51.94202898550725, "cc_tiered_sealed": 53.955192987254144, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 56.800000000000004 }, "score": { "accuracy": 0.6106666666666667, "calibration": 62.3454289761779, "cc": 47.39565350609251, "cc_by_type": { "score": 47.39565350609251 }, "cc_tiered": 44.550864539196226, "cc_tiered_open": 41.10683877740336, "cc_tiered_sealed": 47.994890300989084, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 47.39565350609251 } }, "raw_on_A": { "calibration": 57.69524290701848, "capability": 57.542251645635915, "intelligence": 57.38926038425334 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.013333333333333308, "n": 375 }, "score": { "invalid_rate": 0.021333333333333315, "n": 375 } }, "views": { "Balanced 33:33:33": 59.033372905673865, "Emphasis on Accuracy 60:20:20": 56.86619781829752, "Emphasis on Cost 20:20:60": 54.41034813835743, "Emphasis on Speed 20:60:20": 67.31868662948479, "Intelligence only": 53.898210717803785 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0 (model card metadata); linked Gemma 4 model terms", "repo": "https://huggingface.co/google/diffusiongemma-26B-A4B-it", "gpu": "H100", "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. github.com/Hangzhi/diffusion-jev-sglang@d61f0a9c0c9b6d78631aa250125dc4b20e9ea0a2 serving google/diffusiongemma-26B-A4B-it@f7f5b7f5fa82ffc52addd066915886d497f5517b on SGLang@ef90bf677e4e6b280bf796f1c5f4ef0f05dda27f (their open PR 34061) with their own version-checked readout and context-admission patch. Built from THEIR OWN tested container recipe (deployment/modal_app.py + deployment/gemma-requirements.txt), which their doc names as the recipe to use; their `engine` extra and top-level Dockerfile target the older LLaDA deployment and their doc says plainly not to use them as a DiffusionGemma installer, so they were not used. SGLang is a pure-python install imported through PYTHONPATH, no source compile. Served with their own scripts/launch_diffusiongemma.py: native Gemma4Renoise, a 256-position canvas, up to 48 adaptive denoising steps, seed 42, thinking disabled, 8,192-token context, Triton attention, no CUDA graphs, no prefix cache, T=1 softmax over the options offered - their configuration, unchanged, on one H100 80 GB (Lium) against the A100 80 GB they tested. THE AUTHOR'S OWN STATEMENT, CARRIED UNSOFTENED: these are self-conditioned denoiser scores, NOT independent masked-token likelihoods and NOT calibrated probabilities of correctness; this is not a one-forward-pass or zero-generation method. There are no new trained weights - it is an inference configuration over Google's unchanged checkpoint. A format mismatch returns their own HTTP 503 and is recorded invalid, the same way for everyone. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 55, GitHub #30", "underlying": "google/diffusiongemma-26B-A4B-it", "model_pin": "google/diffusiongemma-26B-A4B-it@f7f5b7f5fa82ffc52addd066915886d497f5517b", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.9297297297297298, "acc_among_decisive": 0.8546511627906976, "valid": 370 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Elon Salfati / salfatigroup", "axes": { "calibration": 90.18677086647014, "cost": 45.963638345890566, "intelligence": 61.72139577013068, "speed": 87.11720932683018 }, "calibration": { "parts": { "choice": { "ece_hard": 0.043257328990227974, "ece_score": 91.34853420195441, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 91.34853420195441 }, "noul": { "abstention_rate": 0.22102425876010778, "brier": 0.06278221024258757, "ece": 0.05196765498652292, "n": 371, "score": 89.60646900269542 }, "score": { "interval90_coverage": 0.986449864498645, "interval90_width": 0.3859530261969287, "n": 369, "nrps": 0.06312416440831078, "score": 89.60530939476061, "top_ece": 0.07238482384823842 } }, "score": 90.18677086647014 }, "capability": 75.9540833183004, "composite_ci95": { "A": [ 54.25035390810508, 56.621792265328125 ], "B": [ 52.791631605081, 56.42470571115517 ], "C": [ 47.82528165394186, 56.621792265328125 ] }, "cost": { "basis": "ESTIMATE (base-model reference, not charged). Unmodified pricing_v15.reference resolves google/gemma-4-31B-it to google/gemma-4-31b-it at USD0.09/M input and USD0.34/M output in frozen snapshot. Measured output tokens are 0 by construction (decision_contract.py and release.json); one forward-pass option readout. Common v1.6.1 cost input range applied unchanged.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.062301899999999986 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.06327207176709546 }, "display": "René-1 31B FP8 (salfatigroup, Gemma 4 31B full fine-tune, one-pass option readout)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 62.91896990676746, "I_sealed": 60.523821633493895, "base": 61.72139577013068, "excess": -0.17823936916468108, "gap": 2.3951482732735627, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 78.65680231931067, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 73.57659641396666, "judge": 84.17930125247197, "standard": 69.8618668899121 } }, "open|noul": { "cc": 53.325051759834366, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 65.21739130434783, "judge": 39.99999999999999, "standard": 33.33333333333333 } }, "open|score": { "cc": 56.77505564115734, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 61.5151515151515, "hard": 65.03703703703704, "judge": 54.15254237288135, "standard": 41.8148148148148 } }, "sealed|choice": { "cc": 78.92231555231959, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 71.49321266968325, "judge": 86.96461824953444, "standard": 81.15531469392782 } }, "sealed|noul": { "cc": 47.36482975523042, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 80.0, "hard": 32.038834951456316, "judge": 50.0, "standard": 57.746478873239425 } }, "sealed|score": { "cc": 55.28431959293169, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 68.97163120567376, "hard": 50.97549019607843, "judge": 64.15568862275447, "standard": 43.75126903553299 } } } }, "jevbench_score": 55.84876743320955, "key": "rene-1-31b-fp8", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 26, "ranked": true, "ranks": { "A": 26, "B": 26, "C": 20, "capability": 12 }, "scores": { "A": 55.84876743320955, "B": 55.06950723833363, "C": 55.84876743320955 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3694145163521171, "p50_s_raw": 0.10970725817605853, "p95_s_adjusted": 0.5257328446954489, "p95_s_raw": 0.18786642234772444 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.96895555807659, 90.68721224415906 ], "I": [ 56.323580074482344, 65.99929373375565 ], "capability": [ 72.10629528396096, 77.53024725815187 ], "composite": { "A": [ 54.25035390810508, 56.621792265328125 ], "B": [ 52.791631605081, 56.42470571115517 ], "C": [ 47.82528165394186, 56.621792265328125 ] }, "seed": 128 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.18677086647014, "capability": 75.9540833183004, "intelligence": 61.72139577013068 }, "per_type": { "choice": { "accuracy": 0.8346666666666667, "calibration": 91.34853420195441, "cc": 78.85622803444389, "cc_by_type": { "choice": 78.85622803444389 }, "cc_tiered": 78.78955893581514, "cc_tiered_open": 78.65680231931067, "cc_tiered_sealed": 78.92231555231959, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 78.85622803444389 }, "noul": { "accuracy": 0.7573333333333333, "calibration": 89.60646900269542, "cc": 51.46666666666666, "cc_by_type": { "noul": 51.46666666666666 }, "cc_tiered": 50.34494075753239, "cc_tiered_open": 53.325051759834366, "cc_tiered_sealed": 47.36482975523042, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 51.46666666666666 }, "score": { "accuracy": 0.6746666666666666, "calibration": 89.60530939476061, "cc": 55.4684210526316, "cc_by_type": { "score": 55.4684210526316 }, "cc_tiered": 56.029687617044516, "cc_tiered_open": 56.77505564115734, "cc_tiered_sealed": 55.28431959293169, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 55.4684210526316 } }, "raw_on_A": { "calibration": 86.48256493925484, "capability": 76.2896760781785, "intelligence": 66.09678721710216 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 51.281219378577866, "Emphasis on Accuracy 60:20:20": 51.62853971578034, "Emphasis on Cost 20:20:60": 45.45809395907335, "Emphasis on Speed 20:60:20": 58.36507393476142, "Intelligence only": 52.15843207508249 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/salfatigroup/rene-1-31b-fp8", "gpu": "RTX PRO6000 Blackwell", "endpoint_condition": "Candidate re-measured on 2026-10-07, current v1.6.0/v1.6.1 1500-item pool (sha256901983ae...). Immutable salfatigroup/rene-1-31b-fp8@bd634489957f8da63ccce858ee33fd6c2929d286, Apache-2.0. Same unmodified reviewed contributor source and run-39 transport shim. Author noul/yesno type mapping only; two-decimal probability rounding retained. One evaluator-owned RTX PRO6000 Blackwell 96GB pod, loopback-only network namespace, offline credential-free inference; no external hosted API, no retry or batching. Unmodified run_v16/typesafe adapter, serial one request at a time, selfhosted x2+0.15s latency adjustment. Missing ATTRIBUTIONS.md and NOTICE disclosed: training-data overlap cannot be checked. Florian GO in DECISIONS.md (go gor it, Apache2.0 confirmed) resolves prior attribution hold; publication remains release lane responsibility. COMPLETE: 1500/1500 distinct input IDs, 1477 HTTP200 native answers,23 HTTP422 refusals under the author's own32768-token state+question /65536-token request budget; every refusal counted wrong. Smoke passed all8 checks. All42 release-manifest files and6 prior source files hash-verified. Raw sha256 a28a8f0068f38b548f064a6e678bf63b13304f9add0a67b448dcc8003c9f38f1. Stack torch2.14.0, transformers5.17.0, compressed-tensors0.19.0, accelerate1.15.0. Provider pod91916684-9980-419c-98b7-c7a714a3912c removed and verified absent; approxUSD0.17. Fresh latest feed places this candidate inside both official caps; material open-board top-five change requires release-lane preview disposition before publication.", "underlying": "google/gemma-4-31B-it", "model_pin": "salfatigroup/rene-1-31b-fp8@bd634489957f8da63ccce858ee33fd6c2929d286", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7789757412398922, "acc_among_decisive": 0.9826989619377162, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "EmpirioLabs (@Adam-Dalloul)", "axes": { "calibration": 89.21801638247211, "cost": 57.86508463809988, "intelligence": 45.81854920739271, "speed": 88.88310989631262 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06972206212499996, "ece_score": 86.055587575, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 86.055587575 }, "noul": { "abstention_rate": 0.31733333333333336, "brier": 0.11392156247660881, "ece": 0.04671065970666656, "n": 375, "score": 90.6578680586667 }, "score": { "interval90_coverage": 0.9706666666666667, "interval90_width": 0.4937777777777778, "n": 375, "nrps": 0.09021695431167416, "score": 90.94059351374963, "top_ece": 0.045485587706666636 } }, "score": 89.21801638247211 }, "capability": 67.51828279493242, "composite_ci95": { "A": [ 40.003677562543594, 66.59206455090654 ], "B": [ 36.14823252368513, 62.39027598829859 ], "C": [ 27.780331640655273, 46.3455307690425 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights): the exact declared base Qwen/Qwen3.5-4B resolves in the frozen pricing snapshot through DEEPINFRA_BASE_REFERENCES to USD 0.03 per 1M input and USD 0.15 per 1M output, read before the pod and asserted in make_registry_r55.py. The same rate run 54 gave messier-one, the other self-hosted Qwen3.5-4B derivative. DELIBERATELY NOT the vendor's own cheaper public tariff: Aplomb 1 is also sold on EmpirioLabs' hosted API at USD 0.02 per 1M input with output free (GitHub #195), which would LOWER this row's cost - but all 106 self-hosted rows in the registry carry cost_kind `estimate` and not one is priced at a vendor tariff, and creating that precedent for one row is the release lane's call, not this job's. The estimate can therefore only OVERSTATE this row, and it is reversible by arithmetic because the raw keeps the tokens. x this row's own measured tokens. INPUT TOKENS ARE EXACT, NOT ESTIMATED, and they include BOTH passes of the author's own --debias flag on every multi-option choice, because that is what the configuration consumes. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the author's answer_logits runs the model with logits_to_keep=1, use_cache=False and reads label logits, and nothing is generated anywhere in their code path. Counts carried in the scorer's own cost_estimate field, never written into usage. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.08184117999999999 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.025380324983073795 }, "display": "Aplomb 1 (5.3B decision model on Qwen3.5-4B, trained readout head, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 48.90245826703744, "I_sealed": 42.73464014774797, "base": 45.81854920739271, "excess": 3.5944304768512274, "gap": 6.167818119289471, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 72.51049065629134, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 73.57659641396666, "judge": 80.22412656558997, "standard": 54.79280033486815 } }, "open|noul": { "cc": 26.777087646652866, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 30.434782608695656, "judge": 0.0, "standard": 44.44444444444444 } }, "open|score": { "cc": 47.419796498168125, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 57.31956009090909, "hard": 46.33070064814815, "judge": 45.652502593220326, "standard": 47.299047259259254 } }, "sealed|choice": { "cc": 61.23201765527931, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 74.72470623013821, "hard": 56.730769230769226, "judge": 62.756052141527014, "standard": 61.20211848749847 } }, "sealed|noul": { "cc": 23.583357683145376, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 43.99999999999999, "hard": -2.9126213592232997, "judge": 36.8421052631579, "standard": 46.47887323943662 } }, "sealed|score": { "cc": 43.38854510481924, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 62.11917736170213, "hard": 39.6622859901961, "judge": 48.63257153293411, "standard": 33.60970756345175 } } } }, "jevbench_score": 54.55838408944504, "key": "aplomb-1", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 27, "ranked": true, "ranks": { "A": 27, "B": 28, "C": 30, "capability": 44 }, "scores": { "A": 54.55838408944504, "B": 50.34915295427923, "C": 37.88776672878128 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.33727095890790226, "p50_s_raw": 0.09363547945395112, "p95_s_adjusted": 0.3834511662367731, "p95_s_raw": 0.11672558311838657 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.76160851883273, 89.84442192937534 ], "I": [ 40.30661294885779, 49.94954146129873 ], "capability": [ 62.95351995583873, 69.45667583287421 ], "composite": { "A": [ 40.003677562543594, 66.59206455090654 ], "B": [ 36.14823252368513, 62.39027598829859 ], "C": [ 27.780331640655273, 46.3455307690425 ] }, "seed": 16 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.21801638247211, "capability": 67.51828279493242, "intelligence": 45.81854920739271 }, "per_type": { "choice": { "accuracy": 0.7213333333333334, "calibration": 86.055587575, "cc": 64.36251338063525, "cc_by_type": { "choice": 64.36251338063525 }, "cc_tiered": 66.87125415578532, "cc_tiered_open": 72.51049065629134, "cc_tiered_sealed": 61.23201765527931, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 64.36251338063525 }, "noul": { "accuracy": 0.6373333333333333, "calibration": 90.6578680586667, "cc": 27.46666666666666, "cc_by_type": { "noul": 27.46666666666666 }, "cc_tiered": 25.18022266489912, "cc_tiered_open": 26.777087646652866, "cc_tiered_sealed": 23.583357683145376, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 27.46666666666666 }, "score": { "accuracy": 0.6106666666666667, "calibration": 90.94059351374963, "cc": 44.34175579824563, "cc_by_type": { "score": 44.34175579824563 }, "cc_tiered": 45.40417080149368, "cc_tiered_open": 47.419796498168125, "cc_tiered_sealed": 43.38854510481924, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 44.34175579824563 } }, "raw_on_A": { "calibration": 89.9318044592949, "capability": 70.86116592289882, "intelligence": 51.790527386502745 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 50.026426143216206, "Emphasis on Accuracy 60:20:20": 44.66300743458741, "Emphasis on Cost 20:20:60": 49.44235580546631, "Emphasis on Speed 20:60:20": 57.6274614302068, "Intelligence only": 38.475475184084495 }, "class": "Jev-compatible decision model", "open": "gated weights", "licence": "EmpirioLabs Model License 1.0", "repo": "https://huggingface.co/empiriolabsai/aplomb-1", "gpu": "H100", "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights empiriolabsai/aplomb-1@f8e2dc81f0505db21a3b7a933ea479658e4ebb8f, bf16, 5.3 B, base Qwen/Qwen3.5-4B plus the shipped 1.3 GB audio encoder and the trained DecisionHead; click-through gate accepted under the EmpirioLabs Model License 1.0 section 2.1, which grants evaluation and the publication of benchmark results without asking. Served behind the author's own TypeSafe wire format by a TRANSPORT SHIM ONLY: run_aplomb.main()'s loading block verbatim, every request handed to THEIR run_aplomb.decide() with THEIR edm_config.json temperatures and abstain_bias and debias=True - the configuration the author names as the reference in #195 and in the model card. Their own aplomb/schema.py documents the request schema as wire-compatible with TypeSafe /v1/systemone and their aplomb/readout.py already emits the answer dicts the unchanged typesafe adapter parses, so no answer field is rewritten and no contributor file is edited. STACK: the author's own documented install line, with transformers pinned to 5.17.0 - the version their own audio sidecar was saved with; their loader compares the saved config to the live one and 5.19.0 fails it on the transformers_version string alone. SPEED CAVEAT: their documented install line does not include causal_conv1d, so transformers falls back to a reference PyTorch implementation it itself calls much slower; the Speed axis is measured on the stack the author documents. One H100 80 GB (Lium). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 55, GitHub #195", "underlying": "Qwen/Qwen3.5-4B", "model_pin": "empiriolabsai/aplomb-1@f8e2dc81f0505db21a3b7a933ea479658e4ebb8f", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.6826666666666666, "acc_among_decisive": 0.93359375, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Bespoke Labs", "axes": { "calibration": 77.04569964036838, "cost": 46.094951684283686, "intelligence": 56.54011796945148, "speed": 89.19502471208428 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1589499873699225, "ece_score": 68.2100025260155, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 68.2100025260155 }, "noul": { "abstention_rate": 0.13477088948787064, "brier": 0.12391951730825534, "ece": 0.10295952013212975, "n": 371, "score": 79.40809597357405 }, "score": { "interval90_coverage": 0.8699186991869918, "interval90_width": 0.20505871725383928, "n": 369, "nrps": 0.07833002311655612, "score": 83.51900042151561, "top_ece": 0.1256449842265658 } }, "score": 77.04569964036838 }, "capability": 66.79290880490993, "composite_ci95": { "A": [ 51.086235675343055, 54.820858958312876 ], "B": [ 49.08081410151091, 54.214427134732105 ], "C": [ 35.737762455481224, 54.819952316161036 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter qwen/qwen3.5-9b, the reference of the published nimble-9b row (same base), USD 0.1 per 1M input and USD 0.15 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.06167713333333334 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.06263757616790792 }, "display": "Bespoke Nimble 9B v3 (Qwen3.5-9B LoRA)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 57.07198447174714, "I_sealed": 56.008251467155816, "base": 56.54011796945148, "excess": -1.5096546378469164, "gap": 1.0637330045913274, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 72.11673425629002, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 60.36489462094999, "judge": 76.26895187870797, "standard": 77.39640016743407 } }, "open|noul": { "cc": 48.59075224292616, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 39.99999999999999, "standard": 44.44444444444444 } }, "open|score": { "cc": 50.508466916025256, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 73.68883649626082, "hard": 50.44357111216904, "judge": 56.6073538275629, "standard": 29.899743366313437 } }, "sealed|choice": { "cc": 63.08410687973317, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 77.38526346907102, "hard": 58.76696832579185, "judge": 60.89385474860336, "standard": 67.8531838896416 } }, "sealed|noul": { "cc": 51.038414647311576, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 28.155339805825253, "judge": 65.78947368421053, "standard": 66.19718309859155 } }, "sealed|score": { "cc": 53.90223287442272, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 75.56838262623131, "hard": 43.523681374236745, "judge": 58.275920598763435, "standard": 57.26572941237931 } } } }, "jevbench_score": 53.47656993239491, "key": "bespoke-nimble-9b-v3", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 28, "ranked": true, "ranks": { "A": 28, "B": 27, "C": 26, "capability": 45 }, "scores": { "A": 53.47656993239491, "B": 52.29615241766553, "C": 47.486970389641556 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.33777692201547327, "p50_s_raw": 0.09388846100773662, "p95_s_adjusted": 0.3563424444932025, "p95_s_raw": 0.10317122224660125 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 72.11138007423693, 79.85549338743986 ], "I": [ 49.993162280566764, 61.190765007174605 ], "capability": [ 61.91008408542518, 69.96614101398373 ], "composite": { "A": [ 51.086235675343055, 54.820858958312876 ], "B": [ 49.08081410151091, 54.214427134732105 ], "C": [ 35.737762455481224, 54.819952316161036 ] }, "seed": 17 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 77.04569964036838, "capability": 66.79290880490993, "intelligence": 56.54011796945148 }, "per_type": { "choice": { "accuracy": 0.7333333333333333, "calibration": 68.2100025260155, "cc": 65.89714199103851, "cc_by_type": { "choice": 65.89714199103851 }, "cc_tiered": 67.6004205680116, "cc_tiered_open": 72.11673425629002, "cc_tiered_sealed": 63.08410687973317, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 65.89714199103851 }, "noul": { "accuracy": 0.7653333333333333, "calibration": 79.40809597357405, "cc": 53.06666666666666, "cc_by_type": { "noul": 53.06666666666666 }, "cc_tiered": 49.81458344511887, "cc_tiered_open": 48.59075224292616, "cc_tiered_sealed": 51.038414647311576, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 53.06666666666666 }, "score": { "accuracy": 0.6586666666666666, "calibration": 83.51900042151561, "cc": 53.90613650563776, "cc_by_type": { "score": 53.90613650563776 }, "cc_tiered": 52.20534989522399, "cc_tiered_open": 50.508466916025256, "cc_tiered_sealed": 53.90223287442272, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 53.90613650563776 } }, "raw_on_A": { "calibration": 79.9777663718046, "capability": 70.03747268625314, "intelligence": 60.09717900070167 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 50.39687437311006, "Emphasis on Accuracy 60:20:20": 49.43254820883513, "Emphasis on Cost 20:20:60": 45.21647399422632, "Emphasis on Speed 20:60:20": 58.20016580649009, "Intelligence only": 48.05332347466076 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "cc-by-nc-4.0", "repo": "https://huggingface.co/bespokelabs/Bespoke-Nimble-9B-v3", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "Qwen/Qwen3.5-9B", "model_pin": "bespokelabs/Bespoke-Nimble-9B-v3@8e927b9b4afdbb14479fac10a7364d1a695be208", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.8652291105121294, "acc_among_decisive": 0.8940809968847352, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "APUS AI-LAB (@cryptoSUN2049 / findoo2049@gmail.com)", "axes": { "calibration": 66.61054590286017, "cost": 48.738123569705756, "intelligence": 47.20514429531085, "speed": 82.18245466956745 }, "calibration": { "parts": { "choice": { "ece_hard": 0.21593843550946115, "ece_score": 56.81231289810778, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 56.81231289810778 }, "noul": { "abstention_rate": 0.1185983827493261, "brier": 0.1586174454486672, "ece": 0.14320810756172592, "n": 371, "score": 71.35837848765482 }, "score": { "interval90_coverage": 0.8401084010840109, "interval90_width": 0.29200542005420044, "n": 369, "nrps": 0.12208457963241476, "score": 71.66094632281795, "top_ece": 0.22234824695561312 } }, "score": 66.61054590286017 }, "capability": 56.907845099085506, "composite_ci95": { "A": [ 33.00331563170442, 56.572404110680004 ], "B": [ 30.747622136678878, 54.687915864792735 ], "C": [ 22.918969188683615, 40.193299875459424 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights): the exact declared base Qwen/Qwen3.5-9B resolves in the frozen pricing snapshot through BASE_REFERENCES to qwen/qwen3.5-9b = USD 0.10 per 1M input and USD 0.15 per 1M output, read before the pod and asserted in make_registry_r56.py. Nothing provisional. x this row's own measured tokens. BECAUSE OUTPUT TOKENS ARE 0 BY CONSTRUCTION this axis rests on the input rate alone. Estimated, not charged.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.0503522 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.05113628977657414 }, "display": "APUS-OpenJev-v1-9B (merged bf16 checkpoint-3000 on Qwen3.5-9B, their own native runtime, full depth, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 52.06268273300428, "I_sealed": 42.347605857617424, "base": 47.20514429531085, "excess": 7.141689232948613, "gap": 9.715076875386856, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 61.13339106992548, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 56.59012268008808, "judge": 48.58272907053395, "standard": 77.39640016743407 } }, "open|noul": { "cc": 59.49758454106281, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 56.52173913043479, "judge": 60.00000000000001, "standard": 44.44444444444444 } }, "open|score": { "cc": 35.55707258802457, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 49.48792439337757, "hard": 38.515870391630514, "judge": 34.91730621356864, "standard": 23.633700639820077 } }, "sealed|choice": { "cc": 56.01292563590547, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 64.08247727440693, "hard": 44.00452488687781, "judge": 59.96275605214154, "standard": 70.07020569035596 } }, "sealed|noul": { "cc": 39.873501407011304, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": 20.388349514563096, "judge": 39.47368421052631, "standard": 63.38028169014085 } }, "sealed|score": { "cc": 31.156390529935493, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 50.76837620384379, "hard": 19.973469764155748, "judge": 28.021772929225364, "standard": 48.418165625606015 } } } }, "jevbench_score": 49.17976892452783, "key": "apus-openjev-v1-9b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 29, "ranked": true, "ranks": { "A": 29, "B": 30, "C": 31, "capability": 77 }, "scores": { "A": 49.17976892452783, "B": 47.01556694203153, "C": 34.15261730869988 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.705569653166458, "p50_s_raw": 0.277784826583229, "p95_s_adjusted": 0.8574615204473958, "p95_s_raw": 0.35373076022369787 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 62.07882152511099, 70.58159568750249 ], "I": [ 39.88317778412442, 50.523530007279774 ], "capability": [ 51.79741923709466, 59.84489784641873 ], "composite": { "A": [ 33.00331563170442, 56.572404110680004 ], "B": [ 30.747622136678878, 54.687915864792735 ], "C": [ 22.918969188683615, 40.193299875459424 ] }, "seed": 18 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 66.61054590286017, "capability": 56.907845099085506, "intelligence": 47.20514429531085 }, "per_type": { "choice": { "accuracy": 0.6666666666666666, "calibration": 56.81231289810778, "cc": 57.37142748879814, "cc_by_type": { "choice": 57.37142748879814 }, "cc_tiered": 58.57315835291547, "cc_tiered_open": 61.13339106992548, "cc_tiered_sealed": 56.01292563590547, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.37142748879814 }, "noul": { "accuracy": 0.7386666666666667, "calibration": 71.35837848765482, "cc": 47.73333333333334, "cc_by_type": { "noul": 47.73333333333334 }, "cc_tiered": 49.685542974037055, "cc_tiered_open": 59.49758454106281, "cc_tiered_sealed": 39.873501407011304, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 47.73333333333334 }, "score": { "accuracy": 0.5306666666666666, "calibration": 71.66094632281795, "cc": 33.347875357101564, "cc_by_type": { "score": 33.347875357101564 }, "cc_tiered": 33.35673155898003, "cc_tiered_open": 35.55707258802457, "cc_tiered_sealed": 31.156390529935493, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 33.347875357101564 } }, "raw_on_A": { "calibration": 70.50531100654376, "capability": 61.66232173095365, "intelligence": 52.819332455363536 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 47.16399064408527, "Emphasis on Accuracy 60:20:20": 44.00058844315432, "Emphasis on Cost 20:20:60": 44.61842645447928, "Emphasis on Speed 20:60:20": 54.145911082892475, "Intelligence only": 39.978415802185516 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/apus-ailab/APUS-OpenJev-v1-9B", "gpu": "H100 PCIe", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights apus-ailab/APUS-OpenJev-v1-9B@82c9c56cfa9de8d36704ed91948d4726ef111635, Apache-2.0, 17.55 GiB bf16 merged weights, checkpoint-3000, declared base Qwen/Qwen3.5-9B, full depth 32 layers. STACK: the authors' own requirements.txt, unmodified - torch==2.8.0 and transformers==5.16.1. The transformers pin is THEIRS, not a workaround: OpenJet.__init__ raises 'Layer execution is audited for transformers==5.16.1' on any other version. Served behind a TRANSPORT SHIM ONLY (r56/serve/apus_systemone_r56.py): loaded with THEIR OpenJet.from_pretrained(device=cuda:0, dtype=bfloat16) and every request handed to THEIR decide(), returning THEIR format_response() output. No contributor file edited. effort='high', the full-depth path, their decide() default and the configuration their own headline Frozen80 number is quoted on; effort='low' is not measured and no routing between the two is invented. One H100 PCIe 80 GB (Lium). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. MAPPING (run 56's, not the authors'; pre-registered in PREREG-R56 s2.2 before any item was sent and repeated here because it is part of the number): their runtime accepts primitive in {choice, noul, score_level} with 2-16 candidates labelled A-P. choice -> their 'choice' primitive, candidates = the item's own labels as id and its own criteria[label] as description, in labels order; their format_response returns probabilities keyed by candidate id, i.e. by the JevBench label already, so nothing is rewritten. All 750 choice items carry 3-9 options, inside their 2-16 window. noul -> their 'noul' primitive; their contract REQUIRES the canonical yes/no criteria there, so the item's own true/false description strings are NOT delivered - the proposition itself travels verbatim in instructions, which is what their noul is built to judge. score -> their 'choice' primitive over the ordinal levels, because their own RUNTIME.md rules score_level out as an ordinal Score API ('score_level is the judgement of a single proposition, it cannot be treated as a complete ordinal Score API'); the 4-5 ordered levels become candidates with id = the level index and description = the item's own level text, which carries the order. Recording all 375 score items as misses instead would understate a system whose published interface answers them. CALIBRATION IS MEASURED AS THE AUTHORS SHIP IT AND THEY DISCLOSED IT IN ADVANCE: their response carries calibrated: False and both their card and RUNTIME.md state the probabilities are an uncalibrated relative softmax over the offered candidates. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, NOT BY ESTIMATE: decide() runs one forward pass with logits_to_keep=1 and reads candidate logits; nothing is generated anywhere in that path. Input tokens are their own prompt_tokens field. THE 23 LONG ITEMS: their compile() raises ValueError('input exceeds runtime limit; no truncation permitted') above 8,192 tokens and their RUNTIME.md states it in advance. Recorded invalid. This is the first row in the series whose refusal on those items is guaranteed in the author's own code with an explicit no-truncation promise. add-requests run 56, GitHub #106, benchmarkheaven.com/submit ref 2033065A", "underlying": "Qwen/Qwen3.5-9B", "model_pin": "apus-ailab/APUS-OpenJev-v1-9B@82c9c56cfa9de8d36704ed91948d4726ef111635", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.8814016172506739, "acc_among_decisive": 0.8470948012232415, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 87.88101858186977, "cost": 63.07359395281681, "intelligence": 42.9626698747687, "speed": 93.83356539601934 }, "calibration": { "parts": { "choice": { "ece_hard": 0.061775972135365, "ece_score": 87.644805572927, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 87.644805572927 }, "noul": { "abstention_rate": 0.29333333333333333, "brier": 0.12494153723477718, "ece": 0.06617846187638739, "n": 375, "score": 86.76430762472252 }, "score": { "interval90_coverage": 0.9573333333333334, "interval90_width": 0.49066666666666664, "n": 375, "nrps": 0.09744572815253887, "score": 89.2339425479598, "top_ece": 0.058937710444132485 } }, "score": 87.88101858186977 }, "capability": 65.42184422831923, "composite_ci95": { "A": [ 34.218232721078074, 62.094303811225494 ], "B": [ 30.257952435880583, 57.342398440511 ], "C": [ 23.762661611859777, 43.121044313351035 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "Plumb-4B (crh225, JevK5 v0.2 + LoRA)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.15367542782093, "I_sealed": 40.77166432171647, "base": 42.9626698747687, "excess": 1.8086234636662133, "gap": 4.382011106104457, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 63.70555942875353, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 54.70273670965714, "judge": 72.31377719182596, "standard": 62.32733361239012 } }, "open|noul": { "cc": 36.11249137336094, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 39.13043478260869, "judge": 10.000000000000009, "standard": 44.44444444444444 } }, "open|score": { "cc": 35.64297548134833, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 60.38519253276966, "hard": 30.154558838586563, "judge": 43.95614933948649, "standard": 21.77893945395395 } }, "sealed|choice": { "cc": 58.16335833730948, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 57.239819004524875, "judge": 57.16945996275605, "standard": 57.876585786426915 } }, "sealed|noul": { "cc": 23.383111546129097, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 56.00000000000001, "hard": 4.854368932038833, "judge": 36.8421052631579, "standard": 23.943661971830977 } }, "sealed|score": { "cc": 40.76852308171083, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 65.90236427761113, "hard": 40.87931496804801, "judge": 36.35703355260749, "standard": 34.597253004741354 } } } }, "jevbench_score": 48.28224126395837, "key": "plumb-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 30, "ranked": true, "ranks": { "A": 30, "B": 31, "C": 32, "capability": 49 }, "scores": { "A": 48.28224126395837, "B": 43.7169924526014, "C": 33.529334211082194 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.18999176411889493, "p50_s_raw": 0.019995882059447467, "p95_s_adjusted": 0.21772519345395266, "p95_s_raw": 0.033862596726976335 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.6814729990224, 88.307491492748 ], "I": [ 37.25495036908704, 47.871420386377224 ], "capability": [ 60.18380887501134, 67.22662477915313 ], "composite": { "A": [ 34.218232721078074, 62.094303811225494 ], "B": [ 30.257952435880583, 57.342398440511 ], "C": [ 23.762661611859777, 43.121044313351035 ] }, "seed": 85 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 87.88101858186977, "capability": 65.42184422831923, "intelligence": 42.9626698747687 }, "per_type": { "choice": { "accuracy": 0.6853333333333333, "calibration": 87.644805572927, "cc": 59.758627549425455, "cc_by_type": { "choice": 59.758627549425455 }, "cc_tiered": 60.93445888303151, "cc_tiered_open": 63.70555942875353, "cc_tiered_sealed": 58.16335833730948, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 59.758627549425455 }, "noul": { "accuracy": 0.6453333333333333, "calibration": 86.76430762472252, "cc": 29.066666666666663, "cc_by_type": { "noul": 29.066666666666663 }, "cc_tiered": 29.74780145974502, "cc_tiered_open": 36.11249137336094, "cc_tiered_sealed": 23.383111546129097, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 29.066666666666663 }, "score": { "accuracy": 0.576, "calibration": 89.2339425479598, "cc": 41.48168877172061, "cc_by_type": { "score": 41.48168877172061 }, "cc_tiered": 38.20574928152958, "cc_tiered_open": 35.64297548134833, "cc_tiered_sealed": 40.76852308171083, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 41.48168877172061 } }, "raw_on_A": { "calibration": 82.25783080586818, "capability": 66.15973395095179, "intelligence": 50.06163709603538 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 44.487894416524036, "Emphasis on Accuracy 60:20:20": 38.31837801553863, "Emphasis on Cost 20:20:60": 45.2973332743533, "Emphasis on Speed 20:60:20": 51.919511056412496, "Intelligence only": 31.720043803902097 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://huggingface.co/crh225/plumb-4b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/plumb-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7066666666666667, "acc_among_decisive": 0.9132075471698113, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": true, "api_subset": false, "author": "SPX AI team (SurdAI, formerly Simplex AI)", "axes": { "calibration": 92.41334857032513, "cost": 43.076275488494026, "intelligence": 64.85693822910903, "speed": 77.96292698098645 }, "calibration": { "parts": { "choice": { "ece_hard": 0.037067490257276604, "ece_score": 92.58650194854468, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 92.58650194854468 }, "noul": { "abstention_rate": 0.1617250673854448, "brier": 0.07013566233777668, "ece": 0.037583902439496455, "n": 371, "score": 92.48321951210072 }, "score": { "interval90_coverage": 0.983739837398374, "interval90_width": 0.26445347786811196, "n": 369, "nrps": 0.05200753435023827, "score": 92.17032425033, "top_ece": 0.052292990321580954 } }, "score": 92.41334857032513 }, "capability": 78.63514339971708, "composite_ci95": { "A": [ 46.40578610997944, 48.21089390268207 ], "B": [ 45.98130956469153, 48.76321124599849 ], "C": [ 45.82343084108345, 48.21089390268207 ] }, "cost": { "basis": "Operator's own public-beta overage tariff, conservative reading USD 0.08 per 1M input tokens, output USD 0: three dated primary readings of the operator's own published overage tariff: 24 Sep 2026 provider console (flash 0.025, pro 0.08 per 1M input), 1 Oct 2026 docs page (flash 0.04, pro 0.02), 6 Oct 2026 docs page AND /v1/models payload agreeing (flash 0.02, pro 0.04). Sources conflict, so the per-model MAXIMUM is used, which is the published fastino-gliner-2-5-decide precedent (\"published sources conflict ... conservative higher\"). Today's agreeing pair would be USD 0.04/M, i.e. half this row's cost. No base-model floor applies: the submitter states the base model stays sealed until a later open release. Priced on usage.input_tokens as the provider reports them. The provider also reports billable_input_tokens = input_tokens / 2 on every single item (297/297 of the public set, exactly 2.0): its default effort 2 renders the prompt twice and its docs bundle the second pass free during public beta (\"default effort 2 is included at the single-pass input price\"). That bundle is a public-beta promotion, and TASK.md forbids a free or promotional tier setting the price, so the two passes actually consumed are charged. The operator's own billed figure is exactly half of this row's cost", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.07775829333333333 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.07896915368991199 }, "display": "SPX-CD Pro (SurdAI, hosted /v1/systemone, Oct-4 checkpoint)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 65.26225762662179, "I_sealed": 64.45161883159628, "base": 64.85693822910903, "excess": -1.7627488474127326, "gap": 0.8106387950255112, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 77.44982804388046, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 79.23875432525952, "judge": 80.22412656558997, "standard": 62.32733361239012 } }, "open|noul": { "cc": 58.726017943409246, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 82.6086956521739, "judge": 19.999999999999996, "standard": 55.55555555555556 } }, "open|score": { "cc": 59.61092689257566, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 62.50227372429029, "hard": 68.13445392047002, "judge": 54.627278742003114, "standard": 48.59367164678844 } }, "sealed|choice": { "cc": 78.81397986232295, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 74.5475113122172, "judge": 77.6536312849162, "standard": 87.80638009607095 } }, "sealed|noul": { "cc": 53.70130193527029, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 84.00000000000001, "hard": 37.86407766990292, "judge": 52.63157894736843, "standard": 71.83098591549295 } }, "sealed|score": { "cc": 60.83957469719559, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 66.86467136136848, "hard": 54.98115972463263, "judge": 65.31737432933835, "standard": 62.82715686202089 } } } }, "jevbench_score": 47.66970820706367, "key": "spx-cd-pro", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 31, "ranked": true, "ranks": { "A": 31, "B": 29, "C": 25, "capability": 7 }, "scores": { "A": 47.66970820706367, "B": 47.762738448829694, "C": 47.66970820706367 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 1.1281277425587177, "p50_s_raw": 1.1281277425587177, "p95_s_adjusted": 1.4169320456683636, "p95_s_raw": 1.4169320456683636 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 87.48842630468798, 92.31235357989142 ], "I": [ 59.60847348558619, 68.98634831582494 ], "capability": [ 74.38248323784742, 79.84438918865506 ], "composite": { "A": [ 46.40578610997944, 48.21089390268207 ], "B": [ 45.98130956469153, 48.76321124599849 ], "C": [ 45.82343084108345, 48.21089390268207 ] }, "seed": 138 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": true, "g_med_used": 2.573387642438244, "lane": "api", "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 92.41334857032513, "capability": 78.63514339971708, "intelligence": 64.85693822910903 }, "per_type": { "choice": { "accuracy": 0.8373333333333334, "calibration": 92.58650194854468, "cc": 79.1972566145335, "cc_by_type": { "choice": 79.1972566145335 }, "cc_tiered": 78.1319039531017, "cc_tiered_open": 77.44982804388046, "cc_tiered_sealed": 78.81397986232295, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 79.1972566145335 }, "noul": { "accuracy": 0.7893333333333333, "calibration": 92.48321951210072, "cc": 57.86666666666667, "cc_by_type": { "noul": 57.86666666666667 }, "cc_tiered": 56.21365993933976, "cc_tiered_open": 58.726017943409246, "cc_tiered_sealed": 53.70130193527029, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 57.86666666666667 }, "score": { "accuracy": 0.6853333333333333, "calibration": 92.17032425033, "cc": 60.615191123661816, "cc_by_type": { "score": 60.615191123661816 }, "cc_tiered": 60.225250794885625, "cc_tiered_open": 59.61092689257566, "cc_tiered_sealed": 60.83957469719559, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 60.615191123661816 } }, "raw_on_A": null, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 43.27025827744028, "Emphasis on Accuracy 60:20:20": 45.09442248470386, "Emphasis on Cost 20:20:60": 37.91159972444074, "Emphasis on Speed 20:60:20": 48.12587216483719, "Intelligence only": 48.138519063832746 }, "class": "Hosted decision API", "open": "closed weights", "licence": null, "repo": "https://surd.ai", "gpu": null, "endpoint_condition": "measured 2026-10-06 on the live v1.6.0 S u P (1,500 rows) under Florian's 5 Oct 2026 full-set amendment, same lane, same adapter and same provider default as spx-cd-flash; add-requests run 53, benchmarkheaven.com/submit ref 3FF00545", "underlying": null, "model_pin": "provider-reported measurement model slug: spx-cd-pro", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.8382749326145552, "acc_among_decisive": 0.9517684887459807, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Current-pool API measurement registry" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Messier AI (@agentmessier-ai, agentmessier.ai@gmail.com)", "axes": { "calibration": 73.90972729526267, "cost": 64.78240136302922, "intelligence": 42.35139759222662, "speed": 92.70909408602279 }, "calibration": { "parts": { "choice": { "ece_hard": 0.16098163180524172, "ece_score": 67.80367363895165, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 67.80367363895165 }, "noul": { "abstention_rate": 0.15200000000000002, "brier": 0.14568508389123155, "ece": 0.10030106317015194, "n": 375, "score": 79.93978736596961 }, "score": { "interval90_coverage": 0.8533333333333334, "interval90_width": 0.378888888888889, "n": 375, "nrps": 0.12363172157496395, "score": 73.98572088086677, "top_ece": 0.1983269304038504 } }, "score": 73.90972729526267 }, "capability": 58.130562443744644, "composite_ci95": { "A": [ 29.542188267758267, 57.66843444819182 ], "B": [ 26.199389362563995, 53.35068096010408 ], "C": [ 20.51540851927657, 40.047523922355424 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights). NO JUDGEMENT OF OURS: pricing_v15.reference() resolves Qwen/Qwen3.5-4B straight through to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the frozen 25 Sep snapshot (the same rate aplomb-1 carries for the same base) x this row's own measured tokens. Estimated, not charged. THE FRONT END REPORTS NO TOKEN USAGE (usage {}), exactly as v0.1 did in run 54, so the counts come from run 54's pre-registered route unchanged (PREREG-R59-AMENDMENT-1 = R54-AMENDMENT-2): the author's own prompt.Prompter (v0.2 source, unmodified) recomputes each item's prompt token ids, output is fixed at 1 by their own max_tokens=1, and the totals were required to match vLLM's own counters or the row would get no cost: 914,665 input and 1,500 output tokens both ways (vllm prompt_tokens_total 915,349 - 684; generation_tokens_total 1,508 - 8). Counts sit in cost_estimate, never in usage.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.0184433 }, "kind": "estimated", "usage_estimated_rows": 1500, "usd_per_1000": 0.014925192958700068 }, "display": "Messier One v0.2 (Qwen3.5-4B fine-tune, one prefill, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 42.671434486238354, "I_sealed": 42.03136069821489, "base": 42.35139759222662, "excess": -1.9333138544147794, "gap": 0.6400737880234644, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 54.85241164824772, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 57.189189189189186, "hard": 60.36489462094999, "judge": 36.71720500988794, "standard": 69.8618668899121 } }, "open|noul": { "cc": 42.58109040717736, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 73.91304347826086, "judge": 0.0, "standard": 22.222222222222232 } }, "open|score": { "cc": 30.580801403290003, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 49.135231705449634, "hard": 32.351343308274984, "judge": 27.483794440784738, "standard": 22.40801288599812 } }, "sealed|choice": { "cc": 55.440313317505925, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 73.3944276106718, "hard": 51.64027149321266, "judge": 54.37616387337058, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 41.10238724117829, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": 22.330097087378633, "judge": 44.736842105263165, "standard": 57.746478873239425 } }, "sealed|score": { "cc": 29.55138153596046, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 66.40617006718976, "hard": 32.98321832976057, "judge": 19.789254292160475, "standard": 18.90350454844556 } } } }, "jevbench_score": 45.29059491870636, "key": "messier-one-v0.2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 32, "ranked": true, "ranks": { "A": 32, "B": 34, "C": 33, "capability": 71 }, "scores": { "A": 45.29059491870636, "B": 41.2441782613488, "C": 31.451802026879417 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.20452698896988294, "p50_s_raw": 0.027263494484941475, "p95_s_adjusted": 0.26202333298570013, "p95_s_raw": 0.056011666492850054 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 68.69942429480604, 76.95757379447907 ], "I": [ 35.53670432007418, 46.90962732194912 ], "capability": [ 53.17607857615277, 61.1523915096487 ], "composite": { "A": [ 29.542188267758267, 57.66843444819182 ], "B": [ 26.199389362563995, 53.35068096010408 ], "C": [ 20.51540851927657, 40.047523922355424 ] }, "seed": 93 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.90972729526267, "capability": 58.130562443744644, "intelligence": 42.35139759222662 }, "per_type": { "choice": { "accuracy": 0.6586666666666666, "calibration": 67.80367363895165, "cc": 56.348341748529286, "cc_by_type": { "choice": 56.348341748529286 }, "cc_tiered": 55.146362482876825, "cc_tiered_open": 54.85241164824772, "cc_tiered_sealed": 55.440313317505925, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 56.348341748529286 }, "noul": { "accuracy": 0.7226666666666667, "calibration": 79.93978736596961, "cc": 44.53333333333334, "cc_by_type": { "noul": 44.53333333333334 }, "cc_tiered": 41.841738824177824, "cc_tiered_open": 42.58109040717736, "cc_tiered_sealed": 41.10238724117829, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 44.53333333333334 }, "score": { "accuracy": 0.504, "calibration": 73.98572088086677, "cc": 32.623848126624935, "cc_by_type": { "score": 32.623848126624935 }, "cc_tiered": 30.06609146962523, "cc_tiered_open": 30.580801403290003, "cc_tiered_sealed": 29.55138153596046, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 32.623848126624935 } }, "raw_on_A": { "calibration": 79.87142853291357, "capability": 63.87968579812095, "intelligence": 47.887943063328336 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 43.19017554763209, "Emphasis on Accuracy 60:20:20": 36.959944674970394, "Emphasis on Cost 20:20:60": 44.448060509071624, "Emphasis on Speed 20:60:20": 50.236730867619606, "Intelligence only": 30.38527918499109 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "cc-by-nc-4.0", "repo": "https://huggingface.co/agentmessier/messier-one", "gpu": "RTX 5090", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. REPLACES the v0.1 request (GitHub #194, measured by run 54); the submission names it so. Weights agentmessier/messier-one tag v0.2 = 8bbddb84140e8fc732b35713f998ed628fc70306 (CC BY-NC 4.0, NON-COMMERCIAL), front end github.com/agentmessier-ai/messier-one tag v0.2 = 1b8c3b80 (Apache-2.0). README install line (vllm==0.30.0, transformers==5.18.0; torch 2.13.0+cu130 resolved by vllm) and the author's two serve commands verbatim; --gate off, their default and the configuration of their own reported single-read run. The author discloses that part of the training material was written in the families of JevBench's hard tier, with no JevBench item used. Their Prompter caps the state at 8,000 tokens (max_total 8,600) and splices long states around a readable marker, so the 23 long items are answered truncated (r54 s on disclosed truncation). UNCHANGED typesafe adapter; the smoke gate refused the first attempt at usage-input-tokens and passed with its existing pre-registered opt-in. One RTX 5090 32 GB (Lium, France). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 455A3944", "underlying": "Qwen/Qwen3.5-4B", "model_pin": "agentmessier/messier-one@8bbddb84140e8fc732b35713f998ed628fc70306", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.848, "acc_among_decisive": 0.8522012578616353, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Abhishek Rai (abhishek085)", "axes": { "calibration": 64.79562625454915, "cost": 60.422063196854964, "intelligence": 43.2364603010071, "speed": 86.95618624402076 }, "calibration": { "parts": { "choice": { "ece_hard": 0.20104071779836963, "ece_score": 59.79185644032607, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 59.79185644032607 }, "noul": { "abstention_rate": 0.16000000000000003, "brier": 0.16425289707950683, "ece": 0.1444944938079116, "n": 375, "score": 71.10110123841768 }, "score": { "interval90_coverage": 0.712, "interval90_width": 0.17133333333333342, "n": 375, "nrps": 0.1542624452699586, "score": 63.493921084903704, "top_ece": 0.2879295665159836 } }, "score": 64.79562625454915 }, "capability": 54.016043277778124, "composite_ci95": { "A": [ 29.776291757874876, 55.39058631138699 ], "B": [ 26.921562908696533, 52.0349558138311 ], "C": [ 20.677980387413104, 38.465684938463184 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.020857610837438423 }, "display": "spark-s1-4b-v6 (Open Spark Jev, abhishek085)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 47.64157364296756, "I_sealed": 38.83134695904664, "base": 43.2364603010071, "excess": 6.236839041482673, "gap": 8.810226683920916, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 63.30038752190768, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 58.47750865051904, "judge": 68.35860250494396, "standard": 54.79280033486815 } }, "open|noul": { "cc": 46.0848861283644, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 56.52173913043479, "judge": 10.000000000000009, "standard": 66.66666666666667 } }, "open|score": { "cc": 33.539447278630604, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 38.99774322763003, "hard": 44.04391027813551, "judge": 21.602744427042197, "standard": 27.706427582503657 } }, "sealed|choice": { "cc": 57.98194607005097, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 72.06414899120539, "hard": 50.113122171945705, "judge": 60.89385474860336, "standard": 62.31062938785566 } }, "sealed|noul": { "cc": 33.87772460002735, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 56.00000000000001, "hard": 2.9126213592232997, "judge": 50.0, "standard": 60.56338028169015 } }, "sealed|score": { "cc": 24.634370207061615, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 51.84151834556672, "hard": 16.688097933910274, "judge": 26.27544811817185, "standard": 24.46172381744639 } } } }, "jevbench_score": 44.90217636528639, "key": "spark-s1-4b-v6", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 33, "ranked": true, "ranks": { "A": 33, "B": 33, "C": 34, "capability": 84 }, "scores": { "A": 44.90217636528639, "B": 41.66207269525555, "C": 31.18206692033776 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.40721648099715824, "p50_s_raw": 0.1286082404985791, "p95_s_adjusted": 0.4949439602918574, "p95_s_raw": 0.17247198014592868 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 59.71050928645707, 68.28479086272019 ], "I": [ 36.48865188869242, 47.17023357174763 ], "capability": [ 48.85711252205833, 57.24976166234473 ], "composite": { "A": [ 29.776291757874876, 55.39058631138699 ], "B": [ 26.921562908696533, 52.0349558138311 ], "C": [ 20.677980387413104, 38.465684938463184 ] }, "seed": 102 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 64.79562625454915, "capability": 54.016043277778124, "intelligence": 43.2364603010071 }, "per_type": { "choice": { "accuracy": 0.6866666666666666, "calibration": 59.79185644032607, "cc": 59.92914183947025, "cc_by_type": { "choice": 59.92914183947025 }, "cc_tiered": 60.64116679597932, "cc_tiered_open": 63.30038752190768, "cc_tiered_sealed": 57.98194607005097, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 59.92914183947025 }, "noul": { "accuracy": 0.6986666666666667, "calibration": 71.10110123841768, "cc": 39.733333333333334, "cc_by_type": { "noul": 39.733333333333334 }, "cc_tiered": 39.981305364195876, "cc_tiered_open": 46.0848861283644, "cc_tiered_sealed": 33.87772460002735, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 39.733333333333334 }, "score": { "accuracy": 0.5386666666666666, "calibration": 63.493921084903704, "cc": 27.536544538601227, "cc_by_type": { "score": 27.536544538601227 }, "cc_tiered": 29.08690874284611, "cc_tiered_open": 33.539447278630604, "cc_tiered_sealed": 24.634370207061615, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 27.536544538601227 } }, "raw_on_A": { "calibration": 68.7404409420719, "capability": 57.96450731310942, "intelligence": 47.18857368414694 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 43.83191134540322, "Emphasis on Accuracy 60:20:20": 38.371611426428935, "Emphasis on Cost 20:20:60": 44.36176060417368, "Emphasis on Speed 20:60:20": 50.402171180021895, "Intelligence only": 32.33034853965725 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (code and weights); base Qwen/Qwen3.5-4B Apache-2.0", "repo": "https://github.com/abhishek085/open-spark-jev", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "Qwen/Qwen3.5-4B with a LoRA, read at the option-letter logits with a fitted temperature (no extra head)", "adapter_id": "images/spark-s1-4b-v6/spec.json:spark_v15.Spark", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.84, "acc_among_decisive": 0.8317460317460318, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "blockbrain", "axes": { "calibration": 88.58151013990448, "cost": 42.15074720483203, "intelligence": 56.15104294255194, "speed": 82.64878475169937 }, "calibration": { "parts": { "choice": { "ece_hard": 0.03565851721103598, "ece_score": 92.8682965577928, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 92.8682965577928 }, "noul": { "abstention_rate": 0.18867924528301883, "brier": 0.10222826353990752, "ece": 0.07688828024001365, "n": 371, "score": 84.62234395199727 }, "score": { "interval90_coverage": 0.924119241192412, "interval90_width": 0.2818428184281841, "n": 369, "nrps": 0.07387893374911293, "score": 88.25388990992337, "top_ece": 0.08052163402620996 } }, "score": 88.58151013990448 }, "capability": 72.36627654122822, "composite_ci95": { "A": [ 42.00000021718559, 44.674228699218794 ], "B": [ 40.50170443761996, 44.404221038840014 ], "C": [ 29.16666681748999, 44.674228699218794 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.08478293103448274 }, "display": "swanOne (blockbrain, Qwen3.8-Flash-Next NVFP4)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 56.23071661013901, "I_sealed": 56.07136927496486, "base": 56.15104294255194, "excess": -2.41404030726409, "gap": 0.15934733517415367, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 76.36851516458754, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 69.80182447310474, "judge": 84.17930125247197, "standard": 69.8618668899121 } }, "open|noul": { "cc": 42.89026915113871, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 39.13043478260869, "judge": 39.99999999999999, "standard": 33.33333333333333 } }, "open|score": { "cc": 49.43336551469081, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 57.945051339203516, "hard": 45.879570270586335, "judge": 54.70148526262065, "standard": 44.382933468748654 } }, "sealed|choice": { "cc": 69.57776531270856, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 74.72470623013821, "hard": 68.43891402714931, "judge": 64.61824953445065, "standard": 76.7212710924991 } }, "sealed|noul": { "cc": 45.48583128818902, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 84.00000000000001, "hard": 18.446601941747566, "judge": 60.52631578947369, "standard": 57.746478873239425 } }, "sealed|score": { "cc": 53.150511223997015, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 70.92864532337575, "hard": 47.07089246791437, "judge": 55.79271749917136, "standard": 52.457372273711435 } } } }, "jevbench_score": 43.78640157551217, "key": "swanone", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 34, "ranked": true, "ranks": { "A": 34, "B": 32, "C": 29, "capability": 24 }, "scores": { "A": 43.78640157551217, "B": 42.950891870154436, "C": 38.348855694306984 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.7114971062168479, "p50_s_raw": 0.28074855310842395, "p95_s_adjusted": 0.7637450117617846, "p95_s_raw": 0.30687250588089227 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.47295009833218, 89.28636150565644 ], "I": [ 49.957046630174666, 60.997721736085786 ], "capability": [ 67.47509953288571, 74.36120880664787 ], "composite": { "A": [ 42.00000021718559, 44.674228699218794 ], "B": [ 40.50170443761996, 44.404221038840014 ], "C": [ 29.16666681748999, 44.674228699218794 ] }, "seed": 104 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.58151013990448, "capability": 72.36627654122822, "intelligence": 56.15104294255194 }, "per_type": { "choice": { "accuracy": 0.7786666666666666, "calibration": 92.8682965577928, "cc": 71.69462785256195, "cc_by_type": { "choice": 71.69462785256195 }, "cc_tiered": 72.97314023864806, "cc_tiered_open": 76.36851516458754, "cc_tiered_sealed": 69.57776531270856, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 71.69462785256195 }, "noul": { "accuracy": 0.744, "calibration": 84.62234395199727, "cc": 48.8, "cc_by_type": { "noul": 48.8 }, "cc_tiered": 44.188050219663864, "cc_tiered_open": 42.89026915113871, "cc_tiered_sealed": 45.48583128818902, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 48.8 }, "score": { "accuracy": 0.6506666666666666, "calibration": 88.25388990992337, "cc": 52.763382802943624, "cc_by_type": { "score": 52.763382802943624 }, "cc_tiered": 51.29193836934391, "cc_tiered_open": 49.43336551469081, "cc_tiered_sealed": 53.150511223997015, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 52.763382802943624 } }, "raw_on_A": { "calibration": 88.23312364196306, "capability": 74.40197927194893, "intelligence": 60.5708349019348 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 39.75216183669048, "Emphasis on Accuracy 60:20:20": 39.813195135198455, "Emphasis on Cost 20:20:60": 35.153484373145204, "Emphasis on Speed 20:60:20": 45.65457340538371, "Intelligence only": 39.90509729608976 }, "class": "system-one-open", "open": "yes", "licence": "patches Apache-2.0, shim MIT; weights under the Qwen licence (LICENSE-NOTICE.md)", "repo": "https://github.com/blockbrain-ai/swanone-recipe", "gpu": "RTXPRO6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "underlying": null, "adapter_id": "images/swanone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8113207547169812, "acc_among_decisive": 0.9269102990033222, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Chinh Nguyen (ncchinh)", "axes": { "calibration": 76.76007761870373, "cost": 63.43457194636937, "intelligence": 41.73350604436517, "speed": 89.46638461924755 }, "calibration": { "parts": { "choice": { "ece_hard": 0.15434999999999968, "ece_score": 69.13000000000007, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 69.13000000000007 }, "noul": { "abstention_rate": 0.24, "brier": 0.16916629888000007, "ece": 0.10974879999999991, "n": 375, "score": 78.05024000000002 }, "score": { "interval90_coverage": 0.92, "interval90_width": 0.4133333333333333, "n": 375, "nrps": 0.10737614287777777, "score": 83.09999285611111, "top_ece": 0.11531199999999998 } }, "score": 76.76007761870373 }, "capability": 59.24679183153445, "composite_ci95": { "A": [ 30.84848869900701, 57.23347326869325 ], "B": [ 27.312995844624446, 52.90407172237297 ], "C": [ 21.422561596532642, 39.74546754770365 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra Qwen/Qwen3.5-4B list price $0.03/M input (the exact base weights; not on OpenRouter; same reference as reflex-4b, jobe and typecastlm), read 2026-09-24 x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01655189039408867 }, "display": "Quyet-1.0-Medium (Chinh Nguyen, Qwen3.5-4B decoder, option-letter logits)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 42.75300136168062, "I_sealed": 40.71401072704972, "base": 41.73350604436517, "excess": -0.5343970078073448, "gap": 2.038990634630899, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 70.4971468014646, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 67.9144385026738, "judge": 72.31377719182596, "standard": 69.8618668899121 } }, "open|noul": { "cc": 16.34230503795721, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 4.347826086956519, "judge": 0.0, "standard": 44.44444444444444 } }, "open|score": { "cc": 41.41955224562005, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 47.18969696969697, "hard": 35.19814814814814, "judge": 54.641694915254234, "standard": 31.14407407407409 } }, "sealed|choice": { "cc": 59.00724128213771, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 55.20361990950226, "judge": 55.30726256983241, "standard": 68.96169478999877 } }, "sealed|noul": { "cc": 25.985534052552406, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 60.00000000000001, "hard": 12.621359223300965, "judge": 26.315789473684205, "standard": 35.2112676056338 } }, "sealed|score": { "cc": 37.14925684645904, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 55.4977304964539, "hard": 32.33151960784312, "judge": 40.893173652694614, "standard": 31.994619289340076 } } } }, "jevbench_score": 43.58925238992216, "key": "quyet-1-0-medium", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 35, "ranked": true, "ranks": { "A": 35, "B": 36, "C": 35, "capability": 66 }, "scores": { "A": 43.58925238992216, "B": 39.63222739545648, "C": 30.270314159668164 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3048233319917927, "p50_s_raw": 0.07741166599589633, "p95_s_adjusted": 0.3709482546968502, "p95_s_raw": 0.1104741273484251 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 71.2671834460676, 80.2774031670074 ], "I": [ 36.06269585061453, 46.7313636734712 ], "capability": [ 54.568687859056396, 62.84407796698955 ], "composite": { "A": [ 30.84848869900701, 57.23347326869325 ], "B": [ 27.312995844624446, 52.90407172237297 ], "C": [ 21.422561596532642, 39.74546754770365 ] }, "seed": 87 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 76.76007761870373, "capability": 59.24679183153445, "intelligence": 41.73350604436517 }, "per_type": { "choice": { "accuracy": 0.7013333333333334, "calibration": 69.13000000000007, "cc": 61.804799029963135, "cc_by_type": { "choice": 61.804799029963135 }, "cc_tiered": 64.75219404180115, "cc_tiered_open": 70.4971468014646, "cc_tiered_sealed": 59.00724128213771, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 61.804799029963135 }, "noul": { "accuracy": 0.64, "calibration": 78.05024000000002, "cc": 28.000000000000004, "cc_by_type": { "noul": 28.000000000000004 }, "cc_tiered": 21.163919545254807, "cc_tiered_open": 16.34230503795721, "cc_tiered_sealed": 25.985534052552406, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 28.000000000000004 }, "score": { "accuracy": 0.56, "calibration": 83.09999285611111, "cc": 38.22100877192973, "cc_by_type": { "score": 38.22100877192973 }, "cc_tiered": 39.28440454603954, "cc_tiered_open": 41.41955224562005, "cc_tiered_sealed": 37.14925684645904, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 38.22100877192973 } }, "raw_on_A": { "calibration": 77.01676225205841, "capability": 61.22371994730187, "intelligence": 45.430677642545334 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 41.05875168344406, "Emphasis on Accuracy 60:20:20": 35.24739388678317, "Emphasis on Cost 20:20:60": 42.25762883934701, "Emphasis on Speed 20:60:20": 47.54937088611952, "Intelligence only": 29.074657383286326 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.76, "acc_among_decisive": 0.8421052631578947, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Medium", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)", "underlying": "Qwen/Qwen3.5-4B" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Interfaze", "axes": { "calibration": 82.32371666823741, "cost": 61.79891447724963, "intelligence": 41.084702071234844, "speed": 88.00251499520694 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09976787636606844, "ece_score": 80.04642472678631, "mean_tvd": null, "n": 735, "n_ece_hard": 305, "n_tvd": 0, "score": 80.04642472678631 }, "noul": { "abstention_rate": 0.15633423180592987, "brier": 0.15547862054178035, "ece": 0.09804350768859321, "n": 371, "score": 80.39129846228136 }, "score": { "interval90_coverage": 0.9728997289972899, "interval90_width": 0.7095754290876237, "n": 369, "nrps": 0.12033260727134382, "score": 86.53342681564456, "top_ece": 0.07449942820788243 } }, "score": 82.32371666823741 }, "capability": 61.70420936973613, "composite_ci95": { "A": [ 28.718912479130207, 50.73774516604421 ], "B": [ 25.322064520088336, 46.59426967064348 ], "C": [ 19.943689221618207, 35.23454525419738 ] }, "cost": { "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3); self-hosted estimate, not measured hosted billing", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01876590517241379 }, "display": "lev (Interfaze AI, Qwen3.5-4B + LoRA r32, label-token readout + candidate-path head, per-bucket temperatures)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.778585099687206, "I_sealed": 36.39081904278248, "base": 41.084702071234844, "excess": 6.81437841446648, "gap": 9.387766056904724, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 61.009098246344166, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 50.92796476879523, "judge": 68.35860250494396, "standard": 62.32733361239012 } }, "open|noul": { "cc": 56.946169772256724, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 73.91304347826086, "judge": 50.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 19.38048728046074, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 22.564349870758527, "hard": 19.589685248514364, "judge": 26.558897550056017, "standard": 6.602544644811692 } }, "sealed|choice": { "cc": 54.52900312820748, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 78.71554208853745, "hard": 48.585972850678715, "judge": 54.37616387337058, "standard": 54.55105308535534 } }, "sealed|noul": { "cc": 29.603595615594433, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 63.99999999999999, "hard": 2.9126213592232997, "judge": 36.8421052631579, "standard": 54.92957746478873 } }, "sealed|score": { "cc": 25.039858384545557, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 47.25762460028372, "hard": 16.968805028678304, "judge": 35.272820946831395, "standard": 14.723638144982221 } } } }, "jevbench_score": 42.177760514319104, "key": "lev", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 36, "ranked": true, "ranks": { "A": 36, "B": 37, "C": 36, "capability": 62 }, "scores": { "A": 42.177760514319104, "B": 38.201113541708786, "C": 29.290111468277157 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3546039809356444, "p50_s_raw": 0.10230199046782218, "p95_s_adjusted": 0.446688625391107, "p95_s_raw": 0.1483443126955535 }, "status": { "answered_ok": 1475, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 76.63846170548479, 84.6851724512148 ], "I": [ 35.04763127374306, 44.44652580243387 ], "capability": [ 56.747890795417284, 63.66051033763406 ], "composite": { "A": [ 28.718912479130207, 50.73774516604421 ], "B": [ 25.322064520088336, 46.59426967064348 ], "C": [ 19.943689221618207, 35.23454525419738 ] }, "seed": 59 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.32371666823741, "capability": 61.70420936973613, "intelligence": 41.084702071234844 }, "per_type": { "choice": { "accuracy": 0.664, "calibration": 80.04642472678631, "cc": 57.030398908708534, "cc_by_type": { "choice": 57.030398908708534 }, "cc_tiered": 57.76905068727582, "cc_tiered_open": 61.009098246344166, "cc_tiered_sealed": 54.52900312820748, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.030398908708534 }, "noul": { "accuracy": 0.6906666666666667, "calibration": 80.39129846228136, "cc": 38.13333333333333, "cc_by_type": { "noul": 38.13333333333333 }, "cc_tiered": 43.274882693925576, "cc_tiered_open": 56.946169772256724, "cc_tiered_sealed": 29.603595615594433, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 38.13333333333333 }, "score": { "accuracy": 0.4533333333333333, "calibration": 86.53342681564456, "cc": 23.66953381268505, "cc_by_type": { "score": 23.66953381268505 }, "cc_tiered": 22.210172832503147, "cc_tiered_open": 19.38048728046074, "cc_tiered_sealed": 25.039858384545557, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 23.66953381268505 } }, "raw_on_A": { "calibration": 79.50973335881918, "capability": 61.67507354014826, "intelligence": 43.84041372147735 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.020000000000000018, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 39.03925349464146, "Emphasis on Accuracy 60:20:20": 33.569486237063714, "Emphasis on Cost 20:20:60": 40.07113476989396, "Emphasis on Speed 20:60:20": 45.246505158344306, "Intelligence only": 27.739614243661805 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/interfaze-ai/lev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h", "underlying": "Qwen/Qwen3.5-4B", "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r39/outputs/meta/specs/lev.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": "interfaze-ai/lev@7bdc748dffebd85b57ee0dbea8f994c6354fed31 + Abhinavexists/lev@cf104b69329302e4eac674a730c71f3511047db8", "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8436657681940701, "acc_among_decisive": 0.8274760383386581, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Igor L. / NInfer contributors", "axes": { "calibration": 90.37232453477458, "cost": 42.53362878447125, "intelligence": 48.980301026876646, "speed": 89.33173525631216 }, "calibration": { "parts": { "choice": { "ece_hard": 0.03727735037984102, "ece_score": 92.54452992403179, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 92.54452992403179 }, "noul": { "abstention_rate": 0.24528301886792447, "brier": 0.10381961467316755, "ece": 0.05486291160058669, "n": 371, "score": 89.02741767988266 }, "score": { "interval90_coverage": 0.94579945799458, "interval90_width": 0.40876242095754284, "n": 369, "nrps": 0.08645101367263491, "score": 89.5450260004093, "top_ece": 0.06132423315958956 } }, "score": 90.37232453477458 }, "capability": 69.67631278082561, "composite_ci95": { "A": [ 30.27504047421702, 44.819854782587115 ], "B": [ 28.319423416872812, 43.496842238713214 ], "C": [ 21.02433366265071, 36.222113908221345 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.08232764778325123 }, "display": "NInfer Qwen3.8-Flash-Next mixed", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 48.827164297755125, "I_sealed": 49.13343775599816, "base": 48.980301026876646, "excess": -2.8796611006812776, "gap": -0.30627345824303376, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 70.22063841159643, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 60.36489462094999, "judge": 76.26895187870797, "standard": 69.8618668899121 } }, "open|noul": { "cc": 36.42788129744652, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 30.434782608695656, "judge": 19.999999999999996, "standard": 55.55555555555556 } }, "open|score": { "cc": 39.83297318422244, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 63.934842929432264, "hard": 29.854097365310107, "judge": 50.77541876771969, "standard": 31.326121574196307 } }, "sealed|choice": { "cc": 68.53578282889805, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 77.38526346907102, "hard": 61.82126696832577, "judge": 68.34264432029796, "standard": 77.82978199285628 } }, "sealed|noul": { "cc": 34.31380022598546, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": -0.9708737864077666, "judge": 52.63157894736843, "standard": 60.56338028169015 } }, "sealed|score": { "cc": 44.55073021311098, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 66.34071381045655, "hard": 36.06656765199164, "judge": 48.53812828291905, "standard": 44.64296643196477 } } } }, "jevbench_score": 41.96785076957165, "key": "ninfer-qwen3.8-flash-next", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 37, "ranked": true, "ranks": { "A": 37, "B": 35, "C": 37, "capability": 38 }, "scores": { "A": 41.96785076957165, "B": 40.09260225022487, "C": 29.144340812202532 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3100576108321548, "p50_s_raw": 0.08002880541607738, "p95_s_adjusted": 0.37616993160918355, "p95_s_raw": 0.11308496580459178 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.49584722295648, 90.3923314283596 ], "I": [ 42.680473865516596, 53.996834913811625 ], "capability": [ 64.4871361894938, 71.33725830271337 ], "composite": { "A": [ 30.27504047421702, 44.819854782587115 ], "B": [ 28.319423416872812, 43.496842238713214 ], "C": [ 21.02433366265071, 36.222113908221345 ] }, "seed": 73 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.37232453477458, "capability": 69.67631278082561, "intelligence": 48.980301026876646 }, "per_type": { "choice": { "accuracy": 0.7613333333333333, "calibration": 92.54452992403179, "cc": 69.47794208197946, "cc_by_type": { "choice": 69.47794208197946 }, "cc_tiered": 69.37821062024724, "cc_tiered_open": 70.22063841159643, "cc_tiered_sealed": 68.53578282889805, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 69.47794208197946 }, "noul": { "accuracy": 0.696, "calibration": 89.02741767988266, "cc": 39.19999999999999, "cc_by_type": { "noul": 39.19999999999999 }, "cc_tiered": 35.37084076171599, "cc_tiered_open": 36.42788129744652, "cc_tiered_sealed": 34.31380022598546, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 39.19999999999999 }, "score": { "accuracy": 0.5653333333333334, "calibration": 89.5450260004093, "cc": 43.98717489610087, "cc_by_type": { "score": 43.98717489610087 }, "cc_tiered": 42.19185169866671, "cc_tiered_open": 39.83297318422244, "cc_tiered_sealed": 44.55073021311098, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 43.98717489610087 } }, "raw_on_A": { "calibration": 86.91372207226564, "capability": 70.43814133335954, "intelligence": 53.962560594453436 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 37.79450787350779, "Emphasis on Accuracy 60:20:20": 36.18545094340192, "Emphasis on Cost 20:20:60": 33.99296424621917, "Emphasis on Speed 20:60:20": 44.79615638245711, "Intelligence only": 34.01333411780027 }, "class": "native-logit", "open": "yes", "licence": "Qwen Community License 1.0 (model); Apache-2.0 (engine)", "repo": "https://github.com/igorls/ninfer", "gpu": "RTXPRO6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "underlying": "Qwen/Qwen3.8-Flash-Next, mixed NVFP4/FP8/INT4 NInfer artifact", "adapter_id": "images/ninfer-qwen3.8-flash-next/spec.json:ninfer_native.NInferNativeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7547169811320755, "acc_among_decisive": 0.9321428571428572, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "us (GitHub)", "axes": { "calibration": 73.06634984647151, "cost": 58.85554343293177, "intelligence": 41.85041350559534, "speed": 75.55955847735143 }, "calibration": { "parts": { "choice": { "ece_hard": 0.16559413680781765, "ece_score": 66.88117263843647, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 66.88117263843647 }, "noul": { "abstention_rate": 0.14555256064690025, "brier": 0.16930220654986547, "ece": 0.14746549865229122, "n": 371, "score": 70.50690026954176 }, "score": { "interval90_coverage": 0.8644986449864499, "interval90_width": 0.325880758807588, "n": 369, "nrps": 0.10296637523035226, "score": 81.81097663143632, "top_ece": 0.1304070460704606 } }, "score": 73.06634984647151 }, "capability": 57.458381676033426, "composite_ci95": { "A": [ 29.136540211681222, 52.90884989606441 ], "B": [ 26.3171992505717, 49.89250384145016 ], "C": [ 20.233708480334183, 36.74225687226694 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.023522413793103446 }, "display": "jev-local (Qwen3.5-9B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 43.39498650106927, "I_sealed": 40.30584051012142, "base": 41.85041350559534, "excess": 0.5157583485096047, "gap": 3.0891459909478485, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 56.18661240978703, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 53.2972972972973, "hard": 49.040578798364265, "judge": 52.537903757415954, "standard": 77.39640016743407 } }, "open|noul": { "cc": 34.189786059351285, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": 30.000000000000004, "standard": 22.222222222222232 } }, "open|score": { "cc": 39.8085610340695, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 40.41969696969697, "hard": 51.80083333333333, "judge": 22.97271186440677, "standard": 40.77222222222221 } }, "sealed|choice": { "cc": 47.63585353447873, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 58.7613627965413, "hard": 39.42307692307691, "judge": 48.78957169459964, "standard": 56.76807488606972 } }, "sealed|noul": { "cc": 35.679356877082625, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 14.563106796116498, "judge": 42.10526315789473, "standard": 52.112676056338024 } }, "sealed|score": { "cc": 37.602311118802916, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 64.23531914893618, "hard": 29.158455882352953, "judge": 39.61335329341317, "standard": 38.15695431472081 } } } }, "jevbench_score": 41.328564334669615, "key": "jev-local", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 38, "ranked": true, "ranks": { "A": 38, "B": 38, "C": 38, "capability": 73 }, "scores": { "A": 41.328564334669615, "B": 38.19938992160457, "C": 28.700391899076113 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.8806604199111462, "p50_s_raw": 0.3653302099555731, "p95_s_adjusted": 3.156717182928696, "p95_s_raw": 1.5033585914643481 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 67.57161963107492, 76.81314888025398 ], "I": [ 36.13732889468463, 46.54627874317067 ], "capability": [ 53.19292305173127, 60.44357521274037 ], "composite": { "A": [ 29.136540211681222, 52.90884989606441 ], "B": [ 26.3171992505717, 49.89250384145016 ], "C": [ 20.233708480334183, 36.74225687226694 ] }, "seed": 45 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.06634984647151, "capability": 57.458381676033426, "intelligence": 41.85041350559534 }, "per_type": { "choice": { "accuracy": 0.6026666666666667, "calibration": 66.88117263843647, "cc": 49.186741566647385, "cc_by_type": { "choice": 49.186741566647385 }, "cc_tiered": 51.91123297213288, "cc_tiered_open": 56.18661240978703, "cc_tiered_sealed": 47.63585353447873, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 49.186741566647385 }, "noul": { "accuracy": 0.696, "calibration": 70.50690026954176, "cc": 39.19999999999999, "cc_by_type": { "noul": 39.19999999999999 }, "cc_tiered": 34.93457146821696, "cc_tiered_open": 34.189786059351285, "cc_tiered_sealed": 35.679356877082625, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 39.19999999999999 }, "score": { "accuracy": 0.5653333333333334, "calibration": 81.81097663143632, "cc": 39.00943859649125, "cc_by_type": { "score": 39.00943859649125 }, "cc_tiered": 38.70543607643621, "cc_tiered_open": 39.8085610340695, "cc_tiered_sealed": 37.602311118802916, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 39.00943859649125 } }, "raw_on_A": { "calibration": 73.6085835138451, "capability": 59.88003789504907, "intelligence": 46.151492276253045 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 38.834988968034565, "Emphasis on Accuracy 60:20:20": 34.37288631458028, "Emphasis on Cost 20:20:60": 39.75999364263786, "Emphasis on Speed 20:60:20": 43.46632094435822, "Intelligence only": 29.31968172619126 }, "class": "jev-rebuild", "open": "yes", "licence": "no licence stated in the repository (public code); Apache-2.0 base weights", "repo": "https://github.com/us/jev-local", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen/Qwen3.5-9B, frozen, per-option mean log-probability", "adapter_id": "images/jev-local/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8544474393530997, "acc_among_decisive": 0.8233438485804416, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mapika", "axes": { "calibration": 89.63735637833331, "cost": 64.54376873042901, "intelligence": 40.12597268917964, "speed": 91.78549329698745 }, "calibration": { "parts": { "choice": { "ece_hard": 0.04618531250000016, "ece_score": 90.76293749999998, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 90.76293749999998 }, "noul": { "abstention_rate": 0.3573333333333333, "brier": 0.1263556454400002, "ece": 0.05272693333333338, "n": 375, "score": 89.45461333333333 }, "score": { "interval90_coverage": 0.9733333333333334, "interval90_width": 0.5584444444444441, "n": 375, "nrps": 0.09968243396666672, "score": 88.69451830166665, "top_ece": 0.06321360000000002 } }, "score": 89.63735637833331 }, "capability": 64.88166453375648, "composite_ci95": { "A": [ 28.727001690110686, 49.92424966763833 ], "B": [ 25.10971569724804, 45.24030143351226 ], "C": [ 19.949306729243542, 34.66961782474885 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.015201077586206898 }, "display": "decider-4b v2 (Mapika)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.24914817619414, "I_sealed": 35.00279720216514, "base": 40.12597268917964, "excess": 7.672963331590756, "gap": 10.246350974029, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 73.21703197407996, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 69.80182447310474, "judge": 76.26895187870797, "standard": 69.8618668899121 } }, "open|noul": { "cc": 26.461697722567287, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 39.13043478260869, "judge": -9.999999999999998, "standard": 33.33333333333333 } }, "open|score": { "cc": 36.06871483193517, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 48.519696969696966, "hard": 36.743888888888875, "judge": 38.64050847457627, "standard": 24.635185185185193 } }, "sealed|choice": { "cc": 56.802191819905744, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 69.40359175227256, "hard": 50.62217194570135, "judge": 57.16945996275605, "standard": 62.31062938785566 } }, "sealed|noul": { "cc": 11.645451862940547, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": -12.621359223300976, "judge": 23.684210526315795, "standard": 23.943661971830977 } }, "sealed|score": { "cc": 36.56074792364913, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 54.016382978723385, "hard": 31.53129901960784, "judge": 40.6503592814371, "standard": 31.75741116751262 } } } }, "jevbench_score": 41.240894350857424, "key": "decider-4b-v2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 39, "ranked": true, "ranks": { "A": 39, "B": 39, "C": 39, "capability": 50 }, "scores": { "A": 41.240894350857424, "B": 36.84955227461344, "C": 28.639509965873213 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.19992743488401174, "p50_s_raw": 0.024963717442005873, "p95_s_adjusted": 0.3315723263658583, "p95_s_raw": 0.09078616318292916 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.18439184191017, 90.03628745741203 ], "I": [ 34.68576750629042, 43.56524556183246 ], "capability": [ 60.38851961863991, 66.00729141285105 ], "composite": { "A": [ 28.727001690110686, 49.92424966763833 ], "B": [ 25.10971569724804, 45.24030143351226 ], "C": [ 19.949306729243542, 34.66961782474885 ] }, "seed": 28 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.63735637833331, "capability": 64.88166453375648, "intelligence": 40.12597268917964 }, "per_type": { "choice": { "accuracy": 0.6933333333333334, "calibration": 90.76293749999998, "cc": 60.781713289694295, "cc_by_type": { "choice": 60.781713289694295 }, "cc_tiered": 65.00961189699285, "cc_tiered_open": 73.21703197407996, "cc_tiered_sealed": 56.802191819905744, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.781713289694295 }, "noul": { "accuracy": 0.592, "calibration": 89.45461333333333, "cc": 18.399999999999995, "cc_by_type": { "noul": 18.399999999999995 }, "cc_tiered": 19.053574792753917, "cc_tiered_open": 26.461697722567287, "cc_tiered_sealed": 11.645451862940547, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 18.399999999999995 }, "score": { "accuracy": 0.512, "calibration": 88.69451830166665, "cc": 36.877482456140385, "cc_by_type": { "score": 36.877482456140385 }, "cc_tiered": 36.31473137779215, "cc_tiered_open": 36.06871483193517, "cc_tiered_sealed": 36.56074792364913, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 36.877482456140385 } }, "raw_on_A": { "calibration": 88.78997857788461, "capability": 66.11763148027418, "intelligence": 43.44528438266377 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 37.65578397809399, "Emphasis on Accuracy 60:20:20": 31.83485819738892, "Emphasis on Cost 20:20:60": 39.12905962819718, "Emphasis on Speed 20:60:20": 44.051943214406094, "Intelligence only": 25.842630080539234 }, "class": "system-one-open", "open": "yes", "licence": "Apache-2.0 (package and weights)", "repo": "https://github.com/Mapika/decider", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/decider-4b-v2/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6426666666666667, "acc_among_decisive": 0.921161825726141, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)", "axes": { "calibration": 58.71667533779978, "cost": 49.33529996402755, "intelligence": 42.225671849648506, "speed": 90.24509258883624 }, "calibration": { "parts": { "choice": { "ece_hard": 0.24400358306188902, "ece_score": 51.199283387622195, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 51.199283387622195 }, "noul": { "abstention_rate": 0.03234501347708896, "brier": 0.20490283425876044, "ece": 0.20667035040431214, "n": 371, "score": 58.66592991913757 }, "score": { "interval90_coverage": 0.7696476964769647, "interval90_width": 0.23373983739837403, "n": 369, "nrps": 0.141136266192412, "score": 66.2848127066396, "top_ece": 0.266583739837398 } }, "score": 58.71667533779978 }, "capability": 50.471173593724146, "composite_ci95": { "A": [ 24.985716007949218, 50.94397024296676 ], "B": [ 22.880770578118387, 48.816339954765404 ], "C": [ 17.35119167218696, 35.37775711317136 ] }, "cost": { "basis": "ESTIMATE (exact base-model market reference; self-hosted open weights, no per-token tariff of its own). THE ONLY ROW OF THIS RUN WHOSE RATE NEEDED NO JUDGEMENT AT ALL: Qwen/Qwen3.5-9B is in the frozen module's BASE_REFERENCES and pricing_v15.reference() resolves it straight through to qwen/qwen3.5-9b at USD 0.10 per 1M input and USD 0.15 per 1M output in the dated 25 Sep snapshot (DeepInfra's listing of the same base agrees at (0.10, 0.15)) x this row's own measured tokens. Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone and any other decision is reversible by arithmetic from the raw.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.0480964 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04884536222071767 }, "display": "ClassOne Qwen 3.5 9B (Qwen3.5-9B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 42.822420470216805, "I_sealed": 41.628923229080215, "base": 42.225671849648506, "excess": -1.3798904013016533, "gap": 1.1934972411365905, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 54.893255386978964, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 49.040578798364265, "judge": 36.71720500988794, "standard": 84.93093344495604 } }, "open|noul": { "cc": 50.0351966873706, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 30.000000000000004, "standard": 66.66666666666667 } }, "open|score": { "cc": 23.538809336300865, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 48.58363636363636, "hard": 33.53250000000001, "judge": 30.56728813559323, "standard": -19.51370370370371 } }, "sealed|choice": { "cc": 50.4691947514562, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 39.42307692307691, "judge": 51.58286778398511, "standard": 63.419140288212844 } }, "sealed|noul": { "cc": 45.96319747817513, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": 20.388349514563096, "judge": 57.89473684210527, "standard": 66.19718309859155 } }, "sealed|score": { "cc": 28.454377457609333, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 65.85865248226952, "hard": 11.203235294117707, "judge": 44.33532934131734, "standard": 20.433096446700493 } } } }, "jevbench_score": 38.54185769307265, "key": "classone-qwen3.5-9b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 40, "ranked": true, "ranks": { "A": 40, "B": 40, "C": 40, "capability": 85 }, "scores": { "A": 38.54185769307265, "B": 36.26089754980279, "C": 26.765178953522675 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2636520860018209, "p50_s_raw": 0.05682604300091043, "p95_s_adjusted": 0.35847554139327253, "p95_s_raw": 0.10423777069663628 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 54.01204599443065, 63.223240175665964 ], "I": [ 35.33390022059835, 47.469588715973586 ], "capability": [ 45.37052539208047, 54.754194446307686 ], "composite": { "A": [ 24.985716007949218, 50.94397024296676 ], "B": [ 22.880770578118387, 48.816339954765404 ], "C": [ 17.35119167218696, 35.37775711317136 ] }, "seed": 27 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 58.71667533779978, "capability": 50.471173593724146, "intelligence": 42.225671849648506 }, "per_type": { "choice": { "accuracy": 0.624, "calibration": 51.199283387622195, "cc": 51.9149702073643, "cc_by_type": { "choice": 51.9149702073643 }, "cc_tiered": 52.68122506921758, "cc_tiered_open": 54.893255386978964, "cc_tiered_sealed": 50.4691947514562, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 51.9149702073643 }, "noul": { "accuracy": 0.752, "calibration": 58.66592991913757, "cc": 50.4, "cc_by_type": { "noul": 50.4 }, "cc_tiered": 47.99919708277287, "cc_tiered_open": 50.0351966873706, "cc_tiered_sealed": 45.96319747817513, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 50.4 }, "score": { "accuracy": 0.5173333333333333, "calibration": 66.2848127066396, "cc": 27.883929824561417, "cc_by_type": { "score": 27.883929824561417 }, "cc_tiered": 25.9965933969551, "cc_tiered_open": 23.538809336300865, "cc_tiered_sealed": 28.454377457609333, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 27.883929824561417 } }, "raw_on_A": { "calibration": 63.67698210538227, "capability": 55.155564978213434, "intelligence": 46.63414785104459 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 37.852042064412586, "Emphasis on Accuracy 60:20:20": 33.90552600237263, "Emphasis on Cost 20:20:60": 36.32702436923972, "Emphasis on Speed 20:60:20": 44.975093090995614, "Intelligence only": 29.320085596648227 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/devops-thiago/classone-qwen3.5-9b", "gpu": "shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...). Weights devops-thiago/classone-qwen3.5-9b (sha a45d16f1f5d1840a420f8ede59fa5cca3f079961), Apache-2.0, public and ungated; code github.com/devops-thiago/class-one, Apache-2.0. ONE DOCUMENTED MAPPING OF OURS, AND IT IS THE ONLY ONE. The author's server keys SCORE probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1'. The UNAMENDED smoke gate REFUSED the first attempt at check score-keys-are-indices, before any sealed item was sent, and the server was torn down with the card verified free. The 1-based convention - which their own schemas.py documents only as 'Probability distribution across each ordered rubric level', without stating the convention - was then established on the AUTHOR'S OWN SERVER with SYNTHETIC requests across every rubric size their own validator allows (2..10): exactly the keys '1'..'n' in order every time, never '0' and never a key above n. It is corroborated independently by their own `score` field being a 1-based probability-weighted mean of the same distribution (1.0185 on a 2-level rubric, 1.6147 on a 10-level one, 3.1155 where the mass sits on key 3; a 0-based convention would have given 0.0185), and a reversed rubric moved the mass to a different key, so 'k' means the k-th level of the criteria array AS SENT - which is what JevBench's 'k-1' means. classone_reindex_r58.ClassOneReindexAdapter therefore relabels score keys k -> k-1 and NOTHING else: the request body, the answer mapping, the noul range check and the choice-must-be-one-of-the-item's-labels rule are the UNCHANGED vendored typesafe adapter's, reused by calling its own run(). An item whose score key set is not exactly {'1'..'n'} is REFUSED and recorded invalid rather than rescored on a guess; none was. choice and noul are untouched. The gate then ran AMENDED in exactly that one check, requiring the key set to be exactly {'1'..'n'} - stricter than the original, because it also pins the count - with the other seven byte-identical, and passed. PREREG-R58-AMENDMENT-1.md (sha256 in PREREG-R58-AMENDMENT-1.md.sha256), receipt PROBE-classone-e2b.json. No file of the author's was edited and no prompt of ours exists. THE SUBMIT FORM CARRIED NO DESCRIPTION FOR EITHER CLASSONE ROW, so everything about how it is run comes from the author's own repository: install line README Quickstart step 1 (`pip install -e \".[dev]\"`, in an isolated venv - torch 2.14.1+cu130, transformers 5.19.0, fastapi 0.142.2), serve line step 3 (`uvicorn classone.server.app:app`), and the checkpoint selected the way their own src/classone/server/app.py reads it (CLASSONE_BASE_MODEL). THEIR SERVER REGISTERS POST /v1/systemone ITSELF (app.py:115, next to /v1/decide and /v1/classone) and their DecisionRequest/DecisionResponse in src/classone/schemas.py are the TypeSafe System One shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. output_tokens is 0 by their own schema comment. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. add-requests run 58, benchmarkheaven.com/submit ref 957CEE0B (bh-submit b45cdc66-df44-4d77-a616-e7c67c83706d)", "underlying": "Qwen/Qwen3.5-9B", "model_pin": "devops-thiago/classone-qwen3.5-9b@a45d16f1f5d1840a420f8ede59fa5cca3f079961", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.967654986522911, "acc_among_decisive": 0.7855153203342619, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 89.88230801873149, "cost": 63.07359395281681, "intelligence": 38.53201254374527, "speed": 93.9472917853559 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05458773337668626, "ece_score": 89.08245332466275, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 89.08245332466275 }, "noul": { "abstention_rate": 0.33692722371967654, "brier": 0.11477714806826486, "ece": 0.053786553580231086, "n": 371, "score": 89.24268928395378 }, "score": { "interval90_coverage": 0.9566395663956639, "interval90_width": 0.4780939476061429, "n": 369, "nrps": 0.09977232020092458, "score": 91.32178144757796, "top_ece": 0.03689602542375804 } }, "score": 89.88230801873149 }, "capability": 64.20716028123837, "composite_ci95": { "A": [ 24.2437041451056, 49.59495089694384 ], "B": [ 20.90887497172086, 45.03359602914599 ], "C": [ 16.835905656323334, 34.440938122877675 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "JevK5 v0.3 (4B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 38.333624208131226, "I_sealed": 38.730400879359316, "base": 38.53201254374527, "excess": -2.970164313666333, "gap": -0.39677667122808913, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.58350447013268, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 60.36489462094999, "judge": 64.40342781806196, "standard": 47.25826705734617 } }, "open|noul": { "cc": 20.41614906832298, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 47.82608695652173, "judge": -9.999999999999998, "standard": 0.0 } }, "open|score": { "cc": 34.00121908593804, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 53.18739024047374, "hard": 34.10450662572488, "judge": 37.96632221813109, "standard": 18.253903730806933 } }, "sealed|choice": { "cc": 58.82061791071595, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 68.07331313280615, "hard": 55.712669683257914, "judge": 59.031657355679705, "standard": 60.09360758714129 } }, "sealed|noul": { "cc": 22.94594773546748, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": 0.9708737864077666, "judge": 39.47368421052631, "standard": 29.5774647887324 } }, "sealed|score": { "cc": 34.42463699189452, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 65.98468618250719, "hard": 27.333586499919292, "judge": 34.53681846177148, "standard": 32.6584411757232 } } } }, "jevbench_score": 37.36493548751909, "key": "jevk5-v0.3-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 41, "ranked": true, "ranks": { "A": 41, "B": 44, "C": 41, "capability": 51 }, "scores": { "A": 37.36493548751909, "B": 33.16714732620591, "C": 25.947871866332704 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.18763320203870534, "p50_s_raw": 0.018816601019352674, "p95_s_adjusted": 0.21476382557302715, "p95_s_raw": 0.03238191278651357 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.86370931431408, 89.71025718681356 ], "I": [ 32.336664057232596, 43.498424875066746 ], "capability": [ 59.05664331423508, 65.72933144554092 ], "composite": { "A": [ 24.2437041451056, 49.59495089694384 ], "B": [ 20.90887497172086, 45.03359602914599 ], "C": [ 16.835905656323334, 34.440938122877675 ] }, "seed": 48 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.88230801873149, "capability": 64.20716028123837, "intelligence": 38.53201254374527 }, "per_type": { "choice": { "accuracy": 0.688, "calibration": 89.08245332466275, "cc": 60.099656129515054, "cc_by_type": { "choice": 60.099656129515054 }, "cc_tiered": 59.70206119042432, "cc_tiered_open": 60.58350447013268, "cc_tiered_sealed": 58.82061791071595, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.099656129515054 }, "noul": { "accuracy": 0.6213333333333333, "calibration": 89.24268928395378, "cc": 24.26666666666666, "cc_by_type": { "noul": 24.26666666666666 }, "cc_tiered": 21.681048401895232, "cc_tiered_open": 20.41614906832298, "cc_tiered_sealed": 22.94594773546748, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 24.26666666666666 }, "score": { "accuracy": 0.552, "calibration": 91.32178144757796, "cc": 35.814600564516994, "cc_by_type": { "score": 35.814600564516994 }, "cc_tiered": 34.21292803891628, "cc_tiered_open": 34.00121908593804, "cc_tiered_sealed": 34.42463699189452, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 35.814600564516994 } }, "raw_on_A": { "calibration": 87.1680130057895, "capability": 65.06324188254719, "intelligence": 42.95847075930487 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 33.96793409727579, "Emphasis on Accuracy 60:20:20": 28.454799805414766, "Emphasis on Cost 20:20:60": 35.283091257812984, "Emphasis on Speed 20:60:20": 40.26908605684886, "Intelligence only": 22.883638070578947 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://huggingface.co/alibiserikbay/JevK5", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/jevk5-v0.3-4b/spec.json:jevk5_v15.JevK5Warm", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6630727762803235, "acc_among_decisive": 0.9471544715447154, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Wayfind (metask-ai)", "axes": { "calibration": 84.41575352936586, "cost": 57.7311327584882, "intelligence": 39.166454612935595, "speed": 90.78239008886905 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06577096449867435, "ece_score": 86.84580710026512, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 86.84580710026512 }, "noul": { "abstention_rate": 0.2506738544474394, "brier": 0.14719486396931378, "ece": 0.09558018432686369, "n": 371, "score": 80.88396313462727 }, "score": { "interval90_coverage": 0.9728997289972899, "interval90_width": 0.6723125564588976, "n": 369, "nrps": 0.11633460342816504, "score": 85.51749035320519, "top_ece": 0.08665779475386558 } }, "score": 84.41575352936586 }, "capability": 61.79110407115073, "composite_ci95": { "A": [ 24.35782606760525, 47.92198520846029 ], "B": [ 21.36935197657226, 43.913080458446345 ], "C": [ 16.915156991392532, 33.27915639476408 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.025642610837438424 }, "display": "metask-jev-4b", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 41.89224435585361, "I_sealed": 36.440664870017585, "base": 39.166454612935595, "excess": 2.878191843397784, "gap": 5.451579485836028, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.50866528870256, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 52.815350739226176, "judge": 60.44825313117996, "standard": 69.8618668899121 } }, "open|noul": { "cc": 37.846790890269155, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 56.52173913043479, "judge": 0.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 27.321276888589132, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 36.68821311926259, "hard": 13.269014128057655, "judge": 46.73482367537818, "standard": 21.622014114131783 } }, "sealed|choice": { "cc": 56.24961082249847, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 73.3944276106718, "hard": 48.076923076923066, "judge": 61.824953445065184, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 22.8393257860911, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 63.99999999999999, "hard": 2.9126213592232997, "judge": 23.684210526315795, "standard": 40.8450704225352 } }, "sealed|score": { "cc": 30.23305800146319, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 50.21133272720072, "hard": 17.595550281640328, "judge": 37.810328121508064, "standard": 34.153030898172844 } } } }, "jevbench_score": 37.34931073616377, "key": "metask-jev-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 42, "ranked": true, "ranks": { "A": 42, "B": 42, "C": 42, "capability": 60 }, "scores": { "A": 37.34931073616377, "B": 33.62317034896201, "C": 25.937021344558172 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.25794456995790827, "p50_s_raw": 0.05397228497895412, "p95_s_adjusted": 0.3237685059779324, "p95_s_raw": 0.08688425298896618 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.04800226228174, 85.90669927321129 ], "I": [ 32.96914634416673, 43.538341958330456 ], "capability": [ 56.20112405588935, 63.92462857658634 ], "composite": { "A": [ 24.35782606760525, 47.92198520846029 ], "B": [ 21.36935197657226, 43.913080458446345 ], "C": [ 16.915156991392532, 33.27915639476408 ] }, "seed": 66 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 84.41575352936586, "capability": 61.79110407115073, "intelligence": 39.166454612935595 }, "per_type": { "choice": { "accuracy": 0.668, "calibration": 86.84580710026512, "cc": 57.541941778842954, "cc_by_type": { "choice": 57.541941778842954 }, "cc_tiered": 58.379138055600514, "cc_tiered_open": 60.50866528870256, "cc_tiered_sealed": 56.24961082249847, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.541941778842954 }, "noul": { "accuracy": 0.6506666666666666, "calibration": 80.88396313462727, "cc": 30.133333333333322, "cc_by_type": { "noul": 30.133333333333322 }, "cc_tiered": 30.34305833818013, "cc_tiered_open": 37.846790890269155, "cc_tiered_sealed": 22.8393257860911, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 30.133333333333322 }, "score": { "accuracy": 0.48, "calibration": 85.51749035320519, "cc": 29.198392850325618, "cc_by_type": { "score": 29.198392850325618 }, "cc_tiered": 28.77716744502616, "cc_tiered_open": 27.321276888589132, "cc_tiered_sealed": 30.23305800146319, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 29.198392850325618 } }, "raw_on_A": { "calibration": 83.18850868556241, "capability": 62.528725908581364, "intelligence": 41.86894313160032 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 34.17198402783528, "Emphasis on Accuracy 60:20:20": 29.237864520407634, "Emphasis on Cost 20:20:60": 34.662048899004674, "Emphasis on Speed 20:60:20": 40.422015539591015, "Intelligence only": 24.032711498388675 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://github.com/metask-ai/metask-jev", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen3.5-4B merged r16 LoRA, candidate-logit readout", "adapter_id": "images/metask-jev-4b/spec.json:metask_v15.MetaskJevWarm", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7493261455525606, "acc_among_decisive": 0.8776978417266187, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "krishna765", "axes": { "calibration": 88.59116686443139, "cost": 63.46145558281372, "intelligence": 38.62465958342338, "speed": 89.93848373337222 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06995357456541221, "ece_score": 86.00928508691756, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 86.00928508691756 }, "noul": { "abstention_rate": 0.3207547169811321, "brier": 0.11609579904406445, "ece": 0.05066343452944609, "n": 371, "score": 89.86731309411078 }, "score": { "interval90_coverage": 0.9539295392953929, "interval90_width": 0.47515808491418265, "n": 369, "nrps": 0.10083978781514244, "score": 89.89690241226586, "top_ece": 0.050611081969770266 } }, "score": 88.59116686443139 }, "capability": 63.60791322392738, "composite_ci95": { "A": [ 23.964977734953006, 50.38055037709117 ], "B": [ 20.750561783148846, 45.824903274791566 ], "C": [ 16.642345649272922, 34.98649331742443 ] }, "cost": { "basis": "ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra Qwen/Qwen3.5-4B list price $0.03/M input, $0.15/M output (base of JevK5, the adapter's base checkpoint; not on OpenRouter; frozen 25 Sep snapshot per I-3, same reference as reflex-4b, jobe, typecastlm and Quyet) x the server's own measured usage.input_tokens; logit readout, 0 output tokens; FP8 weight-only on 1x RTX PRO 6000 96GB (Lium; v1.6 GPU-class deviation from source H100), deck-4b 1.0.0 + jevk5 f944fe37; estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.0162645 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01651777251184834 }, "display": "deck-4B v1.0 (krishna765, JevK5 + LoRA, FP8 weight-only)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 38.79947935146771, "I_sealed": 38.44983981537905, "base": 38.62465958342338, "excess": -2.2237481063495843, "gap": 0.3496395360886595, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 58.68740862543909, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 60.36489462094999, "judge": 64.40342781806196, "standard": 39.723733779824194 } }, "open|noul": { "cc": 25.017253278122844, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 39.13043478260869, "judge": 0.0, "standard": 11.111111111111116 } }, "open|score": { "cc": 32.6937761508412, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 51.877671536622636, "hard": 34.178023262130495, "judge": 35.59214965227232, "standard": 15.785773983225237 } }, "sealed|choice": { "cc": 59.157265682164855, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 69.40359175227256, "hard": 56.22171945701356, "judge": 59.031657355679705, "standard": 60.09360758714129 } }, "sealed|noul": { "cc": 21.704287246216186, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": 0.9708737864077666, "judge": 31.578947368421062, "standard": 35.2112676056338 } }, "sealed|score": { "cc": 34.48796651775611, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 66.00896406653321, "hard": 26.22895515401159, "judge": 35.59993017640196, "standard": 33.57754498288781 } } } }, "jevbench_score": 37.26498528662829, "key": "deck-4b-v1-0", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 43, "ranked": true, "ranks": { "A": 43, "B": 43, "C": 43, "capability": 53 }, "scores": { "A": 37.26498528662829, "B": 33.17299510735835, "C": 25.87846200460298 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.30241732355207207, "p50_s_raw": 0.07620866177603602, "p95_s_adjusted": 0.3353860241360962, "p95_s_raw": 0.09269301206804811 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.64482190603961, 89.1506890221124 ], "I": [ 32.29690931230169, 43.8177901770495 ], "capability": [ 58.30292793082468, 65.68855559803174 ], "composite": { "A": [ 23.98211286851183, 50.42175552568154 ], "B": [ 20.76083692137617, 45.85216931192029 ], "C": [ 16.65424504757766, 35.01510800394553 ] }, "seed": 33 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.59116686443139, "capability": 63.60791322392738, "intelligence": 38.62465958342338 }, "per_type": { "choice": { "accuracy": 0.688, "calibration": 86.00928508691756, "cc": 60.099656129515054, "cc_by_type": { "choice": 60.099656129515054 }, "cc_tiered": 58.922337153801976, "cc_tiered_open": 58.68740862543909, "cc_tiered_sealed": 59.157265682164855, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.099656129515054 }, "noul": { "accuracy": 0.6266666666666667, "calibration": 89.86731309411078, "cc": 25.333333333333343, "cc_by_type": { "noul": 25.333333333333343 }, "cc_tiered": 23.360770262169517, "cc_tiered_open": 25.017253278122844, "cc_tiered_sealed": 21.704287246216186, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 25.333333333333343 }, "score": { "accuracy": 0.5546666666666666, "calibration": 89.89690241226586, "cc": 35.52454801522435, "cc_by_type": { "score": 35.52454801522435 }, "cc_tiered": 33.590871334298654, "cc_tiered_open": 32.6937761508412, "cc_tiered_sealed": 34.48796651775611, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 35.52454801522435 } }, "raw_on_A": { "calibration": 89.0005990160775, "capability": 66.05004010202693, "intelligence": 43.09948118797635 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 33.95960169128873, "Emphasis on Accuracy 60:20:20": 28.553229571667234, "Emphasis on Cost 20:20:60": 35.46476300764428, "Emphasis on Speed 20:60:20": 39.807414011248, "Intelligence only": 23.04910072446208 }, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6792452830188679, "acc_among_decisive": 0.9325396825396826, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": null }, "repo": "https://github.com/krishna-gogineni-765/deck" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Cloudflare", "axes": { "calibration": 85.46479352633332, "cost": 46.79963593205177, "intelligence": 42.18368888957246, "speed": 88.4044196893156 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1119512500000001, "ece_score": 77.60974999999998, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 77.60974999999998 }, "noul": { "abstention_rate": 0.31733333333333336, "brier": 0.11674051933333338, "ece": 0.04965040000000006, "n": 375, "score": 90.06992 }, "score": { "interval90_coverage": 0.9333333333333333, "interval90_width": 0.460888888888889, "n": 375, "nrps": 0.09493298842, "score": 88.71471057900001, "top_ece": 0.06538639999999996 } }, "score": 85.46479352633332 }, "capability": 63.824241207952895, "composite_ci95": { "A": [ 26.640037750072654, 45.22280470107504 ], "B": [ 24.191496498186186, 42.57204972938278 ], "C": [ 18.500026215328234, 31.40472548685768 ] }, "cost": { "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; self-hosted estimate, not measured hosted billing", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.05933971674876847 }, "display": "Clef-Flash (Cloudflare, Qwen3.5-9B post-train with a joint schema head, multimodal, measured on text)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.25550449770323, "I_sealed": 39.11187328144169, "base": 42.18368888957246, "excess": 3.5702435738232956, "gap": 6.143631216261539, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 62.93462232292448, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 56.59012268008808, "judge": 68.35860250494396, "standard": 54.79280033486815 } }, "open|noul": { "cc": 30.42788129744652, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 30.434782608695656, "judge": 0.0, "standard": 55.55555555555556 } }, "open|score": { "cc": 42.4040098727387, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 41.5230303030303, "hard": 37.280925925925935, "judge": 43.06322033898306, "standard": 52.101851851851855 } }, "sealed|choice": { "cc": 54.80150596378907, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 45.53167420814478, "judge": 53.44506517690876, "standard": 70.07020569035596 } }, "sealed|noul": { "cc": 20.824374761599753, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 52.0, "hard": -10.679611650485432, "judge": 31.578947368421062, "standard": 52.112676056338024 } }, "sealed|score": { "cc": 41.70973911893625, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 54.71716312056738, "hard": 35.80301470588234, "judge": 41.72143712574849, "standard": 47.00192893401015 } } } }, "jevbench_score": 36.635050479655426, "key": "clef-flash", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 44, "ranked": true, "ranks": { "A": 44, "B": 41, "C": 44, "capability": 52 }, "scores": { "A": 36.635050479655426, "B": 33.967249202727025, "C": 25.44100727753849 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3556603590026498, "p50_s_raw": 0.10283017950132489, "p95_s_adjusted": 0.405996760725975, "p95_s_raw": 0.12799838036298752 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 79.4975513933287, 86.84610174167315 ], "I": [ 36.95680839342918, 46.203587775711 ], "capability": [ 59.38252934298857, 65.53439425442862 ], "composite": { "A": [ 26.640037750072654, 45.22280470107504 ], "B": [ 24.191496498186186, 42.57204972938278 ], "C": [ 18.500026215328234, 31.40472548685768 ] }, "seed": 21 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.46479352633332, "capability": 63.824241207952895, "intelligence": 42.18368888957246 }, "per_type": { "choice": { "accuracy": 0.6666666666666666, "calibration": 77.60974999999998, "cc": 57.37142748879814, "cc_by_type": { "choice": 57.37142748879814 }, "cc_tiered": 58.868064143356776, "cc_tiered_open": 62.93462232292448, "cc_tiered_sealed": 54.80150596378907, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.37142748879814 }, "noul": { "accuracy": 0.6373333333333333, "calibration": 90.06992, "cc": 27.46666666666666, "cc_by_type": { "noul": 27.46666666666666 }, "cc_tiered": 25.626128029523137, "cc_tiered_open": 30.42788129744652, "cc_tiered_sealed": 20.824374761599753, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 27.46666666666666 }, "score": { "accuracy": 0.6026666666666667, "calibration": 88.71471057900001, "cc": 42.01200877192982, "cc_by_type": { "score": 42.01200877192982 }, "cc_tiered": 42.05687449583748, "cc_tiered_open": 42.4040098727387, "cc_tiered_sealed": 41.70973911893625, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 42.01200877192982 } }, "raw_on_A": { "calibration": 82.15484704393162, "capability": 63.82820945816345, "intelligence": 45.501571872395274 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 33.178006784894365, "Emphasis on Accuracy 60:20:20": 30.03860559786673, "Emphasis on Cost 20:20:60": 31.455768602881705, "Emphasis on Speed 20:60:20": 39.463018165391404, "Intelligence only": 26.30501767147255 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/Cloudflare/clef-flash", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h", "underlying": "Qwen/Qwen3.5-9B", "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r39/outputs/meta/specs/clef-flash.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "alt": { "label": "Workers AI price scenario (latency unmeasured)", "note": "Cloudflare standard launch input price; same self-hosted accuracy and adjusted H100 latency held fixed for this price-only scenario. No Workers AI latency was measured; this is not a hosted API result and does not set official Capability eligibility. Official self-hosted row uses the base-model reference price.", "usd_per_1000": 0.053405745073891626, "axes": { "intelligence": 53.08161490373422, "calibration": 87.5533640278553, "speed": 83.63107135213085, "cost": 48.17236064887202 }, "pricing_only": true }, "model_pin": "Cloudflare/clef-flash@17f0b0ad64efb65d273590632833508766b2aae6", "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6826666666666666, "acc_among_decisive": 0.93359375, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Eval Engine / Chromia", "axes": { "calibration": 80.25524974800695, "cost": 65.87776871337948, "intelligence": 37.93489123230218, "speed": 91.06043780349009 }, "calibration": { "parts": { "choice": { "ece_hard": 0.11555442846600955, "ece_score": 76.88911430679809, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 76.88911430679809 }, "noul": { "abstention_rate": 0.27762803234501343, "brier": 0.14312850498011134, "ece": 0.09433112828523382, "n": 371, "score": 81.13377434295323 }, "score": { "interval90_coverage": 0.907859078590786, "interval90_width": 0.43202348690153575, "n": 369, "nrps": 0.11270639836290829, "score": 82.74286059426956, "top_ece": 0.11621819487585025 } }, "score": 80.25524974800695 }, "capability": 59.09507049015456, "composite_ci95": { "A": [ 21.75506402712618, 47.346177294331085 ], "B": [ 18.702949452781887, 42.97523990261764 ], "C": [ 15.107683352170962, 32.879289787729924 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off, already marked deprecated at that cutoff (replaced by Qwen/Qwen3.5-9B); re-score if it changes (I-3). Frozen market reference deepinfra:Qwen/Qwen3.5-4B.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.013721693349753695 }, "display": "Decision-4B (Eval Engine / Chromia)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 39.60182572574049, "I_sealed": 36.267956738863866, "base": 37.93489123230218, "excess": 0.76048134443838, "gap": 3.333868986876624, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 65.98784234179594, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 50.92796476879523, "judge": 72.31377719182596, "standard": 77.39640016743407 } }, "open|noul": { "cc": 23.933747412008287, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 21.739130434782616, "judge": 0.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 28.883887423417242, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 48.4752733983845, "hard": 37.0901498391321, "judge": 25.649863025499585, "standard": 7.526706201380384 } }, "sealed|choice": { "cc": 58.09043969346601, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 72.06414899120539, "hard": 47.05882352941176, "judge": 60.89385474860336, "standard": 68.96169478999877 } }, "sealed|noul": { "cc": 19.731441484882723, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": -12.621359223300976, "judge": 28.947368421052634, "standard": 46.47887323943662 } }, "sealed|score": { "cc": 30.981989038242872, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 65.61623338387761, "hard": 20.121162500023082, "judge": 32.27985452492822, "standard": 33.439721711837066 } } } }, "jevbench_score": 35.43243109909669, "key": "evalengine-decision-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 45, "ranked": true, "ranks": { "A": 45, "B": 45, "C": 45, "capability": 67 }, "scores": { "A": 35.43243109909669, "B": 31.508666798854446, "C": 24.60585492992826 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.24496516967192292, "p50_s_raw": 0.04748258483596146, "p95_s_adjusted": 0.3197804291266948, "p95_s_raw": 0.08489021456334739 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 75.03864735316435, 82.85641931336254 ], "I": [ 31.11663059494919, 42.88120054683658 ], "capability": [ 54.031339191317976, 62.320753924360694 ], "composite": { "A": [ 21.75506402712618, 47.346177294331085 ], "B": [ 18.702949452781887, 42.97523990261764 ], "C": [ 15.107683352170962, 32.879289787729924 ] }, "seed": 16 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 80.25524974800695, "capability": 59.09507049015456, "intelligence": 37.93489123230218 }, "per_type": { "choice": { "accuracy": 0.6866666666666666, "calibration": 76.88911430679809, "cc": 59.92914183947025, "cc_by_type": { "choice": 59.92914183947025 }, "cc_tiered": 62.039141017630975, "cc_tiered_open": 65.98784234179594, "cc_tiered_sealed": 58.09043969346601, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 59.92914183947025 }, "noul": { "accuracy": 0.632, "calibration": 81.13377434295323, "cc": 26.400000000000002, "cc_by_type": { "noul": 26.400000000000002 }, "cc_tiered": 21.832594448445505, "cc_tiered_open": 23.933747412008287, "cc_tiered_sealed": 19.731441484882723, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 26.400000000000002 }, "score": { "accuracy": 0.5173333333333333, "calibration": 82.74286059426956, "cc": 32.24707656205765, "cc_by_type": { "score": 32.24707656205765 }, "cc_tiered": 29.932938230830057, "cc_tiered_open": 28.883887423417242, "cc_tiered_sealed": 30.981989038242872, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 32.24707656205765 } }, "raw_on_A": { "calibration": 78.19048224326764, "capability": 60.14677409846598, "intelligence": 42.103065953664334 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 32.87874158865029, "Emphasis on Accuracy 60:20:20": 27.346985238704878, "Emphasis on Cost 20:20:60": 34.72560202914835, "Emphasis on Speed 20:60:20": 38.639773156510174, "Intelligence only": 21.836172722245216 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/evalengine/decision-4b-gguf", "gpu": "H100", "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P88 on one evaluator-owned Lium H100. Current staged log verifies evalengine/decision-4b-gguf@08859d1d8307f7b7ea452549e303b7805c35e350; stored GGUF SHA256 f2519bcdc945291974c81a37ff92417b22df27154c9df2564420d08576b907ce. Serial offline loopback-only inference, standard selfhosted x2+0.15 s latency adjustment. Carried published estimate retained.", "underlying": "Qwen/Qwen3.5-4B", "model_pin": "evalengine/decision-4b-gguf@08859d1d8307f7b7ea452549e303b7805c35e350", "last_measured_on": "2026-10-04", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7223719676549866, "acc_among_decisive": 0.8843283582089553, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Current inventory + r20 source metadata; unresolved runtime join" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Icarus AI / cmxu", "axes": { "calibration": 88.12743013642095, "cost": 64.41461114142905, "intelligence": 37.57770128683752, "speed": 93.17471817821269 }, "calibration": { "parts": { "choice": { "ece_hard": 0.08741578110266195, "ece_score": 82.5168437794676, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 82.5168437794676 }, "noul": { "abstention_rate": 0.36657681940700804, "brier": 0.12068971845453581, "ece": 0.04093580710766913, "n": 371, "score": 91.81283857846617 }, "score": { "interval90_coverage": 0.9376693766937669, "interval90_width": 0.4548328816621501, "n": 369, "nrps": 0.10039054297386547, "score": 90.05260805132909, "top_ece": 0.049278647999776395 } }, "score": 88.12743013642095 }, "capability": 62.85256571162924, "composite_ci95": { "A": [ 21.844531574541936, 45.558507177419784 ], "B": [ 18.72406374533524, 41.13032827742159 ], "C": [ 15.169813593431902, 31.63785220654152 ] }, "cost": { "basis": "add-requests run 47 (v1.5 basis, unpublished) base-model price floor deepinfra:Qwen/Qwen3.5-4B", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.015352518472906404 }, "display": "janus 4B (Icarus AI / cmxu, Qwen3.5-4B + LoRA r64 and pointer decision head)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 40.660351519488046, "I_sealed": 34.49505105418699, "base": 37.57770128683752, "excess": 3.5919128228628145, "gap": 6.165300465301058, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 61.67923497711049, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 56.59012268008808, "judge": 60.44825313117996, "standard": 62.32733361239012 } }, "open|noul": { "cc": 23.289164941338854, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 47.82608695652173, "judge": -19.999999999999996, "standard": 22.222222222222232 } }, "open|score": { "cc": 37.01265464001479, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 46.99228809649041, "hard": 42.7176115493677, "judge": 46.3643071982838, "standard": 6.5854452556676435 } }, "sealed|choice": { "cc": 56.292971028212264, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 66.74303451333975, "hard": 49.60407239819003, "judge": 56.238361266294234, "standard": 64.52765118857003 } }, "sealed|noul": { "cc": 12.558893678884758, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": -14.563106796116509, "judge": 23.684210526315795, "standard": 32.3943661971831 } }, "sealed|score": { "cc": 34.633288455463955, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 59.65725589324873, "hard": 29.365017382365, "judge": 35.41609562746013, "standard": 31.483636124775206 } } } }, "jevbench_score": 35.18351981268845, "key": "janus-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 46, "ranked": true, "ranks": { "A": 46, "B": 47, "C": 46, "capability": 54 }, "scores": { "A": 35.18351981268845, "B": 31.093845400105046, "C": 24.432999869922536 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2097848016768694, "p50_s_raw": 0.029892400838434696, "p95_s_adjusted": 0.22948492439463736, "p95_s_raw": 0.039742462197318675 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.8809685848649, 88.36140314243593 ], "I": [ 31.000587963045195, 41.97868333470401 ], "capability": [ 57.23309571773392, 64.21484535464957 ], "composite": { "A": [ 21.844531574541936, 45.558507177419784 ], "B": [ 18.72406374533524, 41.13032827742159 ], "C": [ 15.169813593431902, 31.63785220654152 ] }, "seed": 18 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.12743013642095, "capability": 62.85256571162924, "intelligence": 37.57770128683752 }, "per_type": { "choice": { "accuracy": 0.6733333333333333, "calibration": 82.5168437794676, "cc": 58.22399893902218, "cc_by_type": { "choice": 58.22399893902218 }, "cc_tiered": 58.98610300266138, "cc_tiered_open": 61.67923497711049, "cc_tiered_sealed": 56.292971028212264, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 58.22399893902218 }, "noul": { "accuracy": 0.592, "calibration": 91.81283857846617, "cc": 18.399999999999995, "cc_by_type": { "noul": 18.399999999999995 }, "cc_tiered": 17.924029310111806, "cc_tiered_open": 23.289164941338854, "cc_tiered_sealed": 12.558893678884758, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 18.399999999999995 }, "score": { "accuracy": 0.5653333333333334, "calibration": 90.05260805132909, "cc": 36.47974838634844, "cc_by_type": { "score": 36.47974838634844 }, "cc_tiered": 35.82297154773937, "cc_tiered_open": 37.01265464001479, "cc_tiered_sealed": 34.633288455463955, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 36.47974838634844 } }, "raw_on_A": { "calibration": 84.32063965516178, "capability": 62.99315158120264, "intelligence": 41.6656635072435 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 32.05124045782333, "Emphasis on Accuracy 60:20:20": 26.620100423360302, "Emphasis on Cost 20:20:60": 33.65415776578004, "Emphasis on Speed 20:60:20": 37.99318109571395, "Intelligence only": 21.225142796235538 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/cmxu/janus-4b", "gpu": "RTX PRO6000", "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P83 on one evaluator-owned Lium RTX PRO6000. Offline contributor image, serial loopback-only requests, no batching or retry; standard selfhosted x2+0.15 s latency adjustment. Current stage receipt verifies the same model pin used by the prior add-request run. Cost is the carried documented estimate recorded in current registry.", "underlying": "Qwen/Qwen3.5-4B", "model_pin": "cmxu/janus-4b@fc86a8fbaa99de8dbac5aee46aaa56d1a853b876", "last_measured_on": "2026-10-04", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.633423180592992, "acc_among_decisive": 0.9446808510638298, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "run 47 author metadata + current P83 staged pin/measurement receipt" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "jsaurabh", "axes": { "calibration": 79.78895435560509, "cost": 45.690218065856605, "intelligence": 41.94184606895763, "speed": 85.40793837616167 }, "calibration": { "parts": { "choice": { "ece_hard": 0.14238786731364284, "ece_score": 71.52242653727143, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 71.52242653727143 }, "noul": { "abstention_rate": 0.2911051212938005, "brier": 0.12127289774594222, "ece": 0.07874545275117417, "n": 371, "score": 84.25090944976516 }, "score": { "interval90_coverage": 0.8888888888888888, "interval90_width": 0.31820234869015335, "n": 369, "nrps": 0.0937862889000391, "score": 83.59352707977865, "top_ece": 0.11717158475219394 } }, "score": 79.78895435560509 }, "capability": 60.86540021228136, "composite_ci95": { "A": [ 22.862450507726937, 41.74491188365231 ], "B": [ 20.753639028685242, 39.453425031751074 ], "C": [ 15.87670174147704, 28.989522141425212 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.06461391625615764 }, "display": "Qwen3.5-9B Jev-like data-mix v2", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.69949425428299, "I_sealed": 38.184197883632265, "base": 41.94184606895763, "excess": 4.94190872821248, "gap": 7.515296370650724, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 56.305297450788785, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 49.040578798364265, "judge": 56.49307844429796, "standard": 62.32733361239012 } }, "open|noul": { "cc": 33.4120082815735, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": 19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 47.38117703048668, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 50.73251602617265, "hard": 54.043486375775075, "judge": 35.50551913484291, "standard": 50.19437568553258 } }, "sealed|choice": { "cc": 54.756843153610475, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 64.08247727440693, "hard": 46.04072398190045, "judge": 51.58286778398511, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 20.448763917177054, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 52.0, "hard": -4.854368932038833, "judge": 26.315789473684205, "standard": 46.47887323943662 } }, "sealed|score": { "cc": 39.346986580109274, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 69.00077177598622, "hard": 33.3124721590493, "judge": 38.59270672300753, "standard": 37.720542609943344 } } } }, "jevbench_score": 33.58966697347588, "key": "qwen35-9b-jev-data-mix-v2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 47, "ranked": true, "ranks": { "A": 47, "B": 46, "C": 47, "capability": 64 }, "scores": { "A": 33.58966697347588, "B": 31.31610274190126, "C": 23.326157620469353 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4865717590786517, "p50_s_raw": 0.16828587953932583, "p95_s_adjusted": 0.5916423641145229, "p95_s_raw": 0.22082118205726145 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 75.15659713819453, 82.50836404329671 ], "I": [ 35.70342834084104, 45.9106569165651 ], "capability": [ 56.2798648487553, 63.69018289459294 ], "composite": { "A": [ 22.862450507726937, 41.74491188365231 ], "B": [ 20.753639028685242, 39.453425031751074 ], "C": [ 15.87670174147704, 28.989522141425212 ] }, "seed": 92 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 79.78895435560509, "capability": 60.86540021228136, "intelligence": 41.94184606895763 }, "per_type": { "choice": { "accuracy": 0.6533333333333333, "calibration": 71.52242653727143, "cc": 55.666284588350074, "cc_by_type": { "choice": 55.666284588350074 }, "cc_tiered": 55.53107030219963, "cc_tiered_open": 56.305297450788785, "cc_tiered_sealed": 54.756843153610475, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 55.666284588350074 }, "noul": { "accuracy": 0.6373333333333333, "calibration": 84.25090944976516, "cc": 27.46666666666666, "cc_by_type": { "noul": 27.46666666666666 }, "cc_tiered": 26.930386099375276, "cc_tiered_open": 33.4120082815735, "cc_tiered_sealed": 20.448763917177054, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 27.46666666666666 }, "score": { "accuracy": 0.5733333333333334, "calibration": 83.59352707977865, "cc": 42.24339448080683, "cc_by_type": { "score": 42.24339448080683 }, "cc_tiered": 43.36408180529798, "cc_tiered_open": 47.38117703048668, "cc_tiered_sealed": 39.346986580109274, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 42.24339448080683 } }, "raw_on_A": { "calibration": 80.51953554017331, "capability": 63.66543855723072, "intelligence": 46.811341574288136 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 30.68928023743452, "Emphasis on Accuracy 60:20:20": 27.947003437168036, "Emphasis on Cost 20:20:60": 29.02722060908932, "Emphasis on Speed 20:60:20": 36.335136837664905, "Intelligence only": 24.643879144700183 }, "class": "jev-rebuild", "open": "unknown", "licence": "Apache-2.0 (adapter code and weights; Qwen3.5-9B base is Apache-2.0)", "repo": "https://huggingface.co/jsaurabh/qwen3.5-9b-jev-data-mix-v2", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": null, "adapter_id": "images/qwen35-9b-jev-data-mix-v2/spec.json:qwen35_nimble_lora.Qwen35NimbleLoraAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7088948787061995, "acc_among_decisive": 0.908745247148289, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "allebee", "axes": { "calibration": 89.38710082819252, "cost": 63.07359395281681, "intelligence": 36.261123811123895, "speed": 91.56570599843724 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05722074836783767, "ece_score": 88.55585032643246, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 88.55585032643246 }, "noul": { "abstention_rate": 0.3800539083557951, "brier": 0.12953478188668033, "ece": 0.0506873602652232, "n": 371, "score": 89.86252794695537 }, "score": { "interval90_coverage": 0.981029810298103, "interval90_width": 0.5517163504968384, "n": 369, "nrps": 0.098178280103309, "score": 89.74292421118973, "top_ece": 0.053481617836448225 } }, "score": 89.38710082819252 }, "capability": 62.824112319658205, "composite_ci95": { "A": [ 21.332485805393638, 41.48362112591029 ], "B": [ 18.26634005992914, 37.259117524916284 ], "C": [ 14.814226253745582, 28.808070226326592 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "JevK5 v0.2.0", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 39.85943426682522, "I_sealed": 32.66281335542257, "base": 36.261123811123895, "excess": 4.623233268964404, "gap": 7.196620911402647, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 56.2584494703768, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 57.189189189189186, "hard": 52.815350739226176, "judge": 56.49307844429796, "standard": 62.32733361239012 } }, "open|noul": { "cc": 29.9399585921325, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 47.82608695652173, "judge": -9.999999999999998, "standard": 33.33333333333333 } }, "open|score": { "cc": 33.37989473796636, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.905656531340256, "hard": 27.673703730451283, "judge": 40.84579963079196, "standard": 20.83053851707115 } }, "sealed|choice": { "cc": 55.096468823579634, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 64.08247727440693, "hard": 55.20361990950226, "judge": 51.58286778398511, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 6.636284338632712, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 43.99999999999999, "hard": -14.563106796116509, "judge": 18.421052631578938, "standard": 12.676056338028175 } }, "sealed|score": { "cc": 36.25568690405537, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 64.58927438023056, "hard": 27.57268337567902, "judge": 41.004484216055225, "standard": 32.331704254720705 } } } }, "jevbench_score": 32.09901620353128, "key": "jevk5-v02", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 48, "ranked": true, "ranks": { "A": 48, "B": 50, "C": 48, "capability": 55 }, "scores": { "A": 32.09901620353128, "B": 28.24080105110956, "C": 22.290983474674498 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23078905596630647, "p50_s_raw": 0.04039452798315324, "p95_s_adjusted": 0.30214414960937575, "p95_s_raw": 0.07607207480468788 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.43880788393268, 90.09857764344801 ], "I": [ 30.721600512376973, 40.44080848333523 ], "capability": [ 58.00843038572983, 64.2177747361447 ], "composite": { "A": [ 21.332485805393638, 41.48362112591029 ], "B": [ 18.26634005992914, 37.259117524916284 ], "C": [ 14.814226253745582, 28.808070226326592 ] }, "seed": 49 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.38710082819252, "capability": 62.824112319658205, "intelligence": 36.261123811123895 }, "per_type": { "choice": { "accuracy": 0.6546666666666666, "calibration": 88.55585032643246, "cc": 55.836798878394866, "cc_by_type": { "choice": 55.836798878394866 }, "cc_tiered": 55.67745914697822, "cc_tiered_open": 56.2584494703768, "cc_tiered_sealed": 55.096468823579634, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 55.836798878394866 }, "noul": { "accuracy": 0.5733333333333334, "calibration": 89.86252794695537, "cc": 14.666666666666671, "cc_by_type": { "noul": 14.666666666666671 }, "cc_tiered": 18.288121465382606, "cc_tiered_open": 29.9399585921325, "cc_tiered_sealed": 6.636284338632712, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 14.666666666666671 }, "score": { "accuracy": 0.5546666666666666, "calibration": 89.74292421118973, "cc": 36.384921485746844, "cc_by_type": { "score": 36.384921485746844 }, "cc_tiered": 34.81779082101087, "cc_tiered_open": 33.37989473796636, "cc_tiered_sealed": 36.25568690405537, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 36.384921485746844 } }, "raw_on_A": { "calibration": 85.60375573005521, "capability": 62.620633576812004, "intelligence": 39.637511423568796 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 29.02935442351159, "Emphasis on Accuracy 60:20:20": 24.01393367218798, "Emphasis on Cost 20:20:60": 30.556198079902455, "Emphasis on Speed 20:60:20": 34.51299399305244, "Intelligence only": 19.071452492870637 }, "class": "jev-rebuild", "open": "unknown", "licence": "Apache-2.0 (code/adapter); Apache-2.0 (Qwen3.5-4B base)", "repo": "https://github.com/allebee/jevk5", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/jevk5-v02/spec.json:jevk5_direct.JevK5DirectAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6199460916442049, "acc_among_decisive": 0.9347826086956522, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Juspay", "axes": { "calibration": 91.76581828544445, "cost": 39.84212731591276, "intelligence": 45.955077428728934, "speed": 89.83885080776159 }, "calibration": { "parts": { "choice": { "ece_hard": 0.034452812499999874, "ece_score": 93.10943750000003, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 93.10943750000003 }, "noul": { "abstention_rate": 0.376, "brier": 0.10284091864000006, "ece": 0.03908880000000003, "n": 375, "score": 92.18224 }, "score": { "interval90_coverage": 0.9706666666666667, "interval90_width": 0.3935555555555556, "n": 375, "nrps": 0.08401831953999996, "score": 90.00577735633334, "top_ece": 0.05793306666666674 } }, "score": 91.76581828544445 }, "capability": 68.8604478570867, "composite_ci95": { "A": [ 23.186488200616, 37.838876804389514 ], "B": [ 21.60212409324276, 36.562829588228304 ], "C": [ 16.101727917094447, 27.301863296622916 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.10121908866995075 }, "display": "JevOne", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 47.580446331109215, "I_sealed": 44.329708526348654, "base": 45.955077428728934, "excess": 0.6773501623223175, "gap": 3.2507378047605613, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 80.11686099440308, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 77.35136835482857, "judge": 84.17930125247197, "standard": 77.39640016743407 } }, "open|noul": { "cc": 20.761214630779854, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 30.434782608695656, "judge": -9.999999999999998, "standard": 22.222222222222232 } }, "open|score": { "cc": 41.86326336814473, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 53.82060606060606, "hard": 35.2175, "judge": 43.91203389830509, "standard": 46.102962962962955 } }, "sealed|choice": { "cc": 71.7077246661197, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 82.70637794693668, "hard": 65.89366515837104, "judge": 73.9292364990689, "standard": 74.50424929178472 } }, "sealed|noul": { "cc": 13.421236874491711, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": -8.737864077669899, "judge": 13.157894736842103, "standard": 40.8450704225352 } }, "sealed|score": { "cc": 47.86016403843456, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 63.93652482269504, "hard": 43.95980392156864, "judge": 49.40958083832332, "standard": 45.29857868020299 } } } }, "jevbench_score": 31.14510331640707, "key": "jevone", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 49, "ranked": true, "ranks": { "A": 49, "B": 48, "C": 49, "capability": 39 }, "scores": { "A": 31.14510331640707, "B": 29.58577784402432, "C": 21.62854396972713 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2735971907153726, "p50_s_raw": 0.06179859535768628, "p95_s_adjusted": 0.3793178545311093, "p95_s_raw": 0.11465892726555464 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.59503348121109, 91.11403729704442 ], "I": [ 40.574264452604204, 50.931916001637504 ], "capability": [ 63.974665483283545, 69.89087834774071 ], "composite": { "A": [ 23.186488200616, 37.838876804389514 ], "B": [ 21.60212409324276, 36.562829588228304 ], "C": [ 16.101727917094447, 27.301863296622916 ] }, "seed": 50 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 91.76581828544445, "capability": 68.8604478570867, "intelligence": 45.955077428728934 }, "per_type": { "choice": { "accuracy": 0.7946666666666666, "calibration": 93.10943750000003, "cc": 73.74079933309964, "cc_by_type": { "choice": 73.74079933309964 }, "cc_tiered": 75.91229283026139, "cc_tiered_open": 80.11686099440308, "cc_tiered_sealed": 71.7077246661197, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 73.74079933309964 }, "noul": { "accuracy": 0.5973333333333334, "calibration": 92.18224, "cc": 19.466666666666676, "cc_by_type": { "noul": 19.466666666666676 }, "cc_tiered": 17.091225752635783, "cc_tiered_open": 20.761214630779854, "cc_tiered_sealed": 13.421236874491711, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 19.466666666666676 }, "score": { "accuracy": 0.6, "calibration": 90.00577735633334, "cc": 46.965815789473666, "cc_by_type": { "score": 46.965815789473666 }, "cc_tiered": 44.861713703289645, "cc_tiered_open": 41.86326336814473, "cc_tiered_sealed": 47.86016403843456, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 46.965815789473666 } }, "raw_on_A": { "calibration": 86.03357993512108, "capability": 69.68729793470257, "intelligence": 53.34101593428407 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 27.74831039177889, "Emphasis on Accuracy 60:20:20": 26.4196991707309, "Emphasis on Cost 20:20:60": 24.78908517771933, "Emphasis on Speed 20:60:20": 33.418202927958156, "Intelligence only": 24.64935315350002 }, "class": "jev-rebuild", "open": "yes", "licence": "Component-specific JevOne/Qwen/SGLang terms recorded in RESULT.md", "repo": "https://huggingface.co/juspay/jev-one", "gpu": "RTXPRO6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "underlying": "Qwen3.6-35B-A3B BF16 with JevOne bidirectional option-logit mapping", "adapter_id": "images/jevone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.624, "acc_among_decisive": 0.9572649572649573, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Axel Montzamir (Okura66)", "axes": { "calibration": 83.76747077065221, "cost": 46.37729040230682, "intelligence": 39.71971801341685, "speed": 89.97531755002697 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06004736643320625, "ece_score": 87.99052671335875, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 87.99052671335875 }, "noul": { "abstention_rate": 0.2991913746630728, "brier": 0.1447023593915565, "ece": 0.10598766007071211, "n": 371, "score": 78.80246798585758 }, "score": { "interval90_coverage": 0.9214092140921409, "interval90_width": 0.3678861788617887, "n": 369, "nrps": 0.09917884531883965, "score": 84.50941761274032, "top_ece": 0.10531640121317691 } }, "score": 83.76747077065221 }, "capability": 61.74359439203453, "composite_ci95": { "A": [ 20.65829877133707, 39.3142784887598 ], "B": [ 18.472216595378455, 36.751585049789064 ], "C": [ 14.346040813428516, 27.301582283860974 ] }, "cost": { "basis": "ESTIMATE, ours (the author's server returns no token usage; Florian 7 Oct 2026: use our own estimate): 829,523 input tokens counted with the author's tokenizer and prompt builder over the 1,624 v1.5 items x n_permutations 3 (the author's server default) = 2,488,569 tokens / 1,624 decisions x USD 0.04 per 1M input (deepinfra:Qwen/Qwen3.5-4B, frozen 25 Sep snapshot), 0 output tokens = USD 0.061295 per 1,000 decisions; carried like every Qwen3.5-4B row on this board. Cross-check: the same count on the v1.6 common cost basis (1,477 items, x3) gives USD 0.0597 per 1,000 (within 3 %)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.06129480295566502 }, "display": "Kahn1 4B (Okura66, Qwen3.5-4B LoRA merge, author's sysone engine)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 41.96168150915031, "I_sealed": 37.4777545176834, "base": 39.71971801341685, "excess": 1.910539349028665, "gap": 4.483926991466909, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 62.694715037494376, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 69.80182447310474, "judge": 52.537903757415954, "standard": 54.79280033486815 } }, "open|noul": { "cc": 17.904071773636996, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 30.434782608695656, "judge": -9.999999999999998, "standard": 22.222222222222232 } }, "open|score": { "cc": 45.28625771631957, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 49.624702928143726, "hard": 52.08153129449285, "judge": 35.187676087499, "standard": 44.67436039729178 } }, "sealed|choice": { "cc": 60.13243945421864, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 69.40359175227256, "hard": 56.730769230769226, "judge": 57.16945996275605, "standard": 66.7446729892844 } }, "sealed|noul": { "cc": 16.581379950628655, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -10.679611650485432, "judge": 28.947368421052634, "standard": 40.8450704225352 } }, "sealed|score": { "cc": 35.719444148202925, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 55.76848705914599, "hard": 35.73652095498605, "judge": 30.714342076628732, "standard": 33.16842218652643 } } } }, "jevbench_score": 31.117574813020727, "key": "kahn1", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 50, "ranked": true, "ranks": { "A": 50, "B": 49, "C": 50, "capability": 61 }, "scores": { "A": 31.117574813020727, "B": 28.585119108273094, "C": 21.60942695348661 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2821926829987206, "p50_s_raw": 0.06609634149936028, "p95_s_adjusted": 0.3563875272346194, "p95_s_raw": 0.10319376361730971 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.07087093913225, 85.02256483809754 ], "I": [ 33.5350460501225, 43.90853133816846 ], "capability": [ 56.37025407029597, 63.949534098181175 ], "composite": { "A": [ 20.65829877133707, 39.3142784887598 ], "B": [ 18.472216595378455, 36.751585049789064 ], "C": [ 14.346040813428516, 27.301582283860974 ] }, "seed": 60 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.76747077065221, "capability": 61.74359439203453, "intelligence": 39.71971801341685 }, "per_type": { "choice": { "accuracy": 0.7026666666666667, "calibration": 87.99052671335875, "cc": 61.97531332000794, "cc_by_type": { "choice": 61.97531332000794 }, "cc_tiered": 61.41357724585651, "cc_tiered_open": 62.694715037494376, "cc_tiered_sealed": 60.13243945421864, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 61.97531332000794 }, "noul": { "accuracy": 0.6, "calibration": 78.80246798585758, "cc": 19.999999999999996, "cc_by_type": { "noul": 19.999999999999996 }, "cc_tiered": 17.242725862132826, "cc_tiered_open": 17.904071773636996, "cc_tiered_sealed": 16.581379950628655, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 19.999999999999996 }, "score": { "accuracy": 0.56, "calibration": 84.50941761274032, "cc": 39.16844975336833, "cc_by_type": { "score": 39.16844975336833 }, "cc_tiered": 40.502850932261246, "cc_tiered_open": 45.28625771631957, "cc_tiered_sealed": 35.719444148202925, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 39.16844975336833 } }, "raw_on_A": { "calibration": 81.01531873303975, "capability": 63.27790509971841, "intelligence": 45.54049146639707 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 28.153963481406958, "Emphasis on Accuracy 60:20:20": 25.087824889920398, "Emphasis on Cost 20:20:60": 26.883687773639345, "Emphasis on Speed 20:60:20": 33.89866914190837, "Intelligence only": 21.564986725380077 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/Okura66/Kahn1-Qwen3.5-4B", "gpu": "RTX 5090", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX 5090, the author's own sysone engine (github.com/Okura66/kahn1) on vLLM 0.30.0 with the author's documented max-model-len 4096 and calibration.json, offline (all credentials unset), driven over loopback only by the JevBench client through a thin pre-registered adapter for the author's /v1/evaluate/jev endpoint; label-free inputs; pod destroyed afterwards; JevBench add-request, measured 7 Oct 2026", "underlying": "Qwen/Qwen3.5-4B", "model_pin": "Okura66/Kahn1-Qwen3.5-4B@f3f9910f995b0e345a6e8a9cb0aeed1e06bafaf6", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7008086253369272, "acc_among_decisive": 0.8653846153846154, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a8 (scorer output sha256 7a5b4d4c7b7c, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mikhail Gribov / @MihailGGribov (mihail.gribov.rs@gmail.com)", "axes": { "calibration": 77.35692434429644, "cost": 64.79986008365586, "intelligence": 35.43137209354293, "speed": 92.95077486551261 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10031579026354083, "ece_score": 79.93684194729184, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 79.93684194729184 }, "noul": { "abstention_rate": 0.22666666666666668, "brier": 0.19319243660768046, "ece": 0.16531579253832934, "n": 375, "score": 66.93684149233412 }, "score": { "interval90_coverage": 0.9893333333333333, "interval90_width": 0.7702222222222224, "n": 375, "nrps": 0.1175329689730567, "score": 85.19708959326337, "top_ece": 0.08926261958083807 } }, "score": 77.35692434429644 }, "capability": 56.39414821891969, "composite_ci95": { "A": [ 19.06853775916415, 39.61576669827309 ], "B": [ 16.309735285734533, 35.62882083236784 ], "C": [ 13.242040110530658, 27.510949096022983 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights). NO JUDGEMENT OF OURS: pricing_v15.reference() resolves Qwen/Qwen3.5-4B straight through to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the frozen 25 Sep snapshot (the same rate aplomb-1 carries for the same base) x this row's own measured tokens. Output tokens are 0 in every answered record: one prefill, option-label readout, nothing generated. Estimated, not charged.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.029810319999999994 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.014905206499661474 }, "display": "TypeCastLM 1.4.0 (Mikhail Gribov, Qwen3.5-4B computed head)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 38.13170692188825, "I_sealed": 32.731037265197614, "base": 35.43137209354293, "excess": 2.8272820142523947, "gap": 5.4006696566906385, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 52.552512469264286, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 43.378420887071414, "judge": 60.44825313117996, "standard": 54.79280033486815 } }, "open|noul": { "cc": 25.505175983436857, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 21.739130434782616, "judge": 10.000000000000009, "standard": 33.33333333333333 } }, "open|score": { "cc": 36.33743231296363, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 64.8527119217414, "hard": 32.894905449875765, "judge": 45.884441468733286, "standard": 14.644332501095958 } }, "sealed|choice": { "cc": 47.566937838880506, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 42.986425339366505, "judge": 49.72067039106147, "standard": 44.57445498214066 } }, "sealed|noul": { "cc": 19.912391055582344, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 60.00000000000001, "hard": -6.796116504854366, "judge": 39.47368421052631, "standard": 23.943661971830977 } }, "sealed|score": { "cc": 30.71378290112999, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 52.96711799317693, "hard": 27.922053490937206, "judge": 28.3107755484573, "standard": 28.775085204501117 } } } }, "jevbench_score": 29.827386072784904, "key": "typecastlm-1.4.0", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 51, "ranked": true, "ranks": { "A": 51, "B": 51, "C": 52, "capability": 80 }, "scores": { "A": 29.827386072784904, "B": 26.272910136782546, "C": 20.713462550545067 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.21072090499219484, "p50_s_raw": 0.030360452496097423, "p95_s_adjusted": 0.2405552782700397, "p95_s_raw": 0.04527763913501986 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 71.77140344619215, 79.65897110167947 ], "I": [ 29.58940031351787, 39.89308222540656 ], "capability": [ 51.45876220790242, 59.168420588826265 ], "composite": { "A": [ 19.06853775916415, 39.61576669827309 ], "B": [ 16.309735285734533, 35.62882083236784 ], "C": [ 13.242040110530658, 27.510949096022983 ] }, "seed": 147 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 77.35692434429644, "capability": 56.39414821891969, "intelligence": 35.43137209354293 }, "per_type": { "choice": { "accuracy": 0.6026666666666667, "calibration": 79.93684194729184, "cc": 49.186741566647385, "cc_by_type": { "choice": 49.186741566647385 }, "cc_tiered": 50.059725154072396, "cc_tiered_open": 52.552512469264286, "cc_tiered_sealed": 47.566937838880506, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 49.186741566647385 }, "noul": { "accuracy": 0.624, "calibration": 66.93684149233412, "cc": 24.8, "cc_by_type": { "noul": 24.8 }, "cc_tiered": 22.708783519509602, "cc_tiered_open": 25.505175983436857, "cc_tiered_sealed": 19.912391055582344, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 24.8 }, "score": { "accuracy": 0.5466666666666666, "calibration": 85.19708959326337, "cc": 33.379404446849435, "cc_by_type": { "score": 33.379404446849435 }, "cc_tiered": 33.52560760704681, "cc_tiered_open": 36.33743231296363, "cc_tiered_sealed": 30.71378290112999, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 33.379404446849435 } }, "raw_on_A": { "calibration": 73.20180288665794, "capability": 57.51186004997652, "intelligence": 41.821917213295094 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 27.685082169472313, "Emphasis on Accuracy 60:20:20": 22.64780945981884, "Emphasis on Cost 20:20:60": 29.441987709970327, "Emphasis on Speed 20:60:20": 33.066455632229754, "Intelligence only": 17.791964524810385 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/mihailgribov/typecastlm-qwen3.5-3.8b", "gpu": "RTX 5090", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. A FOLLOW-UP OF THE PUBLISHED typecastlm ROW: the author asks to 'measure 1.4.0 to replace it' and to list it as 'TypeCastLM'; measured here as a CANDIDATE - replace-or-keep-both is the release lane's decision. Weights mihailgribov/typecastlm-qwen3.5-3.8b tag v1.4.0 = commit 0984c3e832f7c28495134bc1d22c66bb0e0a555d (GGUF files excluded), package typecastlm[server]==1.2.2 (latest on PyPI) + flash-linear-attention + fla-core, the model card's own install lines, in an isolated venv (torch 2.14.1+cu130, transformers 5.19.0). Served with the author's own `typecastlm-serve --model --host 127.0.0.1 --port 8000 --device cuda`; the UNCHANGED typesafe adapter with the published row's own wire model; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. THE 23 LONG ITEMS ARE ANSWERED, folded in the middle to the server's own 32,768-token state context (its documented behaviour; max input 32,953 tokens observed). One RTX 5090 32 GB (Lium, France). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 34CA4AED", "underlying": "Qwen/Qwen3.5-4B", "model_pin": "mihailgribov/typecastlm-qwen3.5-3.8b@0984c3e832f7c28495134bc1d22c66bb0e0a555d", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7733333333333333, "acc_among_decisive": 0.8068965517241379, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "HopitAI", "axes": { "calibration": 90.07902805935959, "cost": 62.25627699896333, "intelligence": 34.73970870443837, "speed": 91.11434937226889 }, "calibration": { "parts": { "choice": { "ece_hard": 0.054931646080717024, "ece_score": 89.0136707838566, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 89.0136707838566 }, "noul": { "abstention_rate": 0.43733333333333335, "brier": 0.1320524837128236, "ece": 0.03940530319869939, "n": 375, "score": 92.11893936026013 }, "score": { "interval90_coverage": 0.9733333333333334, "interval90_width": 0.6148888888888887, "n": 375, "nrps": 0.110200461495695, "score": 89.10447403396208, "top_ece": 0.053855028912531695 } }, "score": 90.07902805935959 }, "capability": 62.40936838189898, "composite_ci95": { "A": [ 19.749752505423817, 34.23513789747224 ], "B": [ 16.849888107534454, 30.36407576551454 ], "C": [ 13.715105906544313, 23.774401317689065 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.018118577586206896 }, "display": "Hopper", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 40.99371798867166, "I_sealed": 29.284169358941572, "base": 35.138943673806615, "excess": 9.136160987291847, "gap": 11.70954862973009, "penalty": 0.9886383901270815, "per_type_split": { "open|choice": { "cc": 68.98723802511984, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 64.1396665618119, "judge": 72.31377719182596, "standard": 69.8618668899121 } }, "open|noul": { "cc": 18.554865424430645, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 30.434782608695656, "judge": -19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 35.439050516464505, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 56.1505221295817, "hard": 30.729313281528604, "judge": 33.331159411757824, "standard": 37.66462583683774 } }, "sealed|choice": { "cc": 57.26398771697454, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 68.07331313280615, "hard": 49.60407239819003, "judge": 59.031657355679705, "standard": 64.52765118857003 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 56.00000000000001, "hard": -20.38834951456311, "judge": 0.0, "standard": -1.4084507042253502 } }, "sealed|score": { "cc": 30.58852035985018, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 62.587460120023806, "hard": 30.318479212532644, "judge": 26.893973849966592, "standard": 20.67095253922381 } } } }, "jevbench_score": 28.852652318563656, "key": "hopper", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 52, "ranked": true, "ranks": { "A": 52, "B": 52, "C": 54, "capability": 56 }, "scores": { "A": 28.852652318563656, "B": 25.218775497922103, "C": 20.036564110113652 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.26295550270006063, "p50_s_raw": 0.0564777513500303, "p95_s_adjusted": 0.2942271867766976, "p95_s_raw": 0.07211359338834882 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.58654904175489, 89.37856892562735 ], "I": [ 29.827996307138942, 37.38986029909742 ], "capability": [ 57.58520429455299, 62.579268195863094 ], "composite": { "A": [ 19.749752505423817, 34.23513789747224 ], "B": [ 16.849888107534454, 30.36407576551454 ], "C": [ 13.715105906544313, 23.774401317689065 ] }, "seed": 41 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.07902805935959, "capability": 62.40936838189898, "intelligence": 34.73970870443837 }, "per_type": { "choice": { "accuracy": 0.6866666666666666, "calibration": 89.0136707838566, "cc": 59.92914183947025, "cc_by_type": { "choice": 59.92914183947025 }, "cc_tiered": 63.12561287104719, "cc_tiered_open": 68.98723802511984, "cc_tiered_sealed": 57.26398771697454, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 59.92914183947025 }, "noul": { "accuracy": 0.5306666666666666, "calibration": 92.11893936026013, "cc": 6.133333333333324, "cc_by_type": { "noul": 6.133333333333324 }, "cc_tiered": 9.277432712215322, "cc_tiered_open": 18.554865424430645, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 6.133333333333324 }, "score": { "accuracy": 0.5066666666666667, "calibration": 89.10447403396208, "cc": 33.257420216120906, "cc_by_type": { "score": 33.257420216120906 }, "cc_tiered": 33.01378543815734, "cc_tiered_open": 35.439050516464505, "cc_tiered_sealed": 30.58852035985018, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 33.257420216120906 } }, "raw_on_A": { "calibration": 87.40904942545944, "capability": 64.64647397045381, "intelligence": 41.88389851544817 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 25.942845868326373, "Emphasis on Accuracy 60:20:20": 21.285843260250505, "Emphasis on Cost 20:20:60": 27.444368192058636, "Emphasis on Speed 20:60:20": 31.034805184083435, "Intelligence only": 16.77021030692692 }, "class": "jev-rebuild", "open": "yes", "licence": "Component-specific terms recorded in RESULT.md; submitted adapter release and Qwen base retain their respective terms", "repo": "https://huggingface.co/HopitAI/hopper", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3.5-4B plus HopitAI/hopper LoRA", "adapter_id": "images/hopper/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5626666666666666, "acc_among_decisive": 0.943127962085308, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 89.1627567462182, "cost": 63.26458220826236, "intelligence": 34.62714018309005, "speed": 90.51878000442085 }, "calibration": { "parts": { "choice": { "ece_hard": 0.04836725385924713, "ece_score": 90.32654922815058, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 90.32654922815058 }, "noul": { "abstention_rate": 0.477088948787062, "brier": 0.12273935128941174, "ece": 0.06563823281458846, "n": 371, "score": 86.87235343708231 }, "score": { "interval90_coverage": 0.967479674796748, "interval90_width": 0.43699186991869926, "n": 369, "nrps": 0.08429976663999325, "score": 90.28936757342169, "top_ece": 0.054956440945786535 } }, "score": 89.1627567462182 }, "capability": 61.89494846465412, "composite_ci95": { "A": [ 21.33216708823923, 38.39023119963257 ], "B": [ 18.266955184224454, 34.31649986004929 ], "C": [ 14.814004922388358, 26.659882777522615 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (I-2): measured input tokens; zero generated output tokens for signed logits readout. M2 floor uses the 25 Sep 2026 DeepInfra Qwen3.5-4B snapshot rates (USD 0.03/M input, USD 0.15/M output); the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen3.5-9B.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.016769261083743846 }, "display": "Imajev-4B (RTX 5090)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 35.03758089292987, "I_sealed": 34.21669947325023, "base": 34.62714018309005, "excess": -1.752506222758607, "gap": 0.8208814196796368, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 66.47085555091229, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 58.47750865051904, "judge": 80.22412656558997, "standard": 54.79280033486815 } }, "open|noul": { "cc": 0.6756383712905452, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 4.347826086956519, "judge": -30.000000000000004, "standard": 11.111111111111116 } }, "open|score": { "cc": 37.966248756586765, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 52.45615279420953, "hard": 43.92313144450569, "judge": 33.905546551444175, "standard": 24.898584669651413 } }, "sealed|choice": { "cc": 62.38689563101728, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 70.73387037173899, "hard": 53.676470588235276, "judge": 64.61824953445065, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -30.097087378640776, "judge": 0.0, "standard": 7.042253521126751 } }, "sealed|score": { "cc": 40.26320278873341, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 63.176956160290466, "hard": 35.01688311771597, "judge": 46.365402771352585, "standard": 30.14566547106101 } } } }, "jevbench_score": 28.655751132526984, "key": "imajev-4b-rtx5090-a2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 53, "ranked": true, "ranks": { "A": 53, "B": 53, "C": 55, "capability": 57 }, "scores": { "A": 28.655751132526984, "B": 25.0248976556334, "C": 19.899827175365957 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.29240404299926015, "p50_s_raw": 0.07120202149963006, "p95_s_adjusted": 0.3034859753679484, "p95_s_raw": 0.07674298768397421 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.2060903767957, 89.78727704768245 ], "I": [ 30.707360525579823, 39.19772341159948 ], "capability": [ 58.11557152205388, 63.38857993223046 ], "composite": { "A": [ 21.33216708823923, 38.39023119963257 ], "B": [ 18.266955184224454, 34.31649986004929 ], "C": [ 14.814004922388358, 26.659882777522615 ] }, "seed": 42 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.1627567462182, "capability": 61.89494846465412, "intelligence": 34.62714018309005 }, "per_type": { "choice": { "accuracy": 0.7146666666666667, "calibration": 90.32654922815058, "cc": 63.50994193041121, "cc_by_type": { "choice": 63.50994193041121 }, "cc_tiered": 64.4288755909648, "cc_tiered_open": 66.47085555091229, "cc_tiered_sealed": 62.38689563101728, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 63.50994193041121 }, "noul": { "accuracy": 0.49866666666666665, "calibration": 86.87235343708231, "cc": -0.2666666666666706, "cc_by_type": { "noul": -0.2666666666666706 }, "cc_tiered": 0.3378191856452726, "cc_tiered_open": 0.6756383712905452, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.56, "calibration": 90.28936757342169, "cc": 40.37192786146962, "cc_by_type": { "score": 40.37192786146962 }, "cc_tiered": 39.11472577266009, "cc_tiered_open": 37.966248756586765, "cc_tiered_sealed": 40.26320278873341, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 40.37192786146962 } }, "raw_on_A": { "calibration": 87.91523763851959, "capability": 61.35771088116867, "intelligence": 34.80018412381776 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 25.816712856140082, "Emphasis on Accuracy 60:20:20": 21.130055842968158, "Emphasis on Cost 20:20:60": 27.454793753723415, "Emphasis on Speed 20:60:20": 30.81252811725961, "Intelligence only": 16.607714361099795 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://github.com/fstandhartinger/jevbench/issues/110", "gpu": "RTX 5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX 5090), offline read-only container", "underlying": null, "adapter_id": "A2 reviewed bundle", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.522911051212938, "acc_among_decisive": 0.9639175257731959, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "newfull5 (dhtocks)", "axes": { "calibration": 88.43741591513763, "cost": 55.82433433950372, "intelligence": 33.840703805987175, "speed": 89.51571180654442 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0565084690553745, "ece_score": 88.6983061889251, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 88.6983061889251 }, "noul": { "abstention_rate": 0.3504043126684636, "brier": 0.13494345407008085, "ece": 0.06141401617250676, "n": 371, "score": 87.71719676549866 }, "score": { "interval90_coverage": 0.926829268292683, "interval90_width": 0.5194218608852753, "n": 369, "nrps": 0.1123930174593495, "score": 88.89674479098915, "top_ece": 0.054836043360433705 } }, "score": 88.43741591513763 }, "capability": 61.139059860562405, "composite_ci95": { "A": [ 17.563735957847147, 34.57907875172623 ], "B": [ 15.042433721702132, 31.023588122626016 ], "C": [ 12.197038859616075, 24.013249133143216 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.02968398399014778 }, "display": "Malkuth-4B (newfull5, Kev post-train)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 36.78555463394936, "I_sealed": 30.895852978024994, "base": 33.840703805987175, "excess": 3.316314013486122, "gap": 5.889701655924366, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 57.213904436116636, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 68.86486486486487, "hard": 47.153192827933324, "judge": 68.35860250494396, "standard": 54.79280033486815 } }, "open|noul": { "cc": 19.189786059351285, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": -19.999999999999996, "standard": 22.222222222222232 } }, "open|score": { "cc": 33.95297340638018, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 51.45848484848485, "hard": 32.323055555555534, "judge": 29.900169491525418, "standard": 34.53925925925927 } }, "sealed|choice": { "cc": 53.38404633359997, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 69.40359175227256, "hard": 43.49547511312216, "judge": 51.58286778398511, "standard": 67.8531838896416 } }, "sealed|noul": { "cc": 12.571668333969063, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": -16.504854368932044, "judge": 26.315789473684205, "standard": 32.3943661971831 } }, "sealed|score": { "cc": 26.73184426650596, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 63.09113475177305, "hard": 12.109877450980411, "judge": 28.95095808383231, "standard": 34.467461928933965 } } } }, "jevbench_score": 26.197476126697616, "key": "malkuth-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 54, "ranked": true, "ranks": { "A": 54, "B": 56, "C": 59, "capability": 63 }, "scores": { "A": 26.197476126697616, "B": 23.020714347323178, "C": 18.19269175465112 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2985552769736387, "p50_s_raw": 0.07427763848681934, "p95_s_adjusted": 0.3744588227418717, "p95_s_raw": 0.11222941137093585 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.33637885013427, 89.07368375555646 ], "I": [ 28.800746942905594, 38.02075761880322 ], "capability": [ 56.551766449658025, 62.63523991792939 ], "composite": { "A": [ 17.563735957847147, 34.57907875172623 ], "B": [ 15.042433721702132, 31.023588122626016 ], "C": [ 12.197038859616075, 24.013249133143216 ] }, "seed": 64 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.43741591513763, "capability": 61.139059860562405, "intelligence": 33.840703805987175 }, "per_type": { "choice": { "accuracy": 0.6466666666666666, "calibration": 88.6983061889251, "cc": 54.813713138126026, "cc_by_type": { "choice": 54.813713138126026 }, "cc_tiered": 55.2989753848583, "cc_tiered_open": 57.213904436116636, "cc_tiered_sealed": 53.38404633359997, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 54.813713138126026 }, "noul": { "accuracy": 0.592, "calibration": 87.71719676549866, "cc": 18.399999999999995, "cc_by_type": { "noul": 18.399999999999995 }, "cc_tiered": 15.880727196660175, "cc_tiered_open": 19.189786059351285, "cc_tiered_sealed": 12.571668333969063, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 18.399999999999995 }, "score": { "accuracy": 0.5173333333333333, "calibration": 88.89674479098915, "cc": 29.251991228070175, "cc_by_type": { "score": 29.251991228070175 }, "cc_tiered": 30.34240883644307, "cc_tiered_open": 33.95297340638018, "cc_tiered_sealed": 26.73184426650596, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 29.251991228070175 } }, "raw_on_A": { "calibration": 85.35554104202156, "capability": 62.08736671102251, "intelligence": 38.819192380023466 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 23.437149264983223, "Emphasis on Accuracy 60:20:20": 19.45371080344073, "Emphasis on Cost 20:20:60": 24.246787312645555, "Emphasis on Speed 20:60:20": 28.28433154980761, "Intelligence only": 15.501658014110385 }, "class": "jev-rebuild", "open": "yes", "licence": "CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)", "repo": "https://github.com/newfull5/malkuth", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/malkuth-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6495956873315364, "acc_among_decisive": 0.921161825726141, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "hjmurmur (Octalab)", "axes": { "calibration": 89.87138320201578, "cost": 51.172592607797476, "intelligence": 33.85815624201783, "speed": 83.1032423977654 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06281808353596482, "ece_score": 87.43638329280702, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 87.43638329280702 }, "noul": { "abstention_rate": 0.34231805929919135, "brier": 0.11806081980479606, "ece": 0.04224066622806287, "n": 371, "score": 91.55186675438742 }, "score": { "interval90_coverage": 0.959349593495935, "interval90_width": 0.5268744354110209, "n": 369, "nrps": 0.10409415210041804, "score": 90.6258995588529, "top_ece": 0.04169392836126208 } }, "score": 89.87138320201578 }, "capability": 61.8647697220168, "composite_ci95": { "A": [ 15.505551311407853, 35.88495273799785 ], "B": [ 13.30350456735459, 32.67206876334683 ], "C": [ 10.767743966255452, 24.920106068054068 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.042420935960591134 }, "display": "jqv (Qwen3-32B zero-shot)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 33.07530766335527, "I_sealed": 34.641004820680386, "base": 33.85815624201783, "excess": -4.139084799763362, "gap": -1.5656971573251184, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 58.31263706393481, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 47.153192827933324, "judge": 64.40342781806196, "standard": 62.32733361239012 } }, "open|noul": { "cc": 11.77087646652864, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 4.347826086956519, "judge": -19.999999999999996, "standard": 44.44444444444444 } }, "open|score": { "cc": 29.142409459602362, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 31.012596729013953, "hard": 34.8410871612874, "judge": 43.10234913255198, "standard": -4.129949087897944 } }, "sealed|choice": { "cc": 53.61923821279086, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 62.75219865494053, "hard": 49.60407239819003, "judge": 55.30726256983241, "standard": 54.55105308535534 } }, "sealed|noul": { "cc": 17.979675703685576, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 60.00000000000001, "hard": -4.854368932038833, "judge": 21.052631578947366, "standard": 38.0281690140845 } }, "sealed|score": { "cc": 32.32410054556472, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 52.65739446915423, "hard": 22.31408478189991, "judge": 37.34025956092481, "standard": 34.65324658805946 } } } }, "jevbench_score": 25.391439220985372, "key": "jqv", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 55, "ranked": true, "ranks": { "A": 55, "B": 57, "C": 62, "capability": 58 }, "scores": { "A": 25.391439220985372, "B": 22.528313466257792, "C": 17.63294390346206 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3448032220825553, "p50_s_raw": 0.09740161104127765, "p95_s_adjusted": 1.4193988358601928, "p95_s_raw": 0.6346994179300964 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.42862757082423, 89.91697762368707 ], "I": [ 27.681410439375405, 39.196898115223256 ], "capability": [ 56.87716097755697, 63.63771518645792 ], "composite": { "A": [ 15.505551311407853, 35.88495273799785 ], "B": [ 13.30350456735459, 32.67206876334683 ], "C": [ 10.767743966255452, 24.920106068054068 ] }, "seed": 51 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.87138320201578, "capability": 61.8647697220168, "intelligence": 33.85815624201783 }, "per_type": { "choice": { "accuracy": 0.648, "calibration": 87.43638329280702, "cc": 54.98422742817084, "cc_by_type": { "choice": 54.98422742817084 }, "cc_tiered": 55.96593763836283, "cc_tiered_open": 58.31263706393481, "cc_tiered_sealed": 53.61923821279086, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 54.98422742817084 }, "noul": { "accuracy": 0.6106666666666667, "calibration": 91.55186675438742, "cc": 22.13333333333334, "cc_by_type": { "noul": 22.13333333333334 }, "cc_tiered": 14.875276085107107, "cc_tiered_open": 11.77087646652864, "cc_tiered_sealed": 17.979675703685576, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 22.13333333333334 }, "score": { "accuracy": 0.5066666666666667, "calibration": 90.6258995588529, "cc": 32.28871510053675, "cc_by_type": { "score": 32.28871510053675 }, "cc_tiered": 30.73325500258354, "cc_tiered_open": 29.142409459602362, "cc_tiered_sealed": 32.32410054556472, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 32.28871510053675 } }, "raw_on_A": { "calibration": 85.0933345598925, "capability": 62.946394387426395, "intelligence": 40.79945421496029 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 22.511071395625475, "Emphasis on Accuracy 60:20:20": 19.077648593308684, "Emphasis on Cost 20:20:60": 22.883252890033052, "Emphasis on Speed 20:60:20": 26.917672025192456, "Intelligence only": 15.525654079176814 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (Qwen3-32B weights); serving code public", "repo": "https://github.com/Octalab-Inc/jqv", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen/Qwen3-32B, bf16, read as a direct-logit classifier (no fine-tuning)", "adapter_id": "images/jqv/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6576819407008087, "acc_among_decisive": 0.9385245901639344, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "kirp", "axes": { "calibration": 89.9313809717458, "cost": 33.627578514713974, "intelligence": 51.42208418374838, "speed": 90.51950606084544 }, "calibration": { "parts": { "choice": { "ece_hard": 0.074450619123307, "ece_score": 85.1098761753386, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 85.1098761753386 }, "noul": { "abstention_rate": 0.25876010781671155, "brier": 0.09167755563129923, "ece": 0.04206042293296505, "n": 371, "score": 91.58791541340699 }, "score": { "interval90_coverage": 0.9539295392953929, "interval90_width": 0.3337850045167119, "n": 369, "nrps": 0.06863762470007446, "score": 93.09635132649183, "top_ece": 0.03471767438504456 } }, "score": 89.9313809717458 }, "capability": 70.67673257774709, "composite_ci95": { "A": [ 20.22909286309626, 25.862384694814303 ], "B": [ 19.5180684428127, 25.72730452057616 ], "C": [ 14.047981154927955, 22.379413062160616 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): OpenRouter qwen/qwen3.5-35b-a3b in the frozen 25 Sep snapshot (exact declared base; no explicit listing-decision line in the frozen module: release-lane line), USD 0.3125 per 1M input and USD 1.25 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.160584375 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.16308501184834123 }, "display": "JPT-35B-A3B (Qwen3.5-35B-A3B fine-tune)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 52.77555274931294, "I_sealed": 50.06861561818382, "base": 51.42208418374838, "excess": 0.13354948869087968, "gap": 2.7069371311291235, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 71.66771648987253, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 71.68921044353571, "judge": 72.31377719182596, "standard": 62.32733361239012 } }, "open|noul": { "cc": 33.899930986887505, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 13.043478260869556, "judge": 30.000000000000004, "standard": 55.55555555555556 } }, "open|score": { "cc": 52.759010771178794, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 66.68236487691772, "hard": 47.969282366711376, "judge": 51.97130113741173, "standard": 56.55835497789477 } }, "sealed|choice": { "cc": 64.13898950109635, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 74.72470623013821, "hard": 59.27601809954751, "judge": 64.61824953445065, "standard": 67.8531838896416 } }, "sealed|noul": { "cc": 31.905895053509607, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": 14.563106796116498, "judge": 26.315789473684205, "standard": 54.92957746478873 } }, "sealed|score": { "cc": 54.16096229994551, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 66.36028549114205, "hard": 51.20129834773479, "judge": 55.62940093493124, "standard": 51.77797065629009 } } } }, "jevbench_score": 25.357657195103958, "key": "jpt-35b-a3b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 56, "ranked": true, "ranks": { "A": 56, "B": 54, "C": 58, "capability": 33 }, "scores": { "A": 25.357657195103958, "B": 24.9082855667434, "C": 18.62541574870352 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.29123902405845004, "p50_s_raw": 0.070619512029225, "p95_s_adjusted": 0.30464904906693846, "p95_s_raw": 0.07732452453346922 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.28164810054457, 90.58368457070935 ], "I": [ 45.56151371729817, 55.912127823531534 ], "capability": [ 65.85556031844814, 72.40411689862715 ], "composite": { "A": [ 20.22909286309626, 25.862384694814303 ], "B": [ 19.5180684428127, 25.72730452057616 ], "C": [ 14.047981154927955, 22.379413062160616 ] }, "seed": 58 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.9313809717458, "capability": 70.67673257774709, "intelligence": 51.42208418374838 }, "per_type": { "choice": { "accuracy": 0.7386666666666667, "calibration": 85.1098761753386, "cc": 66.57919915121775, "cc_by_type": { "choice": 66.57919915121775 }, "cc_tiered": 67.90335299548444, "cc_tiered_open": 71.66771648987253, "cc_tiered_sealed": 64.13898950109635, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 66.57919915121775 }, "noul": { "accuracy": 0.688, "calibration": 91.58791541340699, "cc": 37.59999999999999, "cc_by_type": { "noul": 37.59999999999999 }, "cc_tiered": 32.902913020198554, "cc_tiered_open": 33.899930986887505, "cc_tiered_sealed": 31.905895053509607, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 37.59999999999999 }, "score": { "accuracy": 0.6906666666666667, "calibration": 93.09635132649183, "cc": 54.13323074000276, "cc_by_type": { "score": 54.13323074000276 }, "cc_tiered": 53.45998653556215, "cc_tiered_open": 52.759010771178794, "cc_tiered_sealed": 54.16096229994551, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 54.13323074000276 } }, "raw_on_A": { "calibration": 89.37706553915206, "capability": 72.5236455910987, "intelligence": 55.67022564304535 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 22.529266440821832, "Emphasis on Accuracy 60:20:20": 22.81579765829606, "Emphasis on Cost 20:20:60": 18.893073743976853, "Emphasis on Speed 20:60:20": 27.47147293414375, "Intelligence only": 23.25952583784173 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "cc-by-nc-4.0", "repo": "https://huggingface.co/kirp/jpt-35b-a3b", "gpu": "RTX PRO 6000", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "Qwen/Qwen3.5-35B-A3B", "model_pin": "kirp/jpt-35b-a3b@f9b45a14df4f7eaf5551f1ab26607237604a8588", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7412398921832885, "acc_among_decisive": 0.9381818181818182, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mapika", "axes": { "calibration": 83.61830413355554, "cost": 34.38579876859529, "intelligence": 48.72471958402083, "speed": 91.27357492980725 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10659875000000013, "ece_score": 78.68024999999997, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 78.68024999999997 }, "noul": { "abstention_rate": 0.2613333333333333, "brier": 0.11783857269333342, "ece": 0.07045920000000003, "n": 375, "score": 85.90816 }, "score": { "interval90_coverage": 0.9253333333333333, "interval90_width": 0.40466666666666656, "n": 375, "nrps": 0.08455155198666667, "score": 86.26650240066667, "top_ece": 0.09505919999999997 } }, "score": 83.61830413355554 }, "capability": 66.1715118587882, "composite_ci95": { "A": [ 18.69889165560131, 26.594050353427875 ], "B": [ 17.887370941224734, 26.251970770447716 ], "C": [ 12.985341427500913, 20.369267436681472 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.1538650708128079 }, "display": "decider-35b-a3b (Mapika)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 52.787614448504826, "I_sealed": 44.66182471953684, "base": 48.72471958402083, "excess": 5.552402086529744, "gap": 8.125789728967987, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 63.96752213183363, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 60.36489462094999, "judge": 60.44825313117996, "standard": 62.32733361239012 } }, "open|noul": { "cc": 43.862663906142174, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 56.52173913043479, "judge": 10.000000000000009, "standard": 55.55555555555556 } }, "open|score": { "cc": 50.53265730753868, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 41.64121212121212, "hard": 47.127870370370395, "judge": 62.23203389830508, "standard": 44.23888888888891 } }, "sealed|choice": { "cc": 58.35475838263821, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 73.3944276106718, "hard": 49.60407239819003, "judge": 60.89385474860336, "standard": 64.52765118857003 } }, "sealed|noul": { "cc": 27.440215333904295, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": 2.9126213592232997, "judge": 36.8421052631579, "standard": 52.112676056338024 } }, "sealed|score": { "cc": 48.19050044206803, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 62.689858156028365, "hard": 43.85012254901962, "judge": 53.23419161676646, "standard": 42.05604060913702 } } } }, "jevbench_score": 24.772887446577812, "key": "decider-35b-a3b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 57, "ranked": true, "ranks": { "A": 57, "B": 55, "C": 64, "capability": 47 }, "scores": { "A": 24.772887446577812, "B": 24.135656537463312, "C": 17.20339406012348 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2493798840790987, "p50_s_raw": 0.04968994203954935, "p95_s_adjusted": 0.29907567272894087, "p95_s_raw": 0.07453783636447042 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.22742585943612, 84.9038621148463 ], "I": [ 43.19083908199794, 52.63193908662482 ], "capability": [ 61.40117742704215, 67.98807044411026 ], "composite": { "A": [ 18.69889165560131, 26.594050353427875 ], "B": [ 17.887370941224734, 26.251970770447716 ], "C": [ 12.985341427500913, 20.369267436681472 ] }, "seed": 27 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.61830413355554, "capability": 66.1715118587882, "intelligence": 48.72471958402083 }, "per_type": { "choice": { "accuracy": 0.692, "calibration": 78.68024999999997, "cc": 60.611198999649474, "cc_by_type": { "choice": 60.611198999649474 }, "cc_tiered": 61.16114025723592, "cc_tiered_open": 63.96752213183363, "cc_tiered_sealed": 58.35475838263821, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.611198999649474 }, "noul": { "accuracy": 0.6693333333333333, "calibration": 85.90816, "cc": 33.86666666666667, "cc_by_type": { "noul": 33.86666666666667 }, "cc_tiered": 35.65143962002323, "cc_tiered_open": 43.862663906142174, "cc_tiered_sealed": 27.440215333904295, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 33.86666666666667 }, "score": { "accuracy": 0.5893333333333334, "calibration": 86.26650240066667, "cc": 48.45208771929824, "cc_by_type": { "score": 48.45208771929824 }, "cc_tiered": 49.36157887480336, "cc_tiered_open": 50.53265730753868, "cc_tiered_sealed": 48.19050044206803, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 48.45208771929824 } }, "raw_on_A": { "calibration": 80.25053271222222, "capability": 65.45225190461375, "intelligence": 50.65397109700527 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 22.248611212569305, "Emphasis on Accuracy 60:20:20": 22.101309301010737, "Emphasis on Cost 20:20:60": 18.91495293933302, "Emphasis on Speed 20:60:20": 27.229064334478497, "Intelligence only": 21.8839776409655 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Mapika/decider-35b-a3b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen3.5-35B-A3B-Base with a trained decision readout, 34.7B parameters / 3B active", "adapter_id": "images/decider-35b-a3b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7386666666666667, "acc_among_decisive": 0.9061371841155235, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "iamupd / corners-ai (iamupd@gmail.com)", "axes": { "calibration": 81.18327994577871, "cost": 65.17140064644872, "intelligence": 32.65682750200105, "speed": 91.3615355688411 }, "calibration": { "parts": { "choice": { "ece_hard": 0.12863454680642747, "ece_score": 74.2730906387145, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 74.2730906387145 }, "noul": { "abstention_rate": 0.444743935309973, "brier": 0.11602338129808766, "ece": 0.08236478989729613, "n": 371, "score": 83.52704202054078 }, "score": { "interval90_coverage": 0.997289972899729, "interval90_width": 0.6962511291779581, "n": 369, "nrps": 0.09343875797512342, "score": 85.74970717808084, "top_ece": 0.09578354923162988 } }, "score": 81.18327994577871 }, "capability": 56.92005372388988, "composite_ci95": { "A": [ 19.632668233405447, 33.50810171156304 ], "B": [ 16.766044087628412, 29.654016739790904 ], "C": [ 13.633797384309343, 23.269515077474335 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights). NO JUDGEMENT OF OURS: pricing_v15.reference() resolves Qwen/Qwen3.5-4B straight through to deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output in the frozen 25 Sep snapshot (the same rate aplomb-1 carries for the same base) x this row's own measured tokens. Output tokens are 0 in every answered record: one prefill, option-label readout, nothing generated. Estimated, not charged.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.014264039999999997 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.014486161137440754 }, "display": "CoCo-Decision-4B (corners-ai, LoRA on Qwen3.5-4B, option-label logits of one pass, served by oh-my-jev)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 32.635837624396466, "I_sealed": 32.67781737960564, "base": 32.65682750200105, "excess": -2.6153673976474203, "gap": -0.04197975520917652, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 56.102191211482165, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 54.70273670965714, "judge": 68.35860250494396, "standard": 32.18920050230222 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 4.347826086956519, "judge": -30.000000000000004, "standard": 0.0 } }, "open|score": { "cc": 41.80532166170725, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 48.92275346241979, "hard": 29.940973271546778, "judge": 47.18651389666748, "standard": 53.90351418923157 } }, "sealed|choice": { "cc": 60.224217892665166, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 76.05498484960461, "hard": 58.76696832579185, "judge": 56.238361266294234, "standard": 61.20211848749847 } }, "sealed|noul": { "cc": 1.5482925144119708, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": -20.38834951456311, "judge": 7.8947368421052655, "standard": 12.676056338028175 } }, "sealed|score": { "cc": 36.26094173173978, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 57.8083031267016, "hard": 30.22640846893988, "judge": 36.4425387268291, "standard": 37.283932067224676 } } } }, "jevbench_score": 24.64791535308594, "key": "coco-decision-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 58, "ranked": true, "ranks": { "A": 58, "B": 61, "C": 65, "capability": 76 }, "scores": { "A": 24.64791535308594, "B": 21.361319353519175, "C": 17.116607884087454 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23897131998674012, "p50_s_raw": 0.044485659993370064, "p95_s_adjusted": 0.30584448990703095, "p95_s_raw": 0.07792224495351546 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 76.60430858133772, 83.74029534027535 ], "I": [ 29.768152266734436, 37.070992355300035 ], "capability": [ 54.51166132245717, 59.43802140351146 ], "composite": { "A": [ 19.632668233405447, 33.50810171156304 ], "B": [ 16.766044087628412, 29.654016739790904 ], "C": [ 13.633797384309343, 23.269515077474335 ] }, "seed": 31 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 81.18327994577871, "capability": 56.92005372388988, "intelligence": 32.65682750200105 }, "per_type": { "choice": { "accuracy": 0.6946666666666667, "calibration": 74.2730906387145, "cc": 60.9522275797391, "cc_by_type": { "choice": 60.9522275797391 }, "cc_tiered": 58.163204552073665, "cc_tiered_open": 56.102191211482165, "cc_tiered_sealed": 60.224217892665166, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.9522275797391 }, "noul": { "accuracy": 0.528, "calibration": 83.52704202054078, "cc": 5.600000000000005, "cc_by_type": { "noul": 5.600000000000005 }, "cc_tiered": 0.7741462572059854, "cc_tiered_open": 0.0, "cc_tiered_sealed": 1.5482925144119708, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 5.600000000000005 }, "score": { "accuracy": 0.5786666666666667, "calibration": 85.74970717808084, "cc": 37.72075365036465, "cc_by_type": { "score": 37.72075365036465 }, "cc_tiered": 39.033131696723515, "cc_tiered_open": 41.80532166170725, "cc_tiered_sealed": 36.26094173173978, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 37.72075365036465 } }, "raw_on_A": { "calibration": 81.23470206450914, "capability": 59.56576106652099, "intelligence": 37.89682006853285 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 22.48701917025063, "Emphasis on Accuracy 60:20:20": 18.052162943961683, "Emphasis on Cost 20:20:60": 24.348744693137398, "Emphasis on Speed 20:60:20": 27.066970501623096, "Intelligence only": 13.930989601396824 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/corners-ai/CoCo-Decision-4B-Ko", "gpu": "RTX 5090", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,477 of 1,500 answered. LoRA adapter corners-ai/CoCo-Decision-4B-Ko tag v1.0.0 = commit b580ede993493690212a07ed0e6cb972868cc808 (the submission's 'Version 1.0.0', the card's own pin; Apache-2.0) on Qwen/Qwen3.5-4B @ 851bf6e8 as its own adapter_config.json names it. Toolkit github.com/iamupd/oh-my-jev @ f0b8c786 (Apache-2.0) installed with its README's `uv sync` plus the semif extra the README names for --adapter (torch 2.9.1+cu128, transformers 5.17.0, peft 0.21.0). Served by the author's own `omj serve --backend semif --adapter `: the README documents that --adapter runs locally on semif and loads the adapter's base, in 4-bit only when it would not fit; on the 32 GB card omj itself chose bf16 ('Using Qwen/Qwen3.5-4B (the adapter's base model, bf16)'), the card's headline configuration. NO config.toml was authored and no setting of ours was chosen, which removes run 58's blocker. UNCHANGED typesafe adapter; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. THE 23 LONG ITEMS ARE REFUSED with the documented 422 context_length_exceeded at omj serve's default 4,096-token limit (README 'Known limitations'); they are the system's own disclosed refusal, counted wrong, and no other item failed. One RTX 5090 32 GB (Lium, France). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 619F3346 (bh-submit 0195e1e7)", "underlying": "Qwen/Qwen3.5-4B", "model_pin": "corners-ai/CoCo-Decision-4B-Ko@b580ede993493690212a07ed0e6cb972868cc808", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.555256064690027, "acc_among_decisive": 0.9611650485436893, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "oraculumai", "axes": { "calibration": 84.0077997110931, "cost": 64.32996474661243, "intelligence": 32.54303915979197, "speed": 90.7347510499745 }, "calibration": { "parts": { "choice": { "ece_hard": 0.07560407620955467, "ece_score": 84.87918475808907, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 84.87918475808907 }, "noul": { "abstention_rate": 0.37735849056603776, "brier": 0.1368158290573463, "ece": 0.05776783172439211, "n": 371, "score": 88.44643365512158 }, "score": { "interval90_coverage": 0.9051490514905149, "interval90_width": 0.4345076784101176, "n": 369, "nrps": 0.12172060479348325, "score": 78.69778072006868, "top_ece": 0.1521618904025716 } }, "score": 84.0077997110931 }, "capability": 58.27541943544253, "composite_ci95": { "A": [ 17.54626281581445, 31.019351182791876 ], "B": [ 14.871831849106172, 27.358180563302277 ], "C": [ 12.18490473320448, 21.54121609916103 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: frozen 25 Sep DeepInfra Qwen/Qwen3.5-4B market reference, already marked deprecated at that cutoff (replaced by Qwen/Qwen3.5-9B). $0.03/M input, zero generated output; 836,500 measured input tokens across 1,624 decisions. Frozen v1.5 base-model floor applied; no bookable Manchego tariff claimed. Re-score if the basis changes.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.015452586206896551 }, "display": "Manchego v2.1", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 40.1322357986733, "I_sealed": 26.800244420595554, "base": 33.46624010963443, "excess": 10.758603735639502, "gap": 13.331991378077745, "penalty": 0.972413962643605, "per_type_split": { "open|choice": { "cc": 60.640330926520605, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 45.26580685750237, "judge": 68.35860250494396, "standard": 69.8618668899121 } }, "open|noul": { "cc": 19.02691511387163, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 13.043478260869556, "judge": 0.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 40.72946135562766, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 63.415593474232956, "hard": 33.07729129866341, "judge": 27.484035794573337, "standard": 64.558873751835 } }, "sealed|choice": { "cc": 56.6033475264085, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 49.60407239819003, "judge": 56.238361266294234, "standard": 66.7446729892844 } }, "sealed|noul": { "cc": 2.9507869907230795, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -30.097087378640776, "judge": 13.157894736842103, "standard": 35.2112676056338 } }, "sealed|score": { "cc": 20.84659874465509, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 52.898828557922364, "hard": 20.484461869906934, "judge": 15.149834013146535, "standard": 14.089904684780608 } } } }, "jevbench_score": 24.487333472500133, "key": "manchego-v21-v15", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 59, "ranked": true, "ranks": { "A": 59, "B": 62, "C": 66, "capability": 70 }, "scores": { "A": 24.487333472500133, "B": 21.196520189880484, "C": 17.005092689236207 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2574284589994932, "p50_s_raw": 0.05371422949974658, "p95_s_adjusted": 0.32799584188469455, "p95_s_raw": 0.08899792094234726 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.8981045039492, 85.26223280687248 ], "I": [ 28.50329098429543, 35.95139455461947 ], "capability": [ 54.23896847458738, 59.77267571842297 ], "composite": { "A": [ 17.54626281581445, 31.019351182791876 ], "B": [ 14.871831849106172, 27.358180563302277 ], "C": [ 12.18490473320448, 21.54121609916103 ] }, "seed": 65 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 84.0077997110931, "capability": 58.27541943544253, "intelligence": 32.54303915979197 }, "per_type": { "choice": { "accuracy": 0.6693333333333333, "calibration": 84.87918475808907, "cc": 57.712456068887754, "cc_by_type": { "choice": 57.712456068887754 }, "cc_tiered": 58.62183922646455, "cc_tiered_open": 60.640330926520605, "cc_tiered_sealed": 56.6033475264085, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.712456068887754 }, "noul": { "accuracy": 0.5573333333333333, "calibration": 88.44643365512158, "cc": 11.466666666666669, "cc_by_type": { "noul": 11.466666666666669 }, "cc_tiered": 10.988851052297356, "cc_tiered_open": 19.02691511387163, "cc_tiered_sealed": 2.9507869907230795, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 11.466666666666669 }, "score": { "accuracy": 0.49066666666666664, "calibration": 78.69778072006868, "cc": 26.44898115857113, "cc_by_type": { "score": 26.44898115857113 }, "cc_tiered": 30.788030050141376, "cc_tiered_open": 40.72946135562766, "cc_tiered_sealed": 20.84659874465509, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 26.44898115857113 } }, "raw_on_A": { "calibration": 84.55956989627305, "capability": 59.12164774190826, "intelligence": 33.68372558754347 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 22.181162996647437, "Emphasis on Accuracy 60:20:20": 17.83638056849814, "Emphasis on Cost 20:20:60": 23.964666214390906, "Emphasis on Speed 20:60:20": 26.697557732929795, "Intelligence only": 13.785874409327832 }, "class": "system-one-open", "open": "yes", "licence": "Apache-2.0 (code and weights, author-declared)", "repo": "https://github.com/nschlaepfer/manchego-serve", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned offline RTX6000 GPU", "underlying": "Author server manchego-serve v0.1.1 @ 0ee62188af474e3c1d11d47a9c4c8b3a08766db3; oraculumai/Manchego @ 77403228b7dfdf823af99a5f562bdcf80b708d4c. Evaluator-owned RTX6000, offline/read-only container, safetensors with verified weight and support-file checksums, bfloat16, sequential execution, temperature 1.0 and original option order. Stock TypeSafe adapter; all three types native. The author disclosed using public aggregate/family results during development, a prior Jev-assisted phrase filter on two template pools, and one Jev-labelled development group for checkpoint selection. These disclosures do not establish item-text leakage; this is not a claim of fully independent development. Cost is an estimate using measured token usage and the frozen Qwen3.5-4B reference, not a bookable Manchego tariff.", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6226415094339622, "acc_among_decisive": 0.9047619047619048, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 88.42488519968094, "cost": 63.07359395281681, "intelligence": 32.40310972936693, "speed": 93.92936299294371 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0672265434886423, "ece_score": 86.55469130227154, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 86.55469130227154 }, "noul": { "abstention_rate": 0.41239892183288407, "brier": 0.12038668872149434, "ece": 0.03739565815349834, "n": 371, "score": 92.52086836930033 }, "score": { "interval90_coverage": 0.975609756097561, "interval90_width": 0.5718157181571816, "n": 369, "nrps": 0.10515132123479526, "score": 86.19909592747098, "top_ece": 0.08543338010789257 } }, "score": 88.42488519968094 }, "capability": 60.41399746452394, "composite_ci95": { "A": [ 15.392618056749317, 34.34955627190713 ], "B": [ 12.888673039303178, 30.38967783428378 ], "C": [ 10.689318094964802, 23.853858522157736 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "Decision 4B v1.2 (FlyMyJev, Qwen3.5-4B + LoRA)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 32.99062805982875, "I_sealed": 31.81559139890512, "base": 32.40310972936693, "excess": -1.398350981514613, "gap": 1.175036660923631, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 61.39828743553336, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 50.92796476879523, "judge": 68.35860250494396, "standard": 62.32733361239012 } }, "open|noul": { "cc": 6.031055900621121, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 28.57142857142858, "hard": 30.434782608695656, "judge": -30.000000000000004, "standard": 0.0 } }, "open|score": { "cc": 31.542540843331764, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 49.77190646723929, "hard": 38.72644314542413, "judge": 25.494038632486827, "standard": 17.13280674346067 } }, "sealed|choice": { "cc": 58.83327326304771, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 68.07331313280615, "hard": 56.730769230769226, "judge": 56.238361266294234, "standard": 62.31062938785566 } }, "sealed|noul": { "cc": 8.040955184350867, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 28.000000000000004, "hard": -8.737864077669899, "judge": 13.157894736842103, "standard": 23.943661971830977 } }, "sealed|score": { "cc": 28.5725457493168, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 58.99125190199095, "hard": 23.870346180703184, "judge": 26.811123175914563, "standard": 25.409725670310312 } } } }, "jevbench_score": 24.463556699266523, "key": "decision-4b-v12", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 60, "ranked": true, "ranks": { "A": 60, "B": 63, "C": 67, "capability": 65 }, "scores": { "A": 24.463556699266523, "B": 21.09790572169507, "C": 16.988581041157314 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.18829862689599394, "p50_s_raw": 0.019149313447996974, "p95_s_adjusted": 0.2148901690961793, "p95_s_raw": 0.032445084548089656 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.81349929936302, 88.39912781374252 ], "I": [ 26.905961082898493, 37.32153047837871 ], "capability": [ 55.53187740606071, 61.760259132085466 ], "composite": { "A": [ 15.392618056749317, 34.34955627190713 ], "B": [ 12.888673039303178, 30.38967783428378 ], "C": [ 10.689318094964802, 23.853858522157736 ] }, "seed": 31 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.42488519968094, "capability": 60.41399746452394, "intelligence": 32.40310972936693 }, "per_type": { "choice": { "accuracy": 0.688, "calibration": 86.55469130227154, "cc": 60.099656129515054, "cc_by_type": { "choice": 60.099656129515054 }, "cc_tiered": 60.11578034929053, "cc_tiered_open": 61.39828743553336, "cc_tiered_sealed": 58.83327326304771, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.099656129515054 }, "noul": { "accuracy": 0.5493333333333333, "calibration": 92.52086836930033, "cc": 9.866666666666667, "cc_by_type": { "noul": 9.866666666666667 }, "cc_tiered": 7.036005542485994, "cc_tiered_open": 6.031055900621121, "cc_tiered_sealed": 8.040955184350867, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 9.866666666666667 }, "score": { "accuracy": 0.5173333333333333, "calibration": 86.19909592747098, "cc": 30.992722913904714, "cc_by_type": { "score": 30.992722913904714 }, "cc_tiered": 30.057543296324283, "cc_tiered_open": 31.542540843331764, "cc_tiered_sealed": 28.5725457493168, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 30.992722913904714 } }, "raw_on_A": { "calibration": 81.8292718059759, "capability": 58.67224928043624, "intelligence": 35.515226754896574 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 21.964945240595316, "Emphasis on Accuracy 60:20:20": 17.633890573157025, "Emphasis on Cost 20:20:60": 23.575828038111915, "Emphasis on Speed 20:60:20": 26.69802362301605, "Intelligence only": 13.608807339398107 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://benchmarkheaven.com/jev-models/decision-4b-v12", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/decision-4b-v12/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5876010781671159, "acc_among_decisive": 0.944954128440367, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Alibaba Qwen / neutral reproduction", "axes": { "calibration": 35.18175191334879, "cost": 63.36068112101963, "intelligence": 33.95117742100547, "speed": 91.16296741568695 }, "calibration": { "parts": { "choice": { "ece_hard": 0.44119675391398117, "ece_score": 11.760649217203767, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 11.760649217203767 }, "noul": { "abstention_rate": 0.07547169811320753, "brier": 0.26616192586960774, "ece": 0.254956633051564, "n": 371, "score": 49.008673389687196 }, "score": { "interval90_coverage": 0.5501355013550135, "interval90_width": 0.07452574525745256, "n": 369, "nrps": 0.1781187945424487, "score": 44.77593313315542, "top_ece": 0.46318127139722143 } }, "score": 35.18175191334879 }, "capability": 34.56646466717713, "composite_ci95": { "A": [ 13.8369005003162, 28.950164844761904 ], "B": [ 12.606782226433888, 27.192389151058098 ], "C": [ 9.608958680775137, 20.10428114219577 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01664602832512315 }, "display": "Raw Qwen3 4B Instruct 2507 direct logits", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 39.33458111901565, "I_sealed": 28.646049395387923, "base": 33.990315257201786, "excess": 8.115144081189486, "gap": 10.68853172362773, "penalty": 0.9988485591881051, "per_type_split": { "open|choice": { "cc": 53.102399069057086, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 68.86486486486487, "hard": 39.60364894620951, "judge": 44.62755438365195, "standard": 84.93093344495604 } }, "open|noul": { "cc": 40.36853002070393, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 24.532814267285957, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 44.98343409680498, "hard": 0.7007591330399521, "judge": 15.311071908696839, "standard": 75.80422815890212 } }, "sealed|choice": { "cc": 41.64735120428594, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 53.44024831867564, "hard": 30.2601809954751, "judge": 41.34078212290503, "standard": 58.9850966867841 } }, "sealed|noul": { "cc": 25.067667527906327, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 0.9708737864077666, "judge": 31.578947368421062, "standard": 38.0281690140845 } }, "sealed|score": { "cc": 19.223129453971513, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 48.978480535664694, "hard": 7.881002538538395, "judge": 13.283448182092084, "standard": 35.9392296518103 } } } }, "jevbench_score": 21.792436038282993, "key": "raw-qwen3-4b-instruct-2507", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 61, "ranked": true, "ranks": { "A": 61, "B": 65, "C": 70, "capability": 118 }, "scores": { "A": 21.792436038282993, "B": 20.20761454160472, "C": 15.133636137696522 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.24207517188042402, "p50_s_raw": 0.04603758594021201, "p95_s_adjusted": 0.31604797318577765, "p95_s_raw": 0.08302398659288883 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 30.079848447424265, 39.87330204447962 ], "I": [ 28.52380607287802, 37.88233031513499 ], "capability": [ 29.696940965810747, 38.2358105496228 ], "composite": { "A": [ 13.8369005003162, 28.950164844761904 ], "B": [ 12.606782226433888, 27.192389151058098 ], "C": [ 9.608958680775137, 20.10428114219577 ] }, "seed": 96 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 35.18175191334879, "capability": 34.56646466717713, "intelligence": 33.95117742100547 }, "per_type": { "choice": { "accuracy": 0.5626666666666666, "calibration": 11.760649217203767, "cc": 44.07131286530316, "cc_by_type": { "choice": 44.07131286530316 }, "cc_tiered": 47.37487513667151, "cc_tiered_open": 53.102399069057086, "cc_tiered_sealed": 41.64735120428594, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 44.07131286530316 }, "noul": { "accuracy": 0.664, "calibration": 49.008673389687196, "cc": 32.800000000000004, "cc_by_type": { "noul": 32.800000000000004 }, "cc_tiered": 32.718098774305126, "cc_tiered_open": 40.36853002070393, "cc_tiered_sealed": 25.067667527906327, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 32.800000000000004 }, "score": { "accuracy": 0.456, "calibration": 44.77593313315542, "cc": 20.99121192415334, "cc_by_type": { "score": 20.99121192415334 }, "cc_tiered": 21.877971860628733, "cc_tiered_open": 24.532814267285957, "cc_tiered_sealed": 19.223129453971513, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 20.99121192415334 } }, "raw_on_A": { "calibration": 37.75992930947443, "capability": 38.17464116904609, "intelligence": 38.58935302861775 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 24.609772187762267, "Emphasis on Accuracy 60:20:20": 20.02676319678133, "Emphasis on Cost 20:20:60": 26.265558574780524, "Emphasis on Speed 20:60:20": 29.50118725011904, "Intelligence only": 15.653970524552685 }, "class": "raw-logit-control", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-4B-Instruct-2507 BF16", "adapter_id": "images/raw-qwen3-4b-instruct-2507/spec.json:raw_adapter.RawControlAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9245283018867925, "acc_among_decisive": 0.7259475218658892, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "AutoTrust AI Lab", "axes": { "calibration": 90.89967561034447, "cost": 31.02571003473804, "intelligence": 56.828664559360675, "speed": 86.89503266505372 }, "calibration": { "parts": { "choice": { "ece_hard": 0.044592591608484194, "ece_score": 91.08148167830316, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 91.08148167830316 }, "noul": { "abstention_rate": 0.21293800539083563, "brier": 0.0834318516755718, "ece": 0.0453231523993617, "n": 371, "score": 90.93536952012767 }, "score": { "interval90_coverage": 0.9539295392953929, "interval90_width": 0.2831978319783197, "n": 369, "nrps": 0.07252867726920081, "score": 90.68217563260262, "top_ece": 0.05691390503937325 } }, "score": 90.89967561034447 }, "capability": 73.86417008485257, "composite_ci95": { "A": [ 20.660744990021566, 21.639609973811442 ], "B": [ 20.50360595860232, 22.019717921334916 ], "C": [ 15.257925180346783, 21.639609973811442 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.19607951999999995 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.19913289099526063 }, "display": "JEV-27B (AutoTrust, Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 57.748245429372346, "I_sealed": 55.909083689349, "base": 56.828664559360675, "excess": -0.7342259024149009, "gap": 1.8391617400233429, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 77.03865189535385, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 75.46398238439761, "judge": 76.26895187870797, "standard": 69.8618668899121 } }, "open|noul": { "cc": 45.291235334713605, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 56.52173913043479, "judge": 10.000000000000009, "standard": 55.55555555555556 } }, "open|score": { "cc": 50.91484905804961, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 62.85808404482206, "hard": 54.85367186261314, "judge": 58.70222464767498, "standard": 25.384522571098287 } }, "sealed|choice": { "cc": 72.15078699038692, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 77.38526346907102, "hard": 66.4027149321267, "judge": 72.06703910614526, "standard": 81.15531469392782 } }, "sealed|noul": { "cc": 42.779753431164394, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": 24.271844660194162, "judge": 42.10526315789473, "standard": 66.19718309859155 } }, "sealed|score": { "cc": 52.79671064649569, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 58.73482464128581, "hard": 48.652428202150055, "judge": 63.82621512662949, "standard": 41.571961817591195 } } } }, "jevbench_score": 21.291230595611577, "key": "jev-27b-autotrust", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 62, "ranked": true, "ranks": { "A": 62, "B": 60, "C": 56, "capability": 18 }, "scores": { "A": 21.291230595611577, "B": 21.406663290369472, "C": 19.09999105447478 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3604971420194488, "p50_s_raw": 0.1052485710097244, "p95_s_adjusted": 0.5670154692983488, "p95_s_raw": 0.20850773464917438 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 86.12231981093008, 91.47472229481626 ], "I": [ 51.54879559651137, 61.562027595056925 ], "capability": [ 69.671164690791, 75.76898842923057 ], "composite": { "A": [ 20.660744990021566, 21.639609973811442 ], "B": [ 20.50360595860232, 22.019717921334916 ], "C": [ 15.257925180346783, 21.639609973811442 ] }, "seed": 51 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.89967561034447, "capability": 73.86417008485257, "intelligence": 56.828664559360675 }, "per_type": { "choice": { "accuracy": 0.796, "calibration": 91.08148167830316, "cc": 73.91131362314447, "cc_by_type": { "choice": 73.91131362314447 }, "cc_tiered": 74.59471944287039, "cc_tiered_open": 77.03865189535385, "cc_tiered_sealed": 72.15078699038692, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 73.91131362314447 }, "noul": { "accuracy": 0.736, "calibration": 90.93536952012767, "cc": 47.199999999999996, "cc_by_type": { "noul": 47.199999999999996 }, "cc_tiered": 44.035494382939, "cc_tiered_open": 45.291235334713605, "cc_tiered_sealed": 42.779753431164394, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 47.199999999999996 }, "score": { "accuracy": 0.6506666666666666, "calibration": 90.68217563260262, "cc": 51.866475511741164, "cc_by_type": { "score": 51.866475511741164 }, "cc_tiered": 51.85577985227265, "cc_tiered_open": 50.91484905804961, "cc_tiered_sealed": 52.79671064649569, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 51.866475511741164 } }, "raw_on_A": { "calibration": 88.71172832565286, "capability": 73.81510020130236, "intelligence": 58.91847207695186 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 18.83248652432942, "Emphasis on Accuracy 60:20:20": 19.94400418862255, "Emphasis on Cost 20:20:60": 15.303706986099348, "Emphasis on Speed 20:60:20": 22.82314538634501, "Intelligence only": 21.88118814236141 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/autotrust/JEV-27B", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "Qwen/Qwen3.8-27B", "model_pin": "autotrust/JEV-27B@51740a8891c2a8baefd969237fd44187b3e3a115", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7870619946091644, "acc_among_decisive": 0.9452054794520548, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Torchcast AI", "axes": { "calibration": 91.379440941337, "cost": 30.341044859222123, "intelligence": 71.75187578095783, "speed": 87.72436167205926 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05156183229850985, "ece_score": 89.68763354029802, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 89.68763354029802 }, "noul": { "abstention_rate": 0.16800000000000004, "brier": 0.06086846046546967, "ece": 0.03262322669949693, "n": 375, "score": 93.47535466010062 }, "score": { "interval90_coverage": 0.992, "interval90_width": 0.2688888888888889, "n": 375, "nrps": 0.04123358842551135, "score": 90.97533462361235, "top_ece": 0.06962985955112075 } }, "score": 91.379440941337 }, "capability": 81.56565836114741, "composite_ci95": { "A": [ 20.87995157412942, 21.459724866005526 ], "B": [ 21.530802573949977, 22.491279110132325 ], "C": [ 20.87995157412942, 21.459724866005526 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.71455888 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.20987715639810428 }, "display": "torchcast-decision-27b (Torchcast AI, Qwen3.8-27B fine-tune)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 72.61939528208785, "I_sealed": 70.88435627982781, "base": 71.75187578095783, "excess": -0.8383486401782059, "gap": 1.735039002260038, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 84.30724823550054, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 88.67568417741428, "judge": 84.17930125247197, "standard": 69.8618668899121 } }, "open|noul": { "cc": 62.8191856452726, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 73.91304347826086, "judge": 50.0, "standard": 55.55555555555556 } }, "open|score": { "cc": 70.73175196549042, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 72.80894472531861, "hard": 75.13092960510187, "judge": 75.39366934082463, "standard": 53.90192424335207 } }, "sealed|choice": { "cc": 82.73011855635478, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 86.6972138053359, "hard": 79.63800904977374, "judge": 85.1024208566108, "standard": 83.37233649464221 } }, "sealed|noul": { "cc": 58.22591347780089, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 80.0, "hard": 43.689320388349515, "judge": 63.1578947368421, "standard": 69.01408450704226 } }, "sealed|score": { "cc": 71.69703680532778, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 75.23124590065085, "hard": 68.16140987458385, "judge": 75.82659136309096, "standard": 70.80685428250935 } } } }, "jevbench_score": 21.273291266043532, "key": "torchcast-decision-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 63, "ranked": true, "ranks": { "A": 63, "B": 58, "C": 51, "capability": 3 }, "scores": { "A": 21.273291266043532, "B": 22.135897876506068, "C": 21.273291266043532 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3445720289950259, "p50_s_raw": 0.09728601449751295, "p95_s_adjusted": 0.4900989957066485, "p95_s_raw": 0.17004949785332427 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 87.27348639408736, 92.2455647503307 ], "I": [ 66.72487693145953, 75.69548078432408 ], "capability": [ 77.79909897478052, 83.12358210849057 ], "composite": { "A": [ 20.87995157412942, 21.459724866005526 ], "B": [ 21.530802573949977, 22.491279110132325 ], "C": [ 20.87995157412942, 21.459724866005526 ] }, "seed": 120 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 91.379440941337, "capability": 81.56565836114741, "intelligence": 71.75187578095783 }, "per_type": { "choice": { "accuracy": 0.872, "calibration": 89.68763354029802, "cc": 83.63062815569847, "cc_by_type": { "choice": 83.63062815569847 }, "cc_tiered": 83.51868339592767, "cc_tiered_open": 84.30724823550054, "cc_tiered_sealed": 82.73011855635478, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 83.63062815569847 }, "noul": { "accuracy": 0.8053333333333333, "calibration": 93.47535466010062, "cc": 61.06666666666667, "cc_by_type": { "noul": 61.06666666666667 }, "cc_tiered": 60.52254956153674, "cc_tiered_open": 62.8191856452726, "cc_tiered_sealed": 58.22591347780089, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 61.06666666666667 }, "score": { "accuracy": 0.8186666666666667, "calibration": 90.97533462361235, "cc": 71.44725085182297, "cc_by_type": { "score": 71.44725085182297 }, "cc_tiered": 71.2143943854091, "cc_tiered_open": 70.73175196549042, "cc_tiered_sealed": 71.69703680532778, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 71.44725085182297 } }, "raw_on_A": { "calibration": 85.79331025602858, "capability": 80.85694680408841, "intelligence": 75.92058335214823 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 18.95010611000118, "Emphasis on Accuracy 60:20:20": 21.366887812526656, "Emphasis on Cost 20:20:60": 14.822679909386718, "Emphasis on Speed 20:60:20": 22.70411049156899, "Intelligence only": 26.42130811221146 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "cc-by-nc-4.0", "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-27b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "Qwen/Qwen3.8-27B", "model_pin": "torchcast-ai/torchcast-decision-27b@08645834899cb8e00d370b77a3f4e0baa3241dd1", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.832, "acc_among_decisive": 0.967948717948718, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Bespoke Labs", "axes": { "calibration": 69.88762704651819, "cost": 36.75037423027614, "intelligence": 43.14237010986392, "speed": 86.11327010751721 }, "calibration": { "parts": { "choice": { "ece_hard": 0.178988177870932, "ece_score": 64.2023644258136, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 64.2023644258136 }, "noul": { "abstention_rate": 0.2264150943396226, "brier": 0.14509070919724038, "ece": 0.1305339256295796, "n": 371, "score": 73.89321487408408 }, "score": { "interval90_coverage": 0.8401084010840109, "interval90_width": 0.26874435411020764, "n": 369, "nrps": 0.11567566172039558, "score": 71.56730183965689, "top_ece": 0.22648915074323328 } }, "score": 69.88762704651819 }, "capability": 56.514998578191054, "composite_ci95": { "A": [ 15.043066969876142, 26.052957246349575 ], "B": [ 14.110886674014932, 25.283661649629934 ], "C": [ 10.446574284636212, 18.092331421076096 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.1283278325123153 }, "display": "Bespoke Nimble 9B (Bespoke Labs)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 46.96255933707536, "I_sealed": 39.32218088265248, "base": 43.14237010986392, "excess": 5.0669908119846365, "gap": 7.64037845442288, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 59.86495466898259, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 49.040578798364265, "judge": 68.35860250494396, "standard": 62.32733361239012 } }, "open|noul": { "cc": 38.9399585921325, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 47.82608695652173, "judge": 19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 42.08276475011101, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 47.18206312365201, "hard": 44.02495901721014, "judge": 36.708929948504554, "standard": 43.70947923155191 } }, "sealed|choice": { "cc": 57.35467556665829, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 60.0916414160077, "hard": 46.5497737556561, "judge": 60.89385474860336, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 27.490153799650226, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 52.0, "hard": 4.854368932038833, "judge": 36.8421052631579, "standard": 46.47887323943662 } }, "sealed|score": { "cc": 33.12171328164895, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 49.98365384887967, "hard": 27.146627400052527, "judge": 32.82017491999325, "standard": 37.093222303709986 } } } }, "jevbench_score": 21.082616704957896, "key": "nimble-9b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 64, "ranked": true, "ranks": { "A": 64, "B": 64, "C": 71, "capability": 79 }, "scores": { "A": 21.082616704957896, "B": 20.213517776427228, "C": 14.64070604510965 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.43987150778993966, "p50_s_raw": 0.14493575389496982, "p95_s_adjusted": 0.5563488021492958, "p95_s_raw": 0.2031744010746479 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 65.1956012319158, 73.91457472170808 ], "I": [ 37.37380617272022, 47.11791102705384 ], "capability": [ 52.152502588110366, 59.801663230716926 ], "composite": { "A": [ 15.043066969876142, 26.052957246349575 ], "B": [ 14.110886674014932, 25.283661649629934 ], "C": [ 10.446574284636212, 18.092331421076096 ] }, "seed": 71 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 69.88762704651819, "capability": 56.514998578191054, "intelligence": 43.14237010986392 }, "per_type": { "choice": { "accuracy": 0.668, "calibration": 64.2023644258136, "cc": 57.541941778842954, "cc_by_type": { "choice": 57.541941778842954 }, "cc_tiered": 58.60981511782044, "cc_tiered_open": 59.86495466898259, "cc_tiered_sealed": 57.35467556665829, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 57.541941778842954 }, "noul": { "accuracy": 0.664, "calibration": 73.89321487408408, "cc": 32.800000000000004, "cc_by_type": { "noul": 32.800000000000004 }, "cc_tiered": 33.215056195891364, "cc_tiered_open": 38.9399585921325, "cc_tiered_sealed": 27.490153799650226, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 32.800000000000004 }, "score": { "accuracy": 0.536, "calibration": 71.56730183965689, "cc": 35.587315474154515, "cc_by_type": { "score": 35.587315474154515 }, "cc_tiered": 37.60223901587998, "cc_tiered_open": 42.08276475011101, "cc_tiered_sealed": 33.12171328164895, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 35.587315474154515 } }, "raw_on_A": { "calibration": 72.02241547589321, "capability": 59.4874434751425, "intelligence": 46.95247147439178 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 19.46099526058868, "Emphasis on Accuracy 60:20:20": 18.55883910946502, "Emphasis on Cost 20:20:60": 17.273520313490142, "Emphasis on Speed 20:60:20": 23.596180145578664, "Intelligence only": 17.35223877358268 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (weights); repository without a licence file as of 19 Sep", "repo": "https://github.com/bespokelabsai/nimble", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "bespokelabs/Bespoke-Nimble-9B (LoRA, adapter unchanged since 93ec5d6), merged into Qwen/Qwen3.5-9B@c202236 with the author's PEFT safe-merge", "adapter_id": "images/nimble-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7735849056603774, "acc_among_decisive": 0.867595818815331, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "kshetrajna12", "axes": { "calibration": 86.97925577932901, "cost": 63.142712976263866, "intelligence": 30.935558296483705, "speed": 68.7900017304639 }, "calibration": { "parts": { "choice": { "ece_hard": 0.068294434375, "ece_score": 86.341113125, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 86.341113125 }, "noul": { "abstention_rate": 0.41866666666666663, "brier": 0.14083422852570657, "ece": 0.05314781866666669, "n": 375, "score": 89.37043626666666 }, "score": { "interval90_coverage": 0.9866666666666667, "interval90_width": 0.7284444444444443, "n": 375, "nrps": 0.1108956250735923, "score": 85.22621794632039, "top_ece": 0.09229000799999995 } }, "score": 86.97925577932901 }, "capability": 58.95740703790636, "composite_ci95": { "A": [ 14.765436275328822, 29.209404384423753 ], "B": [ 12.581433816044255, 26.114502689493058 ], "C": [ 10.253775191200571, 20.28430860029427 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.016926853448275863 }, "display": "reflex 4B (kshetrajna12)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 33.322266958336286, "I_sealed": 28.548849634631125, "base": 30.935558296483705, "excess": 2.200029681266917, "gap": 4.773417323705161, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 55.076336216021176, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 50.92796476879523, "judge": 48.58272907053395, "standard": 62.32733361239012 } }, "open|noul": { "cc": 10.45962732919255, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 30.434782608695656, "judge": -19.999999999999996, "standard": 0.0 } }, "open|score": { "cc": 34.430837329795125, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 52.20111818181819, "hard": 24.787801851851842, "judge": 33.86929491525422, "standard": 45.67408148148149 } }, "sealed|choice": { "cc": 53.57907452177943, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 65.41275589387332, "hard": 50.113122171945705, "judge": 50.65176908752329, "standard": 58.9850966867841 } }, "sealed|noul": { "cc": 1.7255356358899414, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -14.563106796116509, "judge": 5.263157894736836, "standard": 9.859154929577475 } }, "sealed|score": { "cc": 30.341938746224002, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 54.91348368794326, "hard": 26.727804411764712, "judge": 23.787971856287427, "standard": 35.11538527918782 } } } }, "jevbench_score": 20.637311182702398, "key": "reflex-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 65, "ranked": true, "ranks": { "A": 65, "B": 72, "C": 72, "capability": 68 }, "scores": { "A": 20.637311182702398, "B": 17.968342404009285, "C": 14.33146609909889 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 2.865421538101509, "p50_s_raw": 1.3577107690507546, "p95_s_adjusted": 4.611171829840169, "p95_s_raw": 2.230585914920084 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 80.53682470259905, 87.39459423959244 ], "I": [ 27.05693352837418, 35.79974193094454 ], "capability": [ 54.86352340583442, 60.49053268982462 ], "composite": { "A": [ 14.765436275328822, 29.209404384423753 ], "B": [ 12.581433816044255, 26.114502689493058 ], "C": [ 10.253775191200571, 20.28430860029427 ] }, "seed": 98 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.97925577932901, "capability": 58.95740703790636, "intelligence": 30.935558296483705 }, "per_type": { "choice": { "accuracy": 0.648, "calibration": 86.341113125, "cc": 54.98422742817084, "cc_by_type": { "choice": 54.98422742817084 }, "cc_tiered": 54.327705368900304, "cc_tiered_open": 55.076336216021176, "cc_tiered_sealed": 53.57907452177943, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 54.98422742817084 }, "noul": { "accuracy": 0.5333333333333333, "calibration": 89.37043626666666, "cc": 6.666666666666665, "cc_by_type": { "noul": 6.666666666666665 }, "cc_tiered": 6.092581482541245, "cc_tiered_open": 10.45962732919255, "cc_tiered_sealed": 1.7255356358899414, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 6.666666666666665 }, "score": { "accuracy": 0.5173333333333333, "calibration": 85.22621794632039, "cc": 32.60462780701755, "cc_by_type": { "score": 32.60462780701755 }, "cc_tiered": 32.386388038009564, "cc_tiered_open": 34.430837329795125, "cc_tiered_sealed": 30.341938746224002, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 32.60462780701755 } }, "raw_on_A": { "calibration": 81.52279417962764, "capability": 58.44282614389385, "intelligence": 35.36285810816006 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 18.316133226478374, "Emphasis on Accuracy 60:20:20": 15.029593546249647, "Emphasis on Cost 20:20:60": 20.28126457022741, "Emphasis on Speed 20:60:20": 20.855909642703498, "Intelligence only": 11.842240202134587 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT (code, adapter); Apache-2.0 (base)", "repo": "https://github.com/kshetrajna12/reflex", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen/Qwen3.5-4B + kshetrajna12/reflex-qwen3.5-4b-lora", "adapter_id": "images/reflex-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5813333333333334, "acc_among_decisive": 0.9174311926605505, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Perplexity", "axes": { "calibration": 89.86695733940225, "cost": 29.886617009940878, "intelligence": 75.71677945512734, "speed": 86.79279341461623 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06981468503472926, "ece_score": 86.03706299305415, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 86.03706299305415 }, "noul": { "abstention_rate": 0.0889487870619946, "brier": 0.062186325211303205, "ece": 0.051085406690107325, "n": 371, "score": 89.78291866197854 }, "score": { "interval90_coverage": 0.9566395663956639, "interval90_width": 0.165085817524842, "n": 369, "nrps": 0.04050151348288236, "score": 93.78089036317405, "top_ece": 0.04194033962681827 } }, "score": 89.86695733940225 }, "capability": 82.79186839726479, "composite_ci95": { "A": [ 20.265432194933428, 20.803744956997086 ], "B": [ 21.111865782219617, 21.95821416195702 ], "C": [ 20.265432194933428, 20.803744956997086 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.21399420000000002 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.21732654028436021 }, "display": "Perplexity Decider v1.1 27B (Qwen3.8-27B, noncausal decision readout)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 78.36609502551431, "I_sealed": 73.06746388474036, "base": 75.71677945512734, "excess": 2.7252434983357054, "gap": 5.298631140773949, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 87.75265956440154, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 86.78829820698334, "judge": 88.13447593935398, "standard": 84.93093344495604 } }, "open|noul": { "cc": 77.07729468599035, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 91.30434782608697, "judge": 50.0, "standard": 77.77777777777777 } }, "open|score": { "cc": 70.26833082615104, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 63.42506419505849, "hard": 75.99276310558226, "judge": 80.94030641691248, "standard": 46.23313619669274 } }, "sealed|choice": { "cc": 80.03085922219567, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 86.6972138053359, "hard": 71.49321266968325, "judge": 86.96461824953444, "standard": 83.37233649464221 } }, "sealed|noul": { "cc": 66.96797052113396, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 87.99999999999999, "hard": 53.398058252427184, "judge": 71.05263157894737, "standard": 77.46478873239437 } }, "sealed|score": { "cc": 72.20356191089144, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 73.73143188188055, "hard": 66.84626737629874, "judge": 80.0022436490447, "standard": 70.4561933873524 } } } }, "jevbench_score": 20.617642759667973, "key": "pplx-decider-v1-1-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 66, "ranked": true, "ranks": { "A": 66, "B": 59, "C": 53, "capability": 1 }, "scores": { "A": 20.617642759667973, "B": 21.647466573268815, "C": 20.617642759667973 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.39472174100810664, "p50_s_raw": 0.12236087050405331, "p95_s_adjusted": 0.5301876544195693, "p95_s_raw": 0.19009382720978463 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.86618268245174, 91.29794555028252 ], "I": [ 70.97314659505038, 79.1538588782228 ], "capability": [ 79.18287168257048, 84.70664799115283 ], "composite": { "A": [ 20.265432194933428, 20.803744956997086 ], "B": [ 21.111865782219617, 21.95821416195702 ], "C": [ 20.265432194933428, 20.803744956997086 ] }, "seed": 95 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.86695733940225, "capability": 82.79186839726479, "intelligence": 75.71677945512734 }, "per_type": { "choice": { "accuracy": 0.856, "calibration": 86.03706299305415, "cc": 81.5844566751608, "cc_by_type": { "choice": 81.5844566751608 }, "cc_tiered": 83.8917593932986, "cc_tiered_open": 87.75265956440154, "cc_tiered_sealed": 80.03085922219567, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 81.5844566751608 }, "noul": { "accuracy": 0.856, "calibration": 89.78291866197854, "cc": 71.2, "cc_by_type": { "noul": 71.2 }, "cc_tiered": 72.02263260356216, "cc_tiered_open": 77.07729468599035, "cc_tiered_sealed": 66.96797052113396, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 71.2 }, "score": { "accuracy": 0.8133333333333334, "calibration": 93.78089036317405, "cc": 71.25760707133534, "cc_by_type": { "score": 71.25760707133534 }, "cc_tiered": 71.23594636852124, "cc_tiered_open": 70.26833082615104, "cc_tiered_sealed": 72.20356191089144, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 71.25760707133534 } }, "raw_on_A": { "calibration": 89.4049482205463, "capability": 82.9885681530688, "intelligence": 76.57218808559128 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 18.420307065234873, "Emphasis on Accuracy 60:20:20": 21.115368174311886, "Emphasis on Cost 20:20:60": 14.279000034452622, "Emphasis on Speed 20:60:20": 21.991580334127057, "Intelligence only": 27.052390084345777 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/perplexity-ai/pplx-decider-v1.1-27b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "Qwen/Qwen3.8-27B", "model_pin": "perplexity-ai/pplx-decider-v1.1-27b@3b45dead91dfa6d95aad6b95764a606fab2bf7a6", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.9110512129380054, "acc_among_decisive": 0.9497041420118343, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "theunnecessarythings", "axes": { "calibration": 50.5645403785324, "cost": 33.57969498589053, "intelligence": 47.93439876939806, "speed": 85.53222225487117 }, "calibration": { "parts": { "choice": { "ece_hard": 0.14830087112828266, "ece_score": 70.33982577434347, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 70.33982577434347 }, "noul": { "abstention_rate": 0.05660377358490565, "brier": 0.09551749322805621, "ece": 0.09323102319373136, "n": 371, "score": 81.35379536125373 } }, "score": 50.5645403785324 }, "capability": 49.24946957396523, "composite_ci95": { "A": [ 16.72243182039576, 22.540480675102366 ], "B": [ 16.49103923273388, 22.64957340042747 ], "C": [ 11.612799875274835, 16.71953364433383 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.16117564 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.16368548408937036 }, "display": "JADE (Qwen3.8-27B LoRA)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 47.17172129671271, "I_sealed": 48.6970762420834, "base": 47.93439876939806, "excess": -4.09874258780893, "gap": -1.5253549453706867, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 80.09487403506567, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 73.57659641396666, "judge": 80.22412656558997, "standard": 84.93093344495604 } }, "open|noul": { "cc": 61.42028985507247, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 65.21739130434783, "judge": 39.99999999999999, "standard": 66.66666666666667 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -163.63636363636363, "hard": -161.1111111111111, "judge": -213.55932203389827, "standard": -355.55555555555554 } }, "sealed|choice": { "cc": 75.66415221506249, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 70.47511312217195, "judge": 76.72253258845438, "standard": 82.26382559428504 } }, "sealed|noul": { "cc": 70.42707651118772, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 80.0, "hard": 57.28155339805825, "judge": 76.3157894736842, "standard": 83.09859154929578 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -182.97872340425533, "hard": -183.82352941176472, "judge": -204.79041916167668, "standard": -228.42639593908638 } } } }, "jevbench_score": 20.194459825016597, "key": "jade", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 67, "ranked": true, "ranks": { "A": 67, "B": 66, "C": 73, "capability": 88 }, "scores": { "A": 20.194459825016597, "B": 20.12888689696233, "C": 14.023930434039306 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4045139710127842, "p50_s_raw": 0.12725698550639208, "p95_s_adjusted": 0.6915829036966897, "p95_s_raw": 0.27079145184834486 }, "status": { "answered_ok": 1108, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 46.63888535444405, 53.10214733728139 ], "I": [ 44.307422428017674, 51.72167160495215 ], "capability": [ 45.932872400615175, 51.86244666467468 ], "composite": { "A": [ 16.72243182039576, 22.540480675102366 ], "B": [ 16.49103923273388, 22.64957340042747 ], "C": [ 11.612799875274835, 16.71953364433383 ] }, "seed": 47 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 50.5645403785324, "capability": 49.24946957396523, "intelligence": 47.93439876939806 }, "per_type": { "choice": { "accuracy": 0.8186666666666667, "calibration": 70.33982577434347, "cc": 76.8100565539062, "cc_by_type": { "choice": 76.8100565539062 }, "cc_tiered": 77.87951312506408, "cc_tiered_open": 80.09487403506567, "cc_tiered_sealed": 75.66415221506249, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 76.8100565539062 }, "noul": { "accuracy": 0.8533333333333334, "calibration": 81.35379536125373, "cc": 70.66666666666667, "cc_by_type": { "noul": 70.66666666666667 }, "cc_tiered": 65.9236831831301, "cc_tiered_open": 61.42028985507247, "cc_tiered_sealed": 70.42707651118772, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 70.66666666666667 }, "score": { "accuracy": 0.0, "calibration": null, "cc": -197.36842105263145, "cc_by_type": { "score": -197.36842105263145 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 51.05198021321377, "capability": 50.51094734813176, "intelligence": 49.96991448304975 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 1.0, "n": 375 } }, "views": { "Balanced 33:33:33": 19.951224387926274, "Emphasis on Accuracy 60:20:20": 19.918976230985706, "Emphasis on Cost 20:20:60": 17.004313740085063, "Emphasis on Speed 20:60:20": 24.181035896264405, "Intelligence only": 19.870798988937533 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/theunnecessarythings/JADE", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "Qwen/Qwen3.8-27B", "model_pin": "theunnecessarythings/JADE@8e9f9a3aa4293190fcc4a6ff17ff5f3251f4c270", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.9433962264150944, "acc_among_decisive": 0.9142857142857143, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mikhail Gribov", "axes": { "calibration": 83.29951051808085, "cost": 63.95900966831593, "intelligence": 29.78900656204624, "speed": 92.76354061232337 }, "calibration": { "parts": { "choice": { "ece_hard": 0.14894691278376873, "ece_score": 70.21061744324626, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 70.21061744324626 }, "noul": { "abstention_rate": 0.42666666666666664, "brier": 0.1443460310902931, "ece": 0.03743872703084413, "n": 375, "score": 92.51225459383117 }, "score": { "interval90_coverage": 0.9973333333333333, "interval90_width": 0.8611111111111112, "n": 375, "nrps": 0.11438335298438909, "score": 87.17565951716512, "top_ece": 0.07105172833615442 } }, "score": 83.29951051808085 }, "capability": 56.544258540063545, "composite_ci95": { "A": [ 12.880576558614674, 26.104438970675695 ], "B": [ 10.666557109719218, 22.72472546215454 ], "C": [ 8.944844832371304, 18.12808261852479 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01589887315270936 }, "display": "typecastlm (Mikhail Gribov, Qwen3.5-4B computed head)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 34.211044348094305, "I_sealed": 25.366968775998174, "base": 29.78900656204624, "excess": 6.270687929657887, "gap": 8.84407557209613, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 53.948175356316256, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 45.26580685750237, "judge": 52.537903757415954, "standard": 69.8618668899121 } }, "open|noul": { "cc": 19.30296756383713, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 14.28571428571428, "hard": 39.13043478260869, "judge": 0.0, "standard": 11.111111111111116 } }, "open|score": { "cc": 29.381990124129523, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 59.87815307647144, "hard": 32.63833455597018, "judge": 28.21824515037382, "standard": 9.366837244910798 } }, "sealed|choice": { "cc": 46.35542091610937, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 61.4219200354741, "hard": 37.89592760180995, "judge": 51.58286778398511, "standard": 47.89998768321223 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 19.999999999999996, "hard": -22.330097087378643, "judge": 2.6315789473684292, "standard": 15.492957746478876 } }, "sealed|score": { "cc": 29.745485411885156, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 52.652319160951166, "hard": 25.940906558427802, "judge": 26.848256959038974, "standard": 30.247068923536112 } } } }, "jevbench_score": 19.72256430004839, "key": "typecastlm", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 68, "ranked": true, "ranks": { "A": 68, "B": 75, "C": 74, "capability": 78 }, "scores": { "A": 19.72256430004839, "B": 16.813084809424783, "C": 13.69622520836694 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.20903298105113208, "p50_s_raw": 0.02951649052556604, "p95_s_adjusted": 0.25318100801086985, "p95_s_raw": 0.051590504005434926 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.58498026450093, 85.01640087455884 ], "I": [ 25.093945135273305, 33.49496683838513 ], "capability": [ 52.490530265893305, 58.297640238106396 ], "composite": { "A": [ 12.880576558614674, 26.104438970675695 ], "B": [ 10.666557109719218, 22.72472546215454 ], "C": [ 8.944844832371304, 18.12808261852479 ] }, "seed": 107 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.29951051808085, "capability": 56.544258540063545, "intelligence": 29.78900656204624 }, "per_type": { "choice": { "accuracy": 0.5933333333333334, "calibration": 70.21061744324626, "cc": 47.99314153633374, "cc_by_type": { "choice": 47.99314153633374 }, "cc_tiered": 50.15179813621281, "cc_tiered_open": 53.948175356316256, "cc_tiered_sealed": 46.35542091610937, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 47.99314153633374 }, "noul": { "accuracy": 0.5173333333333333, "calibration": 92.51225459383117, "cc": 3.4666666666666623, "cc_by_type": { "noul": 3.4666666666666623 }, "cc_tiered": 9.651483781918564, "cc_tiered_open": 19.30296756383713, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 3.4666666666666623 }, "score": { "accuracy": 0.48533333333333334, "calibration": 87.17565951716512, "cc": 31.463946158492806, "cc_by_type": { "score": 31.463946158492806 }, "cc_tiered": 29.56373776800734, "cc_tiered_open": 29.381990124129523, "cc_tiered_sealed": 29.745485411885156, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 31.463946158492806 } }, "raw_on_A": { "calibration": 79.3986887571586, "capability": 55.69880635159235, "intelligence": 31.998923946026117 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 17.752271764169233, "Emphasis on Accuracy 60:20:20": 13.960999831173599, "Emphasis on Cost 20:20:60": 19.448558330257274, "Emphasis on Speed 20:60:20": 21.764369978155518, "Intelligence only": 10.57372598609905 }, "class": "system-one-open", "open": "yes", "licence": "Apache-2.0 (package and weights)", "repo": "https://github.com/mihail-gribov/typecastlm", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/typecastlm/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5733333333333334, "acc_among_decisive": 0.9023255813953488, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Frontier Infra", "axes": { "calibration": 83.03844121699153, "cost": 29.935481526523958, "intelligence": 57.58551644935426, "speed": 85.83700487110255 }, "calibration": { "parts": { "choice": { "ece_hard": 0.13434975244299677, "ece_score": 73.13004951140066, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 73.13004951140066 }, "noul": { "abstention_rate": 0.26415094339622647, "brier": 0.06648894231173322, "ece": 0.08077593261455521, "n": 371, "score": 83.84481347708896 }, "score": { "interval90_coverage": 0.981029810298103, "interval90_width": 0.36359530261969286, "n": 369, "nrps": 0.05869267834921634, "score": 92.14046066248498, "top_ece": 0.049249054200542 } }, "score": 83.03844121699153 }, "capability": 70.3119788331729, "composite_ci95": { "A": [ 18.744465890345797, 19.52946206149045 ], "B": [ 18.712637254320782, 20.028302946408985 ], "C": [ 13.997484201611632, 19.52946206149045 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.21319311999999999 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.2165129857819905 }, "display": "Jebadiah 27B (Frontier Infra, Qwen3.8-27B LoRA)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 57.50842605163262, "I_sealed": 57.66260684707591, "base": 57.58551644935426, "excess": -2.727568437881532, "gap": -0.15418079544328833, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 74.63912097217829, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 73.57659641396666, "judge": 68.35860250494396, "standard": 77.39640016743407 } }, "open|noul": { "cc": 37.36853002070393, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 10.000000000000009, "standard": 33.33333333333333 } }, "open|score": { "cc": 60.51762716201564, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 69.48122121212121, "hard": 54.84767685185186, "judge": 70.97751186440678, "standard": 51.685903703703694 } }, "sealed|choice": { "cc": 76.61944220069019, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 67.42081447963801, "judge": 83.24022346368716, "standard": 83.37233649464221 } }, "sealed|noul": { "cc": 39.64965634378576, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 63.99999999999999, "hard": 18.446601941747566, "judge": 42.10526315789473, "standard": 66.19718309859155 } }, "sealed|score": { "cc": 56.71872199675177, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 68.98117588652481, "hard": 44.34649191176467, "judge": 60.52794730538922, "standard": 69.61811725888323 } } } }, "jevbench_score": 19.255307672764186, "key": "jebadiah-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 69, "ranked": true, "ranks": { "A": 69, "B": 67, "C": 61, "capability": 35 }, "scores": { "A": 19.255307672764186, "B": 19.517487118738075, "C": 17.736768346068793 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4285312689666171, "p50_s_raw": 0.13926563448330853, "p95_s_adjusted": 0.6085790472658118, "p95_s_raw": 0.2292895236329059 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.55212332209507, 84.3230183429581 ], "I": [ 51.90122605454064, 62.28147406280156 ], "capability": [ 66.67695825246363, 72.1148551679125 ], "composite": { "A": [ 18.744465890345797, 19.52946206149045 ], "B": [ 18.712637254320782, 20.028302946408985 ], "C": [ 13.997484201611632, 19.52946206149045 ] }, "seed": 49 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.03844121699153, "capability": 70.3119788331729, "intelligence": 57.58551644935426 }, "per_type": { "choice": { "accuracy": 0.8146666666666667, "calibration": 73.13004951140066, "cc": 76.29851368377176, "cc_by_type": { "choice": 76.29851368377176 }, "cc_tiered": 75.62928158643425, "cc_tiered_open": 74.63912097217829, "cc_tiered_sealed": 76.61944220069019, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 76.29851368377176 }, "noul": { "accuracy": 0.7146666666666667, "calibration": 83.84481347708896, "cc": 42.93333333333334, "cc_by_type": { "noul": 42.93333333333334 }, "cc_tiered": 38.50909318224484, "cc_tiered_open": 37.36853002070393, "cc_tiered_sealed": 39.64965634378576, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 42.93333333333334 }, "score": { "accuracy": 0.6853333333333333, "calibration": 92.14046066248498, "cc": 57.405502456140376, "cc_by_type": { "score": 57.405502456140376 }, "cc_tiered": 58.618174579383705, "cc_tiered_open": 60.51762716201564, "cc_tiered_sealed": 56.71872199675177, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 57.405502456140376 } }, "raw_on_A": { "calibration": 80.15230356321821, "capability": 71.64752758243264, "intelligence": 63.14275160164708 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 17.22762964959432, "Emphasis on Accuracy 60:20:20": 18.448138764634205, "Emphasis on Cost 20:20:60": 13.86870216120643, "Emphasis on Speed 20:60:20": 20.908227606199645, "Intelligence only": 20.641713893950236 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/frontier-infra/jebadiah-27b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "Qwen/Qwen3.8-27B", "model_pin": "frontier-infra/jebadiah-27b@4d3ed81640bbc015b47976bedae67212a5fb39cd", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7358490566037735, "acc_among_decisive": 0.9816849816849816, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "vLLM Semantic Router", "axes": { "calibration": 82.20136692745442, "cost": 29.632311028770005, "intelligence": 60.4093162408047, "speed": 83.80610555916236 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10003484913300223, "ece_score": 79.99303017339956, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 79.99303017339956 }, "noul": { "abstention_rate": 0.2668463611859838, "brier": 0.06657517963022924, "ece": 0.0795050319911621, "n": 371, "score": 84.09899360176757 }, "score": { "interval90_coverage": 0.994579945799458, "interval90_width": 0.5117434507678408, "n": 369, "nrps": 0.05676847122486311, "score": 82.51207700719613, "top_ece": 0.14649499431560725 } }, "score": 82.20136692745442 }, "capability": 71.30534158412956, "composite_ci95": { "A": [ 18.452136544131196, 19.07499323293124 ], "B": [ 18.66694813760067, 19.664978813028327 ], "C": [ 15.74438986147704, 19.07499323293124 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.21821211999999998 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.22161014218009478 }, "display": "Decision 2.0 Vega 27B (vLLM-SR, Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 63.49716698880913, "I_sealed": 57.321465492800264, "base": 60.4093162408047, "excess": 3.6023138535706187, "gap": 6.1757014960088625, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 83.11168946691484, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 77.35136835482857, "judge": 80.22412656558997, "standard": 92.46546672247803 } }, "open|noul": { "cc": 48.59696342305038, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 73.91304347826086, "judge": 10.000000000000009, "standard": 44.44444444444444 } }, "open|score": { "cc": 58.78284807646216, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 62.69543007415754, "hard": 61.18205862546275, "judge": 57.862992432344406, "standard": 53.407919445789865 } }, "sealed|choice": { "cc": 75.37523860459501, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 71.49321266968325, "judge": 77.6536312849162, "standard": 76.7212710924991 } }, "sealed|noul": { "cc": 40.19307217860048, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 43.99999999999999, "hard": 26.21359223300972, "judge": 42.10526315789473, "standard": 63.38028169014085 } }, "sealed|score": { "cc": 56.396085695205315, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 62.340074967907654, "hard": 48.314713299858624, "judge": 65.93509972749679, "standard": 55.2783148011103 } } } }, "jevbench_score": 18.883756282485887, "key": "decision-2-0-vega-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 70, "ranked": true, "ranks": { "A": 70, "B": 68, "C": 57, "capability": 28 }, "scores": { "A": 18.883756282485887, "B": 19.308516568607676, "C": 18.883756282485887 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.5303170211613178, "p50_s_raw": 0.1901585105806589, "p95_s_adjusted": 0.784971479140222, "p95_s_raw": 0.317485739570111 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.19756848350097, 83.31136430016561 ], "I": [ 55.369466346348005, 64.07264251912545 ], "capability": [ 67.8132497141865, 72.6791469378467 ], "composite": { "A": [ 18.452136544131196, 19.07499323293124 ], "B": [ 18.66694813760067, 19.664978813028327 ], "C": [ 15.74438986147704, 19.07499323293124 ] }, "seed": 31 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.20136692745442, "capability": 71.30534158412956, "intelligence": 60.4093162408047 }, "per_type": { "choice": { "accuracy": 0.8213333333333334, "calibration": 79.99303017339956, "cc": 77.1510851339958, "cc_by_type": { "choice": 77.1510851339958 }, "cc_tiered": 79.24346403575493, "cc_tiered_open": 83.11168946691484, "cc_tiered_sealed": 75.37523860459501, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 77.1510851339958 }, "noul": { "accuracy": 0.7173333333333334, "calibration": 84.09899360176757, "cc": 43.466666666666676, "cc_by_type": { "noul": 43.466666666666676 }, "cc_tiered": 44.39501780082543, "cc_tiered_open": 48.59696342305038, "cc_tiered_sealed": 40.19307217860048, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 43.466666666666676 }, "score": { "accuracy": 0.688, "calibration": 82.51207700719613, "cc": 56.084109797956, "cc_by_type": { "score": 56.084109797956 }, "cc_tiered": 57.58946688583374, "cc_tiered_open": 58.78284807646216, "cc_tiered_sealed": 56.396085695205315, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 56.084109797956 } }, "raw_on_A": { "calibration": 78.63363678590805, "capability": 71.50200013792696, "intelligence": 64.37036348994586 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 16.931351780927532, "Emphasis on Accuracy 60:20:20": 18.419751896131185, "Emphasis on Cost 20:20:60": 13.537269231212232, "Emphasis on Speed 20:60:20": 20.397174151969452, "Intelligence only": 21.217536521841364 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/vllm-sr/Decision-2.0-Vega-27B", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "Qwen/Qwen3.8-27B", "model_pin": "vllm-sr/Decision-2.0-Vega-27B@7aec49ae11a18741706da549ab626b9052795fe7", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7331536388140162, "acc_among_decisive": 0.9889705882352942, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 85.23934158755043, "cost": 63.07359395281681, "intelligence": 29.134168280172865, "speed": 93.95641941218838 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10557670546665254, "ece_score": 78.88465890666949, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 78.88465890666949 }, "noul": { "abstention_rate": 0.4339622641509434, "brier": 0.11841873108613687, "ece": 0.04782018815907789, "n": 371, "score": 90.43596236818442 }, "score": { "interval90_coverage": 0.9701897018970189, "interval90_width": 0.5756549232158987, "n": 369, "nrps": 0.11671345785768895, "score": 86.39740348779742, "top_ece": 0.07766923619318136 } }, "score": 85.23934158755043 }, "capability": 57.18675493386165, "composite_ci95": { "A": [ 12.444491289404162, 27.25430281789905 ], "B": [ 10.295694887525865, 23.807726232797425 ], "C": [ 8.642007839864, 18.926599179096563 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "Decision 4B v1.1 (FlyMyJev, Qwen3.5-4B + LoRA)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 30.696800084915584, "I_sealed": 27.571536475430143, "base": 29.134168280172865, "excess": 0.5518759670471964, "gap": 3.1252636094854402, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.19131316010312, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 56.59012268008808, "judge": 64.40342781806196, "standard": 54.79280033486815 } }, "open|noul": { "cc": 1.1380262249827418, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 13.043478260869556, "judge": -30.000000000000004, "standard": -11.111111111111116 } }, "open|score": { "cc": 30.761060869660888, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 54.918373499249086, "hard": 31.938975913174172, "judge": 30.29541690533948, "standard": 17.025040414322312 } }, "sealed|choice": { "cc": 57.815173715536396, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 68.07331313280615, "hard": 54.185520361990946, "judge": 56.238361266294234, "standard": 62.31062938785566 } }, "sealed|noul": { "cc": 2.4545445385650666, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 28.000000000000004, "hard": -16.504854368932044, "judge": 10.526315789473696, "standard": 15.492957746478876 } }, "sealed|score": { "cc": 22.444891172188967, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 55.42678299206407, "hard": 20.963579856299262, "judge": 15.25087787708388, "standard": 19.707587836688447 } } } }, "jevbench_score": 18.718356274892123, "key": "decision-4b-v11", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 71, "ranked": true, "ranks": { "A": 71, "B": 77, "C": 76, "capability": 74 }, "scores": { "A": 18.718356274892123, "B": 15.883627667705545, "C": 12.998858524230638 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.188053295109421, "p50_s_raw": 0.019026647554710507, "p95_s_adjusted": 0.21383417290635406, "p95_s_raw": 0.031917086453177035 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 79.26750110051037, 86.2633177195094 ], "I": [ 24.784375611108008, 34.07951912545236 ], "capability": [ 52.99615856993229, 59.129104174768116 ], "composite": { "A": [ 12.444491289404162, 27.25430281789905 ], "B": [ 10.295694887525865, 23.807726232797425 ], "C": [ 8.642007839864, 18.926599179096563 ] }, "seed": 30 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.23934158755043, "capability": 57.18675493386165, "intelligence": 29.134168280172865 }, "per_type": { "choice": { "accuracy": 0.68, "calibration": 78.88465890666949, "cc": 59.07657038924623, "cc_by_type": { "choice": 59.07657038924623 }, "cc_tiered": 59.00324343781976, "cc_tiered_open": 60.19131316010312, "cc_tiered_sealed": 57.815173715536396, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 59.07657038924623 }, "noul": { "accuracy": 0.528, "calibration": 90.43596236818442, "cc": 5.600000000000005, "cc_by_type": { "noul": 5.600000000000005 }, "cc_tiered": 1.7962853817739042, "cc_tiered_open": 1.1380262249827418, "cc_tiered_sealed": 2.4545445385650666, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 5.600000000000005 }, "score": { "accuracy": 0.47733333333333333, "calibration": 86.39740348779742, "cc": 26.584587548466242, "cc_by_type": { "score": 26.584587548466242 }, "cc_tiered": 26.602976020924928, "cc_tiered_open": 30.761060869660888, "cc_tiered_sealed": 22.444891172188967, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 26.584587548466242 } }, "raw_on_A": { "calibration": 85.6538943241602, "capability": 60.62425637183274, "intelligence": 35.59461841950529 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 16.746653513023, "Emphasis on Accuracy 60:20:20": 13.11195414676558, "Emphasis on Cost 20:20:60": 18.346342760998827, "Emphasis on Speed 20:60:20": 20.675164334326784, "Intelligence only": 9.891630033656257 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://benchmarkheaven.com/jev-models/decision-4b-v11", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/decision-4b-v11/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5660377358490566, "acc_among_decisive": 0.9428571428571428, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 87.61544276592178, "cost": 29.365937020749882, "intelligence": 55.62422706919757, "speed": 88.60757446254297 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0794047531024999, "ece_score": 84.11904937950003, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 84.11904937950003 }, "noul": { "abstention_rate": 0.2964959568733153, "brier": 0.06886464230605442, "ece": 0.04566629583733653, "n": 371, "score": 90.8667408325327 }, "score": { "interval90_coverage": 1.0, "interval90_width": 0.41666666666666674, "n": 369, "nrps": 0.0623820892760947, "score": 87.86053808573263, "top_ece": 0.09020357450462646 } }, "score": 87.61544276592178 }, "capability": 71.61983491755967, "composite_ci95": { "A": [ 17.95132364974654, 18.715305818590913 ], "B": [ 17.85214158412037, 19.07667324580802 ], "C": [ 12.749857516255931, 18.58395156303398 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.22618758620689655 }, "display": "AutoJev-27B (denis-pplx, Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 56.79842954079213, "I_sealed": 54.45002459760302, "base": 55.62422706919757, "excess": -0.2249826992491344, "gap": 2.3484049431891094, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 79.34292176773366, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 75.46398238439761, "judge": 80.22412656558997, "standard": 77.39640016743407 } }, "open|noul": { "cc": 34.36853002070393, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 0.0, "standard": 33.33333333333333 } }, "open|score": { "cc": 56.68383683393881, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 65.12454762185703, "hard": 56.31402454129169, "judge": 64.63259495424096, "standard": 41.2799688448207 } }, "sealed|choice": { "cc": 75.98476401702004, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 68.43891402714931, "judge": 78.58472998137803, "standard": 84.48084739499939 } }, "sealed|noul": { "cc": 33.77589728457613, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 12.621359223300965, "judge": 42.10526315789473, "standard": 46.47887323943662 } }, "sealed|score": { "cc": 53.589412491212904, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 68.78624407613681, "hard": 41.90521556676637, "judge": 62.130233048417736, "standard": 56.54815971183675 } } } }, "jevbench_score": 18.463427917707932, "key": "autojev-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 72, "ranked": true, "ranks": { "A": 72, "B": 70, "C": 69, "capability": 27 }, "scores": { "A": 18.463427917707932, "B": 18.603781697586662, "C": 15.868570767928437 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.3447543190326542, "p50_s_raw": 0.09737715951632708, "p95_s_adjusted": 0.3996987941907718, "p95_s_raw": 0.12484939709538592 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 82.93064258139867, 88.05572734371755 ], "I": [ 50.54754866954701, 59.786942555692185 ], "capability": [ 67.70288142885737, 72.92248557654668 ], "composite": { "A": [ 17.95132364974654, 18.715305818590913 ], "B": [ 17.85214158412037, 19.07667324580802 ], "C": [ 12.749857516255931, 18.58395156303398 ] }, "seed": 16 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 87.61544276592178, "capability": 71.61983491755967, "intelligence": 55.62422706919757 }, "per_type": { "choice": { "accuracy": 0.82, "calibration": 84.11904937950003, "cc": 76.98057084395099, "cc_by_type": { "choice": 76.98057084395099 }, "cc_tiered": 77.66384289237685, "cc_tiered_open": 79.34292176773366, "cc_tiered_sealed": 75.98476401702004, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 76.98057084395099 }, "noul": { "accuracy": 0.688, "calibration": 90.8667408325327, "cc": 37.59999999999999, "cc_by_type": { "noul": 37.59999999999999 }, "cc_tiered": 34.07221365264003, "cc_tiered_open": 34.36853002070393, "cc_tiered_sealed": 33.77589728457613, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 37.59999999999999 }, "score": { "accuracy": 0.6506666666666666, "calibration": 87.86053808573263, "cc": 53.92178222051296, "cc_by_type": { "score": 53.92178222051296 }, "cc_tiered": 55.136624662575855, "cc_tiered_open": 56.68383683393881, "cc_tiered_sealed": 53.589412491212904, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 53.92178222051296 } }, "raw_on_A": { "calibration": 85.84423247942522, "capability": 73.42644032760776, "intelligence": 61.00864817579031 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 16.343749600153146, "Emphasis on Accuracy 60:20:20": 17.37362601712716, "Emphasis on Cost 20:20:60": 13.123431991427351, "Emphasis on Speed 20:60:20": 20.08099797467756, "Intelligence only": 19.187204603319074 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://huggingface.co/denis-pplx/autojev-27b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/autojev-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7035040431266847, "acc_among_decisive": 0.9885057471264368, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "unknown", "axes": { "calibration": 89.97714437568723, "cost": 28.689752882469676, "intelligence": 64.40270761245412, "speed": 88.08630815059152 }, "calibration": { "parts": { "choice": { "ece_hard": 0.03950772533975903, "ece_score": 92.09845493204818, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 92.09845493204818 }, "noul": { "abstention_rate": 0.19137466307277629, "brier": 0.05632135167655414, "ece": 0.06181891259769947, "n": 371, "score": 87.6362174804601 }, "score": { "interval90_coverage": 0.964769647696477, "interval90_width": 0.3172990063233966, "n": 369, "nrps": 0.05868305372968562, "score": 90.19676071455342, "top_ece": 0.06869086598962304 } }, "score": 89.97714437568723 }, "capability": 77.18992599407068, "composite_ci95": { "A": [ 17.720899810915583, 18.261473654252256 ], "B": [ 18.087561450267057, 18.993796215955186 ], "C": [ 17.656534367876468, 18.261473654252256 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.23823646551724137 }, "display": "Eikos-27B (caiovicentino1, Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 65.62981582129679, "I_sealed": 63.17559940361144, "base": 64.40270761245412, "excess": -0.1191712247528991, "gap": 2.4542164176853447, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 80.41823040534584, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 77.35136835482857, "judge": 76.26895187870797, "standard": 84.93093344495604 } }, "open|noul": { "cc": 58.24775707384403, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 73.91304347826086, "judge": 30.000000000000004, "standard": 55.55555555555556 } }, "open|score": { "cc": 58.2234599847005, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 61.643002416487, "hard": 61.32359173476558, "judge": 58.76732285181849, "standard": 49.49763096800006 } }, "sealed|choice": { "cc": 78.56119802347493, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 76.05498484960461, "hard": 74.5475113122172, "judge": 78.58472998137803, "standard": 87.80638009607095 } }, "sealed|noul": { "cc": 53.35348874031106, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 37.86407766990292, "judge": 57.89473684210527, "standard": 66.19718309859155 } }, "sealed|score": { "cc": 57.61211144704836, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 67.33566285144035, "hard": 49.71122434442663, "judge": 64.06787863213144, "standard": 58.86845917247121 } } } }, "jevbench_score": 18.077889022432696, "key": "eikos-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 73, "ranked": true, "ranks": { "A": 73, "B": 69, "C": 60, "capability": 10 }, "scores": { "A": 18.077889022432696, "B": 18.627129011951496, "C": 18.077889022432696 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.37523653320968153, "p50_s_raw": 0.11261826660484076, "p95_s_adjusted": 0.4140607553534209, "p95_s_raw": 0.13203037767671047 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 85.83309061783127, 90.19679304741126 ], "I": [ 59.88680163370944, 68.49583860207214 ], "capability": [ 73.72374153392347, 78.55486844196058 ], "composite": { "A": [ 17.720899810915583, 18.261473654252256 ], "B": [ 18.087561450267057, 18.993796215955186 ], "C": [ 17.656534367876468, 18.261473654252256 ] }, "seed": 36 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.97714437568723, "capability": 77.18992599407068, "intelligence": 64.40270761245412 }, "per_type": { "choice": { "accuracy": 0.836, "calibration": 92.09845493204818, "cc": 79.02674232448868, "cc_by_type": { "choice": 79.02674232448868 }, "cc_tiered": 79.48971421441038, "cc_tiered_open": 80.41823040534584, "cc_tiered_sealed": 78.56119802347493, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 79.02674232448868 }, "noul": { "accuracy": 0.784, "calibration": 87.6362174804601, "cc": 56.800000000000004, "cc_by_type": { "noul": 56.800000000000004 }, "cc_tiered": 55.80062290707754, "cc_tiered_open": 58.24775707384403, "cc_tiered_sealed": 53.35348874031106, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 56.800000000000004 }, "score": { "accuracy": 0.6853333333333333, "calibration": 90.19676071455342, "cc": 57.48579408299472, "cc_by_type": { "score": 57.48579408299472 }, "cc_tiered": 57.91778571587443, "cc_tiered_open": 58.2234599847005, "cc_tiered_sealed": 57.61211144704836, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 57.48579408299472 } }, "raw_on_A": { "calibration": 86.77546118929445, "capability": 77.43357682556581, "intelligence": 68.09169246183717 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 15.999272744012083, "Emphasis on Accuracy 60:20:20": 17.74116866920125, "Emphasis on Cost 20:20:60": 12.523731672975002, "Emphasis on Speed 20:60:20": 19.495440346386523, "Intelligence only": 21.20399692738122 }, "class": "unclassified", "open": "unknown", "licence": "not recorded", "repo": "https://huggingface.co/caiovicentino1/eikos-27b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": null, "adapter_id": "images/eikos-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8086253369272237, "acc_among_decisive": 0.98, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Igor L. / NInfer contributors", "axes": { "calibration": 84.18557302696732, "cost": 29.118887844204664, "intelligence": 51.800126951055375, "speed": 90.47734239273063 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09587088648606537, "ece_score": 80.82582270278694, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 80.82582270278694 }, "noul": { "abstention_rate": 0.18328840970350402, "brier": 0.12070867053402283, "ece": 0.0947034093144267, "n": 371, "score": 81.05931813711467 }, "score": { "interval90_coverage": 0.962059620596206, "interval90_width": 0.4135049683830172, "n": 369, "nrps": 0.07983756609019925, "score": 90.67157824100035, "top_ece": 0.05336543454489691 } }, "score": 84.18557302696732 }, "capability": 67.99284998901135, "composite_ci95": { "A": [ 14.657219833459003, 18.09581018423065 ], "B": [ 14.41402694236696, 18.285046805435694 ], "C": [ 10.178624884346531, 15.973963875467245 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.2305174137931034 }, "display": "NInfer Qwen3.8-27B NVFP4", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 51.8273559598382, "I_sealed": 51.77289794227256, "base": 51.800126951055375, "excess": -2.5189296248726003, "gap": 0.0544580175656435, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 68.57305975575287, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 56.59012268008808, "judge": 68.35860250494396, "standard": 84.93093344495604 } }, "open|noul": { "cc": 37.668046928916496, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 39.13043478260869, "judge": 30.000000000000004, "standard": 22.222222222222232 } }, "open|score": { "cc": 49.24096119484526, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 61.871723226731845, "hard": 32.99494797533653, "judge": 63.458675062497626, "standard": 54.0910358164409 } }, "sealed|choice": { "cc": 63.99439201930662, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 70.73387037173899, "hard": 56.730769230769226, "judge": 63.68715083798884, "standard": 75.61276019214189 } }, "sealed|noul": { "cc": 42.068015862163264, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 20.388349514563096, "judge": 50.0, "standard": 60.56338028169015 } }, "sealed|score": { "cc": 49.25628594534781, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 68.47448220301372, "hard": 39.79827642333791, "judge": 53.88993870966854, "standard": 51.612727714053584 } } } }, "jevbench_score": 17.71600496541158, "key": "ninfer-qwen3.8-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 74, "ranked": true, "ranks": { "A": 74, "B": 73, "C": 75, "capability": 42 }, "scores": { "A": 17.71600496541158, "B": 17.686352887683018, "C": 13.20459060190698 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.26002668901346626, "p50_s_raw": 0.05501334450673312, "p95_s_adjusted": 0.3445464937365614, "p95_s_raw": 0.09727324686828069 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.72416061844064, 85.97779338206968 ], "I": [ 46.35923001218984, 56.308300363548845 ], "capability": [ 63.42455870096435, 70.41651644718259 ], "composite": { "A": [ 14.657219833459003, 18.09581018423065 ], "B": [ 14.41402694236696, 18.285046805435694 ], "C": [ 10.178624884346531, 15.973963875467245 ] }, "seed": 72 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 84.18557302696732, "capability": 67.99284998901135, "intelligence": 51.800126951055375 }, "per_type": { "choice": { "accuracy": 0.7266666666666667, "calibration": 80.82582270278694, "cc": 65.04457054081448, "cc_by_type": { "choice": 65.04457054081448 }, "cc_tiered": 66.28372588752974, "cc_tiered_open": 68.57305975575287, "cc_tiered_sealed": 63.99439201930662, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 65.04457054081448 }, "noul": { "accuracy": 0.7226666666666667, "calibration": 81.05931813711467, "cc": 44.53333333333334, "cc_by_type": { "noul": 44.53333333333334 }, "cc_tiered": 39.86803139553988, "cc_tiered_open": 37.668046928916496, "cc_tiered_sealed": 42.068015862163264, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 44.53333333333334 }, "score": { "accuracy": 0.6053333333333333, "calibration": 90.67157824100035, "cc": 49.00847977112264, "cc_by_type": { "score": 49.00847977112264 }, "cc_tiered": 49.24862357009654, "cc_tiered_open": 49.24096119484526, "cc_tiered_sealed": 49.25628594534781, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 49.00847977112264 } }, "raw_on_A": { "calibration": 79.59030598423057, "capability": 67.58012336905948, "intelligence": 55.569940753888396 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 15.726436779262448, "Emphasis on Accuracy 60:20:20": 16.41495956225673, "Emphasis on Cost 20:20:60": 12.713872701078419, "Emphasis on Speed 20:60:20": 19.536094431908698, "Intelligence only": 17.568730575960352 }, "class": "native-logit", "open": "yes", "licence": "Apache-2.0 (engine and submitted 27B artifact/base)", "repo": "https://github.com/igorls/ninfer", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen/Qwen3.8-27B, NInfer NVFP4/FP8 artifact", "adapter_id": "images/ninfer-qwen3.8-27b/spec.json:ninfer_native.NInferNativeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.816711590296496, "acc_among_decisive": 0.8943894389438944, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "razorback16", "axes": { "calibration": 81.41345474033434, "cost": 28.51480924396573, "intelligence": 70.34621384425424, "speed": 72.55270827201397 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1238179439505489, "ece_score": 75.23641120989022, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 75.23641120989022 }, "noul": { "abstention_rate": 0.04312668463611857, "brier": 0.06619006264403647, "ece": 0.07056572946447708, "n": 371, "score": 85.88685410710458 }, "score": { "interval90_coverage": 0.8455284552845529, "interval90_width": 0.06526648599819332, "n": 369, "nrps": 0.05970047185852581, "score": 83.11709890400827, "top_ece": 0.13897877503065434 } }, "score": 81.41345474033434 }, "capability": 75.87983429229429, "composite_ci95": { "A": [ 16.909474945094924, 17.502684272940023 ], "B": [ 17.624044058024868, 18.54119755452704 ], "C": [ 16.909474945094924, 17.502684272940023 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.24145693965517237 }, "display": "OpenJev (thinking, BF16)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 68.15800897274833, "I_sealed": 72.53441871576014, "base": 70.34621384425424, "excess": -6.949797385450049, "gap": -4.376409743011806, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 71.32537528109535, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 67.9144385026738, "judge": 72.31377719182596, "standard": 62.32733361239012 } }, "open|noul": { "cc": 68.51345755693583, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 56.52173913043479, "judge": 80.0, "standard": 66.66666666666667 } }, "open|score": { "cc": 64.63519408021384, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 75.65890690670581, "hard": 83.20743211500854, "judge": 51.76582051786083, "standard": 41.282921940907926 } }, "sealed|choice": { "cc": 72.81145995047186, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 86.6972138053359, "hard": 66.4027149321267, "judge": 74.86033519553072, "standard": 75.61276019214189 } }, "sealed|noul": { "cc": 76.43903646714215, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 84.00000000000001, "hard": 61.165048543689316, "judge": 84.21052631578947, "standard": 91.54929577464787 } }, "sealed|score": { "cc": 68.35275972966642, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 85.29967322252621, "hard": 58.44270568724672, "judge": 72.99731091086852, "standard": 72.73258429627272 } } } }, "jevbench_score": 17.265218966202397, "key": "openjev-thinking", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 75, "ranked": true, "ranks": { "A": 75, "B": 71, "C": 63, "capability": 13 }, "scores": { "A": 17.265218966202397, "B": 18.156241294771178, "C": 17.265218966202397 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.8639664588030427, "p50_s_raw": 0.8569832294015214, "p95_s_adjusted": 2.980513406405225, "p95_s_raw": 1.4152567032026127 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.19017694535097, 83.84266027008114 ], "I": [ 65.12984274564428, 74.6441850225165 ], "capability": [ 71.81263659682128, 78.75991583946256 ], "composite": { "A": [ 16.909474945094924, 17.502684272940023 ], "B": [ 17.624044058024868, 18.54119755452704 ], "C": [ 16.909474945094924, 17.502684272940023 ] }, "seed": 81 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 81.41345474033434, "capability": 75.87983429229429, "intelligence": 70.34621384425424 }, "per_type": { "choice": { "accuracy": 0.7946666666666666, "calibration": 75.23641120989022, "cc": 73.74079933309964, "cc_by_type": { "choice": 73.74079933309964 }, "cc_tiered": 72.0684176157836, "cc_tiered_open": 71.32537528109535, "cc_tiered_sealed": 72.81145995047186, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 73.74079933309964 }, "noul": { "accuracy": 0.8826666666666667, "calibration": 85.88685410710458, "cc": 76.53333333333335, "cc_by_type": { "noul": 76.53333333333335 }, "cc_tiered": 72.47624701203898, "cc_tiered_open": 68.51345755693583, "cc_tiered_sealed": 76.43903646714215, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 76.53333333333335 }, "score": { "accuracy": 0.7786666666666666, "calibration": 83.11709890400827, "cc": 68.45852348469666, "cc_by_type": { "score": 68.45852348469666 }, "cc_tiered": 66.49397690494013, "cc_tiered_open": 64.63519408021384, "cc_tiered_sealed": 68.35275972966642, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 68.45852348469666 } }, "raw_on_A": { "calibration": 79.80009682824573, "capability": 76.44304334504619, "intelligence": 73.08598986184666 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 15.470816250462342, "Emphasis on Accuracy 60:20:20": 17.77278752746646, "Emphasis on Cost 20:20:60": 12.207984243040777, "Emphasis on Speed 20:60:20": 17.94233922790431, "Intelligence only": 22.879243501891544 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://github.com/razorback16/openjev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "google/diffusiongemma-26b-a4b-it, BF16; OpenJev think=512", "adapter_id": "images/openjev-thinking/spec.json:openjev_thinking.ThinkingOpenJev", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9568733153638814, "acc_among_decisive": 0.9323943661971831, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Zefan Cai (@Zefan_Cai)", "axes": { "calibration": 79.80507069734225, "cost": 33.05490478272432, "intelligence": 43.88591848753404, "speed": 74.39486349899005 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0965793794449889, "ece_score": 80.68412411100222, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 80.68412411100222 }, "noul": { "abstention_rate": 0.18867924528301883, "brier": 0.1465943912444352, "ece": 0.11119868678055905, "n": 371, "score": 77.76026264388818 }, "score": { "interval90_coverage": 0.926829268292683, "interval90_width": 0.46047877145438126, "n": 369, "nrps": 0.10981645578089093, "score": 80.97082533713633, "top_ece": 0.13538351873819116 } }, "score": 79.80507069734225 }, "capability": 61.84549459243814, "composite_ci95": { "A": [ 12.01117001715265, 21.4767289933671 ], "B": [ 11.375313778606227, 21.19925799700656 ], "C": [ 8.341090289689344, 14.91439513428271 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.17041317733990147 }, "display": "Open-Jev 9B (Zefan Cai)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 45.51928930503784, "I_sealed": 42.25254767003025, "base": 43.88591848753404, "excess": 0.6933539925693402, "gap": 3.266741635007584, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 61.95574336697866, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 68.86486486486487, "hard": 64.1396665618119, "judge": 56.49307844429796, "standard": 62.32733361239012 } }, "open|noul": { "cc": 35.9399585921325, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 47.82608695652173, "judge": 10.000000000000009, "standard": 33.33333333333333 } }, "open|score": { "cc": 38.66216595600236, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 45.958227241534885, "hard": 49.02462444279136, "judge": 29.24271270673138, "standard": 28.41839821356458 } }, "sealed|choice": { "cc": 56.97677837170926, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 64.08247727440693, "hard": 48.076923076923066, "judge": 59.96275605214154, "standard": 66.7446729892844 } }, "sealed|noul": { "cc": 36.79892764867179, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 48.0, "hard": 20.388349514563096, "judge": 44.736842105263165, "standard": 52.112676056338024 } }, "sealed|score": { "cc": 32.98193698970971, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 51.23313360297058, "hard": 27.657644820832182, "judge": 32.23017056028181, "standard": 35.63257266497618 } } } }, "jevbench_score": 17.045611746249723, "key": "open-jev-zefan-9b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 76, "ranked": true, "ranks": { "A": 76, "B": 76, "C": 77, "capability": 59 }, "scores": { "A": 17.045611746249723, "B": 16.53766290698564, "C": 11.837230379340085 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.3850112770218401, "p50_s_raw": 0.6175056385109201, "p95_s_adjusted": 2.624583175848238, "p95_s_raw": 1.237291587924119 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 73.96736762912185, 81.45578416794778 ], "I": [ 37.82741036517579, 48.601933428170696 ], "capability": [ 56.87009089863343, 64.27945845500723 ], "composite": { "A": [ 12.01117001715265, 21.4767289933671 ], "B": [ 11.375313778606227, 21.19925799700656 ], "C": [ 8.341090289689344, 14.91439513428271 ] }, "seed": 79 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 79.80507069734225, "capability": 61.84549459243814, "intelligence": 43.88591848753404 }, "per_type": { "choice": { "accuracy": 0.6733333333333333, "calibration": 80.68412411100222, "cc": 58.22399893902218, "cc_by_type": { "choice": 58.22399893902218 }, "cc_tiered": 59.466260869343955, "cc_tiered_open": 61.95574336697866, "cc_tiered_sealed": 56.97677837170926, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 58.22399893902218 }, "noul": { "accuracy": 0.6933333333333334, "calibration": 77.76026264388818, "cc": 38.66666666666667, "cc_by_type": { "noul": 38.66666666666667 }, "cc_tiered": 36.369443120402146, "cc_tiered_open": 35.9399585921325, "cc_tiered_sealed": 36.79892764867179, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 38.66666666666667 }, "score": { "accuracy": 0.536, "calibration": 80.97082533713633, "cc": 35.27556488047482, "cc_by_type": { "score": 35.27556488047482 }, "cc_tiered": 35.82205147285603, "cc_tiered_open": 38.66216595600236, "cc_tiered_sealed": 32.98193698970971, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 35.27556488047482 } }, "raw_on_A": { "calibration": 76.09059647397694, "capability": 61.58634472209667, "intelligence": 47.0820929702164 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 15.193819551111247, "Emphasis on Accuracy 60:20:20": 15.024035626101695, "Emphasis on Cost 20:20:60": 13.257314615761418, "Emphasis on Speed 20:60:20": 18.03145694648765, "Intelligence only": 14.776357163022537 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection", "repo": "https://github.com/Zefan-Cai/Open-Jev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen3.5-9B plus rank-8 LoRA and trained scalar decision head", "adapter_id": "images/open-jev-zefan-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.8113207547169812, "acc_among_decisive": 0.8637873754152824, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Standard Thinking (myeongho12)", "axes": { "calibration": 84.01508092906352, "cost": 43.283250331904235, "intelligence": 32.80426604507879, "speed": 92.76442849252426 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1098197498466637, "ece_score": 78.03605003066725, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 78.03605003066725 }, "noul": { "abstention_rate": 0.32345013477088946, "brier": 0.1399053250455738, "ece": 0.07067323668547182, "n": 371, "score": 85.86535266290564 }, "score": { "interval90_coverage": 0.907859078590786, "interval90_width": 0.4114724480578141, "n": 369, "nrps": 0.10288890604189731, "score": 88.14384009361767, "top_ece": 0.06711714604287465 } }, "score": 84.01508092906352 }, "capability": 58.40967348707115, "composite_ci95": { "A": [ 11.018270368438198, 24.24546849303509 ], "B": [ 9.568356102409696, 22.27960999090091 ], "C": [ 7.651576644748748, 16.837130897941034 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.07772456896551724 }, "display": "Standard One 8B (Standard Thinking)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 30.33450693574831, "I_sealed": 35.27402515440927, "base": 32.80426604507879, "excess": -7.512905861099208, "gap": -4.9395182186609645, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 49.295789751985026, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 53.2972972972973, "hard": 52.815350739226176, "judge": 44.62755438365195, "standard": 47.25826705734617 } }, "open|noul": { "cc": 8.598343685300202, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 13.043478260869556, "judge": -30.000000000000004, "standard": 33.33333333333333 } }, "open|score": { "cc": 33.109387369959705, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 52.66883781130674, "hard": 20.17576470368565, "judge": 52.27846833393313, "standard": 20.443286035874163 } }, "sealed|choice": { "cc": 51.12536859237399, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 70.73387037173899, "hard": 39.42307692307691, "judge": 57.16945996275605, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 17.63077432402283, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 56.00000000000001, "hard": -12.621359223300976, "judge": 31.578947368421062, "standard": 38.0281690140845 } }, "sealed|score": { "cc": 37.06593254683102, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 56.312646632383355, "hard": 30.71497074829812, "judge": 39.8913069210783, "standard": 35.90643753974974 } } } }, "jevbench_score": 16.917108299009165, "key": "standardone-8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 77, "ranked": true, "ranks": { "A": 77, "B": 78, "C": 78, "capability": 69 }, "scores": { "A": 16.917108299009165, "B": 15.107984144533011, "C": 11.747991874311923 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.21347399484366178, "p50_s_raw": 0.03173699742183089, "p95_s_adjusted": 0.24786326913163065, "p95_s_raw": 0.04893163456581533 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.9630533878547, 85.6304653046258 ], "I": [ 27.551137017862857, 38.18793043954382 ], "capability": [ 53.64856195871698, 60.83707776399651 ], "composite": { "A": [ 11.018270368438198, 24.24546849303509 ], "B": [ 9.568356102409696, 22.27960999090091 ], "C": [ 7.651576644748748, 16.837130897941034 ] }, "seed": 103 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 84.01508092906352, "capability": 58.40967348707115, "intelligence": 32.80426604507879 }, "per_type": { "choice": { "accuracy": 0.62, "calibration": 78.03605003066725, "cc": 51.40342733722988, "cc_by_type": { "choice": 51.40342733722988 }, "cc_tiered": 50.21057917217951, "cc_tiered_open": 49.295789751985026, "cc_tiered_sealed": 51.12536859237399, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 51.40342733722988 }, "noul": { "accuracy": 0.6026666666666667, "calibration": 85.86535266290564, "cc": 20.53333333333334, "cc_by_type": { "noul": 20.53333333333334 }, "cc_tiered": 13.114559004661515, "cc_tiered_open": 8.598343685300202, "cc_tiered_sealed": 17.63077432402283, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 20.53333333333334 }, "score": { "accuracy": 0.552, "calibration": 88.14384009361767, "cc": 36.63215814216161, "cc_by_type": { "score": 36.63215814216161 }, "cc_tiered": 35.08765995839536, "cc_tiered_open": 33.109387369959705, "cc_tiered_sealed": 37.06593254683102, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 36.63215814216161 } }, "raw_on_A": { "calibration": 79.00702916038831, "capability": 58.75187777403198, "intelligence": 38.49672638767566 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 15.03402443438624, "Emphasis on Accuracy 60:20:20": 12.868191666930013, "Emphasis on Cost 20:20:60": 14.585944618765962, "Emphasis on Speed 20:60:20": 18.769748652230398, "Intelligence only": 10.581584191343778 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (jev-adapter server, LoRA and merged weights; base Ministral 3 8B Apache-2.0)", "repo": "https://huggingface.co/StandardThinking/StandardOne-8B", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/standardone-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6765498652291105, "acc_among_decisive": 0.900398406374502, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Cloudflare", "axes": { "calibration": 91.57703482429629, "cost": 28.102157220114762, "intelligence": 59.08165501447566, "speed": 83.3998209667487 }, "calibration": { "parts": { "choice": { "ece_hard": 0.04191843750000011, "ece_score": 91.61631249999998, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 91.61631249999998 }, "noul": { "abstention_rate": 0.18400000000000005, "brier": 0.08179747616000003, "ece": 0.035773866666666626, "n": 375, "score": 92.84522666666668 }, "score": { "interval90_coverage": 0.92, "interval90_width": 0.31244444444444447, "n": 375, "nrps": 0.07581989387555554, "score": 90.26956530622223, "top_ece": 0.05939439999999997 } }, "score": 91.57703482429629 }, "capability": 75.32934491938597, "composite_ci95": { "A": [ 16.278742772255967, 17.01395343115317 ], "B": [ 16.406079295395777, 17.554401382209043 ], "C": [ 13.014738345348354, 17.01395343115317 ] }, "cost": { "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; self-hosted estimate, not measured hosted billing", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.24922681034482758 }, "display": "Clef (Cloudflare, Qwen3.8-27B post-train with a joint schema head, multimodal, measured on text)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 57.04595985248707, "I_sealed": 61.117350176464235, "base": 59.08165501447566, "excess": -6.644777966415406, "gap": -4.0713903239771625, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 76.21786468827244, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 77.35136835482857, "judge": 72.31377719182596, "standard": 69.8618668899121 } }, "open|noul": { "cc": 43.36853002070393, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 47.82608695652173, "judge": 30.000000000000004, "standard": 33.33333333333333 } }, "open|score": { "cc": 51.55148484848486, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 61.974848484848486, "hard": 50.55657407407409, "judge": 67.49000000000001, "standard": 24.421851851851862 } }, "sealed|choice": { "cc": 74.36979440941545, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 69.9660633484163, "judge": 74.86033519553072, "standard": 78.93829289321346 } }, "sealed|noul": { "cc": 52.76571930304361, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": 32.038834951456316, "judge": 63.1578947368421, "standard": 69.01408450704226 } }, "sealed|score": { "cc": 56.216536816933655, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 56.72936170212767, "hard": 49.606985294117656, "judge": 66.35520958083832, "standard": 53.971218274111656 } } } }, "jevbench_score": 16.753692861922005, "key": "clef", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 78, "ranked": true, "ranks": { "A": 78, "B": 74, "C": 68, "capability": 14 }, "scores": { "A": 16.753692861922005, "B": 17.103724421252515, "C": 16.244762021167798 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.5763746980577708, "p50_s_raw": 0.21318734902888536, "p95_s_adjusted": 0.7930726912803948, "p95_s_raw": 0.3215363456401974 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 86.28138326498332, 91.94843594390738 ], "I": [ 53.626530822718266, 63.68048125704343 ], "capability": [ 70.47069839174009, 77.24671214753253 ], "composite": { "A": [ 16.278742772255967, 17.01395343115317 ], "B": [ 16.406079295395777, 17.554401382209043 ], "C": [ 13.014738345348354, 17.01395343115317 ] }, "seed": 20 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 91.57703482429629, "capability": 75.32934491938597, "intelligence": 59.08165501447566 }, "per_type": { "choice": { "accuracy": 0.8093333333333333, "calibration": 91.61631249999998, "cc": 75.61645652359255, "cc_by_type": { "choice": 75.61645652359255 }, "cc_tiered": 75.29382954884395, "cc_tiered_open": 76.21786468827244, "cc_tiered_sealed": 74.36979440941545, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 75.61645652359255 }, "noul": { "accuracy": 0.768, "calibration": 92.84522666666668, "cc": 53.6, "cc_by_type": { "noul": 53.6 }, "cc_tiered": 48.06712466187377, "cc_tiered_open": 43.36853002070393, "cc_tiered_sealed": 52.76571930304361, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 53.6 }, "score": { "accuracy": 0.6613333333333333, "calibration": 90.26956530622223, "cc": 54.4725614035088, "cc_by_type": { "score": 54.4725614035088 }, "cc_tiered": 53.88401083270926, "cc_tiered_open": 51.55148484848486, "cc_tiered_sealed": 56.216536816933655, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 54.4725614035088 } }, "raw_on_A": { "calibration": 88.38059707442306, "capability": 75.82665436300499, "intelligence": 63.27271165158693 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 14.692533500484354, "Emphasis on Accuracy 60:20:20": 16.05926983891725, "Emphasis on Cost 20:20:60": 11.641983750272722, "Emphasis on Speed 20:60:20": 17.850768289881103, "Intelligence only": 18.663451480355018 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/Cloudflare/clef", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h", "underlying": "Qwen/Qwen3.8-27B", "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r39/outputs/meta/specs/clef.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "alt": { "label": "Workers AI price scenario (latency unmeasured)", "note": "Cloudflare standard launch input price; same self-hosted accuracy and adjusted H100 latency held fixed for this price-only scenario. No Workers AI latency was measured; this is not a hosted API result and does not set official Capability eligibility. Official self-hosted row uses the base-model reference price.", "usd_per_1000": 0.1424153201970443, "axes": { "intelligence": 67.9181689327805, "calibration": 86.72295444340945, "speed": 79.86099006822079, "cost": 35.393298680703595 }, "pricing_only": true }, "model_pin": "Cloudflare/clef@2f3de3dd85f379784083b0814d997ab627200f0c", "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.816, "acc_among_decisive": 0.9411764705882353, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "WilfLin", "axes": { "calibration": 85.03288206087805, "cost": 47.592235403520306, "intelligence": 29.251131763478327, "speed": 93.62779491136789 }, "calibration": { "parts": { "choice": { "ece_hard": 0.05599448584772476, "ece_score": 88.80110283045505, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 88.80110283045505 }, "noul": { "abstention_rate": 0.47439353099730464, "brier": 0.15464738864390087, "ece": 0.10676843098382341, "n": 371, "score": 78.64631380323533 }, "score": { "interval90_coverage": 0.975609756097561, "interval90_width": 0.6402439024390243, "n": 369, "nrps": 0.10255360461757664, "score": 87.65122954894377, "top_ece": 0.07221090220177398 } }, "score": 85.03288206087805 }, "capability": 57.14200691217819, "composite_ci95": { "A": [ 10.554100896173676, 22.576213316295057 ], "B": [ 8.924509602976766, 20.049092209565615 ], "C": [ 7.329236733453941, 15.67792591409379 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (proxy tokens, I-2): exact base-model market reference", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.05583743842364531 }, "display": "JEV Qwen3.5-9B Base NVFP4", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 30.95041263176777, "I_sealed": 27.551850895188885, "base": 29.251131763478327, "excess": 0.8251740941406402, "gap": 3.398561736578884, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 55.43165802161466, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 53.2972972972973, "hard": 47.153192827933324, "judge": 52.537903757415954, "standard": 77.39640016743407 } }, "open|noul": { "cc": 3.6694271911663243, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": -21.739130434782606, "judge": 10.000000000000009, "standard": 11.111111111111116 } }, "open|score": { "cc": 33.750152682522334, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.19655618503351, "hard": 33.2215341463409, "judge": 26.48323152009081, "standard": 33.48456974727688 } }, "sealed|choice": { "cc": 47.891397457772825, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 66.74303451333975, "hard": 41.96832579185519, "judge": 45.065176908752335, "standard": 54.55105308535534 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 24.0, "hard": -26.21359223300971, "judge": -18.42105263157895, "standard": -9.859154929577462 } }, "sealed|score": { "cc": 34.764155227793836, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 59.111326959327606, "hard": 22.60553224460797, "judge": 38.31832027118539, "standard": 41.57656776331136 } } } }, "jevbench_score": 15.97555620862577, "key": "jev-qwen3.5-9b-base-nvfp4", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 79, "ranked": true, "ranks": { "A": 79, "B": 79, "C": 79, "capability": 75 }, "scores": { "A": 15.97555620862577, "B": 13.864517363543307, "C": 11.094136255990119 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.19534559999592602, "p50_s_raw": 0.02267279999796301, "p95_s_adjusted": 0.22203266686410641, "p95_s_raw": 0.03601633343205321 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.95850301005234, 86.17596383374455 ], "I": [ 24.784047457667274, 33.682401804173935 ], "capability": [ 52.832944781851424, 58.83201835103813 ], "composite": { "A": [ 10.554100896173676, 22.576213316295057 ], "B": [ 8.924509602976766, 20.049092209565615 ], "C": [ 7.329236733453941, 15.67792591409379 ] }, "seed": 47 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.03288206087805, "capability": 57.14200691217819, "intelligence": 29.251131763478327 }, "per_type": { "choice": { "accuracy": 0.608, "calibration": 88.80110283045505, "cc": 49.86879872682661, "cc_by_type": { "choice": 49.86879872682661 }, "cc_tiered": 51.66152773969374, "cc_tiered_open": 55.43165802161466, "cc_tiered_sealed": 47.891397457772825, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 49.86879872682661 }, "noul": { "accuracy": 0.464, "calibration": 78.64631380323533, "cc": -7.199999999999996, "cc_by_type": { "noul": -7.199999999999996 }, "cc_tiered": 1.8347135955831622, "cc_tiered_open": 3.6694271911663243, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.5413333333333333, "calibration": 87.65122954894377, "cc": 35.19517866359504, "cc_by_type": { "score": 35.19517866359504 }, "cc_tiered": 34.25715395515809, "cc_tiered_open": 33.750152682522334, "cc_tiered_sealed": 34.764155227793836, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 35.19517866359504 } }, "raw_on_A": { "calibration": 80.94123658739136, "capability": 56.72695595498426, "intelligence": 32.51267532257717 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 14.120527794466737, "Emphasis on Accuracy 60:20:20": 11.548484256330752, "Emphasis on Cost 20:20:60": 14.368613309854894, "Emphasis on Speed 20:60:20": 17.771756183840008, "Intelligence only": 9.070269959573535 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 entrant and upstream checkpoint", "repo": "https://huggingface.co/WIlfLin/JEV-Qwen3.5-9B-Base-NVFP4", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen3.5-9B Base NVFP4 with compact BF16 decision head", "adapter_id": "images/jev-qwen3.5-9b-base-nvfp4/spec.json:jev_choice_http.Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5256064690026954, "acc_among_decisive": 0.8923076923076924, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Theodore Lee (TheoLeeCJ)", "axes": { "calibration": 83.59960184071504, "cost": 63.07359395281681, "intelligence": 27.10113669038628, "speed": 90.76560119421038 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10233013188094879, "ece_score": 79.53397362381024, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 79.53397362381024 }, "noul": { "abstention_rate": 0.38274932614555257, "brier": 0.1432523071034433, "ece": 0.08378278501118164, "n": 371, "score": 83.24344299776368 }, "score": { "interval90_coverage": 0.9701897018970189, "interval90_width": 0.6594399277326105, "n": 369, "nrps": 0.1212670700676031, "score": 88.0213889005712, "top_ece": 0.059152575960486446 } }, "score": 83.59960184071504 }, "capability": 55.35036926555066, "composite_ci95": { "A": [ 9.244572028428712, 23.18413753262927 ], "B": [ 7.522852111652043, 19.995799931382237 ], "C": [ 6.419841686408828, 16.100095508770327 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.01701689039408867 }, "display": "SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 30.391025070826444, "I_sealed": 23.811248309946116, "base": 27.10113669038628, "excess": 4.006389118442085, "gap": 6.579776760880328, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 49.58254501782712, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 35.8288770053476, "judge": 60.44825313117996, "standard": 47.25826705734617 } }, "open|noul": { "cc": 14.439613526570046, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 13.043478260869556, "judge": -9.999999999999998, "standard": 11.111111111111116 } }, "open|score": { "cc": 27.150916668082164, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 62.363221487255814, "hard": 12.79441887527647, "judge": 38.365166132705816, "standard": 21.436385647171228 } }, "sealed|choice": { "cc": 45.71562508055857, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 61.4219200354741, "hard": 39.93212669683257, "judge": 45.99627560521416, "standard": 49.008498583569406 } }, "sealed|noul": { "cc": 2.17292780700555, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 43.99999999999999, "hard": -22.330097087378643, "judge": 2.6315789473684292, "standard": 29.5774647887324 } }, "sealed|score": { "cc": 23.545192042274238, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 63.77035344302047, "hard": 13.542770599427534, "judge": 22.665102695300575, "standard": 24.757588248055008 } } } }, "jevbench_score": 15.517179498801362, "key": "semif-qwen3.5-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 80, "ranked": true, "ranks": { "A": 80, "B": 80, "C": 80, "capability": 81 }, "scores": { "A": 15.517179498801362, "B": 13.04205072010579, "C": 10.775819096389835 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.25408456099685284, "p50_s_raw": 0.05204228049842641, "p95_s_adjusted": 0.32996023471932856, "p95_s_raw": 0.08998011735966428 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.25331982816681, 85.80759674871754 ], "I": [ 22.076409045997522, 31.830294818443246 ], "capability": [ 51.05521406162153, 58.22417544189523 ], "composite": { "A": [ 9.244572028428712, 23.18413753262927 ], "B": [ 7.522852111652043, 19.995799931382237 ], "C": [ 6.419841686408828, 16.100095508770327 ] }, "seed": 99 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.59960184071504, "capability": 55.35036926555066, "intelligence": 27.10113669038628 }, "per_type": { "choice": { "accuracy": 0.5893333333333334, "calibration": 79.53397362381024, "cc": 47.48159866619932, "cc_by_type": { "choice": 47.48159866619932 }, "cc_tiered": 47.64908504919285, "cc_tiered_open": 49.58254501782712, "cc_tiered_sealed": 45.71562508055857, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 47.48159866619932 }, "noul": { "accuracy": 0.552, "calibration": 83.24344299776368, "cc": 10.40000000000001, "cc_by_type": { "noul": 10.40000000000001 }, "cc_tiered": 8.306270666787798, "cc_tiered_open": 14.439613526570046, "cc_tiered_sealed": 2.17292780700555, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 10.40000000000001 }, "score": { "accuracy": 0.49333333333333335, "calibration": 88.0213889005712, "cc": 25.84341933063762, "cc_by_type": { "score": 25.84341933063762 }, "cc_tiered": 25.3480543551782, "cc_tiered_open": 27.150916668082164, "cc_tiered_sealed": 23.545192042274238, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 25.84341933063762 } }, "raw_on_A": { "calibration": 83.04331750428547, "capability": 58.219039699698534, "intelligence": 33.3947618951116 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 13.820858864848555, "Emphasis on Accuracy 60:20:20": 10.67791560389241, "Emphasis on Cost 20:20:60": 15.384875177789313, "Emphasis on Speed 20:60:20": 17.119455792629843, "Intelligence only": 7.962006198162448 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT (code); Qwen3.5 weights Apache-2.0", "repo": "https://github.com/TheoLeeCJ/openjev", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen/Qwen3.5-4B (frozen, BF16)", "adapter_id": "images/semif-qwen3.5-4b/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6172506738544474, "acc_among_decisive": 0.9039301310043668, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Reza Sayar", "axes": { "calibration": 88.50232899798883, "cost": 28.231834531826323, "intelligence": 43.77513601921869, "speed": 72.98222461463935 }, "calibration": { "parts": { "choice": { "ece_hard": 0.08046726298491785, "ece_score": 83.90654740301643, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 83.90654740301643 }, "noul": { "abstention_rate": 0.3045822102425876, "brier": 0.11954030846459725, "ece": 0.04318003176619355, "n": 371, "score": 91.36399364676129 }, "score": { "interval90_coverage": 0.964769647696477, "interval90_width": 0.4652213188798555, "n": 369, "nrps": 0.08597620729262767, "score": 90.23644594418879, "top_ece": 0.0546474369117983 } }, "score": 88.50232899798883 }, "capability": 66.13873250860377, "composite_ci95": { "A": [ 8.55065455714213, 14.587291870035246 ], "B": [ 8.22423019498041, 14.527264406104972 ], "C": [ 5.9379545535709255, 10.130063798635586 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate. Frozen market reference qwen/qwen3.8-27b.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.2467585344827586 }, "display": "Bev / Bonsai 27B", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 46.71700087936099, "I_sealed": 40.833271159076396, "base": 43.77513601921869, "excess": 3.310342077846348, "gap": 5.883729720284592, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 63.31036599788361, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 47.153192827933324, "judge": 72.31377719182596, "standard": 77.39640016743407 } }, "open|noul": { "cc": 30.23947550034507, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 39.13043478260869, "judge": 10.000000000000009, "standard": 22.222222222222232 } }, "open|score": { "cc": 46.6011611398543, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.472258307241056, "hard": 30.60094415356245, "judge": 62.977497036063276, "standard": 48.101542684431145 } }, "sealed|choice": { "cc": 56.58157537232299, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 66.74303451333975, "hard": 49.09502262443438, "judge": 53.44506517690876, "standard": 71.17871659071314 } }, "sealed|noul": { "cc": 20.312021130359053, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 52.0, "hard": -0.9708737864077666, "judge": 26.315789473684205, "standard": 38.0281690140845 } }, "sealed|score": { "cc": 45.60621697454716, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 67.9108636782132, "hard": 33.06846017805201, "judge": 52.300237792599205, "standard": 49.48837598862635 } } } }, "jevbench_score": 11.7393704396132, "key": "bev-bonsai-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 81, "ranked": true, "ranks": { "A": 81, "B": 81, "C": 81, "capability": 48 }, "scores": { "A": 11.7393704396132, "B": 11.515061943796129, "C": 8.152340583064722 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 2.1222116896882652, "p50_s_raw": 0.9861058448441327, "p95_s_adjusted": 2.371312748454511, "p95_s_raw": 1.1106563742272555 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.75270117761126, 88.77503324811279 ], "I": [ 38.25187875678337, 48.17301489659999 ], "capability": [ 60.910547743797885, 67.5609085693478 ], "composite": { "A": [ 8.55065455714213, 14.587291870035246 ], "B": [ 8.22423019498041, 14.527264406104972 ], "C": [ 5.9379545535709255, 10.130063798635586 ] }, "seed": 17 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.50232899798883, "capability": 66.13873250860377, "intelligence": 43.77513601921869 }, "per_type": { "choice": { "accuracy": 0.672, "calibration": 83.90654740301643, "cc": 58.053484648977374, "cc_by_type": { "choice": 58.053484648977374 }, "cc_tiered": 59.9459706851033, "cc_tiered_open": 63.31036599788361, "cc_tiered_sealed": 56.58157537232299, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 58.053484648977374 }, "noul": { "accuracy": 0.6293333333333333, "calibration": 91.36399364676129, "cc": 25.86666666666666, "cc_by_type": { "noul": 25.86666666666666 }, "cc_tiered": 25.27574831535206, "cc_tiered_open": 30.23947550034507, "cc_tiered_sealed": 20.312021130359053, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 25.86666666666666 }, "score": { "accuracy": 0.6, "calibration": 90.23644594418879, "cc": 45.43825770414722, "cc_by_type": { "score": 45.43825770414722 }, "cc_tiered": 46.103689057200725, "cc_tiered_open": 46.6011611398543, "cc_tiered_sealed": 45.60621697454716, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 45.43825770414722 } }, "raw_on_A": { "calibration": 80.9895865212274, "capability": 63.89601132732785, "intelligence": 46.8024361334283 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 10.186877852314403, "Emphasis on Accuracy 60:20:20": 10.385148364433647, "Emphasis on Cost 20:20:60": 8.555937486887114, "Emphasis on Speed 20:60:20": 12.296000304326956, "Intelligence only": 10.697460589059927 }, "class": "system-one-open", "open": "yes", "licence": "Not independently recorded", "repo": "https://huggingface.co/Reza2kn/Bev", "gpu": "H100", "endpoint_condition": "evaluator-owned H100 GPU", "underlying": "Reza2kn/Bev @ f01edf952e3f2da8147aa08ec6d6229c863d5d19. Unchanged Prism ML Ternary-Bonsai-2-27B PQ2_0 quantization of Qwen3.8-27B, with selected-token readout; not a fine-tune. GGUF SHA-256 3907dc1658db1f78a9826bf8d5bcb8dc65db0d466388937af57f2294fae62ec1. H100; CUDA backend compiled from the author-pinned source 9a9394a895b96003ca842a6041cb28ac49a108f7 with the author patch, rather than their prebuilt binary. The wire shim drops noul criteria because the submitted schema rejects them; that loses option descriptions for noul.", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6954177897574124, "acc_among_decisive": 0.9147286821705426, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Zhengxu Yu", "axes": { "calibration": 89.9210925231847, "cost": 28.35809912639418, "intelligence": 43.28549422942267, "speed": 68.17671225113584 }, "calibration": { "parts": { "choice": { "ece_hard": 0.054566021834308726, "ece_score": 89.08679563313827, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 89.08679563313827 }, "noul": { "abstention_rate": 0.33692722371967654, "brier": 0.09143307684062531, "ece": 0.03620776305464309, "n": 371, "score": 92.75844738907139 }, "score": { "interval90_coverage": 0.981029810298103, "interval90_width": 0.534327009936766, "n": 369, "nrps": 0.08453630674044608, "score": 87.91803454734445, "top_ece": 0.07855150115633251 } }, "score": 89.9210925231847 }, "capability": 66.60329337630368, "composite_ci95": { "A": [ 8.746773766553591, 14.766757637183968 ], "B": [ 8.426311974704314, 14.74912210872967 ], "C": [ 6.074148448995549, 10.25469280359998 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.2443787068965517 }, "display": "LitJev (Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 41.98760951977819, "I_sealed": 44.583378939067146, "base": 43.28549422942267, "excess": -5.1691570617272, "gap": -2.595769419288956, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 68.57906399743361, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 84.43243243243244, "hard": 64.1396665618119, "judge": 68.35860250494396, "standard": 69.8618668899121 } }, "open|noul": { "cc": 12.532781228433404, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 4.347826086956519, "judge": 0.0, "standard": 11.111111111111116 } }, "open|score": { "cc": 44.85098333346756, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 58.30288571369996, "hard": 36.97174671674506, "judge": 54.42696015026953, "standard": 39.519540151593404 } }, "sealed|choice": { "cc": 68.65215618863985, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 81.37609932747026, "hard": 60.29411764705882, "judge": 66.4804469273743, "standard": 82.26382559428504 } }, "sealed|noul": { "cc": 24.95501594133015, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": -0.9708737864077666, "judge": 28.947368421052634, "standard": 49.29577464788733 } }, "sealed|score": { "cc": 40.142964687231455, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 64.90525292252141, "hard": 32.24057461852555, "judge": 45.24652806264088, "standard": 35.91125564388415 } } } }, "jevbench_score": 11.458886170296882, "key": "litjev", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 82, "ranked": true, "ranks": { "A": 82, "B": 82, "C": 82, "capability": 46 }, "scores": { "A": 11.458886170296882, "B": 11.238393678896292, "C": 7.957559840483947 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 3.0960148913785814, "p50_s_raw": 1.4730074456892908, "p95_s_adjusted": 4.9150250315666195, "p95_s_raw": 2.3825125157833096 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.4774023239196, 89.78804799226427 ], "I": [ 38.555280103835365, 48.498383980718195 ], "capability": [ 62.48585734643129, 68.14178334863401 ], "composite": { "A": [ 8.746773766553591, 14.766757637183968 ], "B": [ 8.426311974704314, 14.74912210872967 ], "C": [ 6.074148448995549, 10.25469280359998 ] }, "seed": 61 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.9210925231847, "capability": 66.60329337630368, "intelligence": 43.28549422942267 }, "per_type": { "choice": { "accuracy": 0.7613333333333333, "calibration": 89.08679563313827, "cc": 69.47794208197946, "cc_by_type": { "choice": 69.47794208197946 }, "cc_tiered": 68.61561009303674, "cc_tiered_open": 68.57906399743361, "cc_tiered_sealed": 68.65215618863985, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 69.47794208197946 }, "noul": { "accuracy": 0.64, "calibration": 92.75844738907139, "cc": 28.000000000000004, "cc_by_type": { "noul": 28.000000000000004 }, "cc_tiered": 18.743898584881777, "cc_tiered_open": 12.532781228433404, "cc_tiered_sealed": 24.95501594133015, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 28.000000000000004 }, "score": { "accuracy": 0.5626666666666666, "calibration": 87.91803454734445, "cc": 41.34354803654359, "cc_by_type": { "score": 41.34354803654359 }, "cc_tiered": 42.496974010349504, "cc_tiered_open": 44.85098333346756, "cc_tiered_sealed": 40.142964687231455, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 41.34354803654359 } }, "raw_on_A": { "calibration": 89.32478284904047, "capability": 69.19315337175932, "intelligence": 49.06152389447817 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 9.902804220753023, "Emphasis on Accuracy 60:20:20": 10.109112973826948, "Emphasis on Cost 20:20:60": 8.396442160471732, "Emphasis on Speed 20:60:20": 11.774987261719835, "Intelligence only": 10.435213933727047 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (code); Apache-2.0 base weights", "repo": "https://github.com/zhengxuyu/litjev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen/Qwen3.8-27B, frozen, read at the output head", "adapter_id": "images/litjev/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6630727762803235, "acc_among_decisive": 0.975609756097561, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Amith Chandrappa (amithgc)", "axes": { "calibration": 85.84286362718517, "cost": 59.269627658320445, "intelligence": 24.145363218466134, "speed": 86.10068849774444 }, "calibration": { "parts": { "choice": { "ece_hard": 0.039143749999999984, "ece_score": 92.17125, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 92.17125 }, "noul": { "abstention_rate": 0.648, "brier": 0.14289767218666674, "ece": 0.09039973333333337, "n": 375, "score": 81.92005333333331 }, "score": { "interval90_coverage": 0.96, "interval90_width": 0.6557777777777777, "n": 375, "nrps": 0.13420838236888896, "score": 83.4372875482222, "top_ece": 0.0985229333333334 } }, "score": 85.84286362718517 }, "capability": 54.99411342282565, "composite_ci95": { "A": [ 6.925385954427524, 15.24148889625826 ], "B": [ 5.568022650808403, 12.90281361284615 ], "C": [ 4.8092958016857805, 10.584367289068235 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.022786576354679802 }, "display": "local-jev Qwen3.5-4B", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 26.448995848149686, "I_sealed": 21.841730588782582, "base": 24.145363218466134, "excess": 2.03387761692886, "gap": 4.607265259367104, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 52.23659737153346, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 41.49103491664046, "judge": 60.44825313117996, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 28.57142857142858, "hard": -56.52173913043479, "judge": -30.000000000000004, "standard": -33.333333333333336 } }, "open|score": { "cc": 27.1103901729156, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 54.470606060606066, "hard": 15.197777777777777, "judge": 40.368135593220345, "standard": 17.36888888888889 } }, "sealed|choice": { "cc": 46.955426808141326, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 61.4219200354741, "hard": 42.477375565610856, "judge": 45.99627560521416, "standard": 50.1170094839266 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 12.00000000000001, "hard": -70.87378640776699, "judge": -18.42105263157895, "standard": -15.492957746478876 } }, "sealed|score": { "cc": 18.56976495820642, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 56.15659574468085, "hard": 12.02813725490197, "judge": 13.700838323353281, "standard": 20.162994923857813 } } } }, "jevbench_score": 11.438161297684031, "key": "localjev-qwen3.5-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 83, "ranked": true, "ranks": { "A": 83, "B": 84, "C": 83, "capability": 82 }, "scores": { "A": 11.438161297684031, "B": 9.482181005106208, "C": 7.943167567836132 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.39534114915877583, "p50_s_raw": 0.1226705745793879, "p95_s_adjusted": 0.6208106089383364, "p95_s_raw": 0.2354053044691682 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 79.50985112092962, 86.15050973358518 ], "I": [ 19.830618866047317, 27.13596492581447 ], "capability": [ 50.794515955411754, 55.552509505023664 ], "composite": { "A": [ 6.925385954427524, 15.24148889625826 ], "B": [ 5.568022650808403, 12.90281361284615 ], "C": [ 4.8092958016857805, 10.584367289068235 ] }, "seed": 62 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 85.84286362718517, "capability": 54.99411342282565, "intelligence": 24.145363218466134 }, "per_type": { "choice": { "accuracy": 0.6026666666666667, "calibration": 92.17125, "cc": 49.186741566647385, "cc_by_type": { "choice": 49.186741566647385 }, "cc_tiered": 49.596012089837394, "cc_tiered_open": 52.23659737153346, "cc_tiered_sealed": 46.955426808141326, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 49.186741566647385 }, "noul": { "accuracy": 0.34933333333333333, "calibration": 81.92005333333331, "cc": -30.133333333333333, "cc_by_type": { "noul": -30.133333333333333 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.45066666666666666, "calibration": 83.4372875482222, "cc": 22.259017543859695, "cc_by_type": { "score": 22.259017543859695 }, "cc_tiered": 22.84007756556101, "cc_tiered_open": 27.1103901729156, "cc_tiered_sealed": 18.56976495820642, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 22.259017543859695 } }, "raw_on_A": { "calibration": 81.40355743693019, "capability": 54.83642941083503, "intelligence": 28.26930138473988 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 10.008247462291514, "Emphasis on Accuracy 60:20:20": 7.6341780052667865, "Emphasis on Cost 20:20:60": 11.24977392613748, "Emphasis on Speed 20:60:20": 12.519995559254454, "Intelligence only": 5.630684842640914 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT code; Apache-2.0 Qwen weights", "repo": "https://github.com/amithgc/local-jev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen/Qwen3.5-4B text model, zero-shot next-token letter probabilities", "adapter_id": "images/localjev-qwen3.5-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.352, "acc_among_decisive": 0.9924242424242424, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Sean Goedecke", "axes": { "calibration": 30.16473321941294, "cost": 44.96855803039237, "intelligence": 29.271508779158296, "speed": 90.8867694864598 }, "calibration": { "parts": { "choice": { "ece_hard": 0.5207165079484644, "ece_score": 0.0, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 0.0 }, "noul": { "abstention_rate": 0.07547169811320753, "brier": 0.26938609640716954, "ece": 0.26647263490315864, "n": 371, "score": 46.70547301936827 }, "score": { "interval90_coverage": 0.5718157181571816, "interval90_width": 0.08897922312556458, "n": 369, "nrps": 0.18780642533502573, "score": 43.78872663887056, "top_ece": 0.46820952094378154 } }, "score": 30.16473321941294 }, "capability": 29.718120999285617, "composite_ci95": { "A": [ 5.879902468266102, 15.68532110290842 ], "B": [ 5.313278874085685, 14.848330476734906 ], "C": [ 4.08326560296257, 10.892584099241958 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.06829381896551724 }, "display": "system-one (Qwen3-8B, Sean Goedecke)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 32.67027386076885, "I_sealed": 25.87274369754774, "base": 29.271508779158296, "excess": 4.224142520782868, "gap": 6.797530163221111, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 45.21508118936821, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 30.16671909405474, "judge": 48.58272907053395, "standard": 62.32733361239012 } }, "open|noul": { "cc": 28.967563837129056, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": 19.999999999999996, "standard": 11.111111111111116 } }, "open|score": { "cc": 23.8281765558093, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 65.82115115650328, "hard": 4.701913776879685, "judge": 35.713306286008326, "standard": 23.25652021802299 } }, "sealed|choice": { "cc": 35.384978952910906, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 54.77052693814206, "hard": 19.57013574660632, "judge": 35.7541899441341, "standard": 56.76807488606972 } }, "sealed|noul": { "cc": 28.751778735776956, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": -2.9126213592232997, "judge": 44.736842105263165, "standard": 46.47887323943662 } }, "sealed|score": { "cc": 13.48147340395537, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 53.14233360647993, "hard": 0.26200782926150135, "judge": 16.1186769737661, "standard": 16.134169097364737 } } } }, "jevbench_score": 11.027723483913586, "key": "system-one-sg", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 84, "ranked": true, "ranks": { "A": 84, "B": 83, "C": 84, "capability": 126 }, "scores": { "A": 11.027723483913586, "B": 10.28901604793032, "C": 7.658141308273324 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.24992560098180547, "p50_s_raw": 0.04996280049090274, "p95_s_adjusted": 0.3262212941306643, "p95_s_raw": 0.08811064706533216 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 26.715281574276638, 33.82082873066771 ], "I": [ 22.79600693038093, 33.62231882810145 ], "capability": [ 25.123779554066562, 33.19293706149128 ], "composite": { "A": [ 5.879902468266102, 15.68532110290842 ], "B": [ 5.313278874085685, 14.848330476734906 ], "C": [ 4.08326560296257, 10.892584099241958 ] }, "seed": 105 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 30.16473321941294, "capability": 29.718120999285617, "intelligence": 29.271508779158296 }, "per_type": { "choice": { "accuracy": 0.5133333333333333, "calibration": 0.0, "cc": 37.76228413364529, "cc_by_type": { "choice": 37.76228413364529 }, "cc_tiered": 40.30003007113956, "cc_tiered_open": 45.21508118936821, "cc_tiered_sealed": 35.384978952910906, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 37.76228413364529 }, "noul": { "accuracy": 0.6666666666666666, "calibration": 46.70547301936827, "cc": 33.33333333333333, "cc_by_type": { "noul": 33.33333333333333 }, "cc_tiered": 28.859671286453008, "cc_tiered_open": 28.967563837129056, "cc_tiered_sealed": 28.751778735776956, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 33.33333333333333 }, "score": { "accuracy": 0.45066666666666666, "calibration": 43.78872663887056, "cc": 16.56590736182537, "cc_by_type": { "score": 16.56590736182537 }, "cc_tiered": 18.654824979882335, "cc_tiered_open": 23.8281765558093, "cc_tiered_sealed": 13.48147340395537, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 16.56590736182537 } }, "raw_on_A": { "calibration": 29.76734864606182, "capability": 30.080860832336057, "intelligence": 30.394373018610295 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 12.338652673779826, "Emphasis on Accuracy 60:20:20": 10.21232996468921, "Emphasis on Cost 20:20:60": 12.38939107483353, "Emphasis on Speed 20:60:20": 15.503079386401165, "Intelligence only": 8.114714247763947 }, "class": "jev-rebuild", "open": "yes", "licence": "no licence file in the repository as of 19 Sep; Qwen3 weights Apache-2.0", "repo": "https://github.com/sgoedecke/system-one", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen/Qwen3-8B (frozen, BF16), the model of the author's demos", "adapter_id": "images/system-one-sg/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9245283018867925, "acc_among_decisive": 0.7288629737609329, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Jared Palmer", "axes": { "calibration": 70.7561729087212, "cost": 40.42054386073457, "intelligence": 29.26217941280341, "speed": 88.97143473610706 }, "calibration": { "parts": { "choice": { "ece_hard": 0.14254951638954755, "ece_score": 71.49009672209048, "mean_tvd": null, "n": 736, "n_ece_hard": 306, "n_tvd": 0, "score": 71.49009672209048 }, "noul": { "abstention_rate": 0.23450134770889486, "brier": 0.18970835579514836, "ece": 0.16199460916442043, "n": 371, "score": 67.60107816711592 }, "score": { "interval90_coverage": 0.8233695652173914, "interval90_width": 0.382925724637681, "n": 368, "nrps": 0.13575035733208884, "score": 73.17734383695722, "top_ece": 0.20035138296438337 } }, "score": 70.7561729087212 }, "capability": 50.0091761607623, "composite_ci95": { "A": [ 6.289030902475565, 16.488978582767544 ], "B": [ 5.395378045720429, 15.122131514496889 ], "C": [ 4.367382571163586, 11.45067957136635 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.09682374384236453 }, "display": "kev 8B (research preview)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 28.412661059008222, "I_sealed": 30.111697766598596, "base": 29.26217941280341, "excess": -4.272424350028617, "gap": -1.6990367075903734, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 53.56342531883604, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 43.378420887071414, "judge": 48.58272907053395, "standard": 69.8618668899121 } }, "open|noul": { "cc": 15.995169082125601, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 13.043478260869556, "judge": 10.000000000000009, "standard": -11.111111111111116 } }, "open|score": { "cc": 15.679388776063027, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 43.66574839302112, "hard": 33.21276294296097, "judge": 29.41280568734839, "standard": -53.98066473313996 } }, "sealed|choice": { "cc": 48.755934054679, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 61.4219200354741, "hard": 41.96832579185519, "judge": 49.72067039106147, "standard": 54.55105308535534 } }, "sealed|noul": { "cc": 19.06361274442773, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 43.99999999999999, "hard": -18.446601941747577, "judge": 50.0, "standard": 35.2112676056338 } }, "sealed|score": { "cc": 22.51554650068906, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 58.4626547761159, "hard": 14.194388556502712, "judge": 23.600291166841224, "standard": 19.557191252120088 } } } }, "jevbench_score": 10.62283914209564, "key": "kev-8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 85, "ranked": true, "ranks": { "A": 85, "B": 85, "C": 85, "capability": 86 }, "scores": { "A": 10.62283914209564, "B": 9.44790466894511, "C": 7.3769716264553065 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2989678788930178, "p50_s_raw": 0.07448393944650888, "p95_s_adjusted": 0.42386933271773153, "p95_s_raw": 0.13693466635886578 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 65.36869404076147, 74.24437950494625 ], "I": [ 23.662522876918093, 35.13692561855455 ], "capability": [ 45.238684166857645, 53.83400001933101 ], "composite": { "A": [ 6.289030902475565, 16.488978582767544 ], "B": [ 5.395378045720429, 15.122131514496889 ], "C": [ 4.367382571163586, 11.45067957136635 ] }, "seed": 55 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 70.7561729087212, "capability": 50.0091761607623, "intelligence": 29.26217941280341 }, "per_type": { "choice": { "accuracy": 0.612, "calibration": 71.49009672209048, "cc": 50.38034159696103, "cc_by_type": { "choice": 50.38034159696103 }, "cc_tiered": 51.15967968675752, "cc_tiered_open": 53.56342531883604, "cc_tiered_sealed": 48.755934054679, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 50.38034159696103 }, "noul": { "accuracy": 0.6106666666666667, "calibration": 67.60107816711592, "cc": 22.13333333333334, "cc_by_type": { "noul": 22.13333333333334 }, "cc_tiered": 17.529390913276664, "cc_tiered_open": 15.995169082125601, "cc_tiered_sealed": 19.06361274442773, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 22.13333333333334 }, "score": { "accuracy": 0.49866666666666665, "calibration": 73.17734383695722, "cc": 23.802092840863022, "cc_by_type": { "score": 23.802092840863022 }, "cc_tiered": 19.097467638376042, "cc_tiered_open": 15.679388776063027, "cc_tiered_sealed": 22.51554650068906, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 23.802092840863022 } }, "raw_on_A": { "calibration": 67.24332350856244, "capability": 50.76538650167667, "intelligence": 34.2874494947909 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01866666666666672, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 9.572183421009896, "Emphasis on Accuracy 60:20:20": 8.080814281596565, "Emphasis on Cost 20:20:60": 9.355272676593998, "Emphasis on Speed 20:60:20": 12.08216537216834, "Intelligence only": 6.550045158904166 }, "class": "jev-rebuild", "open": true, "licence": "Apache-2.0", "repo": "https://github.com/jaredpalmer/kev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-8B-Base + LoRA + learned pointer head; jaredpalmer/kev-8b", "adapter_id": "images/kev-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7654986522911051, "acc_among_decisive": 0.8063380281690141, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "FlyMy.AI (@denti)", "axes": { "calibration": 86.77033341356835, "cost": 66.37275756963365, "intelligence": 22.875473256337628, "speed": 90.39462801231525 }, "calibration": { "parts": { "choice": { "ece_hard": 0.060081291070015405, "ece_score": 87.98374178599691, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 87.98374178599691 }, "noul": { "abstention_rate": 0.6091644204851752, "brier": 0.15626137202130647, "ece": 0.06929060503851944, "n": 371, "score": 86.1418789922961 }, "score": { "interval90_coverage": 0.9539295392953929, "interval90_width": 0.646115627822945, "n": 369, "nrps": 0.13027713115375125, "score": 86.18537946241204, "top_ece": 0.07300763979900397 } }, "score": 86.77033341356835 }, "capability": 54.82290333495299, "composite_ci95": { "A": [ 6.070784931981477, 14.301728447276645 ], "B": [ 4.787007542130122, 11.888762823793376 ], "C": [ 4.215822869431582, 9.93175586616434 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.013210160098522166 }, "display": "Decision 2B (FlyMy.AI, v59)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 24.479797267586093, "I_sealed": 21.271149245089163, "base": 22.875473256337628, "excess": 0.6352603800586869, "gap": 3.2086480224969307, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 50.388518138122464, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 45.26580685750237, "judge": 40.672379696769944, "standard": 69.8618668899121 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 14.28571428571428, "hard": -56.52173913043479, "judge": -19.999999999999996, "standard": 0.0 } }, "open|score": { "cc": 23.050873664635812, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 20.73971356294858, "hard": 15.558133148961872, "judge": 38.430985047283826, "standard": 16.12176767285528 } }, "sealed|choice": { "cc": 45.220796675917924, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 60.0916414160077, "hard": 37.38687782805428, "judge": 42.271880819366864, "standard": 57.876585786426915 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -4.0000000000000036, "hard": -53.398058252427184, "judge": -23.684210526315784, "standard": -18.309859154929576 } }, "sealed|score": { "cc": 18.59265105934957, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 27.03021479008173, "hard": 11.986008435765372, "judge": 25.699675889534046, "standard": 16.92661719587518 } } } }, "jevbench_score": 10.289708493744515, "key": "decision-2b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 86, "ranked": true, "ranks": { "A": 86, "B": 88, "C": 86, "capability": 83 }, "scores": { "A": 10.289708493744515, "B": 8.367001374285444, "C": 7.14563089843369 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.30028566285036506, "p50_s_raw": 0.07514283142518252, "p95_s_adjusted": 0.30409032410243525, "p95_s_raw": 0.07704516205121763 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 80.98496799808474, 88.16729740402266 ], "I": [ 18.649938213615208, 26.096089404676093 ], "capability": [ 50.712178970237844, 56.11601608037113 ], "composite": { "A": [ 6.070784931981477, 14.301728447276645 ], "B": [ 4.787007542130122, 11.888762823793376 ], "C": [ 4.215822869431582, 9.93175586616434 ] }, "seed": 29 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.77033341356835, "capability": 54.82290333495299, "intelligence": 22.875473256337628 }, "per_type": { "choice": { "accuracy": 0.5853333333333334, "calibration": 87.98374178599691, "cc": 46.9700557960649, "cc_by_type": { "choice": 46.9700557960649 }, "cc_tiered": 47.804657407020194, "cc_tiered_open": 50.388518138122464, "cc_tiered_sealed": 45.220796675917924, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 46.9700557960649 }, "noul": { "accuracy": 0.3626666666666667, "calibration": 86.1418789922961, "cc": -27.46666666666666, "cc_by_type": { "noul": -27.46666666666666 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.4266666666666667, "calibration": 86.18537946241204, "cc": 18.767847334171428, "cc_by_type": { "score": 18.767847334171428 }, "cc_tiered": 20.82176236199269, "cc_tiered_open": 23.050873664635812, "cc_tiered_sealed": 18.59265105934957, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 18.767847334171428 } }, "raw_on_A": { "calibration": 80.16146678308505, "capability": 51.30498083159781, "intelligence": 22.448494880110573 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 8.990679348778704, "Emphasis on Accuracy 60:20:20": 6.6544725899582415, "Emphasis on Cost 20:20:60": 10.468172682915556, "Emphasis on Speed 20:60:20": 11.379622803195113, "Intelligence only": 4.788177641426015 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release", "repo": "https://huggingface.co/flymy-ai/decision-2b-preview", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "openbmb/MiniCPM5-2B with a trained LoRA adapter and pointer head (26.2M trainable parameters)", "adapter_id": "images/decision-2b/spec.json:flymy_v15.FlyMyPackage", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.3908355795148248, "acc_among_decisive": 0.9379310344827586, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "sabeel111", "axes": { "calibration": 73.92484912653177, "cost": 69.13657649738097, "intelligence": 23.217995107845883, "speed": 77.06362388237805 }, "calibration": { "parts": { "choice": { "ece_hard": 0.20911848208469058, "ece_score": 58.17630358306188, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 58.17630358306188 }, "noul": { "abstention_rate": 0.5121293800539084, "brier": 0.1552006734049543, "ece": 0.11625450134770884, "n": 371, "score": 76.74909973045823 }, "score": { "interval90_coverage": 0.9566395663956639, "interval90_width": 0.5600722673893404, "n": 369, "nrps": 0.11625319455925476, "score": 86.84914406607521, "top_ece": 0.07338196205962055 } }, "score": 73.92484912653177 }, "capability": 48.571422117188824, "composite_ci95": { "A": [ 6.483328619758367, 14.579365175086792 ], "B": [ 5.195391797769789, 12.338152010991331 ], "C": [ 4.502311541498866, 10.124559149365826 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0106851539408867 }, "display": "OpenSourceJev (Qwen3.5-4B Q4_K_M, native llama.cpp)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 24.81038701116274, "I_sealed": 21.62560320452903, "base": 23.217995107845883, "excess": 0.6113961641954666, "gap": 3.1847838066337104, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 41.32946193915506, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 24.50456118276188, "judge": 64.40342781806196, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": -4.347826086956519, "judge": -30.000000000000004, "standard": -22.22222222222222 } }, "open|score": { "cc": 33.101699094333156, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 67.34023030303031, "hard": 17.52584074074074, "judge": 47.55760169491523, "standard": 25.450296296296294 } }, "sealed|choice": { "cc": 35.66176229147756, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 48.11913384081, "hard": 23.133484162895915, "judge": 42.271880819366864, "standard": 44.57445498214066 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 24.0, "hard": -30.097087378640776, "judge": -7.8947368421052655, "standard": -4.225352112676061 } }, "sealed|score": { "cc": 29.21504732210953, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 58.979268794326224, "hard": 18.989292892156882, "judge": 31.412349700598806, "standard": 31.488491878172564 } } } }, "jevbench_score": 10.263493713193125, "key": "opensourcejev-qwen35-4b-q4km", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 87, "ranked": true, "ranks": { "A": 87, "B": 86, "C": 87, "capability": 90 }, "scores": { "A": 10.263493713193125, "B": 8.482178412964995, "C": 7.127426189717446 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.7395113246515393, "p50_s_raw": 0.29475566232576966, "p95_s_adjusted": 2.6588435446377843, "p95_s_raw": 1.2544217723188922 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 68.41904601995815, 77.70943106496745 ], "I": [ 19.379013533491335, 26.692735209928347 ], "capability": [ 44.976218992193104, 51.58836189201757 ], "composite": { "A": [ 6.483328619758367, 14.579365175086792 ], "B": [ 5.195391797769789, 12.338152010991331 ], "C": [ 4.502311541498866, 10.124559149365826 ] }, "seed": 84 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.92484912653177, "capability": 48.571422117188824, "intelligence": 23.217995107845883 }, "per_type": { "choice": { "accuracy": 0.5093333333333333, "calibration": 58.17630358306188, "cc": 37.25074126351086, "cc_by_type": { "choice": 37.25074126351086 }, "cc_tiered": 38.495612115316305, "cc_tiered_open": 41.32946193915506, "cc_tiered_sealed": 35.66176229147756, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 37.25074126351086 }, "noul": { "accuracy": 0.45866666666666667, "calibration": 76.74909973045823, "cc": -8.266666666666666, "cc_by_type": { "noul": -8.266666666666666 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.544, "calibration": 86.84914406607521, "cc": 30.80778824561413, "cc_by_type": { "score": 30.80778824561413 }, "cc_tiered": 31.15837320822134, "cc_tiered_open": 33.101699094333156, "cc_tiered_sealed": 29.21504732210953, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 30.80778824561413 } }, "raw_on_A": { "calibration": 73.9791977813484, "capability": 49.500913389632025, "intelligence": 25.022628997915646 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 9.174389111825883, "Emphasis on Accuracy 60:20:20": 6.882520870049428, "Emphasis on Cost 20:20:60": 10.842363781630336, "Emphasis on Speed 20:60:20": 11.176826098501683, "Intelligence only": 5.0064990418048145 }, "class": "jev-rebuild", "open": "unknown", "licence": "MIT (repository code); Apache-2.0 (Qwen/Qwen3.5-4B base and unsloth/Qwen3.5-4B-GGUF Q4_K_M conversion)", "repo": "https://github.com/sabeel111/OpenSourceJev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": null, "adapter_id": "images/opensourcejev-qwen35-4b-q4km/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.48787061994609165, "acc_among_decisive": 0.9502762430939227, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "IkerMoel", "axes": { "calibration": 75.94384178347806, "cost": 63.27859776193043, "intelligence": 22.34610151162883, "speed": 91.30216983336405 }, "calibration": { "parts": { "choice": { "ece_hard": 0.15207997234343346, "ece_score": 69.58400553131331, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 69.58400553131331 }, "noul": { "abstention_rate": 0.4, "brier": 0.1703546052656481, "ece": 0.0957881739676287, "n": 375, "score": 80.84236520647426 }, "score": { "interval90_coverage": 0.8933333333333333, "interval90_width": 0.5735555555555554, "n": 375, "nrps": 0.14203249622577255, "score": 77.40515461264664, "top_ece": 0.1549322057606473 } }, "score": 75.94384178347806 }, "capability": 49.14497164755345, "composite_ci95": { "A": [ 6.323116186163818, 15.819609663846963 ], "B": [ 5.049803290865915, 13.396655191684498 ], "C": [ 4.391052907058209, 10.985840044338168 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (proxy tokens, I-2): exact base-model market reference; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0167512315270936 }, "display": "open-alternative-jev (Qwen3.5-4B, IkerMoel)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 25.032294599047653, "I_sealed": 19.659908424210006, "base": 22.34610151162883, "excess": 2.7989985323994033, "gap": 5.372386174837647, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 51.43479500294909, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 64.97297297297297, "hard": 43.378420887071414, "judge": 60.44825313117996, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.8184955141476893, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 28.57142857142858, "hard": 4.347826086956519, "judge": -19.999999999999996, "standard": 11.111111111111116 } }, "open|score": { "cc": 22.843593280046196, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 40.9867728497478, "hard": 12.570359188934388, "judge": 28.72169648259133, "standard": 25.50131687360131 } }, "sealed|choice": { "cc": 44.97077692151599, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 57.43108417707487, "hard": 39.93212669683257, "judge": 40.409683426443216, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 0.6994422333695622, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 32.00000000000001, "hard": -14.563106796116509, "judge": 2.6315789473684292, "standard": 12.676056338028175 } }, "sealed|score": { "cc": 13.309506117744464, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 44.90364228457862, "hard": 10.653374925310422, "judge": 6.439637126414932, "standard": 13.129503906189765 } } } }, "jevbench_score": 9.435692571873393, "key": "open-alternative-jev", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 88, "ranked": true, "ranks": { "A": 88, "B": 89, "C": 88, "capability": 89 }, "scores": { "A": 9.435692571873393, "B": 7.716442780206986, "C": 6.5525642860231885 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2366679080063477, "p50_s_raw": 0.04333395400317386, "p95_s_adjusted": 0.3130715797946323, "p95_s_raw": 0.08153578989731614 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 70.67388183281524, 78.71286748398254 ], "I": [ 19.129525848760927, 27.488222507229114 ], "capability": [ 45.43710032043718, 52.3768690725618 ], "composite": { "A": [ 6.323116186163818, 15.819609663846963 ], "B": [ 5.049803290865915, 13.396655191684498 ], "C": [ 4.391052907058209, 10.985840044338168 ] }, "seed": 74 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 75.94384178347806, "capability": 49.14497164755345, "intelligence": 22.34610151162883 }, "per_type": { "choice": { "accuracy": 0.5866666666666667, "calibration": 69.58400553131331, "cc": 47.14057008610969, "cc_by_type": { "choice": 47.14057008610969 }, "cc_tiered": 48.20278596223254, "cc_tiered_open": 51.43479500294909, "cc_tiered_sealed": 44.97077692151599, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 47.14057008610969 }, "noul": { "accuracy": 0.52, "calibration": 80.84236520647426, "cc": 4.0000000000000036, "cc_by_type": { "noul": 4.0000000000000036 }, "cc_tiered": 0.7589688737586258, "cc_tiered_open": 0.8184955141476893, "cc_tiered_sealed": 0.6994422333695622, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 4.0000000000000036 }, "score": { "accuracy": 0.4106666666666667, "calibration": 77.40515461264664, "cc": 17.046666794552323, "cc_by_type": { "score": 17.046666794552323 }, "cc_tiered": 18.07654969889533, "cc_tiered_open": 22.843593280046196, "cc_tiered_sealed": 13.309506117744464, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 17.046666794552323 } }, "raw_on_A": { "calibration": 76.45819741974654, "capability": 51.99334325152506, "intelligence": 27.528489083303576 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 8.37993057455771, "Emphasis on Accuracy 60:20:20": 6.202799597209423, "Emphasis on Cost 20:20:60": 9.685497012220603, "Emphasis on Speed 20:60:20": 10.691355012814217, "Intelligence only": 4.463394698403469 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (code and weights)", "repo": "https://github.com/ikermoel/open-alternative-jev", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": "Qwen/Qwen3.5-4B (frozen, BF16)", "adapter_id": "images/open-alternative-jev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6, "acc_among_decisive": 0.8666666666666667, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Alibaba Qwen / neutral reproduction", "axes": { "calibration": 33.721548452619835, "cost": 45.53515592313027, "intelligence": 26.54912920590792, "speed": 88.65282990878164 }, "calibration": { "parts": { "choice": { "ece_hard": 0.43645983930453747, "ece_score": 12.708032139092506, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 12.708032139092506 }, "noul": { "abstention_rate": 0.07816711590296499, "brier": 0.2740805017981965, "ece": 0.2700267098593338, "n": 371, "score": 45.99465802813324 }, "score": { "interval90_coverage": 0.5474254742547425, "interval90_width": 0.14972899728997294, "n": 369, "nrps": 0.21398318753780107, "score": 42.461955190633766, "top_ece": 0.46838885432476185 } }, "score": 33.721548452619835 }, "capability": 30.13533882926388, "composite_ci95": { "A": [ 5.526896473528715, 13.605697544750722 ], "B": [ 4.901937683715855, 12.59857128804635 ], "C": [ 3.838122551061608, 9.448401072743554 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0653875104679803 }, "display": "Raw Qwen3 8B direct logits", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 31.420181182575643, "I_sealed": 21.6780772292402, "base": 26.54912920590792, "excess": 7.168716310897201, "gap": 9.742103953335445, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 51.791553839411165, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 33.941491034916645, "judge": 60.44825313117996, "standard": 69.8618668899121 } }, "open|noul": { "cc": 34.967563837129056, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 30.434782608695656, "judge": 39.99999999999999, "standard": 11.111111111111116 } }, "open|score": { "cc": 7.50142587118671, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 42.05293172396295, "hard": 3.1607638000191285, "judge": 1.3576889332232334, "standard": 8.122602494078969 } }, "sealed|choice": { "cc": 41.65523094061268, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 57.43108417707487, "hard": 31.787330316742068, "judge": 39.47858472998139, "standard": 56.76807488606972 } }, "sealed|noul": { "cc": 22.61439253816204, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 60.00000000000001, "hard": -6.796116504854366, "judge": 31.578947368421062, "standard": 49.29577464788733 } }, "sealed|score": { "cc": 0.7646082089458659, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 50.377225661492794, "hard": -19.926430375972124, "judge": 5.951813095928216, "standard": 9.559569322034855 } } } }, "jevbench_score": 9.30125972789427, "key": "raw-qwen3-8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 89, "ranked": true, "ranks": { "A": 89, "B": 87, "C": 89, "capability": 125 }, "scores": { "A": 9.30125972789427, "B": 8.458419096926727, "C": 6.4592081443710185 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.30506142796948554, "p50_s_raw": 0.07753071398474276, "p95_s_adjusted": 0.4470228368416429, "p95_s_raw": 0.14851141842082144 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 28.041560709983848, 38.19473887695128 ], "I": [ 21.71577559015635, 30.903657696346237 ], "capability": [ 25.49211567892071, 33.98867030681561 ], "composite": { "A": [ 5.526896473528715, 13.605697544750722 ], "B": [ 4.901937683715855, 12.59857128804635 ], "C": [ 3.838122551061608, 9.448401072743554 ] }, "seed": 97 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 33.721548452619835, "capability": 30.13533882926388, "intelligence": 26.54912920590792 }, "per_type": { "choice": { "accuracy": 0.5613333333333334, "calibration": 12.708032139092506, "cc": 43.90079857525836, "cc_by_type": { "choice": 43.90079857525836 }, "cc_tiered": 46.723392390011924, "cc_tiered_open": 51.791553839411165, "cc_tiered_sealed": 41.65523094061268, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 43.90079857525836 }, "noul": { "accuracy": 0.648, "calibration": 45.99465802813324, "cc": 29.600000000000005, "cc_by_type": { "noul": 29.600000000000005 }, "cc_tiered": 28.79097818764555, "cc_tiered_open": 34.967563837129056, "cc_tiered_sealed": 22.61439253816204, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 29.600000000000005 }, "score": { "accuracy": 0.3973333333333333, "calibration": 42.461955190633766, "cc": 3.4025581754630996, "cc_by_type": { "score": 3.4025581754630996 }, "cc_tiered": 4.133017040066288, "cc_tiered_open": 7.50142587118671, "cc_tiered_sealed": 0.7646082089458659, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 3.4025581754630996 } }, "raw_on_A": { "calibration": 38.60246474366818, "capability": 35.16687873306722, "intelligence": 31.73129272246627 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 9.893413746086136, "Emphasis on Accuracy 60:20:20": 7.995042950818957, "Emphasis on Cost 20:20:60": 10.181976971451938, "Emphasis on Speed 20:60:20": 12.509103386849494, "Intelligence only": 6.208182595906101 }, "class": "raw-logit-control", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Qwen/Qwen3-8B", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-8B BF16", "adapter_id": "images/raw-qwen3-8b/spec.json:raw_adapter.RawControlAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.921832884097035, "acc_among_decisive": 0.7105263157894737, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mapika", "axes": { "calibration": 68.52438190259257, "cost": 64.91766177740826, "intelligence": 20.654576369710338, "speed": 94.55945724666857 }, "calibration": { "parts": { "choice": { "ece_hard": 0.18752718750000008, "ece_score": 62.49456249999999, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 62.49456249999999 }, "noul": { "abstention_rate": 0.31999999999999995, "brier": 0.21768861922666663, "ece": 0.1706301333333334, "n": 375, "score": 65.87397333333331 }, "score": { "interval90_coverage": 0.8746666666666667, "interval90_width": 0.5113333333333334, "n": 375, "nrps": 0.14404540251111114, "score": 77.20460987444443, "top_ece": 0.1559312000000001 } }, "score": 68.52438190259257 }, "capability": 44.58947913615145, "composite_ci95": { "A": [ 4.470553979976864, 13.010288397584645 ], "B": [ 3.512028417919115, 10.890754224642697 ], "C": [ 3.1045513749839326, 9.034922498322668 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.014771046798029557 }, "display": "decider-2b (Mapika)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 22.803908585466385, "I_sealed": 18.50524415395429, "base": 20.654576369710338, "excess": 1.7252767890738503, "gap": 4.298664431512094, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 37.01745599452806, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 24.50456118276188, "judge": 28.80685563612393, "standard": 62.32733361239012 } }, "open|noul": { "cc": 4.469289164941341, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 4.347826086956519, "judge": -19.999999999999996, "standard": 22.222222222222232 } }, "open|score": { "cc": 26.92498059692975, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 60.77454545454546, "hard": 25.165648148148158, "judge": 23.80372881355931, "standard": 18.20074074074074 } }, "sealed|choice": { "cc": 39.65284081929382, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 58.7613627965413, "hard": 29.24208144796379, "judge": 35.7541899441341, "standard": 56.76807488606972 } }, "sealed|noul": { "cc": 0.12111092718806482, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -28.155339805825243, "judge": 10.526315789473696, "standard": 21.126760563380277 } }, "sealed|score": { "cc": 15.741780715380987, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 52.834113475177304, "hard": 4.744583333333363, "judge": 10.707844311377212, "standard": 26.740913705583736 } } } }, "jevbench_score": 7.6704332123718, "key": "decider-2b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 90, "ranked": true, "ranks": { "A": 90, "B": 90, "C": 91, "capability": 96 }, "scores": { "A": 7.6704332123718, "B": 6.209608793546842, "C": 5.326689730813751 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.17949020988307893, "p50_s_raw": 0.014745104941539466, "p95_s_adjusted": 0.1949905256042257, "p95_s_raw": 0.022495262802112847 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 62.32738945440369, 71.2629933305287 ], "I": [ 16.778817874747006, 25.441954749747556 ], "capability": [ 40.31419112262012, 47.6366012348925 ], "composite": { "A": [ 4.470553979976864, 13.010288397584645 ], "B": [ 3.512028417919115, 10.890754224642697 ], "C": [ 3.1045513749839326, 9.034922498322668 ] }, "seed": 26 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 68.52438190259257, "capability": 44.58947913615145, "intelligence": 20.654576369710338 }, "per_type": { "choice": { "accuracy": 0.532, "calibration": 62.49456249999999, "cc": 40.149484194272596, "cc_by_type": { "choice": 40.149484194272596 }, "cc_tiered": 38.335148406910946, "cc_tiered_open": 37.01745599452806, "cc_tiered_sealed": 39.65284081929382, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 40.149484194272596 }, "noul": { "accuracy": 0.528, "calibration": 65.87397333333331, "cc": 5.600000000000005, "cc_by_type": { "noul": 5.600000000000005 }, "cc_tiered": 2.295200046064703, "cc_tiered_open": 4.469289164941341, "cc_tiered_sealed": 0.12111092718806482, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 5.600000000000005 }, "score": { "accuracy": 0.41333333333333333, "calibration": 77.20460987444443, "cc": 20.228824561403492, "cc_by_type": { "score": 20.228824561403492 }, "cc_tiered": 21.33338065615537, "cc_tiered_open": 26.92498059692975, "cc_tiered_sealed": 15.741780715380987, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 20.228824561403492 } }, "raw_on_A": { "calibration": 62.755705376823364, "capability": 43.8983266486265, "intelligence": 25.040947920429637 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 6.881302219979439, "Emphasis on Accuracy 60:20:20": 4.98303010129531, "Emphasis on Cost 20:20:60": 8.110241448995462, "Emphasis on Speed 20:60:20": 8.93000700376026, "Intelligence only": 3.5245921294252027 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Mapika/decider-2b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen3.5-2B-Base with a trained decision readout, 1.9B", "adapter_id": "images/decider-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.68, "acc_among_decisive": 0.7764705882352941, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "pst2154", "axes": { "calibration": 74.53740737677316, "cost": 55.49170925809556, "intelligence": 20.38848344571935, "speed": 93.8305724848098 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10832159242229476, "ece_score": 78.33568151554104, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 78.33568151554104 }, "noul": { "abstention_rate": 0.3881401617250674, "brier": 0.22241599402070583, "ece": 0.17236678386479326, "n": 371, "score": 65.52664322704135 }, "score": { "interval90_coverage": 0.9214092140921409, "interval90_width": 0.6684733514001803, "n": 369, "nrps": 0.14545408533817059, "score": 79.74989738773714, "top_ece": 0.12977398345354327 } }, "score": 74.53740737677316 }, "capability": 47.46294541124625, "composite_ci95": { "A": [ 4.1676910325965215, 10.20449077801364 ], "B": [ 3.2802107600784733, 8.482218509265811 ], "C": [ 2.894229883747585, 7.086451929176138 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.03045157019704434 }, "display": "Nemotron Diffusion 8B (pst2154, optimized vLLM)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 22.973528132452522, "I_sealed": 17.803438758986175, "base": 20.38848344571935, "excess": 2.596701731028103, "gap": 5.170089373466347, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 45.141807097909854, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 37.72972972972973, "hard": 32.05410506448569, "judge": 48.58272907053395, "standard": 69.8618668899121 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": -21.739130434782606, "judge": -40.0, "standard": -11.111111111111116 } }, "open|score": { "cc": 23.77877729944772, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 44.848253383055216, "hard": 4.46243778457357, "judge": 33.03604774332046, "standard": 37.99081262158317 } }, "sealed|choice": { "cc": 40.98559356961426, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 61.4219200354741, "hard": 25.169683257918543, "judge": 50.65176908752329, "standard": 47.89998768321223 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 43.99999999999999, "hard": -43.689320388349515, "judge": 5.263157894736836, "standard": -1.4084507042253502 } }, "sealed|score": { "cc": 12.424722707344273, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 42.099894800900906, "hard": 9.652238984201755, "judge": 2.7258103611165985, "standard": 17.6804726261925 } } } }, "jevbench_score": 7.29748476516348, "key": "nemotron-diffusion-8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 91, "ranked": true, "ranks": { "A": 91, "B": 92, "C": 92, "capability": 94 }, "scores": { "A": 7.29748476516348, "B": 5.930430684235979, "C": 5.06769775358575 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1939013559371233, "p50_s_raw": 0.02195067796856165, "p95_s_adjusted": 0.21348231588490307, "p95_s_raw": 0.03174115794245154 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 68.2662042332654, 77.36530331396067 ], "I": [ 16.421783930074305, 23.311693043206645 ], "capability": [ 43.39466614794243, 49.39379257235283 ], "composite": { "A": [ 4.1676910325965215, 10.20449077801364 ], "B": [ 3.2802107600784733, 8.482218509265811 ], "C": [ 2.894229883747585, 7.086451929176138 ] }, "seed": 70 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 74.53740737677316, "capability": 47.46294541124625, "intelligence": 20.38848344571935 }, "per_type": { "choice": { "accuracy": 0.54, "calibration": 78.33568151554104, "cc": 41.17256993454144, "cc_by_type": { "choice": 41.17256993454144 }, "cc_tiered": 43.06370033376206, "cc_tiered_open": 45.141807097909854, "cc_tiered_sealed": 40.98559356961426, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 41.17256993454144 }, "noul": { "accuracy": 0.464, "calibration": 65.52664322704135, "cc": -7.199999999999996, "cc_by_type": { "noul": -7.199999999999996 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.4266666666666667, "calibration": 79.74989738773714, "cc": 16.44673378084317, "cc_by_type": { "score": 16.44673378084317 }, "cc_tiered": 18.101750003396, "cc_tiered_open": 23.77877729944772, "cc_tiered_sealed": 12.424722707344273, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 16.44673378084317 } }, "raw_on_A": { "calibration": 75.92799396845535, "capability": 48.766079474284126, "intelligence": 21.6041649801129 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 6.417819358792639, "Emphasis on Accuracy 60:20:20": 4.7285860819837495, "Emphasis on Cost 20:20:60": 7.307751777390142, "Emphasis on Speed 20:60:20": 8.394346515316924, "Intelligence only": 3.3901175711211273 }, "class": "system-one-open", "open": "yes", "licence": "MIT (application code); modified vLLM runtime Apache-2.0; nvidia/Nemotron-Labs-Diffusion-8B keeps its upstream licence", "repo": "https://github.com/pst2154/Nemotron_Jev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "nvidia/Nemotron-Labs-Diffusion-8B @ 16c67f05 (unchanged weights) served by github.com/pst2154/Nemotron_Jev @ cdc36d60 on the author's digest-pinned modified vLLM runtime: BF16, candidate-only output projection, direct candidate logits, CUDA graphs, one masked forward pass per decision, nothing generated. Disclosed by the author: the inherited option-ordering rule was selected using public benchmark examples in earlier 14B experiments. Our earlier v1.4.3 run of this system used a recipe that left the base image's older 14B application in place; this v1.5 row is a fresh measurement of the 8B.", "adapter_id": "a3/candidates/nemotron-diffusion-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6118598382749326, "acc_among_decisive": 0.7665198237885462, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Jared Palmer", "axes": { "calibration": 68.45285936403559, "cost": 65.78036497095258, "intelligence": 19.510707720813777, "speed": 90.46585183268016 }, "calibration": { "parts": { "choice": { "ece_hard": 0.18711800351703844, "ece_score": 62.576399296592314, "mean_tvd": null, "n": 733, "n_ece_hard": 306, "n_tvd": 0, "score": 62.576399296592314 }, "noul": { "abstention_rate": 0.32884097035040427, "brier": 0.19162911051212952, "ece": 0.13239892183288407, "n": 371, "score": 73.52021563342319 }, "score": { "interval90_coverage": 0.8365122615803815, "interval90_width": 0.4604904632152589, "n": 367, "nrps": 0.15784808091493369, "score": 69.26196316209126, "top_ece": 0.22845632792162052 } }, "score": 68.45285936403559 }, "capability": 43.98178354242468, "composite_ci95": { "A": [ 4.075524448580174, 12.02725721176505 ], "B": [ 3.1848561369918413, 10.0344884794078 ], "C": [ 2.83022531151401, 8.352261952614619 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.013824661330049262 }, "display": "kev 4B (research preview)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 19.939346418532395, "I_sealed": 19.08206902309516, "base": 19.510707720813777, "excess": -1.7161102470010072, "gap": 0.8572773954372366, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 51.235731456250235, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 45.26580685750237, "judge": 44.62755438365195, "standard": 62.32733361239012 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": -4.347826086956519, "judge": -19.999999999999996, "standard": -11.111111111111116 } }, "open|score": { "cc": 8.582307799346953, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 34.24426078971534, "hard": 15.014273649587162, "judge": 21.314617902468203, "standard": -36.211065550999535 } }, "sealed|choice": { "cc": 42.547775845492595, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 56.10080555760846, "hard": 29.751131221719444, "judge": 44.134078212290504, "standard": 58.9850966867841 } }, "sealed|noul": { "cc": 5.208014566705289, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": -30.097087378640776, "judge": 26.315789473684205, "standard": 26.7605633802817 } }, "sealed|score": { "cc": 9.490416657087591, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 60.8431474353109, "hard": -5.464015519199061, "judge": 4.345254884769856, "standard": 21.440658279025847 } } } }, "jevbench_score": 6.611791819903415, "key": "kev-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 92, "ranked": true, "ranks": { "A": 92, "B": 93, "C": 93, "capability": 99 }, "scores": { "A": 6.611791819903415, "B": 5.310194944940352, "C": 4.591522097155148 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.29399301540106537, "p50_s_raw": 0.07199650770053267, "p95_s_adjusted": 0.3055468452628701, "p95_s_raw": 0.07777342263143508 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 62.30814638480004, 71.54002800358278 ], "I": [ 16.228114704223106, 24.71673328753193 ], "capability": [ 39.998560456744215, 47.239203316922335 ], "composite": { "A": [ 4.075524448580174, 12.02725721176505 ], "B": [ 3.1848561369918413, 10.0344884794078 ], "C": [ 2.83022531151401, 8.352261952614619 ] }, "seed": 54 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 68.45285936403559, "capability": 43.98178354242468, "intelligence": 19.510707720813777 }, "per_type": { "choice": { "accuracy": 0.5693333333333334, "calibration": 62.576399296592314, "cc": 44.923884315527204, "cc_by_type": { "choice": 44.923884315527204 }, "cc_tiered": 46.891753650871415, "cc_tiered_open": 51.235731456250235, "cc_tiered_sealed": 42.547775845492595, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 44.923884315527204 }, "noul": { "accuracy": 0.5413333333333333, "calibration": 73.52021563342319, "cc": 8.266666666666666, "cc_by_type": { "noul": 8.266666666666666 }, "cc_tiered": 2.6040072833526446, "cc_tiered_open": 0.0, "cc_tiered_sealed": 5.208014566705289, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 8.266666666666666 }, "score": { "accuracy": 0.41333333333333333, "calibration": 69.26196316209126, "cc": 12.570612763030764, "cc_by_type": { "score": 12.570612763030764 }, "cc_tiered": 9.036362228217271, "cc_tiered_open": 8.582307799346953, "cc_tiered_sealed": 9.490416657087591, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 12.570612763030764 } }, "raw_on_A": { "calibration": 69.97372951951687, "capability": 47.498407356877294, "intelligence": 25.023085194237723 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.022666666666666613, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.021333333333333315, "n": 375 } }, "views": { "Balanced 33:33:33": 5.893456455385603, "Emphasis on Accuracy 60:20:20": 4.229224928104588, "Emphasis on Cost 20:20:60": 7.055012442959745, "Emphasis on Speed 20:60:20": 7.642572180991565, "Intelligence only": 2.970838616432075 }, "class": "jev-rebuild", "open": true, "licence": "Apache-2.0", "repo": "https://github.com/jaredpalmer/kev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-4B-Base + LoRA + learned pointer head; jaredpalmer/kev-4b", "adapter_id": "images/kev-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6711590296495957, "acc_among_decisive": 0.8152610441767069, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Jared Palmer", "axes": { "calibration": 93.71720359195915, "cost": 18.302044660887503, "intelligence": 64.16071886717361, "speed": 83.65684843680415 }, "calibration": { "parts": { "choice": { "ece_hard": 0.034987296416937964, "ece_score": 93.0025407166124, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 93.0025407166124 }, "noul": { "abstention_rate": 0.20485175202156336, "brier": 0.06491574711590294, "ece": 0.03041293800539084, "n": 371, "score": 93.91741239892183 }, "score": { "interval90_coverage": 0.986449864498645, "interval90_width": 0.3089430894308942, "n": 369, "nrps": 0.049457903703703704, "score": 94.23165766034325, "top_ece": 0.03295447154471563 } }, "score": 93.71720359195915 }, "capability": 78.93896122956639, "composite_ci95": { "A": [ 5.665703330300189, 5.813398804963979 ], "B": [ 6.009217533619645, 6.271823804033908 ], "C": [ 5.484111432603801, 5.813398804963979 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.52066024 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.5287680162491537 }, "display": "Kev 27B (Kev 1.0, Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 64.07452328888938, "I_sealed": 64.24691444545786, "base": 64.16071886717361, "excess": -2.745778799006724, "gap": -0.17239115656848014, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 81.24201973326763, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 79.23875432525952, "judge": 80.22412656558997, "standard": 77.39640016743407 } }, "open|noul": { "cc": 47.325051759834366, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 65.21739130434783, "judge": 19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 63.656498373566166, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 67.13878787878788, "hard": 65.75611111111111, "judge": 63.57169491525423, "standard": 57.843333333333334 } }, "sealed|choice": { "cc": 78.44659446797311, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 80.04582070800384, "hard": 73.52941176470587, "judge": 80.44692737430168, "standard": 84.48084739499939 } }, "sealed|noul": { "cc": 50.836710400368496, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 72.0, "hard": 33.98058252427185, "judge": 57.89473684210527, "standard": 63.38028169014085 } }, "sealed|score": { "cc": 63.45743846803195, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 71.6254609929078, "hard": 54.08627450980392, "judge": 71.25748502994011, "standard": 66.4156852791878 } } } }, "jevbench_score": 5.77223551264444, "key": "kev-27b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 93, "ranked": true, "ranks": { "A": 93, "B": 91, "C": 90, "capability": 6 }, "scores": { "A": 5.77223551264444, "B": 6.178201442095797, "C": 5.77223551264444 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.5102782679954544, "p50_s_raw": 0.18013913399772719, "p95_s_adjusted": 0.8443219571985537, "p95_s_raw": 0.34716097859927686 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 88.38693398193061, 92.90424439872974 ], "I": [ 58.981544658104795, 68.3141526180369 ], "capability": [ 74.70682599002546, 79.91812944044146 ], "composite": { "A": [ 5.665703330300189, 5.813398804963979 ], "B": [ 6.009217533619645, 6.271823804033908 ], "C": [ 5.484111432603801, 5.813398804963979 ] }, "seed": 62 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 93.71720359195915, "capability": 78.93896122956639, "intelligence": 64.16071886717361 }, "per_type": { "choice": { "accuracy": 0.8386666666666667, "calibration": 93.0025407166124, "cc": 79.36777090457831, "cc_by_type": { "choice": 79.36777090457831 }, "cc_tiered": 79.84430710062037, "cc_tiered_open": 81.24201973326763, "cc_tiered_sealed": 78.44659446797311, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 79.36777090457831 }, "noul": { "accuracy": 0.7626666666666667, "calibration": 93.91741239892183, "cc": 52.533333333333346, "cc_by_type": { "noul": 52.533333333333346 }, "cc_tiered": 49.080881080101435, "cc_tiered_open": 47.325051759834366, "cc_tiered_sealed": 50.836710400368496, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 52.533333333333346 }, "score": { "accuracy": 0.704, "calibration": 94.23165766034325, "cc": 62.96492982456141, "cc_by_type": { "score": 62.96492982456141 }, "cc_tiered": 63.55696842079906, "cc_tiered_open": 63.656498373566166, "cc_tiered_sealed": 63.45743846803195, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 62.96492982456141 } }, "raw_on_A": { "calibration": 89.47604418949679, "capability": 78.40559149548258, "intelligence": 67.33513880146839 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 4.891297337690537, "Emphasis on Accuracy 60:20:20": 5.910281375617489, "Emphasis on Cost 20:20:60": 3.499139205379304, "Emphasis on Speed 20:60:20": 6.315006752900174, "Intelligence only": 8.596633940261478 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/jaredpalmer/kev-27b", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "Qwen/Qwen3.8-27B", "model_pin": "jaredpalmer/kev-27b@af0e6d551bdc2cc724f3e9d7a8bee1cd4fb8f7bf", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7951482479784366, "acc_among_decisive": 0.9694915254237289, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author model card + config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "newfull5 (dhtocks)", "axes": { "calibration": 79.34885520197874, "cost": 65.5661191189412, "intelligence": 15.917395195670291, "speed": 92.03462875001455 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1264719869706841, "ece_score": 74.70560260586319, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 74.70560260586319 }, "noul": { "abstention_rate": 0.46630727762803237, "brier": 0.1896760988948787, "ece": 0.09166172506738543, "n": 371, "score": 81.6676549865229 }, "score": { "interval90_coverage": 0.9024390243902439, "interval90_width": 0.6144986449864495, "n": 369, "nrps": 0.14966934108401095, "score": 81.67330801355013, "top_ece": 0.10843224932249321 } }, "score": 79.34885520197874 }, "capability": 47.633125198824516, "composite_ci95": { "A": [ 2.1896531510798423, 6.224018858922979 ], "B": [ 1.639660951009494, 4.9293547200832295 ], "C": [ 1.5205924660276677, 4.322235318696512 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.014053873152709357 }, "display": "Malkuth-2B (newfull5, Kev post-train)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 16.94824627575026, "I_sealed": 14.88654411559032, "base": 15.917395195670291, "excess": -0.511685482278303, "gap": 2.061702160159941, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 24.95619850881217, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 61.08108108108109, "hard": 16.955017301038072, "judge": 28.80685563612393, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 28.57142857142858, "hard": -21.739130434782606, "judge": -40.0, "standard": 0.0 } }, "open|score": { "cc": 25.88854031843862, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 49.77636363636364, "hard": 12.745185185185182, "judge": 28.807457627118627, "standard": 35.852962962962955 } }, "sealed|choice": { "cc": 39.92521849097588, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 50.77969107974283, "hard": 26.18778280542985, "judge": 44.134078212290504, "standard": 55.65956398571254 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 28.000000000000004, "hard": -39.80582524271845, "judge": -23.684210526315784, "standard": 18.309859154929576 } }, "sealed|score": { "cc": 4.734413855795083, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 40.8195035460993, "hard": -11.993921568627375, "judge": 11.762814371257434, "standard": 9.605939086294368 } } } }, "jevbench_score": 3.992169201455145, "key": "malkuth-2b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 94, "ranked": true, "ranks": { "A": 94, "B": 96, "C": 95, "capability": 93 }, "scores": { "A": 3.992169201455145, "B": 3.08287077876474, "C": 2.7723397232327396 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23610859100008383, "p50_s_raw": 0.04305429550004192, "p95_s_adjusted": 0.26510952737880866, "p95_s_raw": 0.05755476368940435 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 72.7053529716356, 81.31914491234029 ], "I": [ 12.692506481394203, 18.87608002604669 ], "capability": [ 43.451731383062054, 49.395128138793496 ], "composite": { "A": [ 2.1896531510798423, 6.224018858922979 ], "B": [ 1.639660951009494, 4.9293547200832295 ], "C": [ 1.5205924660276677, 4.322235318696512 ] }, "seed": 63 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 79.34885520197874, "capability": 47.633125198824516, "intelligence": 15.917395195670291 }, "per_type": { "choice": { "accuracy": 0.5133333333333333, "calibration": 74.70560260586319, "cc": 37.76228413364529, "cc_by_type": { "choice": 37.76228413364529 }, "cc_tiered": 32.44070849989403, "cc_tiered_open": 24.95619850881217, "cc_tiered_sealed": 39.92521849097588, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 37.76228413364529 }, "noul": { "accuracy": 0.44533333333333336, "calibration": 81.6676549865229, "cc": -10.933333333333328, "cc_by_type": { "noul": -10.933333333333328 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.38133333333333336, "calibration": 81.67330801355013, "cc": 9.127684210526233, "cc_by_type": { "score": 9.127684210526233 }, "cc_tiered": 15.31147708711685, "cc_tiered_open": 25.88854031843862, "cc_tiered_sealed": 4.734413855795083, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 9.127684210526233 } }, "raw_on_A": { "calibration": 76.45071634453062, "capability": 47.2825063700716, "intelligence": 18.114296395612577 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 3.4183795643165196, "Emphasis on Accuracy 60:20:20": 2.361369226233909, "Emphasis on Cost 20:20:60": 4.242339522263916, "Emphasis on Speed 20:60:20": 4.578614246342653, "Intelligence only": 1.6131545908775944 }, "class": "jev-rebuild", "open": "yes", "licence": "CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)", "repo": "https://github.com/newfull5/malkuth", "gpu": "RTX5090", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container", "underlying": null, "adapter_id": "images/malkuth-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5336927223719676, "acc_among_decisive": 0.8434343434343434, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Qwen", "axes": { "calibration": 73.15350658389404, "cost": 48.39770052432463, "intelligence": 16.337944014530347, "speed": 82.14971164581814 }, "calibration": { "parts": { "choice": { "ece_hard": 0.12818639012273808, "ece_score": 74.36272197545239, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 74.36272197545239 }, "noul": { "abstention_rate": 0.7466666666666666, "brier": 0.23186747724503004, "ece": 0.1932172851672485, "n": 375, "score": 61.3565429665503 }, "score": { "interval90_coverage": 0.9093333333333333, "interval90_width": 0.6708888888888882, "n": 375, "nrps": 0.15088039394905625, "score": 83.74125480967948, "top_ece": 0.08714725492867716 } }, "score": 73.15350658389404 }, "capability": 44.7457252992122, "composite_ci95": { "A": [ 2.2277188584602894, 5.423744955848025 ], "B": [ 1.7231876454208677, 4.422291990633802 ], "C": [ 1.5470269850418672, 3.766489552672239 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: hosted exact-model reference (deepinfra:Qwen/Qwen3-Reranker-4B); no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.05249000923645321 }, "display": "Qwen3-Reranker-4B", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 17.351796606180663, "I_sealed": 15.324091422880029, "base": 16.337944014530347, "excess": -0.5456824591376095, "gap": 2.0277051833006343, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 36.57581104234733, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 53.2972972972973, "hard": 22.617175212330924, "judge": 52.537903757415954, "standard": 32.18920050230222 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -57.14285714285714, "hard": -91.30434782608697, "judge": -40.0, "standard": -66.66666666666667 } }, "open|score": { "cc": 15.479578776194659, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 16.665491764012373, "hard": -1.4208863867574584, "judge": 33.97314797019466, "standard": 20.94719881719004 } }, "sealed|choice": { "cc": 33.418936417285686, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 45.458576601877176, "hard": 22.115384615384603, "judge": 34.823091247672274, "standard": 47.89998768321223 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -43.99999999999999, "hard": -84.46601941747574, "judge": -57.89473684210527, "standard": -43.66197183098591 } }, "sealed|score": { "cc": 12.553337851354403, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 16.14176889876734, "hard": 9.253841484765735, "judge": 15.752141635882, "standard": 12.559909384033885 } } } }, "jevbench_score": 3.715006798009075, "key": "qwen3-reranker-4b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 95, "ranked": true, "ranks": { "A": 95, "B": 97, "C": 96, "capability": 95 }, "scores": { "A": 3.715006798009075, "B": 2.961112195421367, "C": 2.5798658319507455 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.5581826242618263, "p50_s_raw": 0.20409131213091314, "p95_s_adjusted": 1.092075143335387, "p95_s_raw": 0.47103757166769356 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 67.92034254293053, 76.98495141643136 ], "I": [ 13.411885193349915, 18.933037790738602 ], "capability": [ 41.28670365010265, 47.260558123220996 ], "composite": { "A": [ 2.2277188584602894, 5.423744955848025 ], "B": [ 1.7231876454208677, 4.422291990633802 ], "C": [ 1.5470269850418672, 3.766489552672239 ] }, "seed": 91 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.15350658389404, "capability": 44.7457252992122, "intelligence": 16.337944014530347 }, "per_type": { "choice": { "accuracy": 0.4866666666666667, "calibration": 74.36272197545239, "cc": 34.35199833274914, "cc_by_type": { "choice": 34.35199833274914 }, "cc_tiered": 34.99737372981651, "cc_tiered_open": 36.57581104234733, "cc_tiered_sealed": 33.418936417285686, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 34.35199833274914 }, "noul": { "accuracy": 0.18933333333333333, "calibration": 61.3565429665503, "cc": -62.133333333333326, "cc_by_type": { "noul": -62.133333333333326 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.37333333333333335, "calibration": 83.74125480967948, "cc": 12.388563279917397, "cc_by_type": { "score": 12.388563279917397 }, "cc_tiered": 14.016458313774532, "cc_tiered_open": 15.479578776194659, "cc_tiered_sealed": 12.553337851354403, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 12.388563279917397 } }, "raw_on_A": { "calibration": 68.75797234609402, "capability": 42.734177704587836, "intelligence": 16.71038306308165 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 3.1912622823260945, "Emphasis on Accuracy 60:20:20": 2.310804869814862, "Emphasis on Cost 20:20:60": 3.695072544212985, "Emphasis on Speed 20:60:20": 4.224997058133206, "Intelligence only": 1.6344124017617998 }, "class": "reranker", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Qwen/Qwen3-Reranker-4B", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "4B instruction-aware generative reranker", "adapter_id": "images/qwen3-reranker-4b/spec.json:reranker_adapter.RerankerAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.25333333333333335, "acc_among_decisive": 0.7473684210526316, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Featherless", "axes": { "calibration": 91.85610710600702, "cost": 15.10193161538902, "intelligence": 57.92931033189586, "speed": 76.19964460590472 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0559935150394999, "ece_score": 88.80129699210002, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 88.80129699210002 }, "noul": { "abstention_rate": 0.21832884097035044, "brier": 0.07396200744195928, "ece": 0.02357869016917279, "n": 371, "score": 95.28426196616545 }, "score": { "interval90_coverage": 0.9701897018970189, "interval90_width": 0.2899728997289972, "n": 369, "nrps": 0.061950920450199856, "score": 91.48276235975561, "top_ece": 0.05419691617734392 } }, "score": 91.85610710600702 }, "capability": 74.89270871895144, "composite_ci95": { "A": [ 3.3246680487142375, 3.4292430519298938 ], "B": [ 3.5406927144972373, 3.7245951371091817 ], "C": [ 2.5475929721327564, 3.4292430519298938 ] }, "cost": { "basis": "ESTIMATE (base-model reference; self-hosted open weights, no public tariff): frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price, the pair the published autojev-27b, eikos-27b and open-jev-zefan-27b-v1.1 rows use, USD 0.42 per 1M input and USD 3.0 per 1M output x this row's own measured tokens. Estimated, not charged", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.66561768 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.6759827488151658 }, "display": "SimpleJev Qwen3.8-27B (self-hosted, v1.6 pool)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 58.77016996644363, "I_sealed": 57.08845069734809, "base": 57.92931033189586, "excess": -0.8916683733426987, "gap": 1.6817192690955451, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 79.38232847559632, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 92.21621621621622, "hard": 69.80182447310474, "judge": 84.17930125247197, "standard": 84.93093344495604 } }, "open|noul": { "cc": 39.7632850241546, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 39.13043478260869, "judge": 10.000000000000009, "standard": 55.55555555555556 } }, "open|score": { "cc": 57.16489639957998, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 73.0815846586252, "hard": 50.99462651293531, "judge": 64.6276543190174, "standard": 50.35295516419056 } }, "sealed|choice": { "cc": 67.17049093981976, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 74.72470623013821, "hard": 61.312217194570124, "judge": 64.61824953445065, "standard": 78.93829289321346 } }, "sealed|noul": { "cc": 46.31434719713273, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 68.0, "hard": 26.21359223300972, "judge": 52.63157894736843, "standard": 66.19718309859155 } }, "sealed|score": { "cc": 57.78051395509178, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 70.9707943343972, "hard": 48.80809948321188, "judge": 65.15163618443506, "standard": 58.0735193651839 } } } }, "jevbench_score": 3.394848880196433, "key": "simplejev-qwen3.8-27b-selfhost", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 96, "ranked": true, "ranks": { "A": 96, "B": 94, "C": 94, "capability": 16 }, "scores": { "A": 3.394848880196433, "B": 3.656360936982179, "C": 3.164569675341542 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.393547774007311, "p50_s_raw": 0.6217738870036555, "p95_s_adjusted": 1.721526361943688, "p95_s_raw": 0.785763180971844 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 86.56600011746222, 91.91458842585743 ], "I": [ 52.49374537062954, 62.50608217507831 ], "capability": [ 70.32359564579012, 76.5602273459949 ], "composite": { "A": [ 3.3246680487142375, 3.4292430519298938 ], "B": [ 3.5406927144972373, 3.7245951371091817 ], "C": [ 2.5475929721327564, 3.4292430519298938 ] }, "seed": 112 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 91.85610710600702, "capability": 74.89270871895144, "intelligence": 57.92931033189586 }, "per_type": { "choice": { "accuracy": 0.7653333333333333, "calibration": 88.80129699210002, "cc": 69.98948495211388, "cc_by_type": { "choice": 69.98948495211388 }, "cc_tiered": 73.27640970770804, "cc_tiered_open": 79.38232847559632, "cc_tiered_sealed": 67.17049093981976, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 69.98948495211388 }, "noul": { "accuracy": 0.744, "calibration": 95.28426196616545, "cc": 48.8, "cc_by_type": { "noul": 48.8 }, "cc_tiered": 43.038816110643666, "cc_tiered_open": 39.7632850241546, "cc_tiered_sealed": 46.31434719713273, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 48.8 }, "score": { "accuracy": 0.6906666666666667, "calibration": 91.48276235975561, "cc": 57.30970745563964, "cc_by_type": { "score": 57.30970745563964 }, "cc_tiered": 57.47270517733588, "cc_tiered_open": 57.16489639957998, "cc_tiered_sealed": 57.78051395509178, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 57.30970745563964 } }, "raw_on_A": { "calibration": 90.15198763991802, "capability": 76.03520978092624, "intelligence": 61.91843192193445 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 2.833072735614859, "Emphasis on Accuracy 60:20:20": 3.4785771436394777, "Emphasis on Cost 20:20:60": 1.9915507939259591, "Emphasis on Speed 20:60:20": 3.7129751262531117, "Intelligence only": 5.284736623506664 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://github.com/featherless-ai/simple-jev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026", "underlying": "Qwen/Qwen3.8-27B", "model_pin": "Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 + featherless-ai/simple-jev@9c11582d3e631e0b118f3f3132ebbb00c62020c5", "last_measured_on": "2026-10-06", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.7816711590296496, "acc_among_decisive": 0.9620689655172414, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)", "carried_metadata_from": "author repository + base config.json" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "ZeroEntropy", "axes": { "calibration": 83.20040692874471, "cost": 48.39770052432463, "intelligence": 15.596611891228248, "speed": 82.89783918855927 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09594830772201698, "ece_score": 80.8103384555966, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 80.8103384555966 }, "noul": { "abstention_rate": 0.8613333333333333, "brier": 0.2189278600834461, "ece": 0.0929386859803417, "n": 375, "score": 81.41226280393165 }, "score": { "interval90_coverage": 0.984, "interval90_width": 0.9126666666666668, "n": 375, "nrps": 0.14987040001080157, "score": 87.3786195267059, "top_ece": 0.05127860472754006 } }, "score": 83.20040692874471 }, "capability": 49.39850940998648, "composite_ci95": { "A": [ 1.9752898202498443, 4.84420234615273 ], "B": [ 1.507128562340162, 3.903609846255529 ], "C": [ 1.3717290418401693, 3.364029407050507 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-4B); no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.05249000923645321 }, "display": "ZeroEntropy zerank-2", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 15.496935616681888, "I_sealed": 15.696288165774607, "base": 15.596611891228248, "excess": -2.772740191530963, "gap": -0.19935254909271904, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 36.28749068653345, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 49.40540540540541, "hard": 30.16671909405474, "judge": 32.76203032300594, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -82.6086956521739, "judge": -70.0, "standard": -77.77777777777779 } }, "open|score": { "cc": 10.203316163512213, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 13.133808997075057, "hard": 3.6957598546269232, "judge": 27.616853766877792, "standard": -4.367124040546999 } }, "sealed|choice": { "cc": 34.99399091747191, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 50.77969107974283, "hard": 27.205882352941163, "judge": 36.68528864059592, "standard": 40.14041138071193 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -88.0, "hard": -86.40776699029126, "judge": -71.05263157894737, "standard": -52.112676056338024 } }, "sealed|score": { "cc": 12.094873579851914, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 16.780298372211313, "hard": 6.472246249261849, "judge": 17.43542452331028, "standard": 12.986589429664797 } } } }, "jevbench_score": 3.349795360318651, "key": "zerank-2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 97, "ranked": true, "ranks": { "A": 97, "B": 99, "C": 97, "capability": 87 }, "scores": { "A": 3.349795360318651, "B": 2.635183024414276, "C": 2.3262467779990628 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4991468541324139, "p50_s_raw": 0.17457342706620693, "p95_s_adjusted": 1.0279872846323996, "p95_s_raw": 0.4389936423161998 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.23040686190234, 86.00830164706042 ], "I": [ 12.746131677720898, 18.039638464222975 ], "capability": [ 46.43325156721317, 50.75210427056033 ], "composite": { "A": [ 1.9752898202498443, 4.84420234615273 ], "B": [ 1.507128562340162, 3.903609846255529 ], "C": [ 1.3717290418401693, 3.364029407050507 ] }, "seed": 110 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.20040692874471, "capability": 49.39850940998648, "intelligence": 15.596611891228248 }, "per_type": { "choice": { "accuracy": 0.49866666666666665, "calibration": 80.8103384555966, "cc": 35.8866269431524, "cc_by_type": { "choice": 35.8866269431524 }, "cc_tiered": 35.64074080200268, "cc_tiered_open": 36.28749068653345, "cc_tiered_sealed": 34.99399091747191, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 35.8866269431524 }, "noul": { "accuracy": 0.12, "calibration": 81.41226280393165, "cc": -76.0, "cc_by_type": { "noul": -76.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.34933333333333333, "calibration": 87.3786195267059, "cc": 11.246326611904923, "cc_by_type": { "score": 11.246326611904923 }, "cc_tiered": 11.149094871682063, "cc_tiered_open": 10.203316163512213, "cc_tiered_sealed": 12.094873579851914, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 11.246326611904923 } }, "raw_on_A": { "calibration": 80.48954444569493, "capability": 48.704671856760854, "intelligence": 16.919799267826768 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 2.824157802929915, "Emphasis on Accuracy 60:20:20": 2.025225537167865, "Emphasis on Cost 20:20:60": 3.299126425382793, "Emphasis on Speed 20:60:20": 3.7681689882350637, "Intelligence only": 1.4218711229563352 }, "class": "reranker", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/zeroentropy/zerank-2-reranker", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-Reranker-derived 4B cross-encoder", "adapter_id": "images/zerank-2/spec.json:reranker_adapter.RerankerAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.13866666666666666, "acc_among_decisive": 0.8653846153846154, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Dongmin Kim / @carrtesy (tommy.dm.kim@gmail.com) - unofficial personal project", "axes": { "calibration": 80.52861474872404, "cost": 57.919479713067595, "intelligence": 14.831288392065307, "speed": 94.09940251974388 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1631615191239309, "ece_score": 67.36769617521382, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 67.36769617521382 }, "noul": { "abstention_rate": 0.5983827493261455, "brier": 0.1968188345782441, "ece": 0.06151585075654434, "n": 371, "score": 87.69682984869114 }, "score": { "interval90_coverage": 0.948509485094851, "interval90_width": 0.7529358626919599, "n": 369, "nrps": 0.15012627140838788, "score": 86.52131822226714, "top_ece": 0.05972368207313468 } }, "score": 80.52861474872404 }, "capability": 47.67995157039467, "composite_ci95": { "A": [ 1.6959116121891469, 5.270428159114469 ], "B": [ 1.262690264794709, 4.160583281665687 ], "C": [ 1.1777163973535743, 3.660019554940604 ] }, "cost": { "basis": "ESTIMATE (size-class reference; self-hosted open weights, no bookable listing for this base). THE RATE IS RUN 48'S, CARRIED UNCHANGED FOR THE SAME BASE: run 48 priced exaone-4.0-1.2b-jev (v0.2, identical base) at USD 0.02 per 1M input and 0 output, as the HIGHER of the two bracketing published rates, chosen in that direction because the rate was fixed after the measurement existed. Keeping it makes v0.3 directly comparable with v0.2 and takes no new decision. The frozen 25 Sep snapshot still contains NO listing for LGAI-EXAONE/EXAONE-4.0-1.2B (pricing_v15.reference() raises: 'base model lacks an explicit market-listing decision'), and still brackets 1.2B between deepinfra:Qwen/Qwen3.5-0.8B at (0.01, 0.05) and deepinfra:Qwen/Qwen3.5-2B at (0.02, 0.10). Estimated, not charged. RELEASE-LANE LINE, recorded and not acted on, AND IT IS RUN 48'S CARRIED LINE, NOT A NEW ONE: LGAI-EXAONE/EXAONE-4.0-1.2B must be adopted into the frozen UNLISTED_BASE_MODELS table before either EXAONE row may be published. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone and any other decision is reversible by arithmetic from the raw.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.02488704 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.025274583615436694 }, "display": "EXAONE-4.0-1.2B-JEV v0.3 (unofficial fine-tune of LGAI-EXAONE/EXAONE-4.0-1.2B, native option-label softmax, two reads averaged, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 16.786521241837672, "I_sealed": 12.87605554229294, "base": 14.831288392065307, "excess": 1.337078057106488, "gap": 3.9104656995447318, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 39.40654352431394, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 49.40540540540541, "hard": 20.729789241899976, "judge": 40.672379696769944, "standard": 69.8618668899121 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -14.28571428571429, "hard": -47.82608695652174, "judge": -40.0, "standard": -33.333333333333336 } }, "open|score": { "cc": 10.953020201199083, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 36.5784737243812, "hard": -2.3251786244617323, "judge": 27.73524201685903, "standard": -0.4766416325602707 } }, "sealed|choice": { "cc": 28.792593942604675, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 45.458576601877176, "hard": 18.04298642533935, "judge": 32.96089385474861, "standard": 35.70636777928318 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 15.999999999999993, "hard": -63.106796116504846, "judge": -42.10526315789473, "standard": -9.859154929577462 } }, "sealed|score": { "cc": 9.835572684274151, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 36.004469283378306, "hard": -2.325764560565169, "judge": 14.49323221885701, "standard": 14.08730957252643 } } } }, "jevbench_score": 3.266776278413127, "key": "exaone-4.0-1.2b-jev-v0.3", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 98, "ranked": true, "ranks": { "A": 98, "B": 100, "C": 98, "capability": 92 }, "scores": { "A": 3.266776278413127, "B": 2.5116077374172012, "C": 2.2685946377868937 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.19053296000347472, "p50_s_raw": 0.020266480001737364, "p95_s_adjusted": 0.20421593803039284, "p95_s_raw": 0.02710796901519643 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 74.05366116439995, 82.35803770559488 ], "I": [ 11.603293028025588, 17.810851432602874 ], "capability": [ 43.59835541237688, 49.150688153062475 ], "composite": { "A": [ 1.6959116121891469, 5.270428159114469 ], "B": [ 1.262690264794709, 4.160583281665687 ], "C": [ 1.1777163973535743, 3.660019554940604 ] }, "seed": 53 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 80.52861474872404, "capability": 47.67995157039467, "intelligence": 14.831288392065307 }, "per_type": { "choice": { "accuracy": 0.45866666666666667, "calibration": 67.36769617521382, "cc": 30.771198241808182, "cc_by_type": { "choice": 30.771198241808182 }, "cc_tiered": 34.09956873345931, "cc_tiered_open": 39.40654352431394, "cc_tiered_sealed": 28.792593942604675, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 30.771198241808182 }, "noul": { "accuracy": 0.336, "calibration": 87.69682984869114, "cc": -32.8, "cc_by_type": { "noul": -32.8 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3893333333333333, "calibration": 86.52131822226714, "cc": 10.441024816632405, "cc_by_type": { "score": 10.441024816632405 }, "cc_tiered": 10.394296442736618, "cc_tiered_open": 10.953020201199083, "cc_tiered_sealed": 9.835572684274151, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 10.441024816632405 } }, "raw_on_A": { "calibration": 76.1279082946144, "capability": 45.86026412407656, "intelligence": 15.592619953538719 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 2.7692786828242713, "Emphasis on Accuracy 60:20:20": 1.9113656734726492, "Emphasis on Cost 20:20:60": 3.388067484365469, "Emphasis on Speed 20:60:20": 3.7739415394848543, "Intelligence only": 1.304958289921029 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "EXAONE AI Model License 1.2 (non-commercial)", "repo": "https://huggingface.co/carrtesy/EXAONE-4.0-1.2B-JEV", "gpu": "RTX 5090 32 GB", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), 1,477 of 1,500 answered. NEW REVISION of the row run 48 measured: weights carrtesy/EXAONE-4.0-1.2B-JEV at TAG v0.3, server github.com/carrtesy/EXAONE-JEV at tag v0.3 (commit recorded in receipts/POD-R58.json), both the artefacts the author's own submit-form entry names. EXAONE AI Model License 1.2 - NC, NON-COMMERCIAL, and an unofficial personal project - the author states it is 'not an LG AI Research release'; both belong on the row. NO MAPPING DECISION OF OURS: their serve/systemone_server.py speaks the TypeSafe Jev POST /v1/systemone contract, so the UNCHANGED jevbench.adapters.typesafe emits exactly what it parses, and all eight checks of the unamended smoke gate passed on synthetic requests before any sealed item was sent. Read count (permute 2: options in the given and the reversed order, distributions averaged) and the per-type temperatures come from the weights' own jev_config.json, which the author's doc states needs no flags - their shipped default, not a setting of ours. STACK: the author's own docs/REPRODUCE.md install line in an isolated venv, vllm==0.30.0 letting it resolve its own stack (torch 2.13.0+cu130, transformers 5.19.0), served with their own published command verbatim (`python serve/systemone_server.py --model weights --port 8000 --gpu-util 0.9`) and VLLM_USE_FLASHINFER_SAMPLER=0 as their doc sets it. One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW REFUSES THEM, at the limit the author DECLARES: their docs/REPRODUCE.md section 3 states a 65,536-token prompt limit answering 400, and all 23 came back as 'HTTP 400: prompt of 84996 tokens is longer than the maximum context length 65536'. The 23 invalid are therefore the system's own disclosed refusal, counted wrong, and no other item failed. Serial, one decision per request, no retry, no batching, offline, credential-free. add-requests run 58, benchmarkheaven.com/submit ref CA53D793 (bh-submit 5631d258-1eaa-4e2d-a658-f6417b85b351); previously GitHub issue #176", "underlying": "LGAI-EXAONE/EXAONE-4.0-1.2B", "model_pin": "carrtesy/EXAONE-4.0-1.2B-JEV@v0.3", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.40161725067385445, "acc_among_decisive": 0.8456375838926175, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Zefan Cai (@Zefan_Cai)", "axes": { "calibration": 73.61830158128652, "cost": 33.05490478272432, "intelligence": 21.804441188912403, "speed": 77.13731150142542 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1271563639749637, "ece_score": 74.56872720500726, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 74.56872720500726 }, "noul": { "abstention_rate": 0.3261455525606469, "brier": 0.22536579202900195, "ece": 0.1725448435767295, "n": 371, "score": 65.4910312846541 }, "score": { "interval90_coverage": 0.9322493224932249, "interval90_width": 0.6887985546522127, "n": 369, "nrps": 0.14871270327718153, "score": 80.79514625419819, "top_ece": 0.11769218581942746 } }, "score": 73.61830158128652 }, "capability": 47.71137138509946, "composite_ci95": { "A": [ 1.6444373649451645, 4.563766368468016 ], "B": [ 1.361709687197464, 4.0476062027996145 ], "C": [ 1.1419703923230309, 3.169282200325012 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.17041317733990147 }, "display": "Open-Jev 2B (Zefan Cai)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 27.422707291556733, "I_sealed": 16.38819706190457, "base": 21.905452176730652, "excess": 8.46112258721392, "gap": 11.034510229652163, "penalty": 0.9953887741278608, "per_type_split": { "open|choice": { "cc": 41.30120233813892, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 33.837837837837846, "hard": 26.39194715319283, "judge": 44.62755438365195, "standard": 69.8618668899121 } }, "open|noul": { "cc": 17.126293995859218, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 30.434782608695656, "judge": -19.999999999999996, "standard": 33.33333333333333 } }, "open|score": { "cc": 23.84062554067206, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 35.74006952878567, "hard": 15.775595079225702, "judge": 40.5382915597717, "standard": 8.974465440858504 } }, "sealed|choice": { "cc": 35.268479371367654, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 41.46774074347795, "hard": 27.714932126696816, "judge": 38.54748603351957, "standard": 42.35743318142629 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 15.999999999999993, "hard": -28.155339805825243, "judge": -5.263157894736848, "standard": 12.676056338028175 } }, "sealed|score": { "cc": 13.89611181434606, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 33.49212867759059, "hard": -0.08856264182495188, "judge": 18.858024093873603, "standard": 24.62458387577451 } } } }, "jevbench_score": 3.2383990318693976, "key": "open-jev-zefan-2b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 99, "ranked": true, "ranks": { "A": 99, "B": 98, "C": 99, "capability": 91 }, "scores": { "A": 3.2383990318693976, "B": 2.798036205873629, "C": 2.248888216575971 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.0230181839782744, "p50_s_raw": 0.43650909198913723, "p95_s_adjusted": 1.8896679471712559, "p95_s_raw": 0.869833973585628 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 67.27712105156903, 76.2730193438662 ], "I": [ 16.64935744693625, 25.12585533438709 ], "capability": [ 42.961468081385505, 49.91119595082718 ], "composite": { "A": [ 1.6444373649451645, 4.563766368468016 ], "B": [ 1.361709687197464, 4.0476062027996145 ], "C": [ 1.1419703923230309, 3.169282200325012 ] }, "seed": 78 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.61830158128652, "capability": 47.71137138509946, "intelligence": 21.804441188912403 }, "per_type": { "choice": { "accuracy": 0.496, "calibration": 74.56872720500726, "cc": 35.545598363062794, "cc_by_type": { "choice": 35.545598363062794 }, "cc_tiered": 38.284840854753284, "cc_tiered_open": 41.30120233813892, "cc_tiered_sealed": 35.268479371367654, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 35.545598363062794 }, "noul": { "accuracy": 0.49866666666666665, "calibration": 65.4910312846541, "cc": -0.2666666666666706, "cc_by_type": { "noul": -0.2666666666666706 }, "cc_tiered": 8.563146997929609, "cc_tiered_open": 17.126293995859218, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.456, "calibration": 80.79514625419819, "cc": 15.96828593404923, "cc_by_type": { "score": 15.96828593404923 }, "cc_tiered": 18.86836867750906, "cc_tiered_open": 23.84062554067206, "cc_tiered_sealed": 13.89611181434606, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 15.96828593404923 } }, "raw_on_A": { "calibration": 74.44730043893902, "capability": 49.372558434366844, "intelligence": 24.297816429794665 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 2.799165550074912, "Emphasis on Accuracy 60:20:20": 2.2985053544500897, "Emphasis on Cost 20:20:60": 2.7782159796690187, "Emphasis on Speed 20:60:20": 3.6135076182350976, "Intelligence only": 1.812285794889051 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection", "repo": "https://github.com/Zefan-Cai/Open-Jev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "Qwen3.5-2B plus rank-8 LoRA and trained scalar decision head", "adapter_id": "images/open-jev-zefan-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6738544474393531, "acc_among_decisive": 0.748, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Zefan Cai", "axes": { "calibration": 83.46836347105969, "cost": 14.357426070787312, "intelligence": 52.45777412397041, "speed": 72.50871441156775 }, "calibration": { "parts": { "choice": { "ece_hard": 0.09909255777071642, "ece_score": 80.18148844585671, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 80.18148844585671 }, "noul": { "abstention_rate": 0.32884097035040427, "brier": 0.10908145798489985, "ece": 0.08491571210766313, "n": 371, "score": 83.01685757846738 }, "score": { "interval90_coverage": 0.991869918699187, "interval90_width": 0.5313911472448059, "n": 369, "nrps": 0.07069056775911403, "score": 87.20674438885496, "top_ece": 0.09258727223189336 } }, "score": 83.46836347105969 }, "capability": 67.96306879751505, "composite_ci95": { "A": [ 2.556384162487102, 2.91320674604079 ], "B": [ 2.708805295775235, 3.1474885939684447 ], "C": [ 1.7752667795049322, 2.5853832263765106 ] }, "cost": { "basis": "latest published v1.5 row (v1.5.8 combined file) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): Frozen BASE_REFERENCES Qwen/Qwen3.8-27B market price (USD 0.42/M input, 3.00/M output) applied to 2,767,510 input tokens and zero output tokens over 1,624 decisions; self-hosted estimate, not measured hosted billing. Lineage verified by architecture; no pricing gate. The author method encodes each candidate independently.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.7157353448275862 }, "display": "Open-Jev 27B v1.1 (Zefan Cai, LoRA + decision head on Qwen3.8-27B)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 55.382187378509315, "I_sealed": 49.533360869431505, "base": 52.45777412397041, "excess": 3.275438866639566, "gap": 5.84882650907781, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 83.22293323536725, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 75.46398238439761, "judge": 88.13447593935398, "standard": 84.93093344495604 } }, "open|noul": { "cc": 34.03933747412009, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 65.21739130434783, "judge": -9.999999999999998, "standard": 33.33333333333333 } }, "open|score": { "cc": 48.884291426040605, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 50.782717441246206, "hard": 50.66880107494198, "judge": 50.08265733375704, "standard": 42.568510259060396 } }, "sealed|choice": { "cc": 75.06513860323663, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 74.72470623013821, "hard": 70.9841628959276, "judge": 79.51582867783985, "standard": 76.7212710924991 } }, "sealed|noul": { "cc": 22.546963950283207, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 39.99999999999999, "hard": 26.21359223300972, "judge": 18.421052631578938, "standard": 12.676056338028175 } }, "sealed|score": { "cc": 50.987980054774674, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 58.75384354090657, "hard": 44.757967203121375, "judge": 56.3751769445289, "standard": 51.48427868038395 } } } }, "jevbench_score": 2.8808671915746493, "key": "open-jev-zefan-27b-v1.1", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 100, "ranked": true, "ranks": { "A": 100, "B": 95, "C": 100, "capability": 43 }, "scores": { "A": 2.8808671915746493, "B": 3.0870580707350324, "C": 2.202117328789266 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.7305992921814322, "p50_s_raw": 0.7902996460907161, "p95_s_adjusted": 3.242888652905821, "p95_s_raw": 1.5464443264529106 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 79.1715005785796, 85.37530323313025 ], "I": [ 47.50165176644351, 56.56652604734154 ], "capability": [ 64.35002446863409, 69.77751267945519 ], "composite": { "A": [ 2.556384162487102, 2.91320674604079 ], "B": [ 2.708805295775235, 3.1474885939684447 ], "C": [ 1.7752667795049322, 2.5853832263765106 ] }, "seed": 77 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.46836347105969, "capability": 67.96306879751505, "intelligence": 52.45777412397041 }, "per_type": { "choice": { "accuracy": 0.816, "calibration": 80.18148844585671, "cc": 76.46902797381657, "cc_by_type": { "choice": 76.46902797381657 }, "cc_tiered": 79.14403591930194, "cc_tiered_open": 83.22293323536725, "cc_tiered_sealed": 75.06513860323663, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 76.46902797381657 }, "noul": { "accuracy": 0.6266666666666667, "calibration": 83.01685757846738, "cc": 25.333333333333343, "cc_by_type": { "noul": 25.333333333333343 }, "cc_tiered": 28.293150712201648, "cc_tiered_open": 34.03933747412009, "cc_tiered_sealed": 22.546963950283207, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 25.333333333333343 }, "score": { "accuracy": 0.608, "calibration": 87.20674438885496, "cc": 50.311304174784674, "cc_by_type": { "score": 50.311304174784674 }, "cc_tiered": 49.936135740407636, "cc_tiered_open": 48.884291426040605, "cc_tiered_sealed": 50.987980054774674, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 50.311304174784674 } }, "raw_on_A": { "calibration": 81.23345471191158, "capability": 67.87352962184931, "intelligence": 54.51360453178707 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 2.4131836334137757, "Emphasis on Accuracy 60:20:20": 2.9315893503528665, "Emphasis on Cost 20:20:60": 1.7049710123080262, "Emphasis on Speed 20:60:20": 3.1691813006586966, "Intelligence only": 4.3253676470159474 }, "class": "jev-rebuild", "open": "open weights", "licence": "Apache-2.0", "repo": "https://huggingface.co/ZefanCai/Open-Jev-27B-v1.1", "gpu": "RTXPRO6000", "endpoint_condition": "Self-hosted author loader, Transformers reference path on RTX PRO 6000 Blackwell; speed is a LOWER BOUND: flash-linear-attention and causal-conv1d kernels were not installed. The author documents the kernel path as answer-identical. Each candidate is encoded independently, producing high input-token usage.", "underlying": "Qwen3.8-27B (lineage verified by architecture)", "adapter_id": "/home/flori/jobs/jevbench-add-requests-20260919/r45/outputs/meta/specs/open-jev-zefan-27b-v1.1.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": "HF ZefanCai/Open-Jev-27B-v1.1@28cf73067d5b337860bbef3c85b8b82ba8730956; base Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0; author loader Zefan-Cai/Open-Jev@3308a15ccd7eea1df7a37d6ddc39b023b801ba16", "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.6711590296495957, "acc_among_decisive": 0.9437751004016064, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Microsoft / neutral reproduction", "axes": { "calibration": 57.015465878675315, "cost": 53.22051180195924, "intelligence": 13.660237978082595, "speed": 91.89255362835293 }, "calibration": { "parts": { "choice": { "ece_hard": 0.24150351750278642, "ece_score": 51.69929649944272, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 51.69929649944272 }, "noul": { "abstention_rate": 0.49595687331536387, "brier": 0.28688067959784064, "ece": 0.2513310447491633, "n": 371, "score": 49.73379105016734 }, "score": { "interval90_coverage": 0.8970189701897019, "interval90_width": 0.5808491418247513, "n": 369, "nrps": 0.15574884690331758, "score": 69.6133100864159, "top_ece": 0.22599247568418224 } }, "score": 57.015465878675315 }, "capability": 35.33785192837895, "composite_ci95": { "A": [ 0.9687582607873362, 4.12745446682704 ], "B": [ 0.7192841291179243, 3.3096762413612253 ], "C": [ 0.672748792213428, 2.866287824185444 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.03625068965517241 }, "display": "Raw Phi-4 mini direct logits", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 15.681632538277219, "I_sealed": 11.638843417887971, "base": 13.660237978082595, "excess": 1.469401477951004, "gap": 4.042789120389248, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 27.48112242812705, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 26.05405405405406, "hard": 16.955017301038072, "judge": 28.80685563612393, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -57.14285714285714, "hard": -56.52173913043479, "judge": -50.0, "standard": -66.66666666666667 } }, "open|score": { "cc": 19.563775186704607, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 51.64212383139005, "hard": 14.241949100287254, "judge": 14.5533398786592, "standard": 21.683905999264695 } }, "sealed|choice": { "cc": 25.835499598852735, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 37.47690488507872, "hard": 17.533936651583694, "judge": 26.443202979515842, "standard": 35.70636777928318 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 4.0000000000000036, "hard": -66.99029126213591, "judge": -34.210526315789465, "standard": -15.492957746478876 } }, "sealed|score": { "cc": 9.081030654811178, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 50.298756504384265, "hard": 1.511859550820871, "judge": 5.6438828552519915, "standard": 8.766231637344024 } } } }, "jevbench_score": 2.4794266829120515, "key": "raw-phi-4-mini", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 101, "ranked": true, "ranks": { "A": 101, "B": 101, "C": 101, "capability": 116 }, "scores": { "A": 2.4794266829120515, "B": 1.927494035430626, "C": 1.721824085355591 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23357354765757918, "p50_s_raw": 0.04178677382878959, "p95_s_adjusted": 0.27689874032512307, "p95_s_raw": 0.06344937016256152 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 51.56094132127967, 61.43189419566834 ], "I": [ 9.603807805278013, 16.664196620072826 ], "capability": [ 31.514955827472072, 38.142435957298304 ], "composite": { "A": [ 0.9687582607873362, 4.12745446682704 ], "B": [ 0.7192841291179243, 3.3096762413612253 ], "C": [ 0.672748792213428, 2.866287824185444 ] }, "seed": 93 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 57.015465878675315, "capability": 35.33785192837895, "intelligence": 13.660237978082595 }, "per_type": { "choice": { "accuracy": 0.42133333333333334, "calibration": 51.69929649944272, "cc": 25.996798120553574, "cc_by_type": { "choice": 25.996798120553574 }, "cc_tiered": 26.658311013489893, "cc_tiered_open": 27.48112242812705, "cc_tiered_sealed": 25.835499598852735, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 25.996798120553574 }, "noul": { "accuracy": 0.304, "calibration": 49.73379105016734, "cc": -39.2, "cc_by_type": { "noul": -39.2 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.4026666666666667, "calibration": 69.6133100864159, "cc": 13.214800471337696, "cc_by_type": { "score": 13.214800471337696 }, "cc_tiered": 14.322402920757892, "cc_tiered_open": 19.563775186704607, "cc_tiered_sealed": 9.081030654811178, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 13.214800471337696 } }, "raw_on_A": { "calibration": 53.16092441580774, "capability": 33.04410281531054, "intelligence": 12.927281214813327 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 2.1766003082696974, "Emphasis on Accuracy 60:20:20": 1.497083363151444, "Emphasis on Cost 20:20:60": 2.657076827352907, "Emphasis on Speed 20:60:20": 2.994215323690922, "Intelligence only": 1.0196116461240732 }, "class": "raw-logit-control", "open": "yes", "licence": "MIT", "repo": "https://huggingface.co/microsoft/Phi-4-mini-instruct", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Phi-4-mini-instruct BF16", "adapter_id": "images/raw-phi-4-mini/spec.json:raw_adapter.RawControlAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5040431266846361, "acc_among_decisive": 0.6096256684491979, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "sabeel111 / Featherless AI", "axes": { "calibration": 50.197881904888476, "cost": 70.31175519898045, "intelligence": 13.000426205687345, "speed": 58.929227255635794 }, "calibration": { "parts": { "choice": { "ece_hard": 0.22436826460136267, "ece_score": 55.126347079727466, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 55.126347079727466 }, "noul": { "abstention_rate": 0.0, "brier": 0.40897819205718194, "ece": 0.4071351151116132, "n": 371, "score": 18.572976977677357 }, "score": { "interval90_coverage": 0.8807588075880759, "interval90_width": 0.6312104787714541, "n": 369, "nrps": 0.17353025104683517, "score": 76.89432165726062, "top_ece": 0.14429165790397622 } }, "score": 50.197881904888476 }, "capability": 31.59915405528791, "composite_ci95": { "A": [ 0.709004207843539, 4.446455346949053 ], "B": [ 0.5274465681104142, 3.6235557003922736 ], "C": [ 0.49236403322467986, 3.0878162131590647 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.009763559113300492 }, "display": "SimpleJev (Qwen3.5-0.8B, CPU)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 17.1749859973889, "I_sealed": 8.82586641398579, "base": 13.000426205687345, "excess": 5.775731940964869, "gap": 8.349119583403112, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 16.101545239125173, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 2.702702702702708, "hard": 13.180245360176167, "judge": 28.80685563612393, "standard": 9.585600669736289 } }, "open|noul": { "cc": 31.332643202208423, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 57.14285714285714, "hard": 30.434782608695656, "judge": 30.000000000000004, "standard": 22.222222222222232 } }, "open|score": { "cc": 4.0907695508331114, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 21.659111114444617, "hard": -13.376366573006472, "judge": 24.5874654817379, "standard": -0.504172879650655 } }, "sealed|choice": { "cc": 16.39171901424465, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 10.87133249575052, "hard": 9.898190045248851, "judge": 19.92551210428307, "standard": 26.838280576425678 } }, "sealed|noul": { "cc": 10.085880227712721, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 24.0, "hard": 14.563106796116498, "judge": 5.263157894736836, "standard": 1.4084507042253502 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 19.829285077433635, "hard": -8.603274422314232, "judge": 2.232240541966324, "standard": -0.6083842924782035 } } } }, "jevbench_score": 2.112084174105171, "key": "simplejev-qwen3.5-0.8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 102, "ranked": true, "ranks": { "A": 102, "B": 102, "C": 102, "capability": 122 }, "scores": { "A": 2.112084174105171, "B": 1.649257467523907, "C": 1.4667251209063683 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 8.797023160476238, "p50_s_raw": 4.323511580238119, "p95_s_adjusted": 14.545931534096598, "p95_s_raw": 7.197965767048299 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 44.436316357034364, 54.56938440080917 ], "I": [ 8.668681285963862, 17.344962779702836 ], "capability": [ 27.059726936224934, 35.62162347397245 ], "composite": { "A": [ 0.709004207843539, 4.446455346949053 ], "B": [ 0.5274465681104142, 3.6235557003922736 ], "C": [ 0.49236403322467986, 3.0878162131590647 ] }, "seed": 100 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 50.197881904888476, "capability": 31.59915405528791, "intelligence": 13.000426205687345 }, "per_type": { "choice": { "accuracy": 0.3373333333333333, "calibration": 55.126347079727466, "cc": 15.254397847730704, "cc_by_type": { "choice": 15.254397847730704 }, "cc_tiered": 16.24663212668491, "cc_tiered_open": 16.101545239125173, "cc_tiered_sealed": 16.39171901424465, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 15.254397847730704 }, "noul": { "accuracy": 0.576, "calibration": 18.572976977677357, "cc": 15.199999999999992, "cc_by_type": { "noul": 15.199999999999992 }, "cc_tiered": 20.709261714960572, "cc_tiered_open": 31.332643202208423, "cc_tiered_sealed": 10.085880227712721, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 15.199999999999992 }, "score": { "accuracy": 0.33066666666666666, "calibration": 76.89432165726062, "cc": 0.21568120093692356, "cc_by_type": { "score": 0.21568120093692356 }, "cc_tiered": 2.0453847754165557, "cc_tiered_open": 4.0907695508331114, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.21568120093692356 } }, "raw_on_A": { "calibration": 52.11123764887125, "capability": 32.38563147006098, "intelligence": 12.66002529125071 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 1.8759479039856477, "Emphasis on Accuracy 60:20:20": 1.2903897195924547, "Emphasis on Cost 20:20:60": 2.475315897383538, "Emphasis on Speed 20:60:20": 2.3795755523953557, "Intelligence only": 0.8788864373471849 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 server; Apache-2.0 Qwen3.5-0.8B checkpoint", "repo": "https://github.com/featherless-ai/simple-jev", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "Qwen3.5-0.8B through SimpleJev native assistant-prefill option-logit scorer", "adapter_id": "images/simplejev-qwen3.5-0.8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 1.0, "acc_among_decisive": 0.5822102425876011, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Aditya (isHeSatoshi)", "axes": { "calibration": 77.23092648324078, "cost": 60.77469204714053, "intelligence": 8.670533657732655, "speed": 90.21131723708834 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1267499372828752, "ece_score": 74.65001254342496, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 74.65001254342496 }, "noul": { "abstention_rate": 0.7653333333333333, "brier": 0.252171281098217, "ece": 0.15089599377413593, "n": 375, "score": 69.82080124517282 }, "score": { "interval90_coverage": 0.9733333333333334, "interval90_width": 0.8366666666666663, "n": 375, "nrps": 0.15538624289679162, "score": 87.22196566112457, "top_ece": 0.05008722194035847 } }, "score": 77.23092648324078 }, "capability": 42.95073007048671, "composite_ci95": { "A": [ 0.2465766163795243, 1.6433530428400023 ], "B": [ 0.17030179892922623, 1.2180129321235638 ], "C": [ 0.17123376137466964, 1.1412173908611125 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.020300665024630543 }, "display": "smalljev semantic-v9", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 10.044883698522286, "I_sealed": 7.296183616943025, "base": 8.670533657732655, "excess": 0.17531243914101724, "gap": 2.748700081579261, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 18.8258695634495, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 29.945945945945947, "hard": 13.180245360176167, "judge": 28.80685563612393, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -71.42857142857143, "hard": -73.91304347826086, "judge": -80.0, "standard": -66.66666666666667 } }, "open|score": { "cc": 11.308781532117358, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 1.465713661728496, "hard": 15.552501797814056, "judge": 21.821381656800277, "standard": -8.026025251105985 } }, "sealed|choice": { "cc": 12.092078018745205, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 37.47690488507872, "hard": 8.37104072398188, "judge": 6.890130353817524, "standard": 14.644660672496622 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -60.0, "hard": -82.52427184466019, "judge": -52.63157894736843, "standard": -52.112676056338024 } }, "sealed|score": { "cc": 9.796472832083872, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 12.348066695721426, "hard": 6.731683495180563, "judge": 16.491346453277444, "standard": 4.607944142281362 } } } }, "jevbench_score": 0.7719455446134506, "key": "smalljev", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 103, "ranked": true, "ranks": { "A": 103, "B": 103, "C": 103, "capability": 103 }, "scores": { "A": 0.7719455446134506, "B": 0.5545056442894855, "C": 0.5360732948704519 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2831438841298223, "p50_s_raw": 0.06657194206491113, "p95_s_adjusted": 0.33640395337715745, "p95_s_raw": 0.09320197668857871 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 71.4661928849833, 79.90983904292283 ], "I": [ 5.751868300889253, 11.443348446757808 ], "capability": [ 39.42005219873977, 44.84742177133798 ], "composite": { "A": [ 0.2465766163795243, 1.6433530428400023 ], "B": [ 0.17030179892922623, 1.2180129321235638 ], "C": [ 0.17123376137466964, 1.1412173908611125 ] }, "seed": 101 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 77.23092648324078, "capability": 42.95073007048671, "intelligence": 8.670533657732655 }, "per_type": { "choice": { "accuracy": 0.336, "calibration": 74.65001254342496, "cc": 15.0838835576859, "cc_by_type": { "choice": 15.0838835576859 }, "cc_tiered": 15.458973791097353, "cc_tiered_open": 18.8258695634495, "cc_tiered_sealed": 12.092078018745205, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 15.0838835576859 }, "noul": { "accuracy": 0.17066666666666666, "calibration": 69.82080124517282, "cc": -65.86666666666667, "cc_by_type": { "noul": -65.86666666666667 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3626666666666667, "calibration": 87.22196566112457, "cc": 9.603703249850426, "cc_by_type": { "score": 9.603703249850426 }, "cc_tiered": 10.552627182100615, "cc_tiered_open": 11.308781532117358, "cc_tiered_sealed": 9.796472832083872, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 9.603703249850426 } }, "raw_on_A": { "calibration": 71.91610164610462, "capability": 40.0721638349662, "intelligence": 8.22822602382777 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.6314288092197278, "Emphasis on Accuracy 60:20:20": 0.4025186185800094, "Emphasis on Cost 20:20:60": 0.8553620766195851, "Emphasis on Speed 20:60:20": 0.9110150146047675, "Intelligence only": 0.2607338855207101 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://github.com/isHeSatoshi/smalljev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "MiniCPM5-2B-Base, 2.5B dense, with LoRA and native semantic decision heads", "adapter_id": "images/smalljev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.23466666666666666, "acc_among_decisive": 0.7272727272727273, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Jared Palmer", "axes": { "calibration": 66.26655558235618, "cost": 80.09400261254245, "intelligence": 7.598789169415934, "speed": 91.51497362122684 }, "calibration": { "parts": { "choice": { "ece_hard": 0.20778793892461145, "ece_score": 58.44241221507771, "mean_tvd": null, "n": 731, "n_ece_hard": 306, "n_tvd": 0, "score": 58.44241221507771 }, "noul": { "abstention_rate": 0.568733153638814, "brier": 0.2628086253369272, "ece": 0.18765498652291104, "n": 371, "score": 62.4690026954178 }, "score": { "interval90_coverage": 0.9320652173913043, "interval90_width": 0.8045742753623186, "n": 368, "nrps": 0.17410404691603434, "score": 77.88825183657305, "top_ece": 0.1340654581762524 } }, "score": 66.26655558235618 }, "capability": 36.93267237588606, "composite_ci95": { "A": [ 0.13850323393885078, 1.4591062434238613 ], "B": [ 0.09387316749385785, 1.070943045752662 ], "C": [ 0.09618280134642418, 1.0132682245999038 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004608220443349753 }, "display": "kev 0.6B (research preview)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 8.945068603011505, "I_sealed": 6.252509735820362, "base": 7.598789169415934, "excess": 0.1191712247528991, "gap": 2.692558867191143, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 16.179258482292497, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 45.513513513513516, "hard": 9.405473419314262, "judge": 24.851680949241928, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 0.0, "hard": -73.91304347826086, "judge": -60.0, "standard": -44.44444444444444 } }, "open|score": { "cc": 10.655947326742021, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 23.83763830928548, "hard": -1.1034131190897023, "judge": 28.04844552251834, "standard": 1.4950754334692617 } }, "sealed|choice": { "cc": 18.757529207461086, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 34.8163476461459, "hard": 6.334841628959255, "judge": 24.581005586592198, "standard": 26.838280576425678 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -4.0000000000000036, "hard": -66.99029126213591, "judge": -44.73684210526315, "standard": -15.492957746478876 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 11.09665079983172, "hard": -14.21346816054152, "judge": 12.732793638645301, "standard": -1.8098672811444283 } } } }, "jevbench_score": 0.5431213026023877, "key": "kev-0.6b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 104, "ranked": true, "ranks": { "A": 104, "B": 104, "C": 104, "capability": 113 }, "scores": { "A": 0.5431213026023877, "B": 0.382771242886796, "C": 0.377167571251658 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.26348884413018825, "p50_s_raw": 0.05674442206509411, "p95_s_adjusted": 0.2677567361854017, "p95_s_raw": 0.05887836809270084 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 60.54900964119034, 70.35051467735538 ], "I": [ 4.680395319947538, 10.897085922837281 ], "capability": [ 33.22958246000976, 39.72314225130261 ], "composite": { "A": [ 0.13850323393885078, 1.4591062434238613 ], "B": [ 0.09387316749385785, 1.070943045752662 ], "C": [ 0.09618280134642418, 1.0132682245999038 ] }, "seed": 53 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 66.26655558235618, "capability": 36.93267237588606, "intelligence": 7.598789169415934 }, "per_type": { "choice": { "accuracy": 0.368, "calibration": 58.44241221507771, "cc": 19.176226518761275, "cc_by_type": { "choice": 19.176226518761275 }, "cc_tiered": 17.468393844876793, "cc_tiered_open": 16.179258482292497, "cc_tiered_sealed": 18.757529207461086, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 19.176226518761275 }, "noul": { "accuracy": 0.296, "calibration": 62.4690026954178, "cc": -40.800000000000004, "cc_by_type": { "noul": -40.800000000000004 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.30933333333333335, "calibration": 77.88825183657305, "cc": -0.08941736278893497, "cc_by_type": { "score": -0.08941736278893497 }, "cc_tiered": 5.327973663371011, "cc_tiered_open": 10.655947326742021, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 65.17717796738081, "capability": 37.13879390532707, "intelligence": 9.100409843273326 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.02533333333333332, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 0.4469959095569562, "Emphasis on Accuracy 60:20:20": 0.2761353719898703, "Emphasis on Cost 20:20:60": 0.6416336282227073, "Emphasis on Speed 20:60:20": 0.6529389194958221, "Intelligence only": 0.17550648828077675 }, "class": "jev-rebuild", "open": true, "licence": "Apache-2.0", "repo": "https://github.com/jaredpalmer/kev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-0.6B-Base + LoRA + learned pointer head; jaredpalmer/kev-0.6b", "adapter_id": "images/kev-0.6b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.431266846361186, "acc_among_decisive": 0.69375, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Julien Lestel / @jlestel (JuL)", "axes": { "calibration": 78.06387337738309, "cost": 68.9870979950853, "intelligence": 7.4645122320968404, "speed": 90.69403874154062 }, "calibration": { "parts": { "choice": { "ece_hard": 0.049032187499999956, "ece_score": 90.19356250000001, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 90.19356250000001 }, "noul": { "abstention_rate": 0.7013333333333334, "brier": 0.20025538554666658, "ece": 0.08354026666666664, "n": 375, "score": 83.29194666666668 }, "score": { "interval90_coverage": 0.8418230563002681, "interval90_width": 0.7091152815013402, "n": 373, "nrps": 0.22054212385388733, "score": 60.70611096548258, "top_ece": 0.2826678284182305 } }, "score": 78.06387337738309 }, "capability": 42.76419280473996, "composite_ci95": { "A": [ 0.2399155159563835, 1.1190632145905357 ], "B": [ 0.16479417918823877, 0.8102776687445344 ], "C": [ 0.166607997191933, 0.7771272323545388 ] }, "cost": { "basis": "ESTIMATE (size-class reference; self-hosted open weights, no bookable listing for this base). openbmb/MiniCPM5-2B is an EXPLICIT entry in the frozen module's UNLISTED_BASE_MODELS table, so pricing_v15.reference() returns None without raising: the decision that there is no bookable listing for this base is already taken, and the only MiniCPM key in either dated snapshot is deepinfra:openbmb/MiniCPM-Llama3-V-2_5 at (0.34, 0.34), a VISION model and not a comparable reference. The row therefore carries its own labelled size-class estimate: deepinfra:Qwen/Qwen3.5-2B at USD 0.02 per 1M input and USD 0.10 per 1M output in the frozen 25 Sep snapshot - a 2B dense instruct model, the same size class as this 2B base - x this row's own measured tokens. Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone and any other decision is reversible by arithmetic from the raw. RELEASE-LANE LINE, recorded and not acted on: if a bookable MiniCPM5-2B text listing has appeared since 25 Sep, this row reprices by one multiplication from the raw. receipts/COST-ALTERNATIVES-R58.json has the arithmetic for the alternatives already done.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.016866253333333334 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.010808449559918753 }, "display": "jul fast (usejul/jul-decision-minicpm5-2b: MiniCPM5-2B fine-tune with a pointer head, option probabilities from one pass, self-hosted via `jul serve`)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 7.349597785148552, "I_sealed": 7.579426679045129, "base": 7.4645122320968404, "excess": -2.8032165363348214, "gap": -0.22982889389657757, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 22.048793355445657, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 29.945945945945947, "hard": 15.06763133060712, "judge": 16.941331575477918, "standard": 39.723733779824194 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 14.28571428571428, "hard": -100.0, "judge": -50.0, "standard": -55.55555555555556 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 26.921515151515163, "hard": -16.06981481481484, "judge": 27.538983050847456, "standard": -57.756296296296284 } }, "sealed|choice": { "cc": 22.73828003713539, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 54.77052693814206, "hard": 14.479638009049756, "judge": 25.51210428305402, "standard": 19.078704273925375 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -4.0000000000000036, "hard": -74.75728155339806, "judge": -42.10526315789473, "standard": -32.3943661971831 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 2.8917730496454075, "hard": -19.660588235294064, "judge": -11.343473053892229, "standard": -27.880304568527947 } } } }, "jevbench_score": 0.5174168972906218, "key": "jul-fast", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 105, "ranked": true, "ranks": { "A": 105, "B": 106, "C": 105, "capability": 104 }, "scores": { "A": 0.5174168972906218, "B": 0.3638597737908031, "C": 0.35931728978515404 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23712351802387274, "p50_s_raw": 0.043561759011936374, "p95_s_adjusted": 0.3594359368114965, "p95_s_raw": 0.10471796840574825 }, "status": { "answered_ok": 1498, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 72.25639658670305, 80.44357758718796 ], "I": [ 5.676467601887915, 9.888156546142561 ], "capability": [ 39.81441194434129, 44.31702314149329 ], "composite": { "A": [ 0.2399155159563835, 1.1190632145905357 ], "B": [ 0.16479417918823877, 0.8102776687445344 ], "C": [ 0.166607997191933, 0.7771272323545388 ] }, "seed": 74 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 78.06387337738309, "capability": 42.76419280473996, "intelligence": 7.4645122320968404 }, "per_type": { "choice": { "accuracy": 0.404, "calibration": 90.19356250000001, "cc": 23.78011234997108, "cc_by_type": { "choice": 23.78011234997108 }, "cc_tiered": 22.393536696290525, "cc_tiered_open": 22.048793355445657, "cc_tiered_sealed": 22.73828003713539, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 23.78011234997108 }, "noul": { "accuracy": 0.26666666666666666, "calibration": 83.29194666666668, "cc": -46.666666666666664, "cc_by_type": { "noul": -46.666666666666664 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2826666666666667, "calibration": 60.70611096548258, "cc": -13.844131578947305, "cc_by_type": { "score": -13.844131578947305 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 78.36591940088482, "capability": 43.59851726771661, "intelligence": 8.831115134548394 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.005333333333333301, "n": 375 } }, "views": { "Balanced 33:33:33": 0.41923158593664034, "Emphasis on Accuracy 60:20:20": 0.26072023907855174, "Emphasis on Cost 20:20:60": 0.5912460853155722, "Emphasis on Speed 20:60:20": 0.613844381562066, "Intelligence only": 0.16636589222451567 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/usejul/jul-decision-minicpm5-2b", "gpu": "RTX 5090", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,498 of 1,500 answered. Library github.com/usejul/jul tag v0.5.1, PyPI jul==0.5.1, Apache-2.0; weights usejul/jul-decision-minicpm5-2b at f556a6936089693ea68bfed3080c637e4853a3ff, Apache-2.0; base openbmb/MiniCPM5-2B at 12a3808a. NO MAPPING DECISION OF OURS: `jul serve` speaks the TypeSafe /v1/systemone wire format, so the UNCHANGED jevbench.adapters.typesafe applies, and all eight checks of the unamended smoke gate passed on synthetic requests before any sealed item was sent. STACK: the author's own published install line from their request, `pip install \"jul[torch]==0.5.1\"` in an isolated venv (torch 2.14.1+cu130, transformers 5.19.0), served with their own published command verbatim (`jul serve --model jul-decision-minicpm5-2b --host 127.0.0.1 --port 8577`). One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW IS THE FIRST THAT NEITHER REFUSES NOR UNIFORMLY TRUNCATES THEM, AND THE SPLIT IS BY QUESTION TYPE. The author declares a 2,048-token state budget with no refusal - 'the beginning of the text is kept and the end is cut, without an error'. Their server's own usage.input_tokens shows that budget applied to 17 of the 23 (counts 2,096-2,135) and NOT to the other 6, which were read at 107,331-107,997 tokens; those 6 are exactly the 6 score questions among the 23, while all 17 truncated ones are choice or noul. Two of the six full reads then failed with the server's own CUDA OutOfMemoryError on a 32 GB card ('Tried to allocate 5.59 GiB' / '5.41 GiB'), which is this row's only 2 invalid and which a larger card would most likely have answered: the cause is the score path not applying the author's own documented truncation, and the author's note that '24 GB is enough' holds only under that truncation. The 2 invalid are counted wrong, the conservative direction, and are 0.13 % of the pool. PROBABILITIES ARE RETURNED ROUNDED TO FOUR DECIMALS: 426 of 1,498 answered distributions sum to 1 +/- 2e-4 (observed range 0.9998-1.0002); none is off by more than that, so it is the server's rounding and not a partial readout. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 58, GitHub fstandhartinger/jevbench#198", "underlying": "openbmb/MiniCPM5-2B", "model_pin": "usejul/jul-decision-minicpm5-2b@f556a6936089693ea68bfed3080c637e4853a3ff", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.2986666666666667, "acc_among_decisive": 0.8928571428571429, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Deepan Wadhwa", "axes": { "calibration": 68.56606448002526, "cost": 79.10704862374335, "intelligence": 7.425225288997898, "speed": 87.12856375997052 }, "calibration": { "parts": { "choice": { "ece_hard": 0.18936051642522217, "ece_score": 62.12789671495557, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 62.12789671495557 }, "noul": { "abstention_rate": 0.4933333333333333, "brier": 0.2702030712841029, "ece": 0.17767917616168663, "n": 375, "score": 64.46416476766268 }, "score": { "interval90_coverage": 0.9706666666666667, "interval90_width": 0.8675555555555556, "n": 375, "nrps": 0.1718997562232394, "score": 79.10613195745756, "top_ece": 0.12298880231380463 } }, "score": 68.56606448002526 }, "capability": 37.99564488451158, "composite_ci95": { "A": [ 0.20924922812147978, 1.114873836219044 ], "B": [ 0.14364373295734356, 0.808381046176735 ], "C": [ 0.14531196397324977, 0.7742179418187807 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004970862068965518 }, "display": "OpenDecision (ModernBERT-large zero-shot)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 8.199956224645156, "I_sealed": 6.6504943533506395, "base": 7.425225288997898, "excess": -1.0239257711437273, "gap": 1.5494618712945165, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 21.530744131863024, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 37.72972972972973, "hard": -3.8062283737024116, "judge": 32.76203032300594, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -14.28571428571429, "hard": -82.6086956521739, "judge": -30.000000000000004, "standard": -22.22222222222222 } }, "open|score": { "cc": 3.069124542072446, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 29.089033648823246, "hard": -4.629529940171362, "judge": 10.925488522812966, "standard": -6.32806701792612 } }, "sealed|choice": { "cc": 18.786220529284186, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 34.8163476461459, "hard": 9.898190045248851, "judge": 19.92551210428307, "standard": 26.838280576425678 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 12.00000000000001, "hard": -39.80582524271845, "judge": -50.0, "standard": -15.492957746478876 } }, "sealed|score": { "cc": 1.165262530767735, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 23.523223159689433, "hard": -5.374879041668379, "judge": 2.0574839819155533, "standard": 1.728233184457384 } } } }, "jevbench_score": 0.508795165212109, "key": "opendecision", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 106, "ranked": true, "ranks": { "A": 106, "B": 107, "C": 106, "capability": 111 }, "scores": { "A": 0.508795165212109, "B": 0.3579489209247008, "C": 0.3533299758417423 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.29369962783530357, "p50_s_raw": 0.07184981391765177, "p95_s_adjusted": 0.6595386148197576, "p95_s_raw": 0.2547693074098788 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 62.90521170003539, 71.97774479185684 ], "I": [ 5.421041180341186, 9.877186440823028 ], "capability": [ 34.738025189088674, 40.24525966167053 ], "composite": { "A": [ 0.20924922812147978, 1.114873836219044 ], "B": [ 0.14364373295734356, 0.808381046176735 ], "C": [ 0.14531196397324977, 0.7742179418187807 ] }, "seed": 80 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 68.56606448002526, "capability": 37.99564488451158, "intelligence": 7.425225288997898 }, "per_type": { "choice": { "accuracy": 0.36933333333333335, "calibration": 62.12789671495557, "cc": 19.346740808806086, "cc_by_type": { "choice": 19.346740808806086 }, "cc_tiered": 20.158482330573605, "cc_tiered_open": 21.530744131863024, "cc_tiered_sealed": 18.786220529284186, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 19.346740808806086 }, "noul": { "accuracy": 0.3466666666666667, "calibration": 64.46416476766268, "cc": -30.666666666666664, "cc_by_type": { "noul": -30.666666666666664 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3253333333333333, "calibration": 79.10613195745756, "cc": 2.4048951483054193, "cc_by_type": { "score": 2.4048951483054193 }, "cc_tiered": 2.1171935364200904, "cc_tiered_open": 3.069124542072446, "cc_tiered_sealed": 1.165262530767735, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 2.4048951483054193 } }, "raw_on_A": { "calibration": 66.1740117306899, "capability": 37.492707526259565, "intelligence": 8.811403321829234 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.41664409033845246, "Emphasis on Accuracy 60:20:20": 0.25754719918644864, "Emphasis on Cost 20:20:60": 0.5990327904113044, "Emphasis on Speed 20:60:20": 0.6067044473606604, "Intelligence only": 0.1637528610901445 }, "class": "classifier", "open": "yes", "licence": "Apache-2.0", "repo": "https://github.com/deepanwadhwa/OpenDecision", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "MoritzLaurer/ModernBERT-large-zeroshot-v2.0 (~400M) used as a zero-shot NLI decision engine", "adapter_id": "images/opendecision/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5066666666666667, "acc_among_decisive": 0.6842105263157895, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)", "axes": { "calibration": 22.615652963639242, "cost": 70.20313863040258, "intelligence": 7.802105373202749, "speed": 93.2878888255429 }, "calibration": { "parts": { "choice": { "ece_hard": 0.5367426710097716, "ece_score": 0.0, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 0.0 }, "noul": { "abstention_rate": 0.1671159029649596, "brier": 0.3670596395956874, "ece": 0.34489191374663086, "n": 371, "score": 31.021617250673827 }, "score": { "interval90_coverage": 0.5121951219512195, "interval90_width": 0.28523035230352306, "n": 369, "nrps": 0.263493167195122, "score": 36.825341640243906, "top_ece": 0.5007441734417342 } }, "score": 22.615652963639242 }, "capability": 15.208879168420996, "composite_ci95": { "A": [ 0.24070026102449685, 1.3152079969468897 ], "B": [ 0.17590250514236588, 1.0549899064852435 ], "C": [ 0.16715295904478944, 0.9133388867686731 ] }, "cost": { "basis": "ESTIMATE (same-family reference, one size up; self-hosted open weights, no bookable listing for this base). pricing_v15.reference() RAISES on google/gemma-4-E2B and on google/gemma-4-e2b - the base is in none of the three frozen tables - and the published page carries no priced row on this base either: the one Gemma-4-E2B entry on the board, system-one-open, is listed not_measured. The row therefore carries its own labelled estimate from the nearest reference the frozen 25 Sep snapshot itself contains, deepinfra:google/gemma-4-E4B-it at USD 0.02 per 1M input and USD 0.10 per 1M output - the SAME Gemma-4 effective-parameter family, one size LARGER, so it is both the nearest available reference and the direction that cannot flatter this row - x this row's own measured tokens. Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone and any other decision is reversible by arithmetic from the raw. RELEASE-LANE LINE, recorded and not acted on: google/gemma-4-E2B needs either a listing decision or an UNLISTED_BASE_MODELS entry in the frozen table before this row may be published; receipts/COST-ALTERNATIVES-R58.json has the alternatives' arithmetic already done.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.009694333333333333 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.00984529451591063 }, "display": "ClassOne Gemma 4 E2B (Gemma-4-E2B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 8.900914683249555, "I_sealed": 6.703296063155943, "base": 7.802105373202749, "excess": -0.3757690223446315, "gap": 2.1976186200936123, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 26.702744049748667, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 18.270270270270274, "hard": 16.955017301038072, "judge": 28.80685563612393, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": -13.043478260869568, "judge": -40.0, "standard": 11.111111111111116 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 47.08424242424244, "hard": -7.722870370370338, "judge": -13.495593220338996, "standard": -43.26740740740745 } }, "sealed|choice": { "cc": 20.109888189467828, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 30.82551178774667, "hard": 12.952488687782791, "judge": 20.856610800744896, "standard": 27.946791476782867 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 15.999999999999993, "hard": -37.86407766990292, "judge": 2.6315789473684292, "standard": 23.943661971830977 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 14.937446808510646, "hard": -32.15683823529416, "judge": -35.42389221556888, "standard": -33.18258883248737 } } } }, "jevbench_score": 0.49351800738768026, "key": "classone-gemma4-e2b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 107, "ranked": true, "ranks": { "A": 107, "B": 105, "C": 107, "capability": 133 }, "scores": { "A": 0.49351800738768026, "B": 0.37400131309297324, "C": 0.34272083846366685 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.20868116198689676, "p50_s_raw": 0.029340580993448384, "p95_s_adjusted": 0.2247645789931994, "p95_s_raw": 0.0373822894965997 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 19.456713582756297, 25.97574684722229 ], "I": [ 5.95996205912109, 11.37620576911391 ], "capability": [ 13.386066785667875, 17.91536010233214 ], "composite": { "A": [ 0.24070026102449685, 1.3152079969468897 ], "B": [ 0.17590250514236588, 1.0549899064852435 ], "C": [ 0.16715295904478944, 0.9133388867686731 ] }, "seed": 26 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 22.615652963639242, "capability": 15.208879168420996, "intelligence": 7.802105373202749 }, "per_type": { "choice": { "accuracy": 0.38266666666666665, "calibration": 0.0, "cc": 21.051883709254156, "cc_by_type": { "choice": 21.051883709254156 }, "cc_tiered": 23.40631611960825, "cc_tiered_open": 26.702744049748667, "cc_tiered_sealed": 20.109888189467828, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 21.051883709254156 }, "noul": { "accuracy": 0.48533333333333334, "calibration": 31.021617250673827, "cc": -2.9333333333333322, "cc_by_type": { "noul": -2.9333333333333322 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2853333333333333, "calibration": 36.825341640243906, "cc": -21.676596491227972, "cc_by_type": { "score": -21.676596491227972 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 26.647352805544053, "capability": 18.967337110684245, "intelligence": 11.287321415824435 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.4770150324027566, "Emphasis on Accuracy 60:20:20": 0.2973210912399719, "Emphasis on Cost 20:20:60": 0.6703202939761591, "Emphasis on Speed 20:60:20": 0.6973896308140026, "Intelligence only": 0.1899745505796208 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/devops-thiago/classone-gemma4-e2b", "gpu": "shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,477 of 1,500 answered. Weights devops-thiago/classone-gemma4-e2b (sha cd502aa33d288e67f16937c2e125f2297ab7b4ad), Apache-2.0, public and ungated; code github.com/devops-thiago/class-one, Apache-2.0. ONE DOCUMENTED MAPPING OF OURS, AND IT IS THE ONLY ONE. The author's server keys SCORE probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1'. The UNAMENDED smoke gate REFUSED the first attempt at check score-keys-are-indices, before any sealed item was sent, and the server was torn down with the card verified free. The 1-based convention - which their own schemas.py documents only as 'Probability distribution across each ordered rubric level', without stating the convention - was then established on the AUTHOR'S OWN SERVER with SYNTHETIC requests across every rubric size their own validator allows (2..10): exactly the keys '1'..'n' in order every time, never '0' and never a key above n. It is corroborated independently by their own `score` field being a 1-based probability-weighted mean of the same distribution (1.0185 on a 2-level rubric, 1.6147 on a 10-level one, 3.1155 where the mass sits on key 3; a 0-based convention would have given 0.0185), and a reversed rubric moved the mass to a different key, so 'k' means the k-th level of the criteria array AS SENT - which is what JevBench's 'k-1' means. classone_reindex_r58.ClassOneReindexAdapter therefore relabels score keys k -> k-1 and NOTHING else: the request body, the answer mapping, the noul range check and the choice-must-be-one-of-the-item's-labels rule are the UNCHANGED vendored typesafe adapter's, reused by calling its own run(). An item whose score key set is not exactly {'1'..'n'} is REFUSED and recorded invalid rather than rescored on a guess; none was. choice and noul are untouched. The gate then ran AMENDED in exactly that one check, requiring the key set to be exactly {'1'..'n'} - stricter than the original, because it also pins the count - with the other seven byte-identical, and passed. PREREG-R58-AMENDMENT-1.md (sha256 in PREREG-R58-AMENDMENT-1.md.sha256), receipt PROBE-classone-e2b.json. No file of the author's was edited and no prompt of ours exists. THE SUBMIT FORM CARRIED NO DESCRIPTION FOR EITHER CLASSONE ROW, so everything about how it is run comes from the author's own repository: install line README Quickstart step 1 (`pip install -e \".[dev]\"`, in an isolated venv - torch 2.14.1+cu130, transformers 5.19.0, fastapi 0.142.2), serve line step 3 (`uvicorn classone.server.app:app`), and the checkpoint selected the way their own src/classone/server/app.py reads it (CLASSONE_BASE_MODEL). THEIR SERVER REGISTERS POST /v1/systemone ITSELF (app.py:115, next to /v1/decide and /v1/classone) and their DecisionRequest/DecisionResponse in src/classone/schemas.py are the TypeSafe System One shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. output_tokens is 0 by their own schema comment ('In System 1 models this is 0 because there is no autoregressive decode'). THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW REFUSES THEM at a limit far below them: all 23 came back as 'HTTP 400: Prompt length (81338 tokens) exceeds maximum limit (8192)', an 8,192-token ceiling the author's code enforces with an explicit error rather than truncating. The 23 invalid are the system's own refusal, counted wrong, and no other item failed. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 58, benchmarkheaven.com/submit ref C4D7AACD (bh-submit 4567a065-e912-40df-9bc1-68fb37cc7bba)", "underlying": "google/gemma-4-E2B-it", "model_pin": "devops-thiago/classone-gemma4-e2b@cd502aa33d288e67f16937c2e125f2297ab7b4ad", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.8328840970350404, "acc_among_decisive": 0.5889967637540453, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mourad Ghafiri", "axes": { "calibration": 77.95657740696296, "cost": 79.5180630553356, "intelligence": 7.297270616183145, "speed": 55.90747410573378 }, "calibration": { "parts": { "choice": { "ece_hard": 0.16652875000000003, "ece_score": 66.69425, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 66.69425 }, "noul": { "abstention_rate": 0.7813333333333333, "brier": 0.20344673173333316, "ece": 0.05893599999999998, "n": 375, "score": 88.2128 }, "score": { "interval90_coverage": 0.8586666666666667, "interval90_width": 0.6119999999999997, "n": 375, "nrps": 0.16765142224888893, "score": 78.9626822208889, "top_ece": 0.12654746666666666 } }, "score": 77.95657740696296 }, "capability": 42.62692401157305, "composite_ci95": { "A": [ 0.18025356076996352, 1.2178442350406786 ], "B": [ 0.12404410707015784, 0.9011760636632091 ], "C": [ 0.1251760838680302, 0.8457251632226935 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.00481649630541872 }, "display": "Qwen3.5-0.8B Decision Model (Mourad Ghafiri)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 4.79058394567308, "I_sealed": 9.803957286693208, "base": 7.297270616183145, "excess": -7.5867609834583725, "gap": -5.013373341020128, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 13.339251237806783, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -1.1891891891891837, "hard": 13.180245360176167, "judge": 20.896506262359924, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -28.57142857142857, "hard": -73.91304347826086, "judge": -90.0, "standard": -88.88888888888889 } }, "open|score": { "cc": 1.0325005992124563, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 40.642727272727264, "hard": -20.778611111111122, "judge": 25.48779661016949, "standard": -11.83333333333334 } }, "sealed|choice": { "cc": 21.186916712409108, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 24.174118690414627, "hard": 20.079185520361975, "judge": 20.856610800744896, "standard": 22.404236974996934 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -60.0, "hard": -68.93203883495146, "judge": -57.89473684210527, "standard": -43.66197183098591 } }, "sealed|score": { "cc": 8.224955147670517, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 42.56900709219856, "hard": 1.4315441176470767, "judge": 8.896227544910184, "standard": 3.6328426395938807 } } } }, "jevbench_score": 0.47247471382875306, "key": "mghafiri-qwen3.5-0.8b-decision-model", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 108, "ranked": true, "ranks": { "A": 108, "B": 108, "C": 108, "capability": 105 }, "scores": { "A": 0.47247471382875306, "B": 0.3355765591935497, "C": 0.3281074401588563 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 11.252091223932803, "p50_s_raw": 5.551045611966401, "p95_s_adjusted": 22.804436485376208, "p95_s_raw": 11.327218242688105 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 71.68948470123519, 80.02198153079908 ], "I": [ 5.167522017117285, 10.33633269787795 ], "capability": [ 38.95895177314971, 44.525366548573615 ], "composite": { "A": [ 0.18025356076996352, 1.2178442350406786 ], "B": [ 0.12404410707015784, 0.9011760636632091 ], "C": [ 0.1251760838680302, 0.8457251632226935 ] }, "seed": 67 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 77.95657740696296, "capability": 42.62692401157305, "intelligence": 7.297270616183145 }, "per_type": { "choice": { "accuracy": 0.36933333333333335, "calibration": 66.69425, "cc": 19.346740808806086, "cc_by_type": { "choice": 19.346740808806086 }, "cc_tiered": 17.263083975107946, "cc_tiered_open": 13.339251237806783, "cc_tiered_sealed": 21.186916712409108, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 19.346740808806086 }, "noul": { "accuracy": 0.192, "calibration": 88.2128, "cc": -61.6, "cc_by_type": { "noul": -61.6 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.4106666666666667, "calibration": 78.9626822208889, "cc": 7.9552982456141175, "cc_by_type": { "score": 7.9552982456141175 }, "cc_tiered": 4.628727873441487, "cc_tiered_open": 1.0325005992124563, "cc_tiered_sealed": 8.224955147670517, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 7.9552982456141175 } }, "raw_on_A": { "calibration": 67.24894909368233, "capability": 38.149863284895126, "intelligence": 9.050777476107926 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.381493687842595, "Emphasis on Accuracy 60:20:20": 0.24118281186608428, "Emphasis on Cost 20:20:60": 0.5528132957063066, "Emphasis on Speed 20:60:20": 0.5239265172020665, "Intelligence only": 0.15543232661348225 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT code and training data; Apache-2.0 model weights", "repo": "https://huggingface.co/mghafiri/qwen3.5-0.8B-decision-model", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "Qwen3.5-0.8B-Base fine-tuned as a JevLite decision model with per-question calibration", "adapter_id": "images/mghafiri-qwen3.5-0.8b-decision-model/spec.json:mghafiri_adapter.JevLite", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.21866666666666668, "acc_among_decisive": 0.8780487804878049, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": true, "api_subset": false, "author": "Fastino", "axes": { "calibration": 62.13288114639033, "cost": 45.81558630529788, "intelligence": 7.011351364482595, "speed": 84.7052606179839 }, "calibration": { "parts": { "choice": { "basis": "provider_confidence_only", "ece_score": 28.328881967263143, "n": 305, "n_ece": 305, "score": 28.328881967263143, "top_label_ece": 0.3583555901636843 }, "noul": { "basis": "provider_confidence_only", "ece_score": 76.72217340082736, "n": 370, "n_ece": 370, "score": 76.72217340082736, "top_label_ece": 0.11638913299586323 }, "score": { "basis": "provider_confidence_only", "ece_score": 81.3475880710805, "n": 366, "n_ece": 366, "score": 81.3475880710805, "top_label_ece": 0.09326205964459748 } }, "score": 62.13288114639033 }, "capability": 34.57211625543646, "composite_ci95": { "A": [ 0.10777031922591683, 1.0393833813480395 ], "B": [ 0.07441185007614143, 0.7857792694163073 ], "C": [ 0.07484049946244224, 0.7217940148250275 ] }, "cost": { "basis": "ESTIMATE: published Fastino sources conflict (USD 0.03/M vs 0.15/M input); conservative higher USD 0.15/M input, 0/M output x measured tokens; no invoice", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.07366349999999999 }, "kind": "mixed", "usage_estimated_rows": 15, "usd_per_1000": 0.06399515910629654 }, "display": "Fastino GLiNER-2.5-Decide (hosted API)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 6.2445368778313695, "I_sealed": 7.778165851133819, "base": 7.011351364482595, "excess": -4.107016615740694, "gap": -1.5336289733024495, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 18.73361063349411, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 18.270270270270274, "hard": 15.06763133060712, "judge": 24.851680949241928, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -28.57142857142857, "hard": -21.739130434782606, "judge": 10.000000000000009, "standard": 11.111111111111116 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -36.36363636363635, "hard": -28.70370370370372, "judge": -16.949152542372904, "standard": -99.99999999999996 } }, "sealed|choice": { "cc": 15.115509018204584, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 25.504397309881035, "hard": 8.880090497737543, "judge": 18.06331471135942, "standard": 17.970193373568186 } }, "sealed|noul": { "cc": 8.218988535196875, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 4.0000000000000036, "hard": -6.796116504854366, "judge": 7.8947368421052655, "standard": 40.8450704225352 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -38.29787234042554, "hard": -44.60784313725492, "judge": -24.55089820359284, "standard": -63.451776649746215 } } } }, "jevbench_score": 0.34332996901846624, "key": "fastino-gliner-2-5-decide", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 109, "ranked": true, "ranks": { "A": 109, "B": 109, "C": 109, "capability": 117 }, "scores": { "A": 0.34332996901846624, "B": 0.24643533434670511, "C": 0.23842358959615717 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.4409300498664379, "p50_s_raw": 0.4409300498664379, "p95_s_adjusted": 0.7675456959754229, "p95_s_raw": 0.7675456959754229 }, "status": { "answered_ok": 1469, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "confidence", "noul": "confidence", "score": "confidence" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 55.437755987711, 66.14780229160992 ], "I": [ 4.625975407661763, 10.579291699427245 ], "capability": [ 30.439944046543882, 37.65484132935661 ], "composite": { "A": [ 0.10777031922591683, 1.0393833813480395 ], "B": [ 0.07441185007614143, 0.7857792694163073 ], "C": [ 0.07484049946244224, 0.7217940148250275 ] }, "seed": 38 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": true, "g_med_used": 2.573387642438244, "lane": "api", "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 62.13288114639033, "capability": 34.57211625543646, "intelligence": 7.011351364482595 }, "per_type": { "choice": { "accuracy": 0.3426666666666667, "calibration": 28.328881967263143, "cc": 15.936455007909936, "cc_by_type": { "choice": 15.936455007909936 }, "cc_tiered": 16.924559825849347, "cc_tiered_open": 18.73361063349411, "cc_tiered_sealed": 15.115509018204584, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 15.936455007909936 }, "noul": { "accuracy": 0.5333333333333333, "calibration": 76.72217340082736, "cc": 6.666666666666665, "cc_by_type": { "noul": 6.666666666666665 }, "cc_tiered": 4.109494267598437, "cc_tiered_open": 0.0, "cc_tiered_sealed": 8.218988535196875, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 6.666666666666665 }, "score": { "accuracy": 0.248, "calibration": 81.3475880710805, "cc": -42.280701754385944, "cc_by_type": { "score": -42.280701754385944 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": null, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.022666666666666613, "n": 750 }, "noul": { "invalid_rate": 0.013333333333333308, "n": 375 }, "score": { "invalid_rate": 0.02400000000000002, "n": 375 } }, "views": { "Balanced 33:33:33": 0.2810101668608893, "Emphasis on Accuracy 60:20:20": 0.17887060474062133, "Emphasis on Cost 20:20:60": 0.37538092966437175, "Emphasis on Speed 20:60:20": 0.41302239566745963, "Intelligence only": 0.11575817604301195 }, "class": "decision-api", "open": "no", "licence": "proprietary hosted API", "repo": "https://fastino.ai/", "endpoint_condition": "external hosted API (Fastino); received the full set: 1,200 sealed + 300 public items (v1.6.1)", "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).", "model_pin": "fastino/GLiNER-2.5-Decide (requested model id)", "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "noul_decisive": { "supported": true, "decisive_rate": 1.0, "acc_among_decisive": 0.5405405405405406, "valid": 370 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)", "carried_metadata_from": "v1.6.0 release row" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "LibertAI", "axes": { "calibration": 29.6278299611924, "cost": 80.28784730320886, "intelligence": 6.531204174677637, "speed": 85.44505463939521 }, "calibration": { "parts": { "choice": { "ece_hard": 0.5370261325853646, "ece_score": 0.0, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 0.0 }, "noul": { "abstention_rate": 0.2264150943396226, "brier": 0.34735578114832993, "ece": 0.30478924226600856, "n": 371, "score": 39.04215154679829 }, "score": { "interval90_coverage": 0.6233062330623306, "interval90_width": 0.4112466124661248, "n": 369, "nrps": 0.24237776576192704, "score": 49.84133833677892, "top_ece": 0.3803977337512473 } }, "score": 29.6278299611924 }, "capability": 18.07951706793502, "composite_ci95": { "A": [ 0.12746211676468558, 0.7725020253140645 ], "B": [ 0.08887678609852033, 0.587439805908629 ], "C": [ 0.08851535886436501, 0.5364597398014336 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate. Frozen market reference deepinfra:Qwen/Qwen3.5-0.8B.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004540166256157636 }, "display": "Deem 0.8B v1", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 8.352332961291632, "I_sealed": 4.710075388063642, "base": 6.531204174677637, "excess": 1.0688699307897456, "gap": 3.6422575732279894, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 25.056998883874897, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 41.621621621621614, "hard": 9.405473419314262, "judge": 40.672379696769944, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 0.0, "hard": -39.13043478260869, "judge": -9.999999999999998, "standard": 11.111111111111116 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 23.118548982848488, "hard": -12.673038757528188, "judge": 7.843499385378882, "standard": -67.26014388289686 } }, "sealed|choice": { "cc": 14.130226164190928, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 32.15579040721308, "hard": 3.7895927601809762, "judge": 17.132216014897597, "standard": 21.29572607463975 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 28.000000000000004, "hard": -39.80582524271845, "judge": -2.631578947368418, "standard": 4.225352112676051 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -7.688289022450467, "hard": -26.74875998927384, "judge": -33.081610869032005, "standard": -11.930482263268317 } } } }, "jevbench_score": 0.32342918922131697, "key": "deem-0.8-v1", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 110, "ranked": true, "ranks": { "A": 110, "B": 110, "C": 110, "capability": 131 }, "scores": { "A": 0.32342918922131697, "B": 0.23429153017801835, "C": 0.2246036036259146 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.4647796718403697, "p50_s_raw": 0.15738983592018485, "p95_s_adjusted": 0.6141117558348923, "p95_s_raw": 0.23205587791744614 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 25.404113930608833, 32.82152927572935 ], "I": [ 4.660400387407388, 9.084320195556709 ], "capability": [ 15.752622734868103, 20.176991128075684 ], "composite": { "A": [ 0.12746211676468558, 0.7725020253140645 ], "B": [ 0.08887678609852033, 0.587439805908629 ], "C": [ 0.08851535886436501, 0.5364597398014336 ] }, "seed": 35 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 29.6278299611924, "capability": 18.07951706793502, "intelligence": 6.531204174677637 }, "per_type": { "choice": { "accuracy": 0.34933333333333333, "calibration": 0.0, "cc": 16.78902645813397, "cc_by_type": { "choice": 16.78902645813397 }, "cc_tiered": 19.593612524032913, "cc_tiered_open": 25.056998883874897, "cc_tiered_sealed": 14.130226164190928, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 16.78902645813397 }, "noul": { "accuracy": 0.4533333333333333, "calibration": 39.04215154679829, "cc": -9.333333333333337, "cc_by_type": { "noul": -9.333333333333337 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.30933333333333335, "calibration": 49.84133833677892, "cc": -19.150447666018742, "cc_by_type": { "score": -19.150447666018742 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 29.6549349776408, "capability": 17.995438687555687, "intelligence": 6.335942397470573 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.28875743313428337, "Emphasis on Accuracy 60:20:20": 0.17645226737338762, "Emphasis on Cost 20:20:60": 0.4219666221718261, "Emphasis on Speed 20:60:20": 0.4251280933682276, "Intelligence only": 0.11143965867359962 }, "class": "system-one-open", "open": "yes", "licence": "Not independently recorded", "repo": "https://huggingface.co/LibertAIDAI/deem-0.8-v1", "gpu": "H100", "endpoint_condition": "evaluator-owned H100 GPU", "underlying": "LibertAIDAI/deem-0.8-v1 @ 8cbabbb2c4a7ef13c6b43f0ef3ae4157983c6d21; root model.safetensors SHA-256 80438125177c0681e855158d46720cbbd4ff07ea620cd55b0f456aafbfcf778d. Author Python GPU server, default DEEM_N_ORDERS=1, raw softmax (temperature 1.0). A four-field shim translates the wire format. The submitted interface accepts choice labels without descriptions and noul instructions without criteria; score receives full level descriptions. The author Rust/AVX-512 latency claim was not measured.", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.7735849056603774, "acc_among_decisive": 0.5923344947735192, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "FlyMy.AI (@denti)", "axes": { "calibration": 75.07737238688318, "cost": 80.08651867745036, "intelligence": 6.110691078442272, "speed": 91.472185793536 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1702515239808779, "ece_score": 65.94969520382442, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 65.94969520382442 }, "noul": { "abstention_rate": 0.8625336927223719, "brier": 0.2408499885802099, "ece": 0.11364094419260821, "n": 371, "score": 77.27181116147835 }, "score": { "interval90_coverage": 0.959349593495935, "interval90_width": 0.8803071364046972, "n": 369, "nrps": 0.17723101114191298, "score": 82.01061079534679, "top_ece": 0.09127838647557562 } }, "score": 75.07737238688318 }, "capability": 40.59403173266273, "composite_ci95": { "A": [ 0.11705708633042304, 0.8235356910346963 ], "B": [ 0.07857381222727884, 0.5860129975515587 ], "C": [ 0.08128964328501598, 0.5718997854407614 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004610868226600985 }, "display": "Decision Fast (FlyMy.AI, v53a)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 5.162912545807329, "I_sealed": 7.058469611077215, "base": 6.110691078442272, "excess": -4.468944707708131, "gap": -1.8955570652698865, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 15.371451872027388, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 29.945945945945947, "hard": 3.743315508021402, "judge": 24.851680949241928, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -14.28571428571429, "hard": -100.0, "judge": -90.0, "standard": -88.88888888888889 } }, "open|score": { "cc": 0.1172857653946, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 37.515372511743564, "hard": -11.188275382721603, "judge": -4.115171438893084, "standard": 10.37805049488405 } }, "sealed|choice": { "cc": 21.175408833231646, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 32.15579040721308, "hard": 4.298642533936631, "judge": 27.374301675977662, "standard": 40.14041138071193 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -16.000000000000004, "hard": -88.3495145631068, "judge": -86.8421052631579, "standard": -83.09859154929578 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 22.391298609400668, "hard": -12.696561777971827, "judge": 3.397122091897453, "standard": -2.490577451116116 } } } }, "jevbench_score": 0.2981489585308357, "key": "decision-fast", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 111, "ranked": true, "ranks": { "A": 111, "B": 111, "C": 111, "capability": 108 }, "scores": { "A": 0.2981489585308357, "B": 0.20514889729513758, "C": 0.2070478878686359 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2653681320138276, "p50_s_raw": 0.05768406600691378, "p95_s_adjusted": 0.26849280574824663, "p95_s_raw": 0.059246402874123305 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 69.51262824417705, 78.1703871756354 ], "I": [ 4.394274632210615, 8.794694293942209 ], "capability": [ 37.53859057048145, 42.929975748521436 ], "composite": { "A": [ 0.11705708633042304, 0.8235356910346963 ], "B": [ 0.07857381222727884, 0.5860129975515587 ], "C": [ 0.08128964328501598, 0.5718997854407614 ] }, "seed": 32 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 75.07737238688318, "capability": 40.59403173266273, "intelligence": 6.110691078442272 }, "per_type": { "choice": { "accuracy": 0.37333333333333335, "calibration": 65.94969520382442, "cc": 19.85828367894051, "cc_by_type": { "choice": 19.85828367894051 }, "cc_tiered": 18.273430352629518, "cc_tiered_open": 15.371451872027388, "cc_tiered_sealed": 21.175408833231646, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 19.85828367894051 }, "noul": { "accuracy": 0.12266666666666666, "calibration": 77.27181116147835, "cc": -75.46666666666667, "cc_by_type": { "noul": -75.46666666666667 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.31733333333333336, "calibration": 82.01061079534679, "cc": -1.6484760263063114, "cc_by_type": { "score": -1.6484760263063114 }, "cc_tiered": 0.0586428826973, "cc_tiered_open": 0.1172857653946, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 74.21190941158237, "capability": 40.84165753605771, "intelligence": 7.471405660533052 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.23953342944478384, "Emphasis on Accuracy 60:20:20": 0.14519180403822335, "Emphasis on Cost 20:20:60": 0.3522038654235207, "Emphasis on Speed 20:60:20": 0.35744386580296805, "Intelligence only": 0.09127061519323529 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release", "repo": "https://huggingface.co/flymy-ai/decision-fast-preview", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "Qwen/Qwen3-0.6B-Base with a trained LoRA adapter and pointer head (10.6M trainable parameters)", "adapter_id": "images/decision-fast/spec.json:flymy_v15.FlyMyPackage", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.13746630727762804, "acc_among_decisive": 0.9019607843137255, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Chinh Nguyen (ncchinh)", "axes": { "calibration": 73.02134054125926, "cost": 89.11296503596958, "intelligence": 6.054813340422714, "speed": 94.90893356488051 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1464534374999999, "ece_score": 70.70931250000002, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 70.70931250000002 }, "noul": { "abstention_rate": 0.72, "brier": 0.2701432261600001, "ece": 0.15907813333333334, "n": 375, "score": 68.18437333333334 }, "score": { "interval90_coverage": 0.9653333333333334, "interval90_width": 0.8146666666666667, "n": 375, "nrps": 0.1639986175244445, "score": 80.17033579044444, "top_ece": 0.11629733333333336 } }, "score": 73.02134054125926 }, "capability": 39.53807694084099, "composite_ci95": { "A": [ 0.0550759146094291, 0.8570777384335173 ], "B": [ 0.03640384449772412, 0.6085773029152636 ], "C": [ 0.03824716292321466, 0.5951928739121649 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra base-size encoders (bge-base, e5-base, gte-base, all-mpnet-base) $0.005/M input (ModernBERT-base has no market listing; base-size encoder reference) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0023062222906403943 }, "display": "Quyet-1.0-Small-EN (Chinh Nguyen, ModernBERT-base encoder with fixed heads)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 5.966953908358329, "I_sealed": 6.142672772487099, "base": 6.054813340422714, "excess": -2.749106506567014, "gap": -0.17571886412877014, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 11.596876448581234, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 10.486486486486491, "hard": 24.50456118276188, "judge": 1.1206328279499007, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -71.42857142857143, "hard": -65.21739130434783, "judge": -70.0, "standard": -44.44444444444444 } }, "open|score": { "cc": 6.303985276493753, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 12.853939393939395, "hard": 15.763981481481492, "judge": 4.772711864406753, "standard": -13.594074074074047 } }, "sealed|choice": { "cc": 14.421677328746334, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 4.219939398418472, "hard": 11.934389140271476, "judge": 14.338919925512121, "standard": 24.621258775711308 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -48.0, "hard": -84.46601941747574, "judge": -52.63157894736843, "standard": -60.56338028169015 } }, "sealed|score": { "cc": 4.006340988714965, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 17.547730496453894, "hard": 1.5428921568626852, "judge": 5.2876646706586765, "standard": 0.240558375634492 } } } }, "jevbench_score": 0.29239342204088814, "key": "quyet-1-0-small-en", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 112, "ranked": true, "ranks": { "A": 112, "B": 112, "C": 112, "capability": 110 }, "scores": { "A": 0.29239342204088814, "B": 0.20045885649425463, "C": 0.20305098752839457 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1791461439919658, "p50_s_raw": 0.0145730719959829, "p95_s_adjusted": 0.18025992200709878, "p95_s_raw": 0.015129961003549396 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 67.50774946138796, 76.70202320961018 ], "I": [ 3.379327510519746, 8.897820124597144 ], "capability": [ 36.01781590414684, 42.01829735190088 ], "composite": { "A": [ 0.0550759146094291, 0.8570777384335173 ], "B": [ 0.03640384449772412, 0.6085773029152636 ], "C": [ 0.03824716292321466, 0.5951928739121649 ] }, "seed": 89 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 73.02134054125926, "capability": 39.53807694084099, "intelligence": 6.054813340422714 }, "per_type": { "choice": { "accuracy": 0.3253333333333333, "calibration": 70.70931250000002, "cc": 13.719769237327435, "cc_by_type": { "choice": 13.719769237327435 }, "cc_tiered": 13.009276888663784, "cc_tiered_open": 11.596876448581234, "cc_tiered_sealed": 14.421677328746334, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 13.719769237327435 }, "noul": { "accuracy": 0.17866666666666667, "calibration": 68.18437333333334, "cc": -64.26666666666667, "cc_by_type": { "noul": -64.26666666666667 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.312, "calibration": 80.17033579044444, "cc": 5.329298245614, "cc_by_type": { "score": 5.329298245614 }, "cc_tiered": 5.155163132604359, "cc_tiered_open": 6.303985276493753, "cc_tiered_sealed": 4.006340988714965, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 5.329298245614 } }, "raw_on_A": { "calibration": 75.9241654666809, "capability": 42.17966881905028, "intelligence": 8.435172171419651 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.23536198594902974, "Emphasis on Accuracy 60:20:20": 0.14175760897818884, "Emphasis on Cost 20:20:60": 0.3502184575801078, "Emphasis on Speed 20:60:20": 0.3525263048935194, "Intelligence only": 0.08878963459745969 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.28, "acc_among_decisive": 0.638095238095238, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Small-EN", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)", "underlying": "answerdotai/ModernBERT-base" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Convai Innovations", "axes": { "calibration": 82.96996460403702, "cost": 82.52942826196218, "intelligence": 5.471270906587037, "speed": 70.81965680870755 }, "calibration": { "parts": { "choice": { "ece_hard": 0.08964031250000004, "ece_score": 82.07193749999999, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 82.07193749999999 }, "noul": { "abstention_rate": 0.9946666666666667, "brier": 0.24736279077333312, "ece": 0.09575093333333361, "n": 375, "score": 80.84981333333327 }, "score": { "interval90_coverage": 0.968, "interval90_width": 0.8166666666666665, "n": 375, "nrps": 0.15890220709111097, "score": 85.9881429787778, "top_ece": 0.060667466666666586 } }, "score": 82.96996460403702 }, "capability": 44.22061775531203, "composite_ci95": { "A": [ 0.03846019800643322, 0.5898651799347397 ], "B": [ 0.025332676441949146, 0.41662539701360535 ], "C": [ 0.02670847083780085, 0.40962859717690253 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate; no exact base-model floor applies. $0.01/M input, $0/M output; same-class hosted-encoder estimate; no exact-base market floor.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.003822543103448276 }, "display": "Laya typed-decisions", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 5.6928563402143615, "I_sealed": 5.249685472959713, "base": 5.471270906587037, "excess": -2.130216775183595, "gap": 0.4431708672546488, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 10.771453572375108, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": -1.9188424032714584, "judge": 24.851680949241928, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -85.71428571428572, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 6.307115448267977, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 5.935454545454554, "hard": -3.607592592592601, "judge": 9.858813559322, "standard": 20.99481481481481 } }, "sealed|choice": { "cc": 7.102994687643811, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 5.550218017884884, "hard": 2.771493212669664, "judge": 6.890130353817524, "standard": 16.861682473210998 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -96.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 8.646061731235328, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 9.860354609929068, "hard": 2.728602941176461, "judge": 13.044251497005966, "standard": 13.276548223350238 } } } }, "jevbench_score": 0.21666109526429145, "key": "laya-typed-decisions", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 113, "ranked": true, "ranks": { "A": 113, "B": 113, "C": 113, "capability": 97 }, "scores": { "A": 0.21666109526429145, "B": 0.14825243435052415, "C": 0.1504590939335357 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.8777340129803632, "p50_s_raw": 0.8638670064901817, "p95_s_adjusted": 4.409610662701016, "p95_s_raw": 2.1298053313505076 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 77.7245419121787, 86.71414939090002 ], "I": [ 2.9916515598721545, 7.823617519507115 ], "capability": [ 41.079834277143355, 46.819056265617796 ], "composite": { "A": [ 0.03846019800643322, 0.5898651799347397 ], "B": [ 0.025332676441949146, 0.41662539701360535 ], "C": [ 0.02670847083780085, 0.40962859717690253 ] }, "seed": 58 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.96996460403702, "capability": 44.22061775531203, "intelligence": 5.471270906587037 }, "per_type": { "choice": { "accuracy": 0.2733333333333333, "calibration": 82.07193749999999, "cc": 7.069711925579947, "cc_by_type": { "choice": 7.069711925579947 }, "cc_tiered": 8.93722413000946, "cc_tiered_open": 10.771453572375108, "cc_tiered_sealed": 7.102994687643811, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 7.069711925579947 }, "noul": { "accuracy": 0.005333333333333333, "calibration": 80.84981333333327, "cc": -98.93333333333332, "cc_by_type": { "noul": -98.93333333333332 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.36533333333333334, "calibration": 85.9881429787778, "cc": 7.23879824561402, "cc_by_type": { "score": 7.23879824561402 }, "cc_tiered": 7.476588589751652, "cc_tiered_open": 6.307115448267977, "cc_tiered_sealed": 8.646061731235328, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 7.23879824561402 } }, "raw_on_A": { "calibration": 79.92527034876068, "capability": 42.6053892600956, "intelligence": 5.28550817143052 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.17186613033097944, "Emphasis on Accuracy 60:20:20": 0.10420153536931118, "Emphasis on Cost 20:20:60": 0.2566823909067557, "Emphasis on Speed 20:60:20": 0.2523472425383811, "Intelligence only": 0.06551257180569732 }, "class": "system-one-open", "open": "yes", "licence": "apache-2.0, not gated", "repo": "https://huggingface.co/convaiinnovations/laya/tree/1c5edc17a7acd8701df6fc341c0d179f1c62c982/typed-decisions", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU, Sandy (AMD Ryzen 5 3600), 4 of 12 threads, nice -n 5, shared host, offline (HF_HUB_OFFLINE=1)", "underlying": "convaiinnovations/laya @ 1c5edc17a7acd8701df6fc341c0d179f1c62c982, typed-decisions/; ModernBERT-large, max_len 1024, distinct head and calibration from the English root. Fresh full run on 29 Sep; in-process, offline, shared AMD Ryzen 5 3600 CPU, four threads. Measured latency includes shared-host conditions; it is not a controlled GPU comparison. Complete valid probabilities on all types; frequent noul abstention lowers Intelligence to its zero floor and thus the composite to zero. This is not a transport failure.", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.005333333333333333, "acc_among_decisive": 1.0, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Samrat Dutta (samratduttaofficial@gmail.com)", "axes": { "calibration": 65.84236453711111, "cost": 68.67136467637603, "intelligence": 5.305239654184986, "speed": 77.5933742155104 }, "calibration": { "parts": { "choice": { "ece_hard": 0.21968406249999994, "ece_score": 56.06318750000001, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 56.06318750000001 }, "noul": { "abstention_rate": 0.6266666666666667, "brier": 0.25745770685333363, "ece": 0.15462693333333336, "n": 375, "score": 69.07461333333333 }, "score": { "interval90_coverage": 0.904, "interval90_width": 0.7419999999999997, "n": 375, "nrps": 0.18400134444000008, "score": 72.389292778, "top_ece": 0.18410639999999998 } }, "score": 65.84236453711111 }, "capability": 35.57380209564805, "composite_ci95": { "A": [ 0.06779535705435, 0.5949014809956202 ], "B": [ 0.04550364299502931, 0.4255562687724439 ], "C": [ 0.047080109065520834, 0.4131260284691807 ] }, "cost": { "basis": "ESTIMATE, labelled, and RUN 48'S RATE FOR THIS EXACT ROW AND BASE, CARRIED UNCHANGED: answerdotai/ModernBERT-base IS in the frozen pricing_v15.UNLISTED_BASE_MODELS, so no exact base-model floor applies and the module takes the estimate rate from the row's own registry entry; USD 0.03/M input, 0 output is the rate run 47 (quyet-10-small-en) and run 48 (watersheep on v1.5) already assigned to this base. Output 0 is literal: watersheep/infer.py returns output_tokens 0 and the serving path has no generation step (02 Oct deputy rule). x this row's own measured input tokens. Estimated, not charged. No new decision.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.011139300000000001 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.011073574813811781 }, "display": "WaterSheep (Samrat Dutta, ModernBERT-base encoder with calibrated heads, served by the author's own `watersheep --serve`)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 5.71828897246665, "I_sealed": 4.892190335903323, "base": 5.305239654184986, "excess": -1.7472890058749173, "gap": 0.8260986365633265, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 17.15486691739995, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 14.37837837837838, "hard": 11.292859389745214, "judge": 20.896506262359924, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 0.0, "hard": -82.6086956521739, "judge": -50.0, "standard": -33.333333333333336 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 24.005151515151525, "hard": -8.935277777777806, "judge": -0.7272881355932403, "standard": -31.58629629629628 } }, "sealed|choice": { "cc": 14.67657100770997, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 22.843840070948207, "hard": 8.880090497737543, "judge": 15.270018621973943, "standard": 21.29572607463975 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -7.9999999999999964, "hard": -61.165048543689316, "judge": -60.526315789473685, "standard": -46.47887323943662 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 13.4399290780142, "hard": -8.180808823529384, "judge": 3.6488023952095605, "standard": 1.3909137055837406 } } } }, "jevbench_score": 0.19483758736149703, "key": "watersheep", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 114, "ranked": true, "ranks": { "A": 114, "B": 114, "C": 114, "capability": 114 }, "scores": { "A": 0.19483758736149703, "B": 0.13414679955920575, "C": 0.13530388011215072 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.9529444739222527, "p50_s_raw": 0.4014722369611263, "p95_s_adjusted": 1.82639614790678, "p95_s_raw": 0.8381980739533901 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 60.17555613533517, 69.8860267138676 ], "I": [ 3.662660743750301, 7.923346736956256 ], "capability": [ 32.42067095581542, 38.32732634759167 ], "composite": { "A": [ 0.06779535705435, 0.5949014809956202 ], "B": [ 0.04550364299502931, 0.4255562687724439 ], "C": [ 0.047080109065520834, 0.4131260284691807 ] }, "seed": 149 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 65.84236453711111, "capability": 35.57380209564805, "intelligence": 5.305239654184986 }, "per_type": { "choice": { "accuracy": 0.336, "calibration": 56.06318750000001, "cc": 15.0838835576859, "cc_by_type": { "choice": 15.0838835576859 }, "cc_tiered": 15.91571896255496, "cc_tiered_open": 17.15486691739995, "cc_tiered_sealed": 14.67657100770997, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 15.0838835576859 }, "noul": { "accuracy": 0.25866666666666666, "calibration": 69.07461333333333, "cc": -48.266666666666666, "cc_by_type": { "noul": -48.266666666666666 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.32266666666666666, "calibration": 72.389292778, "cc": -1.428035087719337, "cc_by_type": { "score": -1.428035087719337 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 58.9569335572792, "capability": 33.53090791683189, "intelligence": 8.104882276384586 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.15640575110581917, "Emphasis on Accuracy 60:20:20": 0.09493747584576766, "Emphasis on Cost 20:20:60": 0.2296969891012095, "Emphasis on Speed 20:60:20": 0.23287926981068977, "Intelligence only": 0.05972759292809448 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/samratduttaofficial/WaterSheep", "gpu": "CPU (12-core Zen 2; 8 threads)", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. RE-MEASUREMENT on v1.6 of the row run 48 measured on the retired v1.5 basis, same weights: samratduttaofficial/WaterSheep @ ebf3705d2d9dc648d5cc6b4cbca7646c6ab48998 (Apache-2.0), package github.com/SamratDuttaOfficial/WaterSheep @ 3ccf02b3 via the author's own install line in an isolated venv (torch 2.14.1+cpu, transformers 5.19.0), served with the author's own command `watersheep --model samratduttaofficial/WaterSheep@ebf3705d... --serve --device cpu`. Their server speaks POST /v1/systemone with Jev's shapes, so the UNCHANGED typesafe adapter applies; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. Their own max_len 512 (watersheep.json) truncates every longer state, the 23 long items included, without refusing; not overridden. RUN ON SANDY'S SHARED CPU (12-core Zen 2, 8 threads) with server and driver inside a loopback-only network namespace and no credential in the environment; load average 8.6-15.4 (median 13.8) logged every 60 s, so the Speed axis is a DECLARED LOWER BOUND. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline. add-requests run 59, benchmarkheaven.com/submit ref 6F1561FC; previously GitHub #179", "underlying": "answerdotai/ModernBERT-base", "model_pin": "samratduttaofficial/WaterSheep@ebf3705d2d9dc648d5cc6b4cbca7646c6ab48998", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.37333333333333335, "acc_among_decisive": 0.6928571428571428, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Hemant (heman10x)", "axes": { "calibration": 75.69890703057408, "cost": 86.62313527343974, "intelligence": 5.035901218168153, "speed": 81.49274954261848 }, "calibration": { "parts": { "choice": { "ece_hard": 0.13155601347182505, "ece_score": 73.68879730563499, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 73.68879730563499 }, "noul": { "abstention_rate": 1.0, "brier": 0.2495580281877627, "ece": 0.12226520797197521, "n": 375, "score": 75.54695840560495 }, "score": { "interval90_coverage": 0.9893333333333333, "interval90_width": 0.916888888888889, "n": 375, "nrps": 0.17500942351683016, "score": 77.8609653804823, "top_ece": 0.1338856344367619 } }, "score": 75.69890703057408 }, "capability": 40.36740412437111, "composite_ci95": { "A": [ 0.0373899767734964, 0.4672540094953807 ], "B": [ 0.024598584677982253, 0.3262815362283655 ], "C": [ 0.025965261648261392, 0.3244819510384588 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.002791871921182266 }, "display": "openJev Verdict 1.4", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 5.214507036728864, "I_sealed": 4.857295399607441, "base": 5.035901218168153, "excess": -2.216176005316821, "gap": 0.3572116371214227, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 15.643521110186592, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -5.081081081081076, "hard": 13.180245360176167, "judge": 24.851680949241928, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 11.206824837702234, "hard": -8.9038551649125, "judge": -0.9107640863542654, "standard": -33.62275309182012 } }, "sealed|choice": { "cc": 8.257811980618104, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 5.550218017884884, "hard": 6.334841628959255, "judge": 9.683426443202995, "standard": 11.319127971425068 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 6.314074218204219, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 14.960976046364117, "hard": 5.800113224444569, "judge": 5.914716157590583, "standard": 3.617582382564022 } } } }, "jevbench_score": 0.17222636557331997, "key": "openjev-verdict-1.4", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 115, "ranked": true, "ranks": { "A": 115, "B": 116, "C": 115, "capability": 109 }, "scores": { "A": 0.17222636557331997, "B": 0.11682093815444984, "C": 0.11960164275925 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.7352308319881559, "p50_s_raw": 0.2926154159940779, "p95_s_adjusted": 0.9644980074837803, "p95_s_raw": 0.40724900374189016 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 70.20079626876355, 79.44394588474857 ], "I": [ 2.9611688986465445, 7.176652677169861 ], "capability": [ 37.175911743687855, 42.54492005899604 ], "composite": { "A": [ 0.0373899767734964, 0.4672540094953807 ], "B": [ 0.024598584677982253, 0.3262815362283655 ], "C": [ 0.025965261648261392, 0.3244819510384588 ] }, "seed": 83 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 75.69890703057408, "capability": 40.36740412437111, "intelligence": 5.035901218168153 }, "per_type": { "choice": { "accuracy": 0.2866666666666667, "calibration": 73.68879730563499, "cc": 8.774854826028024, "cc_by_type": { "choice": 8.774854826028024 }, "cc_tiered": 11.950666545402349, "cc_tiered_open": 15.643521110186592, "cc_tiered_sealed": 8.257811980618104, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 8.774854826028024 }, "noul": { "accuracy": 0.0, "calibration": 75.54695840560495, "cc": -100.0, "cc_by_type": { "noul": -100.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.344, "calibration": 77.8609653804823, "cc": 4.055284730748376, "cc_by_type": { "score": 4.055284730748376 }, "cc_tiered": 3.1570371091021094, "cc_tiered_open": 0.0, "cc_tiered_sealed": 6.314074218204219, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 4.055284730748376 } }, "raw_on_A": { "calibration": 73.35601823970862, "capability": 39.682189077168275, "intelligence": 6.008359914627925 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.13684263027905905, "Emphasis on Accuracy 60:20:20": 0.08186844689007573, "Emphasis on Cost 20:20:60": 0.2066197471065469, "Emphasis on Speed 20:60:20": 0.20540349835976665, "Intelligence only": 0.05108478843903597 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/heman10x/rlcd-modernbert-151m", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "GLiClass ModernBERT-base fine-tuned decision model, 151M; v1.4 fixed inference engine", "adapter_id": "images/openjev-verdict-1.4/spec.json:warm_adapter.WarmLoaded", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.0, "acc_among_decisive": null, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Alibaba Qwen / neutral reproduction", "axes": { "calibration": 10.94062892769364, "cost": 77.58073177551512, "intelligence": 5.554864880982027, "speed": 93.52703208725202 }, "calibration": { "parts": { "choice": { "ece_hard": 0.6186890197483605, "ece_score": 0.0, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 0.0 }, "noul": { "abstention_rate": 0.0, "brier": 0.580040467448705, "ece": 0.5809468363625269, "n": 371, "score": 0.0 }, "score": { "interval90_coverage": 0.5067750677506775, "interval90_width": 0.31887985546522124, "n": 369, "nrps": 0.34356226433838155, "score": 32.82188678308092, "top_ece": 0.6266298915392519 } }, "score": 10.94062892769364 }, "capability": 8.247746904337834, "composite_ci95": { "A": [ 0.04648667178844407, 0.37042416292233377 ], "B": [ 0.033778521738855774, 0.30178355486696584 ], "C": [ 0.032282410964197275, 0.2572390020293985 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.005588676108374384 }, "display": "Raw Qwen3 0.6B direct logits", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 6.274289147536377, "I_sealed": 4.835440614427676, "base": 5.554864880982027, "excess": -1.1345391093295425, "gap": 1.4388485331087013, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 18.822867442609134, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 29.945945945945947, "hard": 9.405473419314262, "judge": 28.80685563612393, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -57.14285714285714, "hard": -30.434782608695656, "judge": -30.000000000000004, "standard": -22.22222222222222 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 39.408700317953674, "hard": -23.213473292255358, "judge": -47.812096684203475, "standard": -39.963164880126655 } }, "sealed|choice": { "cc": 14.506321843283029, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 28.164954548813856, "hard": -1.8099547511312433, "judge": 26.443202979515842, "standard": 22.404236974996934 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -24.0, "hard": -22.330097087378643, "judge": -5.263157894736848, "standard": -1.4084507042253502 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -8.286778561594211, "hard": -56.317892789427006, "judge": -50.51708927969312, "standard": -75.44853557058413 } } } }, "jevbench_score": 0.1673536172919218, "key": "raw-qwen3-0.6b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 116, "ranked": true, "ranks": { "A": 116, "B": 115, "C": 116, "capability": 135 }, "scores": { "A": 0.1673536172919218, "B": 0.12991425235961862, "C": 0.11621778978605682 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2079640818759799, "p50_s_raw": 0.02898204093798995, "p95_s_adjusted": 0.21345604490488768, "p95_s_raw": 0.03172802245244384 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 10.514844290675109, 11.37896862687295 ], "I": [ 3.4338072843331555, 7.572845892568445 ], "capability": [ 7.166123426414619, 9.279855986833727 ], "composite": { "A": [ 0.04648667178844407, 0.37042416292233377 ], "B": [ 0.033778521738855774, 0.30178355486696584 ], "C": [ 0.032282410964197275, 0.2572390020293985 ] }, "seed": 94 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 10.94062892769364, "capability": 8.247746904337834, "intelligence": 5.554864880982027 }, "per_type": { "choice": { "accuracy": 0.336, "calibration": 0.0, "cc": 15.0838835576859, "cc_by_type": { "choice": 15.0838835576859 }, "cc_tiered": 16.66459464294608, "cc_tiered_open": 18.822867442609134, "cc_tiered_sealed": 14.506321843283029, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 15.0838835576859 }, "noul": { "accuracy": 0.41333333333333333, "calibration": 0.0, "cc": -17.333333333333336, "cc_by_type": { "noul": -17.333333333333336 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.256, "calibration": 32.82188678308092, "cc": -46.098542643497574, "cc_by_type": { "score": -46.098542643497574 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 10.73890139109323, "capability": 8.515857103448049, "intelligence": 6.292812815802869 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.181861744313326, "Emphasis on Accuracy 60:20:20": 0.10948842934445303, "Emphasis on Cost 20:20:60": 0.26903830305770593, "Emphasis on Speed 20:60:20": 0.27429427173292537, "Intelligence only": 0.06856152826445967 }, "class": "raw-logit-control", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Qwen/Qwen3-0.6B", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-0.6B BF16", "adapter_id": "images/raw-qwen3-0.6b/spec.json:raw_adapter.RawControlAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 1.0, "acc_among_decisive": 0.41778975741239893, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Franck Verrot (franckverrot)", "axes": { "calibration": 81.86424479313416, "cost": 80.03797760430007, "intelligence": 4.827562367090417, "speed": 93.98230372494703 }, "calibration": { "parts": { "choice": { "ece_hard": 0.04337133550488594, "ece_score": 91.32573289902281, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 91.32573289902281 }, "noul": { "abstention_rate": 0.921832884097035, "brier": 0.2643957088948787, "ece": 0.12656064690026958, "n": 371, "score": 74.68787061994608 }, "score": { "interval90_coverage": 0.978319783197832, "interval90_width": 0.9372177055103887, "n": 369, "nrps": 0.17965857520325207, "score": 79.5791308604336, "top_ece": 0.11437940379403796 } }, "score": 81.86424479313416 }, "capability": 43.34590358011229, "composite_ci95": { "A": [ 0.036596960876541004, 0.39120491517125294 ], "B": [ 0.024012400167042817, 0.27027613731690037 ], "C": [ 0.02541455616426458, 0.2716700799800367 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.00462807881773399 }, "display": "lev-350m (Franck Verrot, LFM2.5-350M)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 4.690739555547254, "I_sealed": 4.964385178633581, "base": 4.827562367090417, "excess": -2.8470332655245705, "gap": -0.2736456230863267, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 14.072218666641763, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 10.486486486486491, "hard": 11.292859389745214, "judge": 16.941331575477918, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -88.88888888888889 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 16.442424242424224, "hard": -5.066666666666664, "judge": 1.4508474576270802, "standard": -32.55555555555556 } }, "sealed|choice": { "cc": 14.893155535900744, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 21.513561451481806, "hard": 9.898190045248851, "judge": 14.338919925512121, "standard": 22.404236974996934 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -76.0, "hard": -92.23300970873787, "judge": -81.57894736842105, "standard": -94.36619718309859 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 15.455319148936175, "hard": -14.996568627450912, "judge": -3.196407185628769, "standard": -22.131472081218284 } } } }, "jevbench_score": 0.15377137469809216, "key": "lev-350m", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 117, "ranked": true, "ranks": { "A": 117, "B": 118, "C": 117, "capability": 101 }, "scores": { "A": 0.15377137469809216, "B": 0.1036628826651748, "C": 0.10678567687367513 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1946538500036695, "p50_s_raw": 0.022326925001834752, "p95_s_adjusted": 0.2053556374099571, "p95_s_raw": 0.02767781870497856 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 75.71542881918619, 83.56232508492258 ], "I": [ 2.9345502485864183, 6.710083033760257 ], "capability": [ 40.03125074179408, 44.53569592988895 ], "composite": { "A": [ 0.036596960876541004, 0.39120491517125294 ], "B": [ 0.024012400167042817, 0.27027613731690037 ], "C": [ 0.02541455616426458, 0.2716700799800367 ] }, "seed": 60 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 81.86424479313416, "capability": 43.34590358011229, "intelligence": 4.827562367090417 }, "per_type": { "choice": { "accuracy": 0.3333333333333333, "calibration": 91.32573289902281, "cc": 14.74285497759628, "cc_by_type": { "choice": 14.74285497759628 }, "cc_tiered": 14.482687101271253, "cc_tiered_open": 14.072218666641763, "cc_tiered_sealed": 14.893155535900744, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 14.74285497759628 }, "noul": { "accuracy": 0.05333333333333334, "calibration": 74.68787061994608, "cc": -89.33333333333333, "cc_by_type": { "noul": -89.33333333333333 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.248, "calibration": 79.5791308604336, "cc": -8.448333333333302, "cc_by_type": { "score": -8.448333333333302 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 81.0219883791457, "capability": 43.49027409042074, "intelligence": 5.958559801695786 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.12144625080361596, "Emphasis on Accuracy 60:20:20": 0.07231333345467754, "Emphasis on Cost 20:20:60": 0.18259637420585254, "Emphasis on Speed 20:60:20": 0.1852875240477895, "Intelligence only": 0.04500322848109108 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (code); weights under LiquidAI's LFM1.0 licence, following the LFM2.5-350M base", "repo": "https://github.com/franckverrot/lev", "gpu": "RTX6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container", "underlying": "LiquidAI/LFM2.5-350M with a LoRA and a 6.5M-parameter pointer head (a kev clone)", "adapter_id": "images/lev-350m/spec.json:lev_v15.LevAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-05", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.07816711590296496, "acc_among_decisive": 0.6896551724137931, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Alibaba Qwen / neutral reproduction", "axes": { "calibration": 11.516326812547591, "cost": 68.54983190559568, "intelligence": 5.124286554530066, "speed": 93.23530725162705 }, "calibration": { "parts": { "choice": { "ece_hard": 0.7144612706448337, "ece_score": 0.0, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 0.0 }, "noul": { "abstention_rate": 0.0026954177897574594, "brier": 0.5809971640519896, "ece": 0.5815255292937018, "n": 371, "score": 0.0 }, "score": { "interval90_coverage": 0.4959349593495935, "interval90_width": 0.23238482384823847, "n": 369, "nrps": 0.3090203912471445, "score": 34.548980437642776, "top_ece": 0.5704149904947305 } }, "score": 11.516326812547591 }, "capability": 8.320306683538828, "composite_ci95": { "A": [ 0.040682757177666354, 0.3224989584923113 ], "B": [ 0.029352394526125654, 0.25964193163917776 ], "C": [ 0.02825191470671274, 0.22395761006410508 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.011177352216748768 }, "display": "Raw Qwen3 1.7B direct logits", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 7.018912428422589, "I_sealed": 3.2296606806375423, "base": 5.124286554530066, "excess": 1.2158641053468031, "gap": 3.789251747785047, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 21.05673728526777, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 41.621621621621614, "hard": -0.03145643284050508, "judge": 24.851680949241928, "standard": 47.25826705734617 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -57.14285714285714, "hard": -30.434782608695656, "judge": -30.000000000000004, "standard": -22.22222222222222 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 16.019988364571248, "hard": -4.886927335935498, "judge": -32.17506026172301, "standard": -190.1002337898353 } }, "sealed|choice": { "cc": 9.688982041912627, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 21.513561451481806, "hard": -5.3733031674208345, "judge": 21.787709497206727, "standard": 15.753171572853812 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -24.0, "hard": -22.330097087378643, "judge": -5.263157894736848, "standard": -1.4084507042253502 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -19.19653010919571, "hard": -25.671423213473844, "judge": -56.434399118166525, "standard": -31.617468115507542 } } } }, "jevbench_score": 0.13671939226506033, "key": "raw-qwen3-1.7b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 118, "ranked": true, "ranks": { "A": 118, "B": 117, "C": 118, "capability": 134 }, "scores": { "A": 0.13671939226506033, "B": 0.10452214659357986, "C": 0.09494402240629189 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.21396372243762016, "p50_s_raw": 0.03198186121881008, "p95_s_adjusted": 0.221885605994612, "p95_s_raw": 0.03594280299730599 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 11.044864988748188, 11.991879282296361 ], "I": [ 3.263077550449943, 7.14487317844146 ], "capability": [ 7.338168404696409, 9.339947930556471 ], "composite": { "A": [ 0.040682757177666354, 0.3224989584923113 ], "B": [ 0.029352394526125654, 0.25964193163917776 ], "C": [ 0.02825191470671274, 0.22395761006410508 ] }, "seed": 95 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 11.516326812547591, "capability": 8.320306683538828, "intelligence": 5.124286554530066 }, "per_type": { "choice": { "accuracy": 0.30666666666666664, "calibration": 0.0, "cc": 11.332569176700131, "cc_by_type": { "choice": 11.332569176700131 }, "cc_tiered": 15.372859663590198, "cc_tiered_open": 21.05673728526777, "cc_tiered_sealed": 9.688982041912627, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 11.332569176700131 }, "noul": { "accuracy": 0.41333333333333333, "calibration": 0.0, "cc": -17.333333333333336, "cc_by_type": { "noul": -17.333333333333336 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.304, "calibration": 34.548980437642776, "cc": -31.4596497298246, "cc_by_type": { "score": -31.4596497298246 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 11.45520847967844, "capability": 9.226470190380851, "intelligence": 6.997731901083261 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.14292662698061412, "Emphasis on Accuracy 60:20:20": 0.08598558171809113, "Emphasis on Cost 20:20:60": 0.21037073826496294, "Emphasis on Speed 20:60:20": 0.21708826744822254, "Intelligence only": 0.05382204747082655 }, "class": "raw-logit-control", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Qwen/Qwen3-1.7B", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen3-1.7B BF16", "adapter_id": "images/raw-qwen3-1.7b/spec.json:raw_adapter.RawControlAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9973045822102425, "acc_among_decisive": 0.4189189189189189, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Jared Palmer", "axes": { "calibration": 61.20618525995795, "cost": 80.09398520305423, "intelligence": 4.598433868646836, "speed": 92.39904218127765 }, "calibration": { "parts": { "choice": { "ece_hard": 0.22209098434273397, "ece_score": 55.5818031314532, "mean_tvd": null, "n": 729, "n_ece_hard": 307, "n_tvd": 0, "score": 55.5818031314532 }, "noul": { "abstention_rate": 0.7439353099730458, "brier": 0.291543126684636, "ece": 0.1912938005390836, "n": 371, "score": 61.74123989218327 }, "score": { "interval90_coverage": 0.9239130434782609, "interval90_width": 0.835824275362319, "n": 368, "nrps": 0.21140059009020898, "score": 66.29551275623737, "top_ece": 0.23134457739252184 } }, "score": 61.20618525995795 }, "capability": 32.90230956430239, "composite_ci95": { "A": [ 0.024058315307534144, 0.35808460603860887 ], "B": [ 0.015784497925561473, 0.2496641339569836 ], "C": [ 0.016707163408009824, 0.2486698653045895 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004608226600985222 }, "display": "kev 0.5B", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 4.849742544692461, "I_sealed": 4.3471251926012116, "base": 4.598433868646836, "excess": -2.0707702903469944, "gap": 0.5026173520912494, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 14.549227634077383, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": 11.292859389745214, "judge": 24.851680949241928, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -85.71428571428572, "hard": -82.6086956521739, "judge": -100.0, "standard": -66.66666666666667 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 17.000794018795805, "hard": -8.594117745107855, "judge": -32.75818090283609, "standard": -52.88804065591748 } }, "sealed|choice": { "cc": 13.041375577803636, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 17.522725593082576, "hard": 7.352941176470571, "judge": 18.06331471135942, "standard": 14.644660672496622 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -48.0, "hard": -78.64077669902912, "judge": -71.05263157894737, "standard": -66.19718309859155 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 4.208590362582387, "hard": -22.033232980160843, "judge": -8.39299678470844, "standard": -23.154804312918653 } } } }, "jevbench_score": 0.131588623265787, "key": "kev-0.5b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 119, "ranked": true, "ranks": { "A": 119, "B": 119, "C": 119, "capability": 119 }, "scores": { "A": 0.131588623265787, "B": 0.08911346438551963, "C": 0.09138098837901876 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2379663866944611, "p50_s_raw": 0.043983193347230554, "p95_s_adjusted": 0.24186897580511868, "p95_s_raw": 0.04593448790255934 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 55.41312884823437, 65.28644013850979 ], "I": [ 2.5515112413087477, 6.5563351690373795 ], "capability": [ 29.53128001342342, 35.43777813183522 ], "composite": { "A": [ 0.024058315307534144, 0.35808460603860887 ], "B": [ 0.015784497925561473, 0.2496641339569836 ], "C": [ 0.016707163408009824, 0.2486698653045895 ] }, "seed": 52 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 61.20618525995795, "capability": 32.90230956430239, "intelligence": 4.598433868646836 }, "per_type": { "choice": { "accuracy": 0.31866666666666665, "calibration": 55.5818031314532, "cc": 12.8671977871034, "cc_by_type": { "choice": 12.8671977871034 }, "cc_tiered": 13.795301605940509, "cc_tiered_open": 14.549227634077383, "cc_tiered_sealed": 13.041375577803636, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 12.8671977871034 }, "noul": { "accuracy": 0.14133333333333334, "calibration": 61.74123989218327, "cc": -71.73333333333333, "cc_by_type": { "noul": -71.73333333333333 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.272, "calibration": 66.29551275623737, "cc": -15.865897379211603, "cc_by_type": { "score": -15.865897379211603 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 65.8831405050141, "capability": 35.92446865234692, "intelligence": 5.965796799679742 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.028000000000000025, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.01866666666666672, "n": 375 } }, "views": { "Balanced 33:33:33": 0.10538840072414271, "Emphasis on Accuracy 60:20:20": 0.06258833605506466, "Emphasis on Cost 20:20:60": 0.15914261116628434, "Emphasis on Speed 20:60:20": 0.1611592968234279, "Intelligence only": 0.03889464633042005 }, "class": "jev-rebuild", "open": true, "licence": "Apache-2.0", "repo": "https://github.com/jaredpalmer/kev", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "Qwen2.5-0.5B + LoRA + learned pointer head; jaredpalmer/kev-0.5b", "adapter_id": "images/kev-0.5b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.2560646900269542, "acc_among_decisive": 0.5578947368421052, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Hemant (heman10x)", "axes": { "calibration": 49.11476555745841, "cost": 86.62313527343974, "intelligence": 4.1496399816416485, "speed": 79.40046783928 }, "calibration": { "parts": { "choice": { "ece_hard": 0.3268574231249395, "ece_score": 34.6285153750121, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 34.6285153750121 }, "noul": { "abstention_rate": 0.6666666666666667, "brier": 0.28308617598601715, "ece": 0.20554105202175244, "n": 375, "score": 58.89178959564951 }, "score": { "interval90_coverage": 0.816, "interval90_width": 0.6600000000000001, "n": 375, "nrps": 0.21771137902642015, "score": 53.823991701713624, "top_ece": 0.3529043934696537 } }, "score": 49.11476555745841 }, "capability": 26.63220276955003, "composite_ci95": { "A": [ 0.016680739292552642, 0.27473431836523593 ], "B": [ 0.010947780723182459, 0.1924832136378699 ], "C": [ 0.011583846730939337, 0.19078772108696943 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.002791871921182266 }, "display": "openJev Verdict (heman10x, ModernBERT-base 151M)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 5.710480975731317, "I_sealed": 2.5887989875519795, "base": 4.1496399816416485, "excess": 0.5482943457410934, "gap": 3.121681988179337, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 17.13144292719395, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": 13.180245360176167, "judge": 20.896506262359924, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -85.71428571428572, "hard": -56.52173913043479, "judge": -80.0, "standard": -55.55555555555556 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 8.056046691320095, "hard": -27.078088834020786, "judge": -9.574770356665475, "standard": -54.57566450814868 } }, "sealed|choice": { "cc": 7.766396962655938, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 10.87133249575052, "hard": 5.316742081447947, "judge": 6.890130353817524, "standard": 12.427638871782255 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -64.0, "hard": -49.51456310679612, "judge": -63.1578947368421, "standard": -60.56338028169015 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 11.069847893733552, "hard": -5.166740337464448, "judge": -15.492495207027046, "standard": -16.23629733590537 } } } }, "jevbench_score": 0.09650709186186587, "key": "openjev-verdict", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 120, "ranked": true, "ranks": { "A": 120, "B": 120, "C": 120, "capability": 127 }, "scores": { "A": 0.09650709186186587, "B": 0.06541514759096348, "C": 0.0670188137929624 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.6885442301630974, "p50_s_raw": 0.2692721150815487, "p95_s_adjusted": 1.66732926396653, "p95_s_raw": 0.7586646319832651 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 43.627592951745086, 53.02973228229396 ], "I": [ 2.2588266004870072, 6.027430783705419 ], "capability": [ 23.293166910490978, 29.139107914545924 ], "composite": { "A": [ 0.016680739292552642, 0.27473431836523593 ], "B": [ 0.010947780723182459, 0.1924832136378699 ], "C": [ 0.011583846730939337, 0.19078772108696943 ] }, "seed": 82 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 49.11476555745841, "capability": 26.63220276955003, "intelligence": 4.1496399816416485 }, "per_type": { "choice": { "accuracy": 0.2906666666666667, "calibration": 34.6285153750121, "cc": 9.286397696162448, "cc_by_type": { "choice": 9.286397696162448 }, "cc_tiered": 12.448919944924945, "cc_tiered_open": 17.13144292719395, "cc_tiered_sealed": 7.766396962655938, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 9.286397696162448 }, "noul": { "accuracy": 0.2, "calibration": 58.89178959564951, "cc": -60.0, "cc_by_type": { "noul": -60.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2693333333333333, "calibration": 53.823991701713624, "cc": -9.675453706526316, "cc_by_type": { "score": -9.675453706526316 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 49.678974901443674, "capability": 27.759290067036318, "intelligence": 5.839605232628962 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.07793885393113759, "Emphasis on Accuracy 60:20:20": 0.046097376471726674, "Emphasis on Cost 20:20:60": 0.11949201576637343, "Emphasis on Speed 20:60:20": 0.11862755697857134, "Intelligence only": 0.028581910146043114 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://github.com/Heman10x-NGU/openJev-verdict-2.0", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "GLiClass ModernBERT-base (knowledgator/gliclass-modern-base-v2.0) fine-tuned, 151M", "adapter_id": "images/openjev-verdict/spec.json:warm_adapter.WarmLoaded", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.3333333333333333, "acc_among_decisive": 0.6, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Chinh Nguyen (ncchinh)", "axes": { "calibration": 60.96568104722222, "cost": 79.56261245432609, "intelligence": 4.019045994532796, "speed": 94.91846454857637 }, "calibration": { "parts": { "choice": { "ece_hard": 0.22390249999999998, "ece_score": 55.2195, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 55.2195 }, "noul": { "abstention_rate": 0.712, "brier": 0.28332537661333335, "ece": 0.2195469333333333, "n": 375, "score": 56.09061333333334 }, "score": { "interval90_coverage": 0.9253333333333333, "interval90_width": 0.8177777777777777, "n": 375, "nrps": 0.1894891371666667, "score": 71.58692980833334, "top_ece": 0.18938613333333326 } }, "score": 60.96568104722222 }, "capability": 32.492363520877504, "composite_ci95": { "A": [ 0.020959672741331675, 0.3032118958147926 ], "B": [ 0.01370004084448916, 0.2103085758125444 ], "C": [ 0.014555328292591438, 0.21056381653805042 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra same-size encoders (bge-large, e5-large, Qwen3-Embedding-0.6B) $0.01/M input (SEA-LION-ModernBERT-300M has no market listing; the ModernBERT-large-class reference of the Laya/Certo/OpenDecision rows) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.004800055418719211 }, "display": "Quyet-1.0-Small (Chinh Nguyen, SEA-LION-ModernBERT-300M encoder with fixed heads)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 4.071843176603614, "I_sealed": 3.966248812461978, "base": 4.019045994532796, "excess": -2.467793278296608, "gap": 0.10559436414163592, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 12.215529529810842, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 2.702702702702708, "hard": 5.630701478452354, "judge": 20.896506262359924, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -71.42857142857143, "hard": -56.52173913043479, "judge": -40.0, "standard": -55.55555555555556 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 13.416060606060597, "hard": -26.448333333333316, "judge": -3.2088135593220546, "standard": -76.01518518518517 } }, "sealed|choice": { "cc": 11.898746437385935, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 17.522725593082576, "hard": 5.316742081447947, "judge": 19.92551210428307, "standard": 10.210617071067881 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -72.0, "hard": -59.22330097087378, "judge": -65.78947368421053, "standard": -46.47887323943662 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 20.691063829787236, "hard": -7.6461764705882285, "judge": -0.1282634730539245, "standard": 0.04786802030454229 } } } }, "jevbench_score": 0.08963716950381444, "key": "quyet-1-0-small", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 121, "ranked": true, "ranks": { "A": 121, "B": 121, "C": 121, "capability": 121 }, "scores": { "A": 0.08963716950381444, "B": 0.0601437708059232, "C": 0.06224803437764891 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.17903870801674202, "p50_s_raw": 0.01451935400837101, "p95_s_adjusted": 0.17997269080078693, "p95_s_raw": 0.014986345400393473 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 55.688303752892594, 64.99464873956296 ], "I": [ 2.4305093708220933, 6.1818340512057315 ], "capability": [ 29.486356353832495, 35.15105810056002 ], "composite": { "A": [ 0.020959672741331675, 0.3032118958147926 ], "B": [ 0.01370004084448916, 0.2103085758125444 ], "C": [ 0.014555328292591438, 0.21056381653805042 ] }, "seed": 88 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 60.96568104722222, "capability": 32.492363520877504, "intelligence": 4.019045994532796 }, "per_type": { "choice": { "accuracy": 0.30533333333333335, "calibration": 55.2195, "cc": 11.162054886655328, "cc_by_type": { "choice": 11.162054886655328 }, "cc_tiered": 12.057137983598388, "cc_tiered_open": 12.215529529810842, "cc_tiered_sealed": 11.898746437385935, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 11.162054886655328 }, "noul": { "accuracy": 0.20533333333333334, "calibration": 56.09061333333334, "cc": -58.93333333333332, "cc_by_type": { "noul": -58.93333333333332 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2986666666666667, "calibration": 71.58692980833334, "cc": -4.271587719298253, "cc_by_type": { "score": -4.271587719298253 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 55.62018953114671, "capability": 29.685347509548446, "intelligence": 3.750505487950187 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.07128318534334778, "Emphasis on Accuracy 60:20:20": 0.04197968509922768, "Emphasis on Cost 20:20:60": 0.10875180747478416, "Emphasis on Speed 20:60:20": 0.11026128891416027, "Intelligence only": 0.025967427058159954 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.288, "acc_among_decisive": 0.7129629629629629, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Small", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)", "underlying": "aisingapore/SEA-LION-ModernBERT-300M" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Zaitgeist Labs / Christian Oezbek (c.oezbek@gmail.com, GitHub @coezbek / @zaitgeist-charly)", "axes": { "calibration": 79.37888213235459, "cost": 89.00253166733508, "intelligence": 3.759262959387639, "speed": 75.84832436427908 }, "calibration": { "parts": { "choice": { "ece_hard": 0.13684690553745932, "ece_score": 72.63061889250814, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 72.63061889250814 }, "noul": { "abstention_rate": 0.9865229110512129, "brier": 0.2509324734501347, "ece": 0.08358625336927226, "n": 371, "score": 83.28274932614555 }, "score": { "interval90_coverage": 0.967479674796748, "interval90_width": 0.8308491418247516, "n": 369, "nrps": 0.16220543914182478, "score": 82.22327817841011, "top_ece": 0.09666449864498641 } }, "score": 79.37888213235459 }, "capability": 41.569072545871116, "composite_ci95": { "A": [ 0.004016900605731708, 0.3790125753277212 ], "B": [ 0.0025739064184890927, 0.26336355668719996 ], "C": [ 0.0027895143095359085, 0.2632031773109175 ] }, "cost": { "basis": "ESTIMATE (size-class reference; self-hosted open weights, no public tariff). THE SAME CHOICE THE PUBLISHED quyet-1-0-tiny ROW AND RUN 59'S tacet-sonata ROW CARRY FOR THE IDENTICAL mmBERT-small BACKBONE, carried unchanged and not a new judgement: DeepInfra base-size encoders at USD 0.005 per 1M input, 0 output - deepinfra:thenlper/gte-base, deepinfra:BAAI/bge-base-en-v1.5, deepinfra:intfloat/e5-base-v2 and deepinfra:sentence-transformers/all-mpnet-base-v2 are all (0.005, 0.0) in the frozen 25 Sep snapshot - x this row's own measured input tokens. Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, verified in the author's source: watt_flash/serve.py:answer() formats a softmax over the option markers of one encoder pass and the package contains no generation path at all (the 02 Oct deputy rule for classifier read-outs). THE SERVER REPORTS NO usage BLOCK OF ANY KIND, so the input counts come EXACTLY from the author's own unmodified watt_flash.layout.Layout.pack_request - the very number their own WattFlash.decide_batch returns as len(p) and only serve.py drops - under PREREG-R61-AMENDMENT-1, cross-checked three ways (independent reassembly of their documented layout; identity of their packer's Overflow set with the server's own 422 set), with no cost number at all on any disagreement; the counts sit in cost_estimate and never in usage. RELEASE-LANE LINE, recorded and not acted on, AND IT IS THE CARRIED quyet/tacet LINE, NOT A NEW ONE: jhu-clsp/mmBERT-small still has no explicit market-listing decision (pricing_v15.reference() raises), so an UNLISTED_BASE_MODELS entry is the release lane's.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.00229019 }, "kind": "mixed", "usage_estimated_rows": 1477, "usd_per_1000": 0.0023258530805687202 }, "display": "watt-flash-0.1 (Zaitgeist Labs, 140M mmBERT-small encoder with an option-marker scorer, one forward pass per request, self-hosted)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 5.1066210946590145, "I_sealed": 2.4119048241162644, "base": 3.759262959387639, "excess": 0.12132862810450629, "gap": 2.69471627054275, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 9.15774230012725, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 26.05405405405406, "hard": -1.9188424032714584, "judge": 12.986156888595913, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -85.71428571428572, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 6.162120983849794, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 20.743030303030295, "hard": -4.7528703703703545, "judge": 12.24322033898303, "standard": 11.579999999999991 } }, "sealed|choice": { "cc": 1.205164808546094, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 38.80718350454514, "hard": -2.3190045248868976, "judge": -8.9385474860335, "standard": 4.668062569281944 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -92.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 6.0305496638027005, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 12.903262411347516, "hard": -5.752573529411742, "judge": 16.985089820359278, "standard": 9.728629441624314 } } } }, "jevbench_score": 0.07461802002995101, "key": "watt-flash-0.1", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 122, "ranked": true, "ranks": { "A": 122, "B": 122, "C": 122, "capability": 107 }, "scores": { "A": 0.07461802002995101, "B": 0.049670099364146944, "C": 0.05181806946524376 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.0144243478775024, "p50_s_raw": 0.4322121739387512, "p95_s_adjusted": 2.564176402240991, "p95_s_raw": 1.2070882011204955 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 74.23510907541693, 82.71222317398025 ], "I": [ 1.3821722420456268, 6.668265912418397 ], "capability": [ 38.3327917105868, 43.940510516442146 ], "composite": { "A": [ 0.004016900605731708, 0.3790125753277212 ], "B": [ 0.0025739064184890927, 0.26336355668719996 ], "C": [ 0.0027895143095359085, 0.2632031773109175 ] }, "seed": 150 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 79.37888213235459, "capability": 41.569072545871116, "intelligence": 3.759262959387639 }, "per_type": { "choice": { "accuracy": 0.25866666666666666, "calibration": 72.63061889250814, "cc": 5.194054735087065, "cc_by_type": { "choice": 5.194054735087065 }, "cc_tiered": 5.181453554336672, "cc_tiered_open": 9.15774230012725, "cc_tiered_sealed": 1.205164808546094, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 5.194054735087065 }, "noul": { "accuracy": 0.008, "calibration": 83.28274932614555, "cc": -98.4, "cc_by_type": { "noul": -98.4 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.368, "calibration": 82.22327817841011, "cc": 4.764543859649162, "cc_by_type": { "score": 4.764543859649162 }, "cc_tiered": 6.096335323826247, "cc_tiered_open": 6.162120983849794, "cc_tiered_sealed": 6.0305496638027005, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 4.764543859649162 } }, "raw_on_A": { "calibration": 80.11428356201952, "capability": 42.17074394730905, "intelligence": 4.227204332598573 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 0.05839101580851302, "Emphasis on Accuracy 60:20:20": 0.03436581390194284, "Emphasis on Cost 20:20:60": 0.09032934623751702, "Emphasis on Speed 20:60:20": 0.08921814434421181, "Intelligence only": 0.02125044886845491 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/zaitlabs/watt-flash-0.1", "gpu": "CPU (12-core Zen 2, AVX2; 8 threads)", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights zaitlabs/watt-flash-0.1 @ d49a3a880b0201b325f2d2299d4c36ffc8735477 (Apache-2.0, public and ungated), model.safetensors sha256 099e611bd27c9542732b89be47bcace54964c00d9c7161ebccb061648c9780bf = THE SUBMITTER'S OWN DECLARED HASH, loaded from a local snapshot with the author's own --model flag (no network during the run). Code github.com/zaitgeist-charly/zaitlabs-watt-flash-0.1 @ e9863f06230ad874a5a2cfca3af922e45161a4d7 (Apache-2.0, = HEAD of main), installed with the author's own line `uv sync` against their own uv.lock in an isolated venv (torch 2.14.0, transformers 5.17.0, safetensors 0.8.0) and served with their own documented command `python -m watt_flash.serve --port 8942`. No file of theirs was edited. Their server answers POST /v1/systemone with Jev's shapes (noul -> {noul: P(true)}; choice probabilities keyed by the criteria keys with choice = argmax; score probabilities keyed '0'..'n-1'), so the UNCHANGED typesafe adapter applies and no mapping is ours; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. THE AUTHOR'S FREE HOSTED ENDPOINT api.inference.zaitlabs.com WAS NEVER USED - the submission states it may train on the data, so no sealed item reached it. THE LONG ITEMS ARE REFUSED, NOT TRUNCATED, by the author's own code: watt_flash/layout.py raises Overflow past its documented 8,192-token window and serve.py answers that with HTTP 422, which their own commit e9863f06 says is deliberate ('answer an over-long request with 422, which JevBench scores as a refusal, not an outage'); those items are counted wrong and no other item failed. A RESUBMISSION of GitHub #113 via the form, which the author asks to dedupe with it. RUN ON SANDY'S SHARED CPU (12-core Zen 2, AVX2, fp32 - the package itself calls model.float() off-accelerator, 8 threads, nice), server and driver inside a loopback-only network namespace (unshare -rn) with no credential in the environment; load average logged every 60 s, so the Speed axis is a DECLARED LOWER BOUND. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline. add-requests run 61, benchmarkheaven.com/submit ref 0874B655 (bh-submit 4cf0bdfb-b6d7-4777-a8ac-350d29c5ac3f)", "underlying": "jhu-clsp/mmBERT-small", "model_pin": "zaitlabs/watt-flash-0.1@d49a3a880b0201b325f2d2299d4c36ffc8735477", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.013477088948787063, "acc_among_decisive": 0.6, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Chinh Nguyen (ncchinh)", "axes": { "calibration": 58.46458026762962, "cost": 88.43590631389968, "intelligence": 3.6964771541368573, "speed": 95.09610910731021 }, "calibration": { "parts": { "choice": { "ece_hard": 0.2315684375, "ece_score": 53.6863125, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 53.6863125 }, "noul": { "abstention_rate": 0.8026666666666666, "brier": 0.28768576031999976, "ece": 0.19567466666666677, "n": 375, "score": 60.86506666666664 }, "score": { "interval90_coverage": 0.8933333333333333, "interval90_width": 0.7453333333333331, "n": 375, "nrps": 0.1976125006088888, "score": 60.84236163622223, "top_ece": 0.2927701333333333 } }, "score": 58.46458026762962 }, "capability": 31.08052871088324, "composite_ci95": { "A": [ 0.019766287637893492, 0.2734027471253872 ], "B": [ 0.012911567780248491, 0.18895613284128507 ], "C": [ 0.013726588637426037, 0.1898630188370744 ] }, "cost": { "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra base-size encoders (bge-base, e5-base, gte-base, all-mpnet-base) $0.005/M input (mmBERT-small 16-layer, 183M; base-size class reference, errs high vs the small-encoder $0.004 class) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0024292364532019703 }, "display": "Quyet-1.0-Tiny (Chinh Nguyen, mmBERT-small encoder (16 layers kept) with fixed heads)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 1.2549826648038989, "I_sealed": 6.1379716434698155, "base": 3.6964771541368573, "excess": -7.456376621104161, "gap": -4.882988978665916, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 3.764947994411697, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -12.86486486486486, "hard": 5.630701478452354, "judge": 12.986156888595913, "standard": -5.4834658853076625 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -71.42857142857143, "hard": -56.52173913043479, "judge": -80.0, "standard": -88.88888888888889 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -8.307575757575748, "hard": -21.650555555555574, "judge": -15.678474576271228, "standard": -51.14888888888891 } }, "sealed|choice": { "cc": 16.468453763405932, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 12.201611115216933, "hard": 14.479638009049756, "judge": 18.06331471135942, "standard": 20.18721517428256 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -84.0, "hard": -68.93203883495146, "judge": -71.05263157894737, "standard": -83.09859154929578 } }, "sealed|score": { "cc": 1.9454611670035158, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 16.69851063829787, "hard": -2.889509803921597, "judge": -1.9068263473054348, "standard": 10.017309644669991 } } } }, "jevbench_score": 0.07064766318734213, "key": "quyet-1-0-tiny", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 123, "ranked": true, "ranks": { "A": 123, "B": 123, "C": 123, "capability": 123 }, "scores": { "A": 0.07064766318734213, "B": 0.04711839739648976, "C": 0.04906087721343202 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1753336349996971, "p50_s_raw": 0.012666817499848548, "p95_s_adjusted": 0.17641026510682423, "p95_s_raw": 0.013205132553412113 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 52.6105536767426, 62.31255872829814 ], "I": [ 2.382388771227772, 5.956289831309647 ], "capability": [ 27.937688273976466, 33.75728623357733 ], "composite": { "A": [ 0.019766287637893492, 0.2734027471253872 ], "B": [ 0.012911567780248491, 0.18895613284128507 ], "C": [ 0.013726588637426037, 0.1898630188370744 ] }, "seed": 90 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 58.46458026762962, "capability": 31.08052871088324, "intelligence": 3.6964771541368573 }, "per_type": { "choice": { "accuracy": 0.32133333333333336, "calibration": 53.6863125, "cc": 13.20822636719302, "cc_by_type": { "choice": 13.20822636719302 }, "cc_tiered": 10.116700878908814, "cc_tiered_open": 3.764947994411697, "cc_tiered_sealed": 16.468453763405932, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 13.20822636719302 }, "noul": { "accuracy": 0.12533333333333332, "calibration": 60.86506666666664, "cc": -74.93333333333334, "cc_by_type": { "noul": -74.93333333333334 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.33866666666666667, "calibration": 60.84236163622223, "cc": -1.831508771929835, "cc_by_type": { "score": -1.831508771929835 }, "cc_tiered": 0.9727305835017579, "cc_tiered_open": 0.0, "cc_tiered_sealed": 1.9454611670035158, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 54.460514049074064, "capability": 29.223962167243283, "intelligence": 3.987410285412503 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.05608574548219273, "Emphasis on Accuracy 60:20:20": 0.03279057174059279, "Emphasis on Cost 20:20:60": 0.08676445199059092, "Emphasis on Speed 20:60:20": 0.08720297577636826, "Intelligence only": 0.020203381773039073 }, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.19733333333333333, "acc_among_decisive": 0.6351351351351351, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "author submission" }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Tiny", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (order 3643732b)", "underlying": "jhu-clsp/mmBERT-small" }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Kotoba Labs", "axes": { "calibration": 67.96597988813339, "cost": 77.5922354035203, "intelligence": 2.8293414417857914, "speed": 67.62138193768119 }, "calibration": { "parts": { "choice": { "ece_hard": 0.20927949454905329, "ece_score": 58.14410109018935, "mean_tvd": null, "n": 682, "n_ece_hard": 276, "n_tvd": 0, "score": 58.14410109018935 }, "noul": { "abstention_rate": 0.848, "brier": 0.2599137925955484, "ece": 0.20243010437488554, "n": 375, "score": 59.5139791250229 }, "score": { "interval90_coverage": 0.8945945945945946, "interval90_width": 0.7720720720720718, "n": 370, "nrps": 0.16620611847475783, "score": 86.23985944918795, "top_ece": 0.05449834627074163 } }, "score": 67.96597988813339 }, "capability": 35.39766066495959, "composite_ci95": { "A": [ 0.0016094499426836406, 0.16453983269325992 ], "B": [ 0.0010275985408014102, 0.11262740891592737 ], "C": [ 0.0011176735713080835, 0.11426377270365268 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.005583743842364532 }, "display": "open-jev-deberta-v3-large (local CPU)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 3.3193692126143426, "I_sealed": 2.3393136709572406, "base": 2.8293414417857914, "excess": -1.5933321007811418, "gap": 0.980055541657102, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 9.958107637843028, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 22.16216216216216, "hard": 1.855929537590448, "judge": 16.941331575477918, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -71.42857142857143, "hard": -73.91304347826086, "judge": -70.0, "standard": -77.77777777777779 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -3.536711678360427, "hard": -10.998114736543751, "judge": 10.982108090893682, "standard": -6.176287708459061 } }, "sealed|choice": { "cc": 5.620402697655405, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 17.522725593082576, "hard": -1.3009049773755852, "judge": 7.8212290502793484, "standard": 10.210617071067881 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -68.0, "hard": -88.3495145631068, "judge": -84.21052631578947, "standard": -91.54929577464789 } }, "sealed|score": { "cc": 1.3975383152163166, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -2.206062525510788, "hard": -5.817775736909847, "judge": 9.848256453782477, "standard": 4.953889631982955 } } } }, "jevbench_score": 0.03235814777127837, "key": "open-jev-deberta-v3-large", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 124, "ranked": true, "ranks": { "A": 124, "B": 124, "C": 124, "capability": 115 }, "scores": { "A": 0.03235814777127837, "B": 0.021367993730971636, "C": 0.02247093595227665 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 3.5280169868841766, "p50_s_raw": 1.6890084934420884, "p95_s_adjusted": 4.901524059008808, "p95_s_raw": 2.3757620295044037 }, "status": { "answered_ok": 1427, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 62.83136705170831, 72.03960745264857 ], "I": [ 1.0164743841847181, 4.99302167761034 ], "capability": [ 32.41699387669788, 37.92156574534849 ], "composite": { "A": [ 0.0016094499426836406, 0.16453983269325992 ], "B": [ 0.0010275985408014102, 0.11262740891592737 ], "C": [ 0.0011176735713080835, 0.11426377270365268 ] }, "seed": 75 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 67.96597988813339, "capability": 35.39766066495959, "intelligence": 2.8293414417857914 }, "per_type": { "choice": { "accuracy": 0.272, "calibration": 58.14410109018935, "cc": 6.899197635535144, "cc_by_type": { "choice": 6.899197635535144 }, "cc_tiered": 7.789255167749216, "cc_tiered_open": 9.958107637843028, "cc_tiered_sealed": 5.620402697655405, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 6.899197635535144 }, "noul": { "accuracy": 0.088, "calibration": 59.5139791250229, "cc": -82.4, "cc_by_type": { "noul": -82.4 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.31733333333333336, "calibration": 86.23985944918795, "cc": -0.7784697361159942, "cc_by_type": { "score": -0.7784697361159942 }, "cc_tiered": 0.6987691576081583, "cc_tiered_open": 0.0, "cc_tiered_sealed": 1.3975383152163166, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 68.63668964823074, "capability": 36.14242333704565, "intelligence": 3.64815702586055 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.09066666666666667, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.013333333333333308, "n": 375 } }, "views": { "Balanced 33:33:33": 0.0252055189654814, "Emphasis on Accuracy 60:20:20": 0.014715479135725748, "Emphasis on Cost 20:20:60": 0.0393479951952157, "Emphasis on Speed 20:60:20": 0.038983856596301915, "Intelligence only": 0.00905974708046916 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (model card); DeBERTa-v3 keeps its own terms", "repo": "https://github.com/kotoba-lang/typed-decisions", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "microsoft/deberta-v3-large", "adapter_id": "images/open-jev-deberta-v3-large/spec.json:warm_adapter.WarmLoaded", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.152, "acc_among_decisive": 0.5789473684210527, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Manavarya09 (Manav)", "axes": { "calibration": 72.92000413192591, "cost": 100.0, "intelligence": 2.3417095945172277, "speed": 89.48731594094681 }, "calibration": { "parts": { "choice": { "ece_hard": 0.13790718750000003, "ece_score": 72.4185625, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 72.4185625 }, "noul": { "abstention_rate": 0.5546666666666666, "brier": 0.28117829917333326, "ece": 0.18824453333333335, "n": 375, "score": 62.35109333333333 }, "score": { "interval90_coverage": 0.984, "interval90_width": 0.9684444444444443, "n": 375, "nrps": 0.18136406875111122, "score": 83.99035656244443, "top_ece": 0.06941440000000007 } }, "score": 72.92000413192591 }, "capability": 37.63085686322157, "composite_ci95": { "A": [ 0.00034752623458969486, 0.12857270817562724 ], "B": [ 0.00021950048318305755, 0.08623222919712645 ], "C": [ 0.0002413376629095103, 0.08928660289974111 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.000866152709359606 }, "display": "verdict-small (Manavarya09, multilingual-e5-small 118M)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 4.005009650747796, "I_sealed": 0.6784095382866591, "base": 2.3417095945172277, "excess": 0.7532124700228935, "gap": 3.3266001124611373, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 12.01502895224339, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -8.972972972972968, "hard": 13.180245360176167, "judge": 9.03098220171391, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -28.57142857142857, "hard": -39.13043478260869, "judge": -40.0, "standard": -77.77777777777779 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 6.291818181818199, "hard": -6.231481481481493, "judge": 7.341694915254216, "standard": -11.9677777777778 } }, "sealed|choice": { "cc": 2.035228614859977, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -7.75256817677922, "hard": -4.355203619909522, "judge": 6.890130353817524, "standard": 12.427638871782255 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -31.999999999999996, "hard": -45.63106796116505, "judge": -47.36842105263158, "standard": -46.47887323943662 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -0.7665248226950139, "hard": -6.760098039215645, "judge": 6.645688622754465, "standard": -2.297512690355341 } } } }, "jevbench_score": 0.01899384266854915, "key": "verdict-small", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 125, "ranked": true, "ranks": { "A": 125, "B": 125, "C": 125, "capability": 112 }, "scores": { "A": 0.01899384266854915, "B": 0.012337048142708213, "C": 0.013190168519825799 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.24820413775742053, "p50_s_raw": 0.04910206887871027, "p95_s_adjusted": 0.4533769000321627, "p95_s_raw": 0.15168845001608133 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 67.28225057545556, 76.20018850659073 ], "I": [ 0.6053648936331004, 4.532806367679299 ], "capability": [ 34.685562893137785, 39.80647930109692 ], "composite": { "A": [ 0.00034752623458969486, 0.12857270817562724 ], "B": [ 0.00021950048318305755, 0.08623222919712645 ], "C": [ 0.0002413376629095103, 0.08928660289974111 ] }, "seed": 108 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 72.92000413192591, "capability": 37.63085686322157, "intelligence": 2.3417095945172277 }, "per_type": { "choice": { "accuracy": 0.23733333333333334, "calibration": 72.4185625, "cc": 2.4658260943701484, "cc_by_type": { "choice": 2.4658260943701484 }, "cc_tiered": 7.0251287835516845, "cc_tiered_open": 12.01502895224339, "cc_tiered_sealed": 2.035228614859977, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 2.4658260943701484 }, "noul": { "accuracy": 0.2773333333333333, "calibration": 62.35109333333333, "cc": -44.53333333333334, "cc_by_type": { "noul": -44.53333333333334 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2986666666666667, "calibration": 83.99035656244443, "cc": -2.2494035087719633, "cc_by_type": { "score": -2.2494035087719633 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 74.70758393102564, "capability": 38.932253022553525, "intelligence": 3.156922114081418 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.014681237741460948, "Emphasis on Accuracy 60:20:20": 0.008421478349576572, "Emphasis on Cost 20:20:60": 0.02342353351655187, "Emphasis on Speed 20:60:20": 0.02330657899907492, "Intelligence only": 0.005136403075864165 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 (verdictml code and the Manav2op/verdict-small checkpoint over intfloat/multilingual-e5-small)", "repo": "https://github.com/Manavarya09/verdict", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "intfloat/multilingual-e5-small (118M multilingual bi-encoder) fine-tuned on a typed-decision mix; each option is scored against the rendered state by cosine similarity, with temperature scaling and a conformal abstain set on top. One encoder pass per option, no tokens generated.", "adapter_id": "images/verdict-small/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.44533333333333336, "acc_among_decisive": 0.6227544910179641, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "CodePawl / @anxuanng (founder@codepawl.com)", "axes": { "calibration": 46.37837260299999, "cost": 89.01467380421724, "intelligence": 2.2436273530567146, "speed": 80.28211114890141 }, "calibration": { "parts": { "choice": { "ece_hard": 0.43076843750000016, "ece_score": 13.846312499999968, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 13.846312499999968 }, "noul": { "abstention_rate": 0.6746666666666667, "brier": 0.28588573402666656, "ece": 0.15048053333333336, "n": 375, "score": 69.90389333333333 }, "score": { "interval90_coverage": 0.736, "interval90_width": 0.5186666666666669, "n": 375, "nrps": 0.21227189381999992, "score": 55.38491197566667, "top_ece": 0.3400149333333333 } }, "score": 46.37837260299999 }, "capability": 24.310999978028352, "composite_ci95": { "A": [ 0.0012369873259327144, 0.11178055042619336 ], "B": [ 0.0007893069841145717, 0.07607180414333671 ], "C": [ 0.0008590189763421627, 0.07762538224041209 ] }, "cost": { "basis": "ESTIMATE (size-class reference; self-hosted open weights, no public tariff). THE SAME CHOICE THE PUBLISHED quyet-1-0-tiny ROW CARRIES FOR THE IDENTICAL mmBERT-small BACKBONE: DeepInfra base-size encoders at USD 0.005 per 1M input, 0 output - deepinfra:thenlper/gte-base, deepinfra:BAAI/bge-base-en-v1.5, deepinfra:intfloat/e5-base-v2 and deepinfra:sentence-transformers/all-mpnet-base-v2 are all (0.005, 0.0) in the frozen 25 Sep snapshot - x this row's own measured usage.input_tokens (the package's packed length, capped at 4,096). Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, verified in the package source (tacet/decoding.py usage_of sets output_tokens 0; the readout is a softmax over option [MASK] markers from one encoder pass and nothing is generated), the 02 Oct deputy rule for classifier read-outs. RELEASE-LANE LINE, recorded and not acted on, AND IT IS THE QUYET ROWS' CARRIED LINE, NOT A NEW ONE: jhu-clsp/mmBERT-small still has no explicit market-listing decision (pricing_v15.reference() raises), so an UNLISTED_BASE_MODELS entry is the release lane's.", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.0026020833333333334 }, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0023236865267433986 }, "display": "Tacet Sonata (CodePawl, 144M packed encoder on mmBERT-small, option-marker softmax in one pass, in-process via the author's tacet package)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 1.6856027951868566, "I_sealed": 2.801651910926573, "base": 2.2436273530567146, "excess": -3.68943675817796, "gap": -1.1160491157397163, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 5.05680838556057, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": 11.292859389745214, "judge": -6.789716545814108, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -85.71428571428572, "hard": -65.21739130434783, "judge": -70.0, "standard": -66.66666666666667 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 21.489090909090912, "hard": -14.746481481481478, "judge": 14.17932203389829, "standard": -58.12777777777776 } }, "sealed|choice": { "cc": 8.404955732779719, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 16.19244697361616, "hard": 5.316742081447947, "judge": 5.027932960893876, "standard": 15.753171572853812 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -96.0, "hard": -47.57281553398058, "judge": -52.63157894736843, "standard": -69.01408450704226 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 2.2302836879432686, "hard": -10.51115196078436, "judge": -12.453053892215603, "standard": -2.9353299492385965 } } } }, "jevbench_score": 0.016405006835540154, "key": "tacet-sonata", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 126, "ranked": true, "ranks": { "A": 126, "B": 126, "C": 126, "capability": 129 }, "scores": { "A": 0.016405006835540154, "B": 0.010748475876643845, "C": 0.011392365858013995 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.6127247303724289, "p50_s_raw": 0.23136236518621445, "p95_s_adjusted": 1.5294084161520003, "p95_s_raw": 0.6897042080760002 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 40.52958575528056, 49.9066658083324 ], "I": [ 0.9307057036499335, 4.3706191326925845 ], "capability": [ 21.016174186281347, 26.68388532804497 ], "composite": { "A": [ 0.0012369873259327144, 0.11178055042619336 ], "B": [ 0.0007893069841145717, 0.07607180414333671 ], "C": [ 0.0008590189763421627, 0.07762538224041209 ] }, "seed": 143 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 46.37837260299999, "capability": 24.310999978028352, "intelligence": 2.2436273530567146 }, "per_type": { "choice": { "accuracy": 0.284, "calibration": 13.846312499999968, "cc": 8.433826245938405, "cc_by_type": { "choice": 8.433826245938405 }, "cc_tiered": 6.730882059170145, "cc_tiered_open": 5.05680838556057, "cc_tiered_sealed": 8.404955732779719, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 8.433826245938405 }, "noul": { "accuracy": 0.18133333333333335, "calibration": 69.90389333333333, "cc": -63.733333333333334, "cc_by_type": { "noul": -63.733333333333334 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.304, "calibration": 55.38491197566667, "cc": -7.235394736842027, "cc_by_type": { "score": -7.235394736842027 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 45.61976295895299, "capability": 24.54098913610771, "intelligence": 3.4622153132624316 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.012868928732057194, "Emphasis on Accuracy 60:20:20": 0.007398330998569957, "Emphasis on Cost 20:20:60": 0.02046846868350025, "Emphasis on Speed 20:60:20": 0.020367269262702412, "Intelligence only": 0.004517645714999133 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/codepawl/tacet-sonata", "gpu": "CPU (12-core Zen 2, AVX2; 8 threads)", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. Weights codepawl/tacet-sonata @ 26f447c35881a80020fe557c95ceb8c61bc3fd12, Apache-2.0, public and ungated, loaded from a local snapshot (no network during the run). Runtime tacet==0.3.0 from PyPI, the author's own install line (`pip install \"tacet>=0.3\"`) in an isolated venv (torch 2.14.1+cpu, transformers 5.19.0). ADAPTER: the author's own PR fstandhartinger/jevbench#197 file tacet_local.py at head 76eb75da, byte-identical (sha256 c63a4fe0...), reviewed before running: it maps answers only (noul P(true) -> yes/no; choice and score probabilities used as returned, score keys '0'..'n-1') and reaches nothing but the Hub. It ran through the UNCHANGED v1.6 driver run_v16.py; the only addition, in a wrapper, registers the file as jevbench.adapters.tacet_local and lists it in IN_PROCESS_MODULES as laya_local is listed - lane verification and the runtime egress guard unchanged. Synthetic smoke gate passed before any sealed item was sent (PREREG-R59.md, receipts/SMOKE-tacet-sonata.json). THE 23 LONG ITEMS ARE TRUNCATED, NOT REFUSED, by the package's own code at its default and documented max_length 4,096 (tacet/packing.py keeps the questions and cuts the state; it refuses only states over 200,000 characters, which no item here reaches): exactly 23 records report input_tokens = 4,096 and they are answered and scored. RUN ON SANDY'S SHARED CPU (12-core Zen 2, AVX2, fp32 as the package runs off-accelerator, 8 threads), load average 6.6-24.2 (median ~14) logged every 60 s: the Speed axis is a DECLARED LOWER BOUND, and the author reports raw p50 0.042 s on an RTX 3060 against this run's 0.265 s. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per call, no retry, no batching, offline, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 716D1957 (bh-submit 07d384eb-fc4e-4536-ba61-888567efdf34)", "underlying": "jhu-clsp/mmBERT-small", "model_pin": "codepawl/tacet-sonata@26f447c35881a80020fe557c95ceb8c61bc3fd12", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "not_ranked_because": null, "noul_decisive": { "supported": true, "decisive_rate": 0.3253333333333333, "acc_among_decisive": 0.5573770491803278, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Convai Innovations", "axes": { "calibration": 63.22326564159259, "cost": 84.89402784895724, "intelligence": 1.7910341507793501, "speed": 73.26302890816496 }, "calibration": { "parts": { "choice": { "ece_hard": 0.21969249999999993, "ece_score": 56.06150000000001, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 56.06150000000001 }, "noul": { "abstention_rate": 0.808, "brier": 0.28256342088000014, "ece": 0.18899546666666664, "n": 375, "score": 62.20090666666667 }, "score": { "interval90_coverage": 0.8826666666666667, "interval90_width": 0.7128888888888881, "n": 375, "nrps": 0.18963512817111114, "score": 71.4073902581111, "top_ece": 0.19110853333333327 } }, "score": 63.22326564159259 }, "capability": 32.50714989618597, "composite_ci95": { "A": [ 0.00020217712264809318, 0.06712460770140848 ], "B": [ 0.00012766304420524027, 0.044955197581294834 ], "C": [ 0.00014040077961673136, 0.046614310903755875 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0031881034482758625 }, "display": "Laya (Convai Innovations, ModernBERT-large 421M)", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 1.8026531464810367, "I_sealed": 1.7794151550776638, "base": 1.7910341507793501, "excess": -2.5501496510348707, "gap": 0.023237991403372904, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 5.40795943944311, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 14.37837837837838, "hard": -3.8062283737024116, "judge": 16.941331575477918, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -57.14285714285714, "hard": -91.30434782608697, "judge": -80.0, "standard": -88.88888888888889 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 14.020606060606056, "hard": -11.245833333333334, "judge": -16.895084745762734, "standard": -50.04185185185186 } }, "sealed|choice": { "cc": 5.338245465232991, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 1.5593821594856503, "hard": -3.846153846153868, "judge": 9.683426443202995, "standard": 19.078704273925375 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -56.00000000000001, "hard": -84.46601941747574, "judge": -76.31578947368422, "standard": -74.64788732394366 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 4.452695035460996, "hard": -11.828823529411746, "judge": -8.140898203592851, "standard": -0.2669035532995023 } } } }, "jevbench_score": 0.008560099345041717, "key": "laya", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 127, "ranked": true, "ranks": { "A": 127, "B": 127, "C": 127, "capability": 120 }, "scores": { "A": 0.008560099345041717, "B": 0.005540634651880784, "C": 0.005944513434056748 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.800616904720664, "p50_s_raw": 0.825308452360332, "p95_s_adjusted": 2.619846137147397, "p95_s_raw": 1.2349230685736985 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 57.988437430874065, 67.35373438390093 ], "I": [ 0.5052598551452469, 3.643432797894196 ], "capability": [ 29.732944854873807, 34.91391985018961 ], "composite": { "A": [ 0.00020217712264809318, 0.06712460770140848 ], "B": [ 0.00012766304420524027, 0.044955197581294834 ], "C": [ 0.00014040077961673136, 0.046614310903755875 ] }, "seed": 56 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 63.22326564159259, "capability": 32.50714989618597, "intelligence": 1.7910341507793501 }, "per_type": { "choice": { "accuracy": 0.25466666666666665, "calibration": 56.06150000000001, "cc": 4.682511864952642, "cc_by_type": { "choice": 4.682511864952642 }, "cc_tiered": 5.373102452338051, "cc_tiered_open": 5.40795943944311, "cc_tiered_sealed": 5.338245465232991, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 4.682511864952642 }, "noul": { "accuracy": 0.11733333333333333, "calibration": 62.20090666666667, "cc": -76.53333333333333, "cc_by_type": { "noul": -76.53333333333333 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2826666666666667, "calibration": 71.4073902581111, "cc": -7.6405701754385325, "cc_by_type": { "score": -7.6405701754385325 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 62.99195454673789, "capability": 33.098085372560654, "intelligence": 3.20421619838342 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.006594024627233607, "Emphasis on Accuracy 60:20:20": 0.0037729124634726325, "Emphasis on Cost 20:20:60": 0.010563725142022983, "Emphasis on Speed 20:60:20": 0.010499068205592536, "Intelligence only": 0.0022981141246738437 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/convaiinnovations/laya", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "ModernBERT-large encoder + option-marker decision head, 421M, RLCD-trained", "adapter_id": "images/laya/spec.json:warm_adapter.WarmLoaded", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.192, "acc_among_decisive": 0.6111111111111112, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Mixedbread", "axes": { "calibration": 86.60772459153094, "cost": 60.33589314238483, "intelligence": 1.140707327109041, "speed": 92.58758170313658 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06855480504522303, "ece_score": 86.28903899095539, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 86.28903899095539 }, "noul": { "abstention_rate": 1.0, "brier": 0.25057053855990563, "ece": 0.07454411727378144, "n": 375, "score": 85.09117654524371 }, "score": { "interval90_coverage": 1.0, "interval90_width": 1.0, "n": 375, "nrps": 0.17161399632774915, "score": 88.44295823839374, "top_ece": 0.029763419452187988 } }, "score": 86.60772459153094 }, "capability": 43.87421595931999, "composite_ci95": { "A": [ 4.213826902762912e-06, 0.0321843481372824 ], "B": [ 2.6407677436324784e-06, 0.02116413476940045 ], "C": [ 2.9262686824742456e-06, 0.022350241762001666 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-0.6B); no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.020996016009852216 }, "display": "Mixedbread mxbai-rerank-base-v2", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 1.5821442040322362, "I_sealed": 0.6992704501858459, "base": 1.140707327109041, "excess": -1.6905138885918536, "gap": 0.8828737538463903, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 4.746432612096709, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -5.081081081081076, "hard": -0.03145643284050508, "judge": 1.1206328279499007, "standard": 24.654667224780237 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -11.030065754992213, "hard": -1.5426620524559675, "judge": 8.497895555670832, "standard": -21.144844232023672 } }, "sealed|choice": { "cc": 0.0, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -7.75256817677922, "hard": -4.355203619909522, "judge": -1.4897579143388995, "standard": 0.23401896785319723 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 2.0978113505575378, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -4.830772427457819, "hard": 1.5240605072453395, "judge": 6.561942168694756, "standard": 0.013408698983785605 } } } }, "jevbench_score": 0.002273929358844764, "key": "mxbai-rerank-base-v2", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 128, "ranked": true, "ranks": { "A": 128, "B": 128, "C": 128, "capability": 100 }, "scores": { "A": 0.002273929358844764, "B": 0.0014520695133694007, "C": 0.001579117610308864 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.21285173026844859, "p50_s_raw": 0.031425865134224296, "p95_s_adjusted": 0.2589194315019995, "p95_s_raw": 0.05445971575099974 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 81.5709999496809, 89.28164941290872 ], "I": [ 0.13834674179555617, 2.8157994354473197 ], "capability": [ 41.27690031534251, 45.59940972334693 ], "composite": { "A": [ 4.213826902762912e-06, 0.0321843481372824 ], "B": [ 2.6407677436324784e-06, 0.02116413476940045 ], "C": [ 2.9262686824742456e-06, 0.022350241762001666 ] }, "seed": 69 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.60772459153094, "capability": 43.87421595931999, "intelligence": 1.140707327109041 }, "per_type": { "choice": { "accuracy": 0.2, "calibration": 86.28903899095539, "cc": -2.3085740268844592, "cc_by_type": { "choice": -2.3085740268844592 }, "cc_tiered": 2.3732163060483544, "cc_tiered_open": 4.746432612096709, "cc_tiered_sealed": 0.0, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.0, "calibration": 85.09117654524371, "cc": -100.0, "cc_by_type": { "noul": -100.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.3413333333333333, "calibration": 88.44295823839374, "cc": 0.3851131600845714, "cc_by_type": { "score": 0.3851131600845714 }, "cc_tiered": 1.0489056752787689, "cc_tiered_open": 0.0, "cc_tiered_sealed": 2.0978113505575378, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.3851131600845714 } }, "raw_on_A": { "calibration": 84.06228474550946, "capability": 42.68728856938789, "intelligence": 1.312292393266321 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.0017272292257436567, "Emphasis on Accuracy 60:20:20": 0.0009793418989392745, "Emphasis on Cost 20:20:60": 0.002776895162759242, "Emphasis on Speed 20:60:20": 0.002811535213497941, "Intelligence only": 0.0005937213753409678 }, "class": "reranker", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/mixedbread-ai/mxbai-rerank-base-v2", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "494M cross-encoder", "adapter_id": "images/mxbai-rerank-base-v2/spec.json:reranker_adapter.RerankerAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.0, "acc_among_decisive": null, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Convai Innovations", "axes": { "calibration": 33.655846666999985, "cost": 82.1551651086044, "intelligence": 0.2879184668737643, "speed": 77.82044954853706 }, "calibration": { "parts": { "choice": { "ece_hard": 0.44879250000000004, "ece_score": 10.241499999999991, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 10.241499999999991 }, "noul": { "abstention_rate": 0.28, "brier": 0.4157348964266666, "ece": 0.3921216000000001, "n": 375, "score": 21.575679999999974 }, "score": { "interval90_coverage": 0.896, "interval90_width": 0.7111111111111107, "n": 375, "nrps": 0.18850373331333325, "score": 69.150360001, "top_ece": 0.21424453333333335 } }, "score": 33.655846666999985 }, "capability": 16.971882566936873, "composite_ci95": { "A": [ 0.0, 0.016100086645554724 ], "B": [ 0.0, 0.010673895641907892 ], "C": [ 0.0, 0.01118061572607967 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate; no exact base-model floor applies. $0.01/M input, $0/M output; same-class hosted-encoder estimate; no exact-base market floor.", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.003933940886699508 }, "display": "Laya multilingual", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 0.3687761432977424, "I_sealed": 0.20706079044978623, "base": 0.2879184668737643, "excess": -2.4116722895902876, "gap": 0.16171535284795616, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 1.1063284298932272, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 2.702702702702708, "hard": -13.243158225857174, "judge": 9.03098220171391, "standard": 17.120133947258267 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -14.28571428571429, "hard": -30.434782608695656, "judge": -50.0, "standard": -44.44444444444444 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 13.373939393939372, "hard": 2.3077777777777753, "judge": -8.492881355932202, "standard": -37.68148148148147 } }, "sealed|choice": { "cc": 0.6211823713493587, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 2.889660778952063, "hard": -4.86425339366518, "judge": 5.959031657355701, "standard": 2.451040768567572 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 0.0, "hard": -30.097087378640776, "judge": -36.8421052631579, "standard": -29.5774647887324 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -0.387943262411361, "hard": -0.8122549019607739, "judge": -1.6932335329341575, "standard": -3.1987817258883444 } } } }, "jevbench_score": 3.7595668430455566e-05, "key": "laya-multilingual", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 129, "ranked": true, "ranks": { "A": 129, "B": 129, "C": 129, "capability": 132 }, "scores": { "A": 3.7595668430455566e-05, "B": 2.368099374667976e-05, "C": 2.6108103076705254e-05 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.8654558090260253, "p50_s_raw": 0.3577279045130126, "p95_s_adjusted": 1.908579030103283, "p95_s_raw": 0.8792895150516415 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 28.738769095461095, 38.12861857012684 ], "I": [ 0.0, 2.2482651295080083 ], "capability": [ 14.566920276697426, 19.746046606060705 ], "composite": { "A": [ 0.0, 0.016100086645554724 ], "B": [ 0.0, 0.010673895641907892 ], "C": [ 0.0, 0.01118061572607967 ] }, "seed": 57 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 33.655846666999985, "capability": 16.971882566936873, "intelligence": 0.2879184668737643 }, "per_type": { "choice": { "accuracy": 0.21733333333333332, "calibration": 10.241499999999991, "cc": -0.09188825630196516, "cc_by_type": { "choice": -0.09188825630196516 }, "cc_tiered": 0.863755400621293, "cc_tiered_open": 1.1063284298932272, "cc_tiered_sealed": 0.6211823713493587, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.35733333333333334, "calibration": 21.575679999999974, "cc": -28.53333333333333, "cc_by_type": { "noul": -28.53333333333333 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.296, "calibration": 69.150360001, "cc": -1.8657280701754209, "cc_by_type": { "score": -1.8657280701754209 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 32.7172206036182, "capability": 17.460072535779506, "intelligence": 2.202924467940814 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 2.843624303166903e-05, "Emphasis on Accuracy 60:20:20": 1.5873606797432713e-05, "Emphasis on Cost 20:20:60": 4.7066204506481234e-05, "Emphasis on Speed 20:60:20": 4.704809343432325e-05, "Intelligence only": 9.547035876864281e-06 }, "class": "system-one-open", "open": "yes", "licence": "apache-2.0, not gated", "repo": "https://huggingface.co/convaiinnovations/laya/tree/1c5edc17a7acd8701df6fc341c0d179f1c62c982/multilingual", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU, Sandy (AMD Ryzen 5 3600), 4 of 12 threads, nice -n 5, shared host, offline (HF_HUB_OFFLINE=1)", "underlying": "convaiinnovations/laya @ 1c5edc17a7acd8701df6fc341c0d179f1c62c982, multilingual/; mmBERT-base 322M, max_len 1024. Distinct from the published English root checkpoint. Fresh run from item 1 on 29 Sep; interrupted earlier runs were not stitched. In-process, offline, shared AMD Ryzen 5 3600 CPU, four threads. Measured latency includes shared-host conditions; it is not a controlled GPU comparison. The release owner independently adopted mmBERT as unlisted in the frozen pricing snapshots; the labelled same-class encoder estimate has zero cost delta against BAAI/bge-m3.", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-04", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.72, "acc_among_decisive": 0.4962962962962963, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "AltSlate Labs", "axes": { "calibration": 86.14595667337036, "cost": 96.9130801359906, "intelligence": 0.20278658812856143, "speed": 92.53493469610669 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06436937500000006, "ece_score": 87.12612499999999, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 87.12612499999999 }, "noul": { "abstention_rate": 0.9893333333333333, "brier": 0.25290737730666696, "ece": 0.08273039999999988, "n": 375, "score": 83.45392000000002 }, "score": { "interval90_coverage": 0.9946666666666667, "interval90_width": 0.9942222222222223, "n": 375, "nrps": 0.17416936626444454, "score": 87.8578250201111, "top_ece": 0.0343370666666668 } }, "score": 86.14595667337036 }, "capability": 43.17437163074946, "composite_ci95": { "A": [ 0.0, 0.004427570625789243 ], "B": [ 0.0, 0.0028324322673094096 ], "C": [ 0.0, 0.003074701823464752 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0012673522167487683 }, "display": "Certo v1 (AltSlate Labs)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 0.12963682017919345, "I_sealed": 0.27593635607792943, "base": 0.20278658812856143, "excess": -2.71968717833698, "gap": -0.14629953589873598, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 0.0, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 10.486486486486491, "hard": 1.855929537590448, "judge": -18.655240606460122, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -91.30434782608697, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.3889104605375804, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 5.111515151515156, "hard": -0.6524999999999892, "judge": 1.8106779661016925, "standard": -2.0222222222222364 } }, "sealed|choice": { "cc": 0.6168084160817238, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -6.42228955731281, "hard": -4.86425339366518, "judge": -0.5586592178770757, "standard": 16.861682473210998 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 0.21100065215206465, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -0.7955319148936102, "hard": 1.9993137254901705, "judge": -1.0858682634730998, "standard": -0.9170558375635629 } } } }, "jevbench_score": 1.3254531668229452e-05, "key": "certo", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 130, "ranked": true, "ranks": { "A": 130, "B": 130, "C": 130, "capability": 102 }, "scores": { "A": 1.3254531668229452e-05, "B": 8.311485824043969e-06, "C": 9.204535880714898e-06 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.23407380385324358, "p50_s_raw": 0.042036901926621795, "p95_s_adjusted": 0.2383162835612893, "p95_s_raw": 0.04415814178064466 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 80.84167298672317, 89.02661690205646 ], "I": [ 0.0, 1.4261709005081855 ], "capability": [ 40.54746106861931, 44.91215541214051 ], "composite": { "A": [ 0.0, 0.004427570625789243 ], "B": [ 0.0, 0.0028324322673094096 ], "C": [ 0.0, 0.003074701823464752 ] }, "seed": 19 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 86.14595667337036, "capability": 43.17437163074946, "intelligence": 0.20278658812856143 }, "per_type": { "choice": { "accuracy": 0.21733333333333332, "calibration": 87.12612499999999, "cc": -0.09188825630196516, "cc_by_type": { "choice": -0.09188825630196516 }, "cc_tiered": 0.3084042080408619, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.6168084160817238, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.0026666666666666666, "calibration": 83.45392000000002, "cc": -99.46666666666667, "cc_by_type": { "noul": -99.46666666666667 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.27466666666666667, "calibration": 87.8578250201111, "cc": 0.4315701754385892, "cc_by_type": { "score": 0.4315701754385892 }, "cc_tiered": 0.2999555563448225, "cc_tiered_open": 0.3889104605375804, "cc_tiered_sealed": 0.21100065215206465, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.4315701754385892 } }, "raw_on_A": { "calibration": 84.87407902121794, "capability": 42.46944871565377, "intelligence": 0.06481841008959673 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 9.964199692814982e-06, "Emphasis on Accuracy 60:20:20": 5.551453514139297e-06, "Emphasis on Cost 20:20:60": 1.653808420915537e-05, "Emphasis on Speed 20:60:20": 1.6534837758955257e-05, "Intelligence only": 3.3356285030110653e-06 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT", "repo": "https://huggingface.co/altslate/certo-decision-model", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "ModernBERT-large with a per-option query/scoring head, ~400M parameters", "adapter_id": "images/certo/spec.json:certo_v15.CertoLocalV15Adapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.010666666666666666, "acc_among_decisive": 0.25, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Contrastive-LM (Kwok, Kang, Suresh, Saad-Falcon, Pavone, Ré, Mirhoseini)", "axes": { "calibration": 40.041900621338065, "cost": 50.501662240594015, "intelligence": 0.13075619159747265, "speed": 92.63548383949004 }, "calibration": { "parts": { "choice": { "ece_hard": 0.3993510766836583, "ece_score": 20.129784663268335, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 20.129784663268335 }, "noul": { "abstention_rate": 0.4906666666666667, "brier": 0.26050031793956074, "ece": 0.18460455903608142, "n": 375, "score": 63.079088192783715 }, "score": { "interval90_coverage": 0.688, "interval90_width": 0.5184444444444443, "n": 375, "nrps": 0.2846437031480146, "score": 36.91682900796216, "top_ece": 0.48850985834637106 } }, "score": 40.041900621338065 }, "capability": 20.086328406467768, "composite_ci95": { "A": [ 0.0, 0.010708585645302497 ], "B": [ 0.0, 0.007040756824095157 ], "C": [ 0.0, 0.007436517809237845 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): price floor: base-model reference price applied", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.044662660714285714 }, "display": "CLM-8B (Contrastive-LM, clm-latest)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 0.2615123831949453, "I_sealed": 0.0, "base": 0.13075619159747265, "excess": -2.3118752592432985, "gap": 0.2615123831949453, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 0.7845371495848359, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": -16.75675675675675, "hard": -7.5810003145643154, "judge": 16.941331575477918, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -28.57142857142857, "hard": -56.52173913043479, "judge": -19.999999999999996, "standard": -44.44444444444444 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -21.94096032005719, "hard": -1.9559175706170606, "judge": -11.452540725941729, "standard": -95.63691251131768 } }, "sealed|choice": { "cc": 0.0, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -6.42228955731281, "hard": -4.355203619909522, "judge": 0.3724394785847514, "standard": 7.993595270353507 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -31.999999999999996, "hard": -33.98058252427184, "judge": -31.57894736842105, "standard": -9.859154929577462 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -10.75981978481908, "hard": -29.341375883900135, "judge": -28.232663703069093, "standard": -55.86396111215961 } } } }, "jevbench_score": 3.5510969933114796e-06, "key": "clm-8b", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 131, "ranked": true, "ranks": { "A": 131, "B": 131, "C": 131, "capability": 130 }, "scores": { "A": 3.5510969933114796e-06, "B": 2.2274698008760027e-06, "C": 2.466039578688528e-06 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.19238147847354412, "p50_s_raw": 0.02119073923677206, "p95_s_adjusted": 0.2833272576332092, "p95_s_raw": 0.0666636288166046 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 35.39195795358793, 44.454662318208044 ], "I": [ 0.0, 1.9510432807841849 ], "capability": [ 17.69968780119306, 22.526421025586746 ], "composite": { "A": [ 0.0, 0.010708585645302497 ], "B": [ 0.0, 0.007040756824095157 ], "C": [ 0.0, 0.007436517809237845 ] }, "seed": 22 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 40.041900621338065, "capability": 20.086328406467768, "intelligence": 0.13075619159747265 }, "per_type": { "choice": { "accuracy": 0.20533333333333334, "calibration": 20.129784663268335, "cc": -1.6265168667052297, "cc_by_type": { "choice": -1.6265168667052297 }, "cc_tiered": 0.39226857479241795, "cc_tiered_open": 0.7845371495848359, "cc_tiered_sealed": 0.0, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.352, "calibration": 63.079088192783715, "cc": -29.600000000000005, "cc_by_type": { "noul": -29.600000000000005 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.26666666666666666, "calibration": 36.91682900796216, "cc": -29.29970040054666, "cc_by_type": { "score": -29.29970040054666 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 38.495636537708954, "capability": 19.549069061819143, "intelligence": 0.6025015859293292 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 2.67198512793619e-06, "Emphasis on Accuracy 60:20:20": 1.488390058441499e-06, "Emphasis on Cost 20:20:60": 4.430457758214625e-06, "Emphasis on Speed 20:60:20": 4.4408219452037955e-06, "Intelligence only": 8.942249433748766e-07 }, "class": "system-one-open", "open": "yes", "licence": "Apache-2.0 (code and CLM-v0.1-8B head); Qwen3-8B encoder Apache-2.0", "repo": "https://github.com/Contrastive-LM/CLM", "gpu": "RTXPRO6000", "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container", "underlying": null, "adapter_id": "images/clm-8b/spec.json:clm_adapter.CLMAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.5093333333333333, "acc_among_decisive": 0.6910994764397905, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "BAAI", "axes": { "calibration": 88.10046932372114, "cost": 59.325605115070715, "intelligence": 0.06512222845080358, "speed": 93.86976236118387 }, "calibration": { "parts": { "choice": { "ece_hard": 0.027473000546109336, "ece_score": 94.50539989077814, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 94.50539989077814 }, "noul": { "abstention_rate": 1.0, "brier": 0.25096294008809744, "ece": 0.0684901503152521, "n": 375, "score": 86.30196993694959 }, "score": { "interval90_coverage": 1.0, "interval90_width": 1.0, "n": 375, "nrps": 0.1745560185107233, "score": 83.49403814343572, "top_ece": 0.07778160931028115 } }, "score": 88.10046932372114 }, "capability": 44.08279577608597, "composite_ci95": { "A": [ 0.0, 0.008573668302290527 ], "B": [ 0.0, 0.005538697219146524 ], "C": [ 0.0, 0.005953936321035087 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: base-model market reference (deepinfra:BAAI/bge-m3)", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.022688885467980296 }, "display": "BAAI bge-reranker-v2-m3", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 0.018692443590466585, "I_sealed": 0.11155201331114058, "base": 0.06512222845080358, "excess": -2.666247212158918, "gap": -0.092859569720674, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 0.05607733077139976, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 10.486486486486491, "hard": 7.518087448883308, "judge": -14.700065919578117, "standard": 2.051067392214312 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 5.306919494288776, "hard": -1.440659315203363, "judge": -2.980317227364382, "standard": -2.397579483854617 } }, "sealed|choice": { "cc": 0.33465603993342175, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -5.092010937846398, "hard": -0.791855203619931, "judge": 2.234636871508399, "standard": 2.451040768567572 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 2.319622122935927, "hard": -0.38388489349363564, "judge": 0.07005074041354087, "standard": -0.8557298803223867 } } } }, "jevbench_score": 4.4076803151681546e-07, "key": "bge-reranker-v2-m3", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 132, "ranked": true, "ranks": { "A": 132, "B": 132, "C": 132, "capability": 98 }, "scores": { "A": 4.4076803151681546e-07, "B": 2.758281497029271e-07, "C": 3.060889107755663e-07 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.1841465687379241, "p50_s_raw": 0.01707328436896205, "p95_s_adjusted": 0.2227717584464699, "p95_s_raw": 0.03638587922323495 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.14061191725726, 90.57274307882632 ], "I": [ 0.0, 1.7895886627489592 ], "capability": [ 41.80444040346114, 45.59879463409008 ], "composite": { "A": [ 0.0, 0.008573668302290527 ], "B": [ 0.0, 0.005538697219146524 ], "C": [ 0.0, 0.005953936321035087 ] }, "seed": 18 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 88.10046932372114, "capability": 44.08279577608597, "intelligence": 0.06512222845080358 }, "per_type": { "choice": { "accuracy": 0.22133333333333333, "calibration": 94.50539989077814, "cc": 0.4196546138324576, "cc_by_type": { "choice": 0.4196546138324576 }, "cc_tiered": 0.19536668535241075, "cc_tiered_open": 0.05607733077139976, "cc_tiered_sealed": 0.33465603993342175, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.4196546138324576 }, "noul": { "accuracy": 0.0, "calibration": 86.30196993694959, "cc": -100.0, "cc_by_type": { "noul": -100.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.2826666666666667, "calibration": 83.49403814343572, "cc": -0.18199548701309798, "cc_by_type": { "score": -0.18199548701309798 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 85.43472005366168, "capability": 43.01708899298245, "intelligence": 0.5994579323032229 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 3.308199423158738e-07, "Emphasis on Accuracy 60:20:20": 1.8400822603920523e-07, "Emphasis on Cost 20:20:60": 5.501608977147286e-07, "Emphasis on Speed 20:60:20": 5.506039727293195e-07, "Intelligence only": 1.1047086427994778e-07 }, "class": "reranker", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/BAAI/bge-reranker-v2-m3", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "568M multilingual cross-encoder", "adapter_id": "images/bge-reranker-v2-m3/spec.json:reranker_adapter.RerankerAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.0, "acc_among_decisive": null, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Alibaba-NLP", "axes": { "calibration": 83.4556233651092, "cost": 69.26965431492843, "intelligence": 0.0, "speed": 93.76120791185616 }, "calibration": { "parts": { "choice": { "ece_hard": 0.0790764439307023, "ece_score": 84.18471121385954, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 84.18471121385954 }, "noul": { "abstention_rate": 1.0, "brier": 0.25231019651494707, "ece": 0.08962639509020176, "n": 375, "score": 82.07472098195964 }, "score": { "interval90_coverage": 1.0, "interval90_width": 1.0, "n": 375, "nrps": 0.17667560258952839, "score": 84.10743789950841, "top_ece": 0.07058781971015167 } }, "score": 83.4556233651092 }, "capability": 41.7278116825546, "composite_ci95": { "A": [ 0.0, 0.003662687483805933 ], "B": [ 0.0, 0.0023440984279851505 ], "C": [ 0.0, 0.002543532974865231 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:thenlper/gte-base); no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.010576570197044334 }, "display": "Alibaba GTE Reranker ModernBERT-base", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 0.0, "I_sealed": 0.0, "base": 0.0, "excess": -2.573387642438244, "gap": 0.0, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 0.0, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": -3.8062283737024116, "judge": -6.789716545814108, "standard": 9.585600669736289 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": -1.920002214440264, "hard": -1.3526386292960213, "judge": 1.2639659476201492, "standard": 0.3612908498087819 } }, "sealed|choice": { "cc": 0.0, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": -5.092010937846398, "hard": -6.391402714932147, "judge": -5.214152700186197, "standard": -7.525557334647108 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": -100.0, "hard": -100.0, "judge": -100.0, "standard": -100.0 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -1.317182445372178, "hard": 0.03511410774166279, "judge": -0.2802076730282099, "standard": 0.30690766431314964 } } } }, "jevbench_score": 0.0, "key": "gte-reranker-modernbert-base", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 133, "ranked": true, "ranks": { "A": 133, "B": 133, "C": 133, "capability": 106 }, "scores": { "A": 0.0, "B": 0.0, "C": 0.0 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.19835810186341404, "p50_s_raw": 0.024179050931707025, "p95_s_adjusted": 0.2120455999393016, "p95_s_raw": 0.031022799969650804 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 78.59536793549519, 87.51314141306226 ], "I": [ 0.0, 1.339161261347401 ], "capability": [ 39.372795544944715, 43.89015023558699 ], "composite": { "A": [ 0.0, 0.003662687483805933 ], "B": [ 0.0, 0.0023440984279851505 ], "C": [ 0.0, 0.002543532974865231 ] }, "seed": 39 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 83.4556233651092, "capability": 41.7278116825546, "intelligence": 0.0 }, "per_type": { "choice": { "accuracy": 0.17866666666666667, "calibration": 84.18471121385954, "cc": -5.036802667601379, "cc_by_type": { "choice": -5.036802667601379 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.0, "calibration": 82.07472098195964, "cc": -100.0, "cc_by_type": { "noul": -100.0 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.304, "calibration": 84.10743789950841, "cc": -0.24811073820258933, "cc_by_type": { "score": -0.24811073820258933 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 88.93979403901187, "capability": 44.469897019505936, "intelligence": 0.0 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 0.0, "Emphasis on Accuracy 60:20:20": 0.0, "Emphasis on Cost 20:20:60": 0.0, "Emphasis on Speed 20:60:20": 0.0, "Intelligence only": 0.0 }, "class": "reranker", "open": "yes", "licence": "Apache-2.0", "repo": "https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base", "gpu": "A6000", "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container", "underlying": "149M ModernBERT cross-encoder", "adapter_id": "images/gte-reranker-modernbert-base/spec.json:reranker_adapter.RerankerAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.0, "acc_among_decisive": null, "valid": 375 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "Bluusun", "axes": { "calibration": 51.61277239297377, "cost": 89.27300805786774, "intelligence": 0.0, "speed": 73.47211485993975 }, "calibration": { "parts": { "choice": { "ece_hard": 0.2204942558257705, "ece_score": 55.90114883484589, "mean_tvd": null, "n": 379, "n_ece_hard": 100, "n_tvd": 0, "score": 55.90114883484589 }, "noul": { "abstention_rate": 0.07773851590106007, "brier": 0.3825718867287919, "ece": 0.37640864622953524, "n": 283, "score": 24.718270754092952 }, "score": { "interval90_coverage": 0.9471544715447154, "interval90_width": 0.8113143631436313, "n": 246, "nrps": 0.18235540291655192, "score": 74.21889758998248, "top_ece": 0.1666333226418992 } }, "score": 51.61277239297377 }, "capability": 25.806386196486883, "composite_ci95": { "A": [ 0.0, 9.853474730694675e-09 ], "B": [ 0.0, 6.161443172813211e-09 ], "C": [ 0.0, 6.8426907852046335e-09 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.0022780665024630543 }, "display": "Mirror", "endpoint_kind": "cpu", "full_coverage": true, "intelligence": { "I_open": 0.0, "I_sealed": 0.0, "base": 0.0, "excess": -2.573387642438244, "gap": 0.0, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 0.0, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 6.5945945945945965, "hard": -7.5810003145643154, "judge": -14.700065919578117, "standard": -5.4834658853076625 } }, "open|noul": { "cc": 0.0, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 0.0, "hard": -73.91304347826086, "judge": -19.999999999999996, "standard": 22.222222222222232 } }, "open|score": { "cc": 0.0, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 14.02621656810803, "hard": -86.48465684738163, "judge": -131.29938184642268, "standard": -61.876452149889175 } }, "sealed|choice": { "cc": 0.0, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 14.86216835414975, "hard": -16.063348416289617, "judge": -12.662942271880798, "standard": -6.417046434289924 } }, "sealed|noul": { "cc": 0.0, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 12.00000000000001, "hard": -33.98058252427184, "judge": -15.789473684210531, "standard": -7.042253521126762 } }, "sealed|score": { "cc": 0.0, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": -1.6525187057042423, "hard": -113.52957658615944, "judge": -73.84737475308705, "standard": -29.673128809498152 } } } }, "jevbench_score": 0.0, "key": "mirror", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 134, "ranked": true, "ranks": { "A": 134, "B": 134, "C": 134, "capability": 128 }, "scores": { "A": 0.0, "B": 0.0, "C": 0.0 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 1.4516588259488343, "p50_s_raw": 0.6508294129744172, "p95_s_adjusted": 3.096876988979056, "p95_s_raw": 1.473438494489528 }, "status": { "answered_ok": 908, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 42.56771416804497, 56.74054715693854 ], "I": [ 0.0, 0.01760273906633102 ], "capability": [ 21.283857084022486, 28.426215016960548 ], "composite": { "A": [ 0.0, 9.853474730694675e-09 ], "B": [ 0.0, 6.161443172813211e-09 ], "C": [ 0.0, 6.8426907852046335e-09 ] }, "seed": 68 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 51.61277239297377, "capability": 25.806386196486883, "intelligence": 0.0 }, "per_type": { "choice": { "accuracy": 0.156, "calibration": 55.90114883484589, "cc": -7.935545598363106, "cc_by_type": { "choice": -7.935545598363106 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 0.0 }, "noul": { "accuracy": 0.416, "calibration": 24.718270754092952, "cc": -16.800000000000004, "cc_by_type": { "noul": -16.800000000000004 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 0.0 }, "score": { "accuracy": 0.18666666666666668, "calibration": 74.21889758998248, "cc": -72.82982094179775, "cc_by_type": { "score": -72.82982094179775 }, "cc_tiered": 0.0, "cc_tiered_open": 0.0, "cc_tiered_sealed": 0.0, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 0.0 } }, "raw_on_A": { "calibration": 48.60190522372855, "capability": 24.300952611864275, "intelligence": 0.0 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.4946666666666667, "n": 750 }, "noul": { "invalid_rate": 0.2453333333333333, "n": 375 }, "score": { "invalid_rate": 0.344, "n": 375 } }, "views": { "Balanced 33:33:33": 0.0, "Emphasis on Accuracy 60:20:20": 0.0, "Emphasis on Cost 20:20:60": 0.0, "Emphasis on Speed 20:60:20": 0.0, "Intelligence only": 0.0 }, "class": "jev-rebuild", "open": "yes", "licence": "Apache-2.0 wrapper and Mirror release; upstream DeBERTa/model assets retain their own terms", "repo": "https://github.com/Bluusun/Decision-API", "gpu": "cpu", "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores", "underlying": "Mirror DeBERTa-v3-large 436M classification-span scorer", "adapter_id": "images/mirror/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 0.9222614840989399, "acc_among_decisive": 0.5977011494252874, "valid": 283 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "JoshuaSP", "axes": { "calibration": 0.0, "cost": 49.295219536461964, "intelligence": 61.19221807027989, "speed": 85.53332924416175 }, "calibration": { "parts": {}, "score": 0.0 }, "capability": 30.596109035139946, "composite_ci95": { "A": [ 0.0, 0.0 ], "B": [ 0.0, 0.0 ], "C": [ 0.0, 0.0 ] }, "cost": { "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies", "common_basis": null, "kind": "estimate", "usage_estimated_rows": 0, "usd_per_1000": 0.04899585591133005 }, "display": "Open Jev JSON Canvas (JoshuaSP)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 61.36340599482574, "I_sealed": 61.02103014573405, "base": 61.19221807027989, "excess": -2.2310117933465556, "gap": 0.34237584909168817, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.21617418117772, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 100.00000000000001, "hard": 49.040578798364265, "judge": 60.44825313117996, "standard": 62.32733361239012 } }, "open|noul": { "cc": 72.15458937198068, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 82.6086956521739, "judge": 60.00000000000001, "standard": 55.55555555555556 } }, "open|score": { "cc": 51.71945443131884, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 54.54545454545454, "hard": 38.88888888888889, "judge": 67.79661016949152, "standard": 51.85185185185186 } }, "sealed|choice": { "cc": 68.74482965663773, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 77.38526346907102, "hard": 60.80316742081447, "judge": 71.13594040968344, "standard": 76.7212710924991 } }, "sealed|noul": { "cc": 64.47674149136002, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 84.00000000000001, "hard": 45.63106796116505, "judge": 76.3157894736842, "standard": 74.64788732394365 } }, "sealed|score": { "cc": 49.84151928920441, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 70.92198581560282, "hard": 46.81372549019608, "judge": 50.29940119760479, "standard": 44.6700507614213 } } } }, "jevbench_score": 0.0, "key": "open-jev-json-canvas-joshuasp", "listing": "ranked", "planned_weight_denominator": 1.0, "rank": 135, "ranked": true, "ranks": { "A": 135, "B": 135, "C": 135, "capability": 124 }, "scores": { "A": 0.0, "B": 0.0, "C": 0.0 }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.44667022498324516, "p50_s_raw": 0.14833511249162257, "p95_s_adjusted": 0.6261524326633662, "p95_s_raw": 0.23807621633168308 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "label", "noul": "label", "score": "label" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 0.0, 0.0 ], "I": [ 54.979058028212734, 65.63316060204029 ], "capability": [ 27.489529014106367, 32.816580301020146 ], "composite": { "A": [ 0.0, 0.0 ], "B": [ 0.0, 0.0 ], "C": [ 0.0, 0.0 ] }, "seed": 76 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 0.0, "capability": 30.596109035139946, "intelligence": 61.19221807027989 }, "per_type": { "choice": { "accuracy": 0.7493333333333333, "calibration": null, "cc": 67.9433134715762, "cc_by_type": { "choice": 67.9433134715762 }, "cc_tiered": 64.48050191890772, "cc_tiered_open": 60.21617418117772, "cc_tiered_sealed": 68.74482965663773, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 67.9433134715762 }, "noul": { "accuracy": 0.84, "calibration": null, "cc": 68.0, "cc_by_type": { "noul": 68.0 }, "cc_tiered": 68.31566543167034, "cc_tiered_open": 72.15458937198068, "cc_tiered_sealed": 64.47674149136002, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 68.0 }, "score": { "accuracy": 0.624, "calibration": null, "cc": 50.6140350877193, "cc_by_type": { "score": 50.6140350877193 }, "cc_tiered": 50.780486860261625, "cc_tiered_open": 51.71945443131884, "cc_tiered_sealed": 49.84151928920441, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 50.6140350877193 } }, "raw_on_A": { "calibration": 0.0, "capability": 31.028102732437542, "intelligence": 62.056205464875084 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": 60.34909628466887, "Emphasis on Accuracy 60:20:20": 59.99813997005745, "Emphasis on Cost 20:20:60": 54.674045042588084, "Emphasis on Speed 20:60:20": 67.78111653907041, "Intelligence only": 59.47929292694184 }, "class": "jev-rebuild", "open": "yes", "licence": "MIT code; Apache-2.0 DiffusionGemma weights", "repo": "https://github.com/JoshuaSP/open-jev", "gpu": "H100", "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container", "underlying": "google/diffusiongemma-26B-A4B-it BF16, one-step JSON canvas", "adapter_id": "images/open-jev-json-canvas-joshuasp/spec.json:joshuasp_adapter.JoshuaSPCanvasAdapter", "api_exposure_note": null, "model_pin": null, "last_measured_on": "2026-10-02", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.0", "noul_decisive": { "supported": true, "decisive_rate": 1.0, "acc_among_decisive": 0.8490566037735849, "valid": 371 }, "not_ranked_because": null, "provenance": { "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)", "carried_metadata_from": "v1.5.7" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "APUS AI-LAB (@cryptoSUN2049 / findoo2049@gmail.com)", "axes": { "calibration": 67.91594971090585, "cost": null, "intelligence": 52.808647095759525, "speed": 80.50610262660216 }, "calibration": { "parts": { "choice": { "ece_hard": 0.21087790603358122, "ece_score": 57.82441879328375, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 57.82441879328375 }, "noul": { "abstention_rate": 0.0835579514824798, "brier": 0.14666984115552173, "ece": 0.13630441067160756, "n": 371, "score": 72.73911786567848 }, "score": { "interval90_coverage": 0.8238482384823849, "interval90_width": 0.22018970189701897, "n": 369, "nrps": 0.10709067085641949, "score": 73.18431247375533, "top_ece": 0.21461153983423703 } }, "score": 67.91594971090585 }, "capability": 60.36229840333269, "composite_ci95": { "A": null, "B": null, "C": null }, "cost": { "basis": "UNPRICED, and deliberately not invented. pricing_v15.reference() RAISES on the exact declared base Qwen/Qwen3.5-35B-A3B ('base model lacks an explicit market-listing decision'). Run 46's bongard-mini rule applies unchanged: the frozen module raises rather than estimating, and the FROZEN-TABLE ROW is the release lane's (board #11 / #10132) - not this job's and not Florian's, because his standing 24 Sep 20:30 rule already decides the principle and the 26 Sep rule fixes the method. Complete but unpriced - shown, not ranked. RELEASE-LANE LINE, and a tight one: the dated snapshot ALREADY lists the exact base as qwen/qwen3.5-35b-a3b at (0.3125, 1.25), and BASE_REFERENCES ALREADY carries 'Qwen/Qwen3.5-35B-A3B-Base' -> 'qwen/qwen3.5-35b-a3b'. Only the non--Base (instruct) spelling of the same model is missing, so every Qwen3.5-35B-A3B derivative raises today. BECAUSE OUTPUT TOKENS ARE 0 BY CONSTRUCTION this row's Cost depends on the input rate alone, so the number is one multiplication over tokens the raw keeps; r56/receipts/COST-ARITHMETIC-R56.json carries it already computed.", "common_basis": null, "kind": "unpriced", "usage_estimated_rows": 0, "usd_per_1000": null }, "display": "APUS-OpenJev-v1-35B-A3B (merged bf16 checkpoint-5949 MoE on Qwen3.5-35B-A3B, their own native runtime, full depth, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 54.71860410185919, "I_sealed": 50.898690089659866, "base": 52.808647095759525, "excess": 1.2465263697610829, "gap": 3.8199140121993267, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.14590221055973, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 76.64864864864865, "hard": 54.70273670965714, "judge": 60.44825313117996, "standard": 62.32733361239012 } }, "open|noul": { "cc": 64.24775707384403, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 73.91304347826086, "judge": 50.0, "standard": 55.55555555555556 } }, "open|score": { "cc": 39.762153021173845, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 51.86605733086419, "hard": 29.028440368940068, "judge": 66.44632488030888, "standard": 15.151368382093667 } }, "sealed|choice": { "cc": 59.02771188966037, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 70.73387037173899, "hard": 51.131221719457, "judge": 59.031657355679705, "standard": 68.96169478999877 } }, "sealed|noul": { "cc": 49.333250807861994, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 76.0, "hard": 32.038834951456316, "judge": 57.89473684210527, "standard": 57.746478873239425 } }, "sealed|score": { "cc": 44.33510757145725, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 70.35456392472854, "hard": 33.80629692129137, "judge": 52.81314502924963, "standard": 39.66594450846479 } } } }, "jevbench_score": null, "key": "apus-openjev-v1-35b-a3b", "listing": "listed", "not_ranked_because": "registry ranked=false or no composite", "planned_weight_denominator": 1.0, "rank": null, "ranked": false, "ranks": { "capability": 66 }, "scores": { "A": null, "B": null, "C": null }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.859432493080385, "p50_s_raw": 0.3547162465401925, "p95_s_adjusted": 1.0355664409697056, "p95_s_raw": 0.4427832204848528 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 63.096906433455544, 71.75588820395149 ], "I": [ 45.94798828612946, 57.24766533646037 ], "capability": [ 55.374556366190006, 63.9661711269007 ], "composite": { "A": null, "B": null, "C": null }, "seed": 17 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 67.91594971090585, "capability": 60.36229840333269, "intelligence": 52.808647095759525 }, "per_type": { "choice": { "accuracy": 0.6866666666666666, "calibration": 57.82441879328375, "cc": 59.92914183947025, "cc_by_type": { "choice": 59.92914183947025 }, "cc_tiered": 59.58680705011005, "cc_tiered_open": 60.14590221055973, "cc_tiered_sealed": 59.02771188966037, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 59.92914183947025 }, "noul": { "accuracy": 0.7733333333333333, "calibration": 72.73911786567848, "cc": 54.666666666666664, "cc_by_type": { "noul": 54.666666666666664 }, "cc_tiered": 56.790503940853014, "cc_tiered_open": 64.24775707384403, "cc_tiered_sealed": 49.333250807861994, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 54.666666666666664 }, "score": { "accuracy": 0.5946666666666667, "calibration": 73.18431247375533, "cc": 43.44124606258307, "cc_by_type": { "score": 43.44124606258307 }, "cc_tiered": 42.04863029631555, "cc_tiered_open": 39.762153021173845, "cc_tiered_sealed": 44.33510757145725, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 43.44124606258307 } }, "raw_on_A": { "calibration": 68.85867558536114, "capability": 63.06963064948997, "intelligence": 57.280585713618805 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": null, "Emphasis on Accuracy 60:20:20": null, "Emphasis on Cost 20:20:60": null, "Emphasis on Speed 20:60:20": null, "Intelligence only": 52.808647095759525 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/apus-ailab/APUS-OpenJev-v1-35B-A3B", "gpu": "H100 PCIe", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights apus-ailab/APUS-OpenJev-v1-35B-A3B@090388abe2237117cc3b4df6f4250e81cbb36864, Apache-2.0, 66.99 GiB bf16 merged weights, checkpoint-5949, Qwen3_5MoeForConditionalGeneration (40 layers, 256 experts, 8 active), declared base Qwen/Qwen3.5-35B-A3B, full depth 40 layers (their depth_config declares exit_depth 20 / full_depth 40). accelerate>=1.10 additionally, per this repo's own requirements.txt. STACK: the authors' own requirements.txt, unmodified - torch==2.8.0 and transformers==5.16.1. The transformers pin is THEIRS, not a workaround: OpenJet.__init__ raises 'Layer execution is audited for transformers==5.16.1' on any other version. Served behind a TRANSPORT SHIM ONLY (r56/serve/apus_systemone_r56.py): loaded with THEIR OpenJet.from_pretrained(device=cuda:0, dtype=bfloat16) and every request handed to THEIR decide(), returning THEIR format_response() output. No contributor file edited. effort='high', the full-depth path, their decide() default and the configuration their own headline Frozen80 number is quoted on; effort='low' is not measured and no routing between the two is invented. One H100 PCIe 80 GB (Lium). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. MAPPING (run 56's, not the authors'; pre-registered in PREREG-R56 s2.2 before any item was sent and repeated here because it is part of the number): their runtime accepts primitive in {choice, noul, score_level} with 2-16 candidates labelled A-P. choice -> their 'choice' primitive, candidates = the item's own labels as id and its own criteria[label] as description, in labels order; their format_response returns probabilities keyed by candidate id, i.e. by the JevBench label already, so nothing is rewritten. All 750 choice items carry 3-9 options, inside their 2-16 window. noul -> their 'noul' primitive; their contract REQUIRES the canonical yes/no criteria there, so the item's own true/false description strings are NOT delivered - the proposition itself travels verbatim in instructions, which is what their noul is built to judge. score -> their 'choice' primitive over the ordinal levels, because their own RUNTIME.md rules score_level out as an ordinal Score API ('score_level is the judgement of a single proposition, it cannot be treated as a complete ordinal Score API'); the 4-5 ordered levels become candidates with id = the level index and description = the item's own level text, which carries the order. Recording all 375 score items as misses instead would understate a system whose published interface answers them. CALIBRATION IS MEASURED AS THE AUTHORS SHIP IT AND THEY DISCLOSED IT IN ADVANCE: their response carries calibrated: False and both their card and RUNTIME.md state the probabilities are an uncalibrated relative softmax over the offered candidates. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, NOT BY ESTIMATE: decide() runs one forward pass with logits_to_keep=1 and reads candidate logits; nothing is generated anywhere in that path. Input tokens are their own prompt_tokens field. THE 23 LONG ITEMS: their compile() raises ValueError('input exceeds runtime limit; no truncation permitted') above 8,192 tokens and their RUNTIME.md states it in advance. Recorded invalid. This is the first row in the series whose refusal on those items is guaranteed in the author's own code with an explicit no-truncation promise. add-requests run 56, GitHub #105, benchmarkheaven.com/submit ref AE709BBB", "underlying": "Qwen/Qwen3.5-35B-A3B", "model_pin": "apus-ailab/APUS-OpenJev-v1-35B-A3B@090388abe2237117cc3b4df6f4250e81cbb36864", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "noul_decisive": { "supported": true, "decisive_rate": 0.9164420485175202, "acc_among_decisive": 0.8529411764705882, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": true, "api_subset": false, "author": "Babak Barazandeh (bbarazandeh@cribl.io, GitHub @bbarazandeh-crypto)", "axes": { "calibration": 82.13857452060611, "cost": null, "intelligence": 38.67355961154223, "speed": 81.57813729165589 }, "calibration": { "parts": { "choice": { "ece_hard": 0.1281668719369918, "ece_score": 74.36662561260164, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 74.36662561260164 }, "noul": { "abstention_rate": 0.28800000000000003, "brier": 0.1440167151699402, "ece": 0.07805321715454966, "n": 375, "score": 84.38935656909007 }, "score": { "interval90_coverage": 0.9146666666666666, "interval90_width": 0.45133333333333325, "n": 375, "nrps": 0.10596156489230459, "score": 87.65974138012665, "top_ece": 0.07042180375258127 } }, "score": 82.13857452060611 }, "capability": 60.40606706607417, "composite_ci95": { "A": null, "B": null, "C": null }, "cost": { "basis": "UNPRICED - NOT A MEASUREMENT FAILURE. The base is disclosed (Qwen/Qwen3.5-4B-Base; reference() resolves it to deepinfra:Qwen/Qwen3.5-4B at 0.03/0.15), but the endpoint reports usage.input_tokens ONLY - no output_tokens on any answer - and the system is API-only, so 'nothing is generated' cannot be verified in its code; the 02 Oct deputy rule keeps an unverified null unpriced. The frozen scorer therefore gives no Cost axis and no composite; the row is reported on Capability. RELEASE-LANE LINE: if the author confirms (or the release lane accepts) a one-pass readout with 0 output tokens, the row prices by one multiplication from the raw at 0.03/M input (receipts/COST-ALTERNATIVES-R59.json).", "common_basis": null, "kind": "estimated", "usage_estimated_rows": 1500, "usd_per_1000": null }, "display": "BB-Qwen3.5-4B-LoRA (Babak Barazandeh, LoRA on Qwen3.5-4B-Base, hosted /v1/systemone)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 40.54310423897533, "I_sealed": 36.80401498410914, "base": 38.67355961154223, "excess": 1.16570161242795, "gap": 3.739089254866194, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 54.16068441724518, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 72.75675675675674, "hard": 52.815350739226176, "judge": 64.40342781806196, "standard": 32.18920050230222 } }, "open|noul": { "cc": 28.017253278122844, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 71.42857142857142, "hard": 39.13043478260869, "judge": 10.000000000000009, "standard": 11.111111111111116 } }, "open|score": { "cc": 39.45137502155798, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 35.5237907386309, "hard": 38.20598063967837, "judge": 35.150958941159246, "standard": 50.35658004737882 } }, "sealed|choice": { "cc": 53.65959260673962, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 68.07331313280615, "hard": 49.09502262443438, "judge": 50.65176908752329, "standard": 60.09360758714129 } }, "sealed|noul": { "cc": 19.15862307210663, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 56.00000000000001, "hard": -4.854368932038833, "judge": 26.315789473684205, "standard": 38.0281690140845 } }, "sealed|score": { "cc": 37.59382927348118, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 61.46292444619053, "hard": 28.220009445462267, "judge": 41.5085830641898, "standard": 38.53479065710138 } } } }, "jevbench_score": null, "key": "bb-qwen3.5-4b-lora", "listing": "listed", "not_ranked_because": "registry ranked=false or no composite", "planned_weight_denominator": 1.0, "rank": null, "ranked": false, "ranks": { "capability": 65 }, "scores": { "A": null, "B": null, "C": null }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.6954362019896507, "p50_s_raw": 0.6954362019896507, "p95_s_adjusted": 0.9998364746570587, "p95_s_raw": 0.9998364746570587 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 76.70685263219829, 84.35075129046632 ], "I": [ 32.59884752255904, 43.02764829916808 ], "capability": [ 55.59198238994904, 62.87426662252782 ], "composite": { "A": null, "B": null, "C": null }, "seed": 20 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": true, "g_med_used": 2.573387642438244, "lane": "api", "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 82.13857452060611, "capability": 60.40606706607417, "intelligence": 38.67355961154223 }, "per_type": { "choice": { "accuracy": 0.6493333333333333, "calibration": 74.36662561260164, "cc": 55.15474171821564, "cc_by_type": { "choice": 55.15474171821564 }, "cc_tiered": 53.9101385119924, "cc_tiered_open": 54.16068441724518, "cc_tiered_sealed": 53.65959260673962, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 55.15474171821564 }, "noul": { "accuracy": 0.624, "calibration": 84.38935656909007, "cc": 24.8, "cc_by_type": { "noul": 24.8 }, "cc_tiered": 23.587938175114736, "cc_tiered_open": 28.017253278122844, "cc_tiered_sealed": 19.15862307210663, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 24.8 }, "score": { "accuracy": 0.5413333333333333, "calibration": 87.65974138012665, "cc": 38.101218047454275, "cc_by_type": { "score": 38.101218047454275 }, "cc_tiered": 38.52260214751958, "cc_tiered_open": 39.45137502155798, "cc_tiered_sealed": 37.59382927348118, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 38.101218047454275 } }, "raw_on_A": null, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": null, "Emphasis on Accuracy 60:20:20": null, "Emphasis on Cost 20:20:60": null, "Emphasis on Speed 20:60:20": null, "Intelligence only": 23.13675425207104 }, "class": "Hosted decision API", "open": "closed weights", "licence": null, "repo": "https://github.com/fstandhartinger/jevbench/issues/156", "gpu": null, "endpoint_condition": "Hosted full-set API measured 2026-10-07; submitted LoRA checkpoint-030500 on Qwen3.5-4B-Base, weights not released. Serial requests over author ngrok tunnel, no retry; latency includes transit. Complete 1500/1500 answers. Input usage is reported but output usage is absent, so Cost and composite remain unpriced; Capability is reported. Exposure ledger recorded before dispatch.", "underlying": "Qwen/Qwen3.5-4B-Base", "model_pin": "checkpoint-030500 (author declaration); base revision 1001bb4d", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "noul_decisive": { "supported": true, "decisive_rate": 0.712, "acc_among_decisive": 0.8764044943820225, "valid": 375 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Current registry and GitHub #156; no credential or private endpoint URL copied" } }, { "addendum": null, "api_flag": false, "api_subset": false, "author": "@ironbcc / stasv@whatsdo.com", "axes": { "calibration": 90.06815367376362, "cost": null, "intelligence": 43.512227190846936, "speed": 90.02719232031932 }, "calibration": { "parts": { "choice": { "ece_hard": 0.06836203698728327, "ece_score": 86.32759260254335, "mean_tvd": null, "n": 737, "n_ece_hard": 307, "n_tvd": 0, "score": 86.32759260254335 }, "noul": { "abstention_rate": 0.3638814016172507, "brier": 0.09472415548142367, "ece": 0.035727812709697306, "n": 371, "score": 92.85443745806053 }, "score": { "interval90_coverage": 0.975609756097561, "interval90_width": 0.3764679313459801, "n": 369, "nrps": 0.07213300558995515, "score": 91.02243096068702, "top_ece": 0.05370918759815233 } }, "score": 90.06815367376362 }, "capability": 66.79019043230528, "composite_ci95": { "A": null, "B": null, "C": null }, "cost": { "basis": "UNPRICED, and deliberately not invented. The author declares NO base model anywhere in the repository, so pricing_v15.reference() has nothing to resolve: it returns None without raising, and the frozen module records the row unpriced. Complete but unpriced - shown, not ranked - exactly as run 52 left ines-1. RELEASE-LANE LINE: whether an UNDECLARED base that is architecturally identified may supply a size-class estimate. The repo ships Qwen3_5ForConditionalGeneration with 32 layers, hidden 4096 and the Qwen3.5 tokenizer under the name '9B', and qwen/qwen3.5-9b = (0.10, 0.15) is the hosted model of that exact shape; r56/receipts/COST-ARITHMETIC-R56.json carries the resulting cost per 1,000 over this row's own measured tokens, clearly labelled as OUR candidate reference and NOT the author's claim, so the release lane needs no re-measurement either way.", "common_basis": null, "kind": "unpriced", "usage_estimated_rows": 0, "usd_per_1000": null }, "display": "Seb-9B (Qwen3.5-9B-shaped multimodal decision model, one forward pass, stock vLLM, self-hosted)", "endpoint_kind": "gpu", "full_coverage": true, "intelligence": { "I_open": 44.886936829605986, "I_sealed": 42.137517552087886, "base": 43.512227190846936, "excess": 0.17603163507985675, "gap": 2.7494192775181006, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 69.07949695507523, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 88.32432432432431, "hard": 62.25228059138095, "judge": 76.26895187870797, "standard": 62.32733361239012 } }, "open|noul": { "cc": 17.61421670117322, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 42.85714285714286, "hard": 13.043478260869556, "judge": -9.999999999999998, "standard": 55.55555555555556 } }, "open|score": { "cc": 47.967096832569524, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 63.56858306985941, "hard": 46.62698762332502, "judge": 48.46182494567677, "standard": 42.104479962752684 } }, "sealed|choice": { "cc": 60.02946620132359, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 72.06414899120539, "hard": 51.64027149321266, "judge": 59.031657355679705, "standard": 72.28722749107035 } }, "sealed|noul": { "cc": 15.21016646635048, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 60.00000000000001, "hard": -12.621359223300976, "judge": 18.421052631578938, "standard": 43.66197183098593 } }, "sealed|score": { "cc": 51.1729199885896, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 72.60014322050257, "hard": 41.09886296789279, "judge": 57.18523582822763, "standard": 51.58894865456964 } } } }, "jevbench_score": null, "key": "seb-9b", "listing": "listed", "not_ranked_because": "registry ranked=false or no composite", "planned_weight_denominator": 1.0, "rank": null, "ranked": false, "ranks": { "capability": 45 }, "scores": { "A": null, "B": null, "C": null }, "speed": { "adjustment": "x2 + 0.15 s (assumption, not measured)", "n": 124, "p50_s_adjusted": 0.2702792370459065, "p50_s_raw": 0.06013961852295324, "p95_s_adjusted": 0.36767837166553363, "p95_s_raw": 0.1088391858327668 }, "status": { "answered_ok": 1477, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 83.95463343142413, 89.7683403725512 ], "I": [ 37.84443104855967, 48.05431373613838 ], "capability": [ 62.04135739756021, 68.30735969098161 ], "composite": { "A": null, "B": null, "C": null }, "seed": 130 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "g_med_used": 2.573387642438244, "lane": "selfhosted", "metric_basis": "S u P", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 90.06815367376362, "capability": 66.79019043230528, "intelligence": 43.512227190846936 }, "per_type": { "choice": { "accuracy": 0.708, "calibration": 86.32759260254335, "cc": 62.65737048018717, "cc_by_type": { "choice": 62.65737048018717 }, "cc_tiered": 64.55448157819941, "cc_tiered_open": 69.07949695507523, "cc_tiered_sealed": 60.02946620132359, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 62.65737048018717 }, "noul": { "accuracy": 0.6053333333333333, "calibration": 92.85443745806053, "cc": 21.066666666666656, "cc_by_type": { "noul": 21.066666666666656 }, "cc_tiered": 16.41219158376185, "cc_tiered_open": 17.61421670117322, "cc_tiered_sealed": 15.21016646635048, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 21.066666666666656 }, "score": { "accuracy": 0.624, "calibration": 91.02243096068702, "cc": 50.743386042692286, "cc_by_type": { "score": 50.743386042692286 }, "cc_tiered": 49.57000841057956, "cc_tiered_open": 47.967096832569524, "cc_tiered_sealed": 51.1729199885896, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 50.743386042692286 } }, "raw_on_A": { "calibration": 86.29901585869655, "capability": 66.82769737889441, "intelligence": 47.35637889909227 }, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.01733333333333331, "n": 750 }, "noul": { "invalid_rate": 0.010666666666666713, "n": 375 }, "score": { "invalid_rate": 0.016000000000000014, "n": 375 } }, "views": { "Balanced 33:33:33": null, "Emphasis on Accuracy 60:20:20": null, "Emphasis on Cost 20:20:60": null, "Emphasis on Speed 20:60:20": null, "Intelligence only": 32.95292208710638 }, "class": "Jev-compatible decision model", "open": "open weights", "licence": "apache-2.0", "repo": "https://huggingface.co/ironbcc/seb-9b", "gpu": "H100 PCIe 80 GB", "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights ironbcc/seb-9b@ac57b36bbe978762032d0ad7197778b4bc4a0209, Apache-2.0, 18.0 GiB bf16. Served with the AUTHOR'S OWN PUBLISHED vLLM COMMAND, verbatim from their model card, on vLLM 0.30 - the version their card states it is tested with - at the 8,192 --max-model-len their own flag note names for long documents (the larger of the two values they give). r56/serve/seb_systemone_r56.py supplies TRANSPORT ONLY: it builds their own documented chat request per decision (their system string verbatim, max_tokens 1, temperature 0, logprobs true, top_logprobs = the option count, structured_outputs.choice = the option keys, enable_thinking false) inside their own user-message envelope {task,state,question,type,options}, and reads the probabilities exactly where their card says they are - exp of choices[0].logprobs.content[0].top_logprobs - renormalised over the option keys. Their own key convention is used unchanged (noul Y/N, choice A..T, score 0..4) and every criterion description travels verbatim; no prompt wording of theirs is altered and no file of theirs is edited. THE 23 LONG ITEMS are rejected server-side by vLLM because the author's own published command sets --max-model-len 8192; recorded invalid and recorded as a CONFIGURATION limit under their own published flags, NOT as a model refusal - their config declares 262,144 positions and raising the flag would be our change, not theirs. One H100 PCIe 80 GB (Lium). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 56, benchmarkheaven.com/submit ref C566D021", "underlying": null, "model_pin": "ironbcc/seb-9b@ac57b36bbe978762032d0ad7197778b4bc4a0209", "last_measured_on": "2026-10-07", "adapter_id": null, "alt": null, "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "noul_decisive": { "supported": true, "decisive_rate": 0.6361185983827493, "acc_among_decisive": 0.961864406779661, "valid": 371 }, "provenance": { "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)", "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record" } }, { "addendum": null, "api_flag": true, "api_subset": false, "author": "Wity (alphanimble)", "axes": { "calibration": 89.25002486738025, "cost": 58.8349720563948, "intelligence": 69.25112551166924, "speed": 71.20296182763504 }, "calibration": { "parts": { "choice": { "ece_hard": 0.07469390266334129, "ece_score": 85.06121946733174, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 85.06121946733174 }, "noul": { "abstention_rate": 0.07999999999999996, "brier": 0.07909001541945995, "ece": 0.05007116234318407, "n": 375, "score": 89.98576753136318 }, "score": { "interval90_coverage": 0.9493333333333334, "interval90_width": 0.21622222222222207, "n": 375, "nrps": 0.051449317612291365, "score": 92.70308760344582, "top_ece": 0.047244465159396204 } }, "score": 89.25002486738025 }, "capability": 79.25057518952474, "composite_ci95": { "A": [ 68.6626707423781, 71.62549018189077 ], "B": [ 67.67306476984521, 71.90451506429471 ], "C": [ 68.6626707423781, 71.62549018189077 ] }, "cost": { "basis": "Wity's stated API tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.074026736 }, "kind": "tariff", "usage_estimated_rows": 0, "usd_per_1000": 0.023559582938388626 }, "display": "wity-1 (Wity, reasoning always)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 67.76360137882253, "I_sealed": 70.73864964451597, "base": 69.25112551166924, "excess": -5.5484359081316885, "gap": -2.975048265693445, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 82.46797884719487, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 96.10810810810811, "hard": 73.57659641396666, "judge": 88.13447593935398, "standard": 84.93093344495604 } }, "open|noul": { "cc": 58.719806763285035, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 100.0, "hard": 56.52173913043479, "judge": 50.0, "standard": 55.55555555555556 } }, "open|score": { "cc": 62.10301852598769, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 75.97327196189137, "hard": 60.98101749360573, "judge": 66.87830350526892, "standard": 50.24896640387794 } }, "sealed|choice": { "cc": 75.47557599484546, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 85.3669351858695, "hard": 67.42081447963801, "judge": 77.6536312849162, "standard": 83.37233649464221 } }, "sealed|noul": { "cc": 68.28141233707817, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 92.0, "hard": 51.45631067961165, "judge": 71.05263157894737, "standard": 85.91549295774648 } }, "sealed|score": { "cc": 68.45896060162431, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 83.95007438141664, "hard": 70.44083680919044, "judge": 63.191458207841954, "standard": 64.65090488726939 } } } }, "jevbench_score": 70.56449594359258, "key": "wity-1-always", "listing": "listed", "not_ranked_because": "registry ranked=false or no composite", "planned_weight_denominator": 1.0, "rank": null, "ranked": false, "scores": { "A": 70.56449594359258, "B": 70.29785155628596, "C": 70.56449594359258 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 1.6885605081915855, "p50_s_raw": 1.6885605081915855, "p95_s_adjusted": 4.489388496428728, "p95_s_raw": 4.489388496428728 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 84.74337331208723, 90.81676960358106 ], "I": [ 63.93278524147689, 73.29022847890529 ], "capability": [ 75.07164660500513, 81.44981666247915 ], "composite": { "A": [ 68.6626707423781, 71.62549018189077 ], "B": [ 67.67306476984521, 71.90451506429471 ], "C": [ 68.6626707423781, 71.62549018189077 ] }, "seed": 112 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": true, "g_med_used": 2.573387642438244, "lane": "api", "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 89.25002486738025, "capability": 79.25057518952474, "intelligence": 69.25112551166924 }, "per_type": { "choice": { "accuracy": 0.8213333333333334, "calibration": 85.06121946733174, "cc": 77.1510851339958, "cc_by_type": { "choice": 77.1510851339958 }, "cc_tiered": 78.97177742102016, "cc_tiered_open": 82.46797884719487, "cc_tiered_sealed": 75.47557599484546, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 77.1510851339958 }, "noul": { "accuracy": 0.848, "calibration": 89.98576753136318, "cc": 69.6, "cc_by_type": { "noul": 69.6 }, "cc_tiered": 63.5006095501816, "cc_tiered_open": 58.719806763285035, "cc_tiered_sealed": 68.28141233707817, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 69.6 }, "score": { "accuracy": 0.7946666666666666, "calibration": 92.70308760344582, "cc": 68.65055574841978, "cc_by_type": { "score": 68.65055574841978 }, "cc_tiered": 65.280989563806, "cc_tiered_open": 62.10301852598769, "cc_tiered_sealed": 68.45896060162431, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 68.65055574841978 } }, "raw_on_A": null, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 65.96124511263525, "Emphasis on Accuracy 60:20:20": 67.23896089186712, "Emphasis on Cost 20:20:60": 62.91315123993267, "Emphasis on Speed 20:60:20": 67.96251148296399, "Intelligence only": 69.25112551166924 }, "model_pin": null, "last_measured_on": "2026-10-06", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "noul_decisive": { "supported": true, "decisive_rate": 0.92, "acc_among_decisive": 0.9217391304347826, "valid": 375 }, "provenance": { "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)", "carried_metadata_from": "v1.6.0 release row" }, "class": "decision-api", "open": "no", "licence": "proprietary hosted API", "repo": "https://wity.alphanimble.com/", "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=always; labelled configuration variant.", "underlying": "Base model: undisclosed (author request)", "adapter_id": "run/wity-1-always.spec.json:wity_reasoning_adapter.WityReasoningAdapter", "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).", "alt": { "label": "Base-model reference price (base undisclosed at the author's request)", "note": "base model undisclosed at the author's request; frozen base-model reference price applied to the same measured tokens", "usd_per_1000": 0.08721059113300492, "axes": { "calibration": 91.0421772337923, "cost": 41.78292309262718, "intelligence": 79.49960112075699, "speed": 74.36141604169042 } } }, { "addendum": null, "api_flag": true, "api_subset": false, "author": "Wity (alphanimble)", "axes": { "calibration": 81.82569250484788, "cost": 58.8349720563948, "intelligence": 44.11116118075945, "speed": 89.55912090143326 }, "calibration": { "parts": { "choice": { "ece_hard": 0.10943535174564842, "ece_score": 78.11292965087031, "mean_tvd": null, "n": 750, "n_ece_hard": 320, "n_tvd": 0, "score": 78.11292965087031 }, "noul": { "abstention_rate": 0.20266666666666666, "brier": 0.14300007101908263, "ece": 0.10507128193494888, "n": 375, "score": 78.98574361301023 }, "score": { "interval90_coverage": 0.9493333333333334, "interval90_width": 0.5226666666666664, "n": 375, "nrps": 0.09960810647445889, "score": 88.37840425066312, "top_ece": 0.06641190425613937 } }, "score": 81.82569250484788 }, "capability": 62.968426842803666, "composite_ci95": { "A": [ 32.27069947350198, 65.6460656322266 ], "B": [ 28.862486308497225, 61.52715969788266 ], "C": [ 22.410207967709706, 45.58754557793514 ] }, "cost": { "basis": "Wity's stated API tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage", "common_basis": { "n_items": 1477, "rule": "common", "usd_per_1000_all_items": 0.074026736 }, "kind": "tariff", "usage_estimated_rows": 0, "usd_per_1000": 0.023559582938388626 }, "display": "wity-1 (Wity, reasoning off)", "endpoint_kind": "api", "full_coverage": true, "intelligence": { "I_open": 41.233518722452544, "I_sealed": 46.98880363906636, "base": 44.11116118075945, "excess": -8.32867255905206, "gap": -5.755284916613817, "penalty": 1.0, "per_type_split": { "open|choice": { "cc": 60.211735029468755, "n": { "easy": 33, "hard": 68, "judge": 32, "standard": 17 }, "tiers": { "easy": 80.54054054054055, "hard": 50.92796476879523, "judge": 64.40342781806196, "standard": 62.32733361239012 } }, "open|noul": { "cc": 29.155969634230512, "n": { "easy": 14, "hard": 23, "judge": 20, "standard": 18 }, "tiers": { "easy": 85.71428571428572, "hard": 21.739130434782616, "judge": 10.000000000000009, "standard": 44.44444444444444 } }, "open|score": { "cc": 34.332851503658375, "n": { "easy": 9, "hard": 29, "judge": 21, "standard": 16 }, "tiers": { "easy": 66.55807900089734, "hard": 31.747271308753067, "judge": 45.35548485808892, "standard": 6.857448113203668 } }, "sealed|choice": { "cc": 60.2170501413345, "n": { "easy": 96, "hard": 252, "judge": 137, "standard": 115 }, "tiers": { "easy": 72.06414899120539, "hard": 49.60407239819003, "judge": 60.89385474860336, "standard": 74.50424929178472 } }, "sealed|noul": { "cc": 37.60945396446127, "n": { "easy": 50, "hard": 103, "judge": 76, "standard": 71 }, "tiers": { "easy": 63.99999999999999, "hard": 20.388349514563096, "judge": 42.10526315789473, "standard": 52.112676056338024 } }, "sealed|score": { "cc": 43.13990681140334, "n": { "easy": 43, "hard": 125, "judge": 57, "standard": 75 }, "tiers": { "easy": 63.76401506785865, "hard": 42.31306223924574, "judge": 40.34902565317462, "standard": 38.66786356483397 } } } }, "jevbench_score": 49.37499462500363, "key": "wity-1-off", "listing": "listed", "not_ranked_because": "registry ranked=false or no composite", "planned_weight_denominator": 1.0, "rank": null, "ranked": false, "scores": { "A": 49.37499462500363, "B": 45.396938654270464, "C": 34.28819071180808 }, "speed": { "adjustment": "none (API)", "n": 124, "p50_s_adjusted": 0.28173011541366577, "p50_s_raw": 0.28173011541366577, "p95_s_adjusted": 0.39287521950900556, "p95_s_raw": 0.39287521950900556 }, "status": { "answered_ok": 1500, "missing": 0, "rows": 1500, "status": "complete" }, "support": { "choice": "native", "noul": "native", "score": "native" }, "v16": { "api_subset": false, "api_subset_tag": null, "breakdowns_equated": false, "ci95": { "B": 1000, "C": [ 75.68787622816288, 83.1859182390883 ], "I": [ 37.039067859171155, 49.769681165865016 ], "capability": [ 57.45466681572874, 65.81540071488247 ], "composite": { "A": [ 32.27069947350198, 65.6460656322266 ], "B": [ 28.862486308497225, 61.52715969788266 ], "C": [ 22.410207967709706, 45.58754557793514 ] }, "seed": 113 }, "ci95_n": 1200, "complete": true, "equated": null, "equating_flag": null, "equating_offset": null, "full_set_api": true, "g_med_used": 2.573387642438244, "lane": "api", "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)", "n_items": 1500, "n_public": 300, "n_sealed_side": 1200, "on_S": { "calibration": 81.82569250484788, "capability": 62.968426842803666, "intelligence": 44.11116118075945 }, "per_type": { "choice": { "accuracy": 0.6933333333333334, "calibration": 78.11292965087031, "cc": 60.781713289694295, "cc_by_type": { "choice": 60.781713289694295 }, "cc_tiered": 60.214392585401626, "cc_tiered_open": 60.211735029468755, "cc_tiered_sealed": 60.2170501413345, "low_n": false, "n": 750, "n_by_type": { "choice": 750 }, "score": 60.781713289694295 }, "noul": { "accuracy": 0.6986666666666667, "calibration": 78.98574361301023, "cc": 39.733333333333334, "cc_by_type": { "noul": 39.733333333333334 }, "cc_tiered": 33.38271179934589, "cc_tiered_open": 29.155969634230512, "cc_tiered_sealed": 37.60945396446127, "low_n": false, "n": 375, "n_by_type": { "noul": 375 }, "score": 39.733333333333334 }, "score": { "accuracy": 0.5706666666666667, "calibration": 88.37840425066312, "cc": 43.067156612799565, "cc_by_type": { "score": 43.067156612799565 }, "cc_tiered": 38.73637915753086, "cc_tiered_open": 34.332851503658375, "cc_tiered_sealed": 43.13990681140334, "low_n": false, "n": 375, "n_by_type": { "score": 375 }, "score": 43.067156612799565 } }, "raw_on_A": null, "set": "S+P" }, "validity": { "choice": { "invalid_rate": 0.0, "n": 750 }, "noul": { "invalid_rate": 0.0, "n": 375 }, "score": { "invalid_rate": 0.0, "n": 375 } }, "views": { "Balanced 33:33:33": 45.93426190954148, "Emphasis on Accuracy 60:20:20": 40.46467602098598, "Emphasis on Cost 20:20:60": 45.877374451824245, "Emphasis on Speed 20:60:20": 53.189857817529536, "Intelligence only": 34.3325026440474 }, "model_pin": null, "last_measured_on": "2026-10-06", "measurement_date_status": "run output completion day (UTC)", "measured_in": "v1.6.1", "noul_decisive": { "supported": true, "decisive_rate": 0.7973333333333333, "acc_among_decisive": 0.8762541806020067, "valid": 375 }, "provenance": { "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)", "carried_metadata_from": "v1.6.0 release row" }, "class": "decision-api", "open": "no", "licence": "proprietary hosted API", "repo": "https://wity.alphanimble.com/", "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=off; labelled configuration variant. The first approximately 70 decisions ran alongside two other streams; the remainder ran alone.", "underlying": "Base model: undisclosed (author request)", "adapter_id": "run/wity-1-off.spec.json:wity_reasoning_adapter.WityReasoningAdapter", "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).", "alt": { "label": "Base-model reference price (base undisclosed at the author's request)", "note": "base model undisclosed at the author's request; frozen base-model reference price applied to the same measured tokens", "usd_per_1000": 0.08721059113300492, "axes": { "calibration": 83.45289319036758, "cost": 41.78292309262718, "intelligence": 60.40492797149856, "speed": 87.66397534612332 } } } ], "not_measured": [ { "key": "autojev-27b-rtxpro6000-a2", "display": "AutoJev-27B (RTX PRO 6000)", "author": "unknown", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/denis-pplx/autojev-27b" }, { "key": "bosun-v31-0.6b", "display": "Bosun v3.1 0.6B", "author": "Clause Logic", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/Hanno-Labs/bosun-v3.1-0.6b" }, { "key": "classifier-dev-fast", "display": "classifier.dev (fast tier)", "author": "mrmps (@michael_chomsky)", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://classifier.dev" }, { "key": "decision-machine-1", "display": "decision-machine-1 (milliseconds.ai)", "author": "milliseconds.ai (Baptiste Laget)", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://www.milliseconds.ai" }, { "key": "deepseek-flash", "display": "DeepSeek V4.1 Flash (thinking default)", "author": "DeepSeek", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/deepseek-flash" }, { "key": "gemini-3.1-flash-lite", "display": "Gemini 3.1 Flash-Lite", "author": "Google", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/gemini-3.1-flash-lite" }, { "key": "gliner2", "display": "GLiNER2 (Fastino, gliner2.5-base)", "author": "Fastino", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/fastino-ai/GLiNER2" }, { "key": "gliner2-large", "display": "GLiNER2 large (Fastino)", "author": "Fastino", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/fastino/gliner2-large-v1" }, { "key": "gliner2.5-multi", "display": "GLiNER2.5 multi (Fastino, 287M)", "author": "Fastino", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/fastino/gliner2.5-multi-v1" }, { "key": "gliner2.5-small", "display": "GLiNER2.5 small (Fastino, 74M)", "author": "Fastino", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/fastino/gliner2.5-small-v1" }, { "key": "gpt-5.6-luna", "display": "GPT-5.6 Luna (low reasoning effort)", "author": "OpenAI", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/gpt-5.6-luna" }, { "key": "gpt-6-luna", "display": "GPT-6 Luna (default medium reasoning effort)", "author": "OpenAI", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/gpt-6-luna" }, { "key": "gpt-6-luna-low", "display": "GPT-6 Luna (low reasoning effort)", "author": "OpenAI", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/gpt-6-luna-low" }, { "key": "instinct", "display": "Instinct (ZooWork, Qwen3.8-27B)", "author": "rayrain-srp (ZooWork)", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/fstandhartinger/jevbench/issues/69" }, { "key": "instinct-dual-4b", "display": "Instinct Dual 4B", "author": "ZooWork / pierre-srp", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/fstandhartinger/jevbench/issues/123" }, { "key": "jeff", "display": "jeff (Logan Markewich, GLiFormer 400M)", "author": "Logan Markewich", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/logan-markewich/jeff" }, { "key": "jevact", "display": "JevAct (einptein, jev1-2b-v2)", "author": "einptein", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/fstandhartinger/jevbench/issues/66" }, { "key": "jobe-qwen3.5-4b", "display": "Jobe Qwen3.5-4B (frozen)", "author": "MantisShrimpdev", "addendum": null, "status": "not measured", "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: no official score was published.", "repo": "https://benchmarkheaven.com/jev-models/jobe-qwen3.5-4b" }, { "key": "kushal-gemma4-31b-it-autoloops", "display": "Autoloops – Gemma 4 31B IT", "author": "Autoloops", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/kushal-gemma4-31b-it-autoloops" }, { "key": "mica-v01-4b", "display": "mica-v01-4b", "author": "unknown", "addendum": { "id": "A1", "release": "v1.5", "label": "v1.5 roster addendum A1" }, "status": "not measured", "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: The frozen refusal policy stopped the run after 1,088 of 1,624 rows: 27 documented refusals were mapped to HTTP 422, then three consecutive passthrough HTTP 400 refusals triggered exit 6. The remaining 536 rows have no scores, so this system is not eligible for an official rank.", "repo": "https://benchmarkheaven.com/jev-models/mica-v01-4b" }, { "key": "needle-3", "display": "Needle 3 (Cactus, 2-bit, local CPU)", "author": "Cactus Compute", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/cactus-compute/needle" }, { "key": "needle-3-tools", "display": "Needle 3, options as tools (post-hoc adapter mode)", "author": "Cactus Compute", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/cactus-compute/needle" }, { "key": "ninfer-qwen3.8-27b-t1.5", "display": "NInfer Qwen3.8-27B NVFP4 (T=1.5)", "author": "Igor L. / NInfer contributors", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/igorls/ninfer" }, { "key": "openjev-razorback16", "display": "OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)", "author": "razorback16 / Codiv", "addendum": null, "status": "not measured", "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: no official score was published.", "repo": "https://benchmarkheaven.com/jev-models/openjev-razorback16" }, { "key": "openjev-sglang", "display": "openjev-sglang (Qwen3.6-35B-A3B on SGLang)", "author": "ekzhang", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/ekzhang/openjev-sglang" }, { "key": "qwen3.8-27b", "display": "Qwen3.8 27B (Chutes TEE)", "author": "Qwen / Chutes", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://benchmarkheaven.com/jev-models/qwen3.8-27b" }, { "key": "reflex-27b", "display": "reflex-27b (Qwen3.8-27B)", "author": "kshetrajna12", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/kshetrajna12/reflex" }, { "key": "simplejev-qwen3.6-35b-a3b", "display": "SimpleJev Qwen3.6-35B-A3B", "author": "Featherless AI", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/featherless-ai/simple-jev" }, { "key": "simplejev-qwen3.8-27b", "display": "SimpleJev Qwen3.8-27B", "author": "Featherless AI", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/featherless-ai/simple-jev" }, { "key": "surogate-rune-26b-a4b-v3-rtxpro6000-a2", "display": "Surogate Rune 26B-A4B v3 (RTX PRO 6000)", "author": "unknown", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://huggingface.co/api/models/surogate/rune-26b-a4b-GGUF" }, { "key": "system-one-open", "display": "system-one-open (Gemma 4 E2B LoRA on an L4)", "author": "mithalouni", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/mithalouni/system-one-open" }, { "key": "vansa-3.4", "display": "Vansa-3.4 (Vansa, hosted System One API)", "author": "Vansa", "addendum": null, "status": "not measured", "reason": "Not measured on v1.6.0; the latest published score is shown separately with its date.", "repo": "https://vansa.org" }, { "key": "von-395m", "display": "Von (wfzyx, Option-Marker 395M)", "author": "wfzyx (Victor Hugo)", "addendum": null, "status": "not measured", "reason": "Not measured on the v1.6.0 pool.", "repo": "https://github.com/wfzyx/von" } ], "roster_count": 173, "overnight": { "round": "score-v161-3 (v1.6.1)", "scored_utc": "2026-10-06 00:39:36 UTC", "a2_note": "History: hosted APIs were first measured on API subsets (A, then A2 and A3) and equated to the self-hosted scale in v1.6.0. From v1.6.1 they answer the same full set as self-hosted systems, so no equating is applied; the earlier subsets A, A2 and A3 are retired.", "notes": [ "Scores are the official v1.6.0 scorer (score_v16.py, method option B / O1S, bootstrap B = 1,000) over complete outputs only; each scoring round is kept separately.", "Hosted APIs are scored on the same full item set as self-hosted systems (S 1,200 + P 300, v1.6.1) and are not equated.", "GPU-class deviations: reproducible recipes used the hardware listed per row, including H100 for large fast-lane decoders and RTX 6000 for the baseline; hardware differences remain in measured latency. The standard x2 + 0.15 s self-hosted adjustment is an assumption, not a hardware normalization.", "Jev-class caps use a fixed reference: Jev 1.13.0 as measured in v1.5 (p50 0.62 s, USD 0.0323 per 1k answers); caps = 2x (1.23 s, USD 0.0646). Jev's own v1.6 p50 is 0.24 s. Wity auto remains outside the latency cap and stays ranked in Composite A. OFF and ALWAYS are unranked variants of the AUTO main row.", "6 further measured candidates await a separate publication decision.", "Sage 1.3.0 (Levanto Labs) was measured on 5 Oct from Sandy (Helsinki), text only. Cost uses the Levanto list tariff (USD 0.05/M input, USD 10/M output; levanto.ai/pricing, read 5 Oct) over all 600 answered rows: USD 0.0766/1,000 decisions. It exceeds the frozen Jev-class cost cap and is excluded from the Capability headline.", "A fresh Monday Jev 1.13.0 re-check on A3 measured p50 0.239 s and equated Capability 77.5 versus the official 76.5, within the confidence interval; the official v1.6.0 Jev row is retained." ], "exposure": { "wity-1-auto": { "display": "wity-1-auto (wity via railway)", "sealed_items_exposed": 1200, "draws": "S (1,200 sealed) + P (300 public)", "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired" }, "wity-1-off": { "display": "wity-1-off (wity via railway)", "sealed_items_exposed": 1200, "draws": "S (1,200 sealed) + P (300 public)", "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired" }, "wity-1-always": { "display": "wity-1-always (wity via railway)", "sealed_items_exposed": 1200, "draws": "S (1,200 sealed) + P (300 public)", "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired" }, "jev-1.13.0": { "display": "jev-1.13.0 (typesafe)", "sealed_items_exposed": 1200, "draws": "S (1,200 sealed) + P (300 public)", "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired" }, "fastino-gliner-2-5-decide": { "display": "fastino-gliner-2-5-decide (fastino)", "sealed_items_exposed": 1200, "draws": "S (1,200 sealed) + P (300 public)", "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired" }, "gpt-6-luna": { "display": "gpt-6-luna (openai)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "gpt-6-luna-low": { "display": "gpt-6-luna-low (openai)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "gpt-5.6-luna": { "display": "gpt-5.6-luna (openai)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "gemini-3.1-flash-lite": { "display": "gemini-3.1-flash-lite (google)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "deepseek-flash": { "display": "deepseek-flash (deepseek)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "qwen3.8-27b": { "display": "qwen3.8-27b (chutes)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "decision-machine-1": { "display": "decision-machine-1 (milliseconds.ai)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "classifier-dev-fast": { "display": "classifier-dev-fast (classifier.dev)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "vansa-3.4": { "display": "vansa-3.4 (vansa)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "kushal-gemma4-31b-it-autoloops": { "display": "kushal-gemma4-31b-it-autoloops (autoloops)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "system-one-open": { "display": "system-one-open (modal via modal)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "openjev-sglang": { "display": "openjev-sglang (modal via modal)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "instinct": { "display": "instinct (zoowork)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "instinct-dual-4b": { "display": "instinct-dual-4b (zoowork)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "simplejev-qwen3.8-27b": { "display": "simplejev-qwen3.8-27b (featherless)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "simplejev-qwen3.6-35b-a3b": { "display": "simplejev-qwen3.6-35b-a3b (featherless)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "jevact": { "display": "jevact (jevact)", "sealed_items_exposed": 0, "draws": "v1.5 sealed-720 (retired for scoring)", "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)" }, "sage-1.3.0": { "display": "Sage 1.3.0 (Levanto Labs)", "sealed_items_exposed": 1200, "draws": "S (1,200 sealed) + P (300 public)", "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired" } } }, "amendments": [ { "date": "2026-10-05", "revision": "v1.6.1", "text": "5 Oct 2026, v1.6.1 (Florian's decision): hosted/API systems now answer the same full item set as self-hosted systems (1,200 sealed + 300 public). Their earlier answers on the API subsets were reused and only the missing sealed items were sent; every item was logged in the exposure ledger before it was sent. API rows are no longer equated. Consequence: every v1.6.0 sealed item has now been seen by at least two providers, so the whole v1.6.0 sealed draw is retired for future releases; v1.6.2 onward draws fresh items from the reserve." }, { "date": "2026-10-05", "revision": "v1.6.1", "text": "5 Oct 2026, v1.6.1 cost rule (Florian's decision): cost per 1,000 decisions is now measured on one common item set for every system: all items except the 23 very long items that lie outside the Jev reference's accepted input range. Before, a system that processed those items paid for their tokens while a system that refused them did not. Refusals still count against Intelligence exactly as before. Hosted APIs with a per-token tariff are priced from their measured tokens on this common set; carried per-decision prices of self-hosted systems are unchanged. Effect: Sage 1.3.0 USD 0.0247 per 1,000 decisions on the common set (0.0650 on all 1,500 items); Jev 1.13.0 0.0323 (unchanged)." } ], "revision_history": [ { "revision": "v1.6.1", "date": "2026-10-05", "summary": "Hosted APIs (fastino-gliner-2-5-decide, jev-1.13.0, sage-1.3.0, wity-1, wity-1-always, wity-1-off) answer the full 1,500-item set instead of the 600-item API subset and are no longer equated; self-hosted rows unchanged; whole v1.6.0 sealed draw retired. Cost per 1,000 decisions is measured on one common item set (all items except the 23 outside the Jev reference input range) for every token-priced row." }, { "revision": "v1.6.0", "date": "2026-10-05", "summary": "Rotating item draw (1,200 sealed + 300 public); hosted APIs on API subsets equated to the self-hosted scale; /jev-models/v1.6.0 stays available unchanged." } ], "additions": [ { "date": "2026-10-07", "label": "a6", "keys": [ "kev-27b", "pplx-decider-v1-1-27b", "decider-chat-gemma4-31b", "jpt-35b-a3b", "jebadiah-27b", "simplejev-qwen3.8-27b-selfhost", "bespoke-nimble-9b-v3", "gev-26b-decide", "torchcast-decision-27b", "decision-2-0-vega-27b", "jade", "jev-27b-autotrust" ], "note": "Twelve open-weights systems from the top 20 of the community Jev Decision Index on Hugging Face, measured on the same sealed v1.6 pool, scorer and cost basis as every other row.", "scorer_source_sha256": "9d512dc13d7d876b233560e8710ed5f64d83f555be90141fa62644f22d5c9421", "baseline_systems_sha256": "6255c1c1014f148ad59eb43f78ded416d3dbb6604ebd6af3d530935edabc8f6b", "baseline_categories_sha256": "55c86278fb2ca0e8407f58eeb5338406f298a1bca426efd2ae7b52dad2752f6f", "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank." }, { "date": "2026-10-07", "label": "a7", "keys": [ "surogate-rune-26b-a4b-v3-v16", "xor-26b-a4b" ], "note": "Surogate Rune 26B-A4B v3 and Xor 26B-A4B from the top 20 of the community Jev Decision Index on Hugging Face, measured on the same sealed v1.6 pool, scorer and cost basis as every other row.", "scorer_source_sha256": "9d512dc13d7d876b233560e8710ed5f64d83f555be90141fa62644f22d5c9421", "baseline_systems_sha256": "2ff7130b5dc1946f005044b773c703835d902b808dec9a7f1556f2b9452b2657", "baseline_categories_sha256": "a2fe49209522d498b3be81e884f9751392c9c135669f11fbb5b8bdda0b2765ce", "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank." }, { "date": "2026-10-07", "label": "a8", "keys": [ "h2o-lightning-4b", "kahn1" ], "note": "H2O-Lightning-4B v1.1 (H2O.ai) and Kahn1 4B (Okura66), two Qwen3.5-4B fine-tunes submitted as JevBench add-requests, measured on the same sealed v1.6 pool, scorer and cost basis as every other row; both are priced with labelled cost estimates (Kahn1's server reports no token usage).", "scorer_source_sha256": "7a5b4d4c7b7ce966c1d7048d5106d50a6a700c8d89372e223ac276f41540be9d", "baseline_systems_sha256": "67d9aca6921a4f148b1137b2fcd438e582463b6e09a1e9904bf00a60c0b82812", "baseline_categories_sha256": "1922c9d5f5391b2e280288f4b5414edf24ac82e06b06cb6fb8bda3d34bf6342e", "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank." }, { "date": "2026-10-08", "label": "a11", "keys": [ "aplomb-1", "apus-openjev-v1-35b-a3b", "apus-openjev-v1-9b", "bb-qwen3.5-4b-lora", "classone-gemma4-e2b", "classone-qwen3.5-9b", "coco-decision-4b", "decisio-gemma-4-12b-v080", "decisio-gemma-4-12b-v090", "diffusion-jev", "evalengine-decision-4b", "exaone-4.0-1.2b-jev-v0.3", "hopper-12b-trained", "janus-4b", "jul-fast", "messier-one-v0.2", "seb-9b", "spx-cd-omni", "tacet-sonata", "typecastlm-1.4.0", "watersheep", "watt-flash-0.1" ], "note": "Completed current-pool submissions added; historical versions and self-hosted/API conditions are disclosed.", "scorer_source_sha256": "4d2833926c2daf284ce98e1fea6d9223dfc08341905778eb77391734638c27d2", "baseline_systems_sha256": "20b282fdf8597453965eb1bb6fe2d2fd47fbd07963adb5c9da97df862c309318", "baseline_categories_sha256": "07d69bc9f0638886a19c5d63f2473f9b2677b66950b7d2ea79f0d7c7902f4d69", "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank." }, { "date": "2026-10-08", "label": "a12", "keys": [ "bobcat-flash-1.2", "decider-12b", "decider-12b-v1", "decisio-gemma-4-31b-v080", "mercury-decide", "rene-1-31b-fp8", "spx-cd-flash", "spx-cd-pro" ], "note": "Approval-required completed candidates; current open/API top-five impact reviewed separately.", "scorer_source_sha256": "4d2833926c2daf284ce98e1fea6d9223dfc08341905778eb77391734638c27d2", "baseline_systems_sha256": "e3b67483db81438b8ec1c8adfab8dc8b12d9ef0ddec32617b0dec1d59766fbb2", "baseline_categories_sha256": "25dddf8fab1b6e240bbe68429f5986f74ef6407ee935adc99972390458c05263", "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank." } ] }