{ "analysis_version": 1, "generated_at_utc": "2026-10-02T18:20:10.891438+00:00", "analysis_spec_sha256": "e9adf0af72ba995d759f21e776e7f3d892e31fdcc51bee24606818322cde9054", "analysis_code_sha256": "373c479406245aed0d33c52db83b10cb83b3218327767c387e38e4267b6c6b44", "scoring_dependency_sha256": "0fbe6b55563ecdaa21f81553903df5f1d8c2b598ddd10c389cfd2a819b52a801", "cases_sha256": "e271790b2ee38a1c231e312fa15ece2a56c95f6bf705294f1637d43b2edcf256", "models": [ "Jev", "Luna", "Qwen3.8 Flash", "DeepSeek V4.1 Flash", "MiMo V2.6 Flash", "Hy4 Preview" ], "bootstrap_replicates": 5000, "bootstrap_seed": 20261002, "quality_sources": [ { "run_dir": "benchmark/expanded/runs/quality-20261002T093341Z", "metadata": { "status": "complete", "created_at": "2026-10-02T09:33:41.649117+00:00", "models": { "Jev": "typesafe/jev-1.13", "Luna": "openai/gpt-6-luna" }, "cases": 4785, "planned_calls": 9570, "concurrency_pairs": 16, "phase": "quality", "seed": 20261002, "budget_usd": 0.8, "cases_sha256": "e271790b2ee38a1c231e312fa15ece2a56c95f6bf705294f1637d43b2edcf256", "runner_sha256": "992c7c8ce88be5a4b7928957690a239cee5cac510f7882a8be61636eaa2c79e8", "adapter_sha256": "3a05cb20816fbc5b7c30a065429a7365afa9583f273d683bf69784f4af3e9ad7", "protocol": "Fresh full quality evaluation. Random case and within-pair model order. Thread workers each perform one pair sequentially. No retries or repairs. Concurrent latency is diagnostic only, excluded from speed claims. Shared answer contract and Luna settings unchanged from pilot.", "calls_completed": 9570, "valid": 9569, "invalid": 1, "charged_usd": null, "known_charge_subtotal_usd": 0.28895565000000006, "unknown_billing": 1, "budget_pending_usd": 0.0, "budget_stop_reason": null, "finished_at": "2026-10-02T09:49:23.338894+00:00", "protocol_amendments": [ { "timestamp": "2026-10-02T09:42:28.671416+00:00", "reason": "Continue only unattempted calls after a transient HTTP 520 and unknown bill. Retain the failed attempt without retry. Separate budget allowances permit continuation without fabricating billing.", "code_sha256": "9e13adeed4f44c0aa376c4a25e14beeb3770861e21daba65854aff32fb558768", "initial_code_sha256": "992c7c8ce88be5a4b7928957690a239cee5cac510f7882a8be61636eaa2c79e8", "initial_metadata_sha256": "a7c3d8243bd30a7da333de4736ce5630a5307f1236d8a53cb4aed54238937a8d", "initial_metadata_snapshot": "metadata-before-amendment-1.json", "initial_raw_prefix_sha256": "74bd79d76e29d2bc6f747e4d3632e56c2d7bd4ca0badf89fbbe5a22946c5fc94", "initial_raw_prefix_bytes": 899939, "initial_attempts": 308, "unattempted_calls": 9262, "frozen_plan_sha256": "4a9deec659f08d1698e150b41b0b32ad01f55dde55c2c93fc4e288ef89a67c36", "workers": 16, "unknown_billing_policy": "$0.01 accounting allowance per unknown attempt; reported cost remains null. In-flight reservations are the larger of the original request estimate and $0.01.", "failure_circuit": "Five consecutive invalid completions, or at least 5% invalid among global completed attempts once 100 have completed. Completion order is append order in responses.jsonl.", "retry_policy": "No retries, repairs, or replays; every original (case_id,model) is attempted at most once." } ], "continuation_started_at": "2026-10-02T09:42:28.672123+00:00", "consecutive_invalid": 0, "unknown_charge_allowance_per_attempt_usd": 0.01, "unknown_charge_allowance_total_usd": 0.01, "budget_accounted_usd": 0.2989556500000001 }, "responses_sha256": "c508f6173a27bed9834a294329414cf499bdc078381eb3f41e4d66c08f9f3f8b", "attempts": 9570, "cost_all_attempts": { "requests": 9570, "known_billed_usd": 0.28895565, "observed_bills": 9569, "missing_bills": 1, "total_billed_usd": null, "usd_per_1000_attempts": null } }, { "run_dir": "benchmark/multimodel/runs/quality-20261002T155349573021Z", "metadata": { "schema_version": 1, "created_at": "2026-10-02T15:53:49.997645+00:00", "models": [ { "label": "Qwen3.8 Flash", "id": "qwen/qwen3.8-flash", "provider": "alibaba", "provider_name": "Alibaba", "prices": { "prompt": "0.00000015", "completion": "0.00000047", "input_cache_read": "0.000000016", "input_cache_write": "0.0000002" }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null } ], "phase": "quality", "cases": 4785, "planned_calls": 4785, "calls_planned": 4785, "concurrency_pairs": 16, "seed": 20261002, "source_hashes": { "cases_sha256": "e271790b2ee38a1c231e312fa15ece2a56c95f6bf705294f1637d43b2edcf256", "timing_cases_sha256": "aa034a8fd9eedcf2ef2614fd3861d81af9d500eaed94629fb9f72c7d1df504f8" }, "cases_sha256": "e271790b2ee38a1c231e312fa15ece2a56c95f6bf705294f1637d43b2edcf256", "code_hashes": { "original-runner.py": "1c6d2ea8084da6e671d41a6d89f089edbf13729c75990bf4be0458db5df4242c", "original-adapter.py": "47c300bd57b53f669dd9c707b4317cf47cf2b5888c5233e17b94d1310e6f691b", "original-answer-contract.py": "3a05cb20816fbc5b7c30a065429a7365afa9583f273d683bf69784f4af3e9ad7" }, "plan_sha256": "b6f47e7d5913a3be6c34e0ac8b59ee7e8e70953907a53a002818286c6887327f", "requests_sha256": "eca30790f8e18d580b19662c3d61cee27f4f8de6db4a3c3b8ba31e45a4b23911", "automatic_retries": 0, "repeats": 1, "warmups_per_model_per_block": 0, "unknown_bill_allowance_usd": 0.01, "protocol": "Frozen full quality split; randomized case and within-case model order. Each worker runs a case's models sequentially. Bulk latency is diagnostic only.", "stop_rule": "Stop new dispatches after 5 consecutive invalid or >=5% invalid after 100 completed calls, overall or per model, in completion order. In-flight attempts are retained.", "billing_guard": "Known bills use observed values. Missing bills remain null and retain their reservation, at least $0.01, against the local guard. A local estimate cannot enforce a remote account-level cap.", "resume_policy": "Quality only: identical source, adapter, plan and exact request hashes; dispatch journal prevents every attempted sequence from being retried, including lost outcomes.", "status": "complete", "budget_usd": 4.99986885, "calls_completed": 4785, "valid": 4671, "invalid": 114, "charged_usd": null, "known_charge_subtotal_usd": 0.22825445199999933, "unknown_billing": 114, "unknown_billing_guard_usd": 1.1400000000000008, "budget_pending_usd": 0.0, "budget_accounted_usd": 1.3682544520000002, "budget_stop_reason": null, "responses_sha256": "a0bc839aaf80fb359b48e808962f21249bde21ef00d058ce163a6bab6ca750b6", "attempts_sha256": "595f028be707ecd9b1b9451ef9b8eceebea0aa1d75bdf6d606ec60eedcc99f22", "operational_amendments": [ { "id": "20261002T164150404672Z", "decided_at_utc": "2026-10-02T16:41:50.415128+00:00", "status": "continuation_stopped", "continuation_sha256": "439e844fd74b119eef354ba2ff367fb0039b7f6438c8e0c89270f0b16e29de46", "pre_continuation": { "responses.jsonl": { "sha256": "ea56e931c3509298d6f34ad7fd68c6dcc678cd41be34b50b1c1281541d113998", "bytes": 11831768 }, "attempts.jsonl": { "sha256": "63597db4441ebdbfe4758b4003cf1c6cb5a920a7eb0f029bc065ea944d802864", "bytes": 460797 }, "plan.json": { "sha256": "b6f47e7d5913a3be6c34e0ac8b59ee7e8e70953907a53a002818286c6887327f", "bytes": 1342472 }, "requests.jsonl": { "sha256": "eca30790f8e18d580b19662c3d61cee27f4f8de6db4a3c3b8ba31e45a4b23911", "bytes": 9149297 } }, "metadata_before_sha256": "537352319561e3368e72879891a89233dc103e1c820282ac7c9c31f9e2f16b96", "metadata_before_path": "benchmark/multimodel/runs/quality-20261002T155349573021Z/continuation-history/20261002T164150404672Z/metadata-before.json", "recovery_run_dir": "benchmark/multimodel/runs/probes-recovery-20261002T164150404672Z", "original_attempts": 3019, "original_invalid": 47, "unattempted_requests": 1766, "models": [ { "label": "Qwen3.8 Flash", "id": "qwen/qwen3.8-flash", "provider": "alibaba", "provider_name": "Alibaba", "prices": { "prompt": "0.00000015", "completion": "0.00000047", "input_cache_read": "0.000000016", "input_cache_write": "0.0000002" }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null } ], "workers": 16, "seed": 20261002, "total_quality_and_recovery_budget_usd": 5, "change": "After two valid synthetic service-health probes with observed zero reasoning, clear only the replayed historical consecutive-failure stop and streaks. Preserve cumulative invalid counts, every original attempt and bill, all unknown-bill reservations, request hashes, worker16 and the original live circuit. Dispatch only previously unattempted sequences. No benchmark output was selected for removal or retry.", "recovery_finished_at": "2026-10-02T16:41:52.652944+00:00", "recovery_budget_accounted_usd": 6.534e-05, "historical_recovery_budget_accounted_usd": 0, "recovery_calls": 2, "finished_at_utc": "2026-10-02T16:42:49.088229+00:00", "final_quality_calls": 3651, "final_quality_unknown_bills": 71 }, { "id": "20261002T165108634637Z", "decided_at_utc": "2026-10-02T16:51:08.646544+00:00", "status": "completed", "continuation_sha256": "de869edee16e83efa9651d8298ddadd6b3f2b19c3750f483322ad3cb86a88661", "continuation_file": "continuation_paced.py", "pre_continuation": { "responses.jsonl": { "sha256": "07a7991e64b7381d2c077c564a968f6192c255952b752f55b66ef727ae32f723", "bytes": 14310934 }, "attempts.jsonl": { "sha256": "682223d0c0f497848ff86310cbc3fb2dec7ab94aa374c95b972af21e595c7e89", "bytes": 557493 }, "plan.json": { "sha256": "b6f47e7d5913a3be6c34e0ac8b59ee7e8e70953907a53a002818286c6887327f", "bytes": 1342472 }, "requests.jsonl": { "sha256": "eca30790f8e18d580b19662c3d61cee27f4f8de6db4a3c3b8ba31e45a4b23911", "bytes": 9149297 } }, "metadata_before_sha256": "293b715756ee17039812b6f2b5329bdd07be3a571ed7d30343c38bfb63202ef0", "metadata_before_path": "benchmark/multimodel/runs/quality-20261002T155349573021Z/continuation-history/20261002T165108634637Z/metadata-before.json", "recovery_run_dir": "benchmark/multimodel/runs/probes-recovery-20261002T165108634637Z", "original_attempts": 3651, "original_invalid": 71, "unattempted_requests": 1134, "models": [ { "label": "Qwen3.8 Flash", "id": "qwen/qwen3.8-flash", "provider": "alibaba", "provider_name": "Alibaba", "prices": { "prompt": "0.00000015", "completion": "0.00000047", "input_cache_read": "0.000000016", "input_cache_write": "0.0000002" }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null } ], "workers": 16, "original_worker_setting": 16, "effective_worker_ceiling": 2, "effective_max_active_requests": 2, "min_dispatch_spacing_s": 0.25, "first_unattempted_sequence": 3651, "seed": 20261002, "total_quality_and_recovery_budget_usd": 5, "change": "Prospectively limit active outbound calls to two after two worker-16 episodes hit the shared provider quota. The saved original worker setting remains 16; actual executor workers are prospectively capped at two before the dispatch journal and adapter timer. Every new row records the effective ceiling. A minimum 0.25-second spacing is enforced before dispatch claims, with the original stop and budget guards rechecked after waiting. This changes the remaining bulk dispatch schedule; bulk timing remains diagnostic only. No provider, prompt, schema, reasoning setting, case, score or attempted request changes. After two valid synthetic service-health probes with observed zero reasoning, clear only the replayed historical consecutive-failure stop and streaks. Preserve cumulative invalid counts, every original attempt and bill, all unknown-bill reservations, request hashes, worker16 and the original live circuit. Dispatch only previously unattempted sequences. No benchmark output was selected for removal or retry.", "recovery_finished_at": "2026-10-02T16:51:12.059331+00:00", "recovery_budget_accounted_usd": 6.581e-05, "historical_recovery_budget_accounted_usd": 6.534e-05, "recovery_calls": 2, "finished_at_utc": "2026-10-02T17:05:22.955181+00:00", "final_quality_calls": 4785, "final_quality_unknown_bills": 114 } ], "resume_count": 2, "continuation_runtime": [ { "amendment_id": "20261002T164150404672Z", "armed_at": "2026-10-02T16:41:53.159836+00:00", "cleared_reason": "Five consecutive invalid responses (all models)", "prior_consecutive_counts": { "all models": 0, "Qwen3.8 Flash": 0 }, "cumulative_counts_retained": true, "unknown_bill_guard_retained": 0.47000000000000025 }, { "amendment_id": "20261002T165108634637Z", "armed_at": "2026-10-02T16:51:12.576520+00:00", "cleared_reason": "Five consecutive invalid responses (all models)", "prior_consecutive_counts": { "all models": 0, "Qwen3.8 Flash": 0 }, "cumulative_counts_retained": true, "unknown_bill_guard_retained": 0.7100000000000004, "effective_max_active_requests": 2 } ], "original_worker_setting": 16, "effective_worker_ceiling": 2, "effective_max_active_requests": 2, "min_dispatch_spacing_s": 0.25, "finished_at": "2026-10-02T17:05:22.921561+00:00" }, "responses_sha256": "a0bc839aaf80fb359b48e808962f21249bde21ef00d058ce163a6bab6ca750b6", "attempts": 4785, "cost_all_attempts": { "requests": 4785, "known_billed_usd": 0.228254452, "observed_bills": 4671, "missing_bills": 114, "total_billed_usd": null, "usd_per_1000_attempts": null } }, { "run_dir": "benchmark/multimodel/runs/quality-20261002T160544592288Z", "metadata": { "schema_version": 1, "created_at": "2026-10-02T16:05:45.675189+00:00", "models": [ { "label": "DeepSeek V4.1 Flash", "id": "deepseek/deepseek-v4.1-flash", "provider": "deepinfra/fp8", "provider_name": "DeepInfra", "prices": { "prompt": "0.00000014", "completion": "0.00000042", "input_cache_read": "0.0000000042", "discount": 0.3 }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null, "cache_reason": "No verified matched OpenRouter cache-enable/cache-disable treatment for this provider in this study." }, { "label": "MiMo V2.6 Flash", "id": "xiaomi/mimo-v2.6-flash", "provider": "xiaomi/fp8", "provider_name": "Xiaomi", "prices": { "prompt": "0.00000014", "completion": "0.00000028", "input_cache_read": "0.0000000028", "discount": 0 }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null, "cache_reason": "No verified matched OpenRouter cache-enable/cache-disable treatment for this provider in this study." }, { "label": "Hy4 Preview", "id": "tencent/hy4-preview", "provider": "tencent/fp8", "provider_name": "Tencent", "prices": { "prompt": "0.000000834", "completion": "0.000002501", "input_cache_read": "0.000000042", "discount": 0, "overrides": [ { "utc_start": 0, "utc_end": 1600, "prompt": "0.000000834", "completion": "0.000002501", "input_cache_read": "0.000000042" }, { "utc_start": 1600, "utc_end": 0, "prompt": "0.0000007506", "completion": "0.0000022509", "input_cache_read": "0.0000000378" } ] }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null, "cache_reason": "No verified matched OpenRouter cache-enable/cache-disable treatment for this provider in this study." } ], "phase": "quality", "cases": 4785, "planned_calls": 14355, "calls_planned": 14355, "concurrency_pairs": 16, "seed": 20261002, "source_hashes": { "cases_sha256": "e271790b2ee38a1c231e312fa15ece2a56c95f6bf705294f1637d43b2edcf256", "timing_cases_sha256": "aa034a8fd9eedcf2ef2614fd3861d81af9d500eaed94629fb9f72c7d1df504f8" }, "cases_sha256": "e271790b2ee38a1c231e312fa15ece2a56c95f6bf705294f1637d43b2edcf256", "code_hashes": { "original-runner.py": "1c6d2ea8084da6e671d41a6d89f089edbf13729c75990bf4be0458db5df4242c", "original-adapter.py": "47c300bd57b53f669dd9c707b4317cf47cf2b5888c5233e17b94d1310e6f691b", "original-answer-contract.py": "3a05cb20816fbc5b7c30a065429a7365afa9583f273d683bf69784f4af3e9ad7" }, "plan_sha256": "7beaa575ad9b58776da1be18e8c5a451d0a5199990f61b3f593bb65607fb06a8", "requests_sha256": "edbb6e6de3f59b7b5dceb81479a683f923a28351bdf93624fcd1c01e475a11ce", "automatic_retries": 0, "repeats": 1, "warmups_per_model_per_block": 0, "unknown_bill_allowance_usd": 0.01, "protocol": "Frozen full quality split; randomized case and within-case model order. Each worker runs a case's models sequentially. Bulk latency is diagnostic only.", "stop_rule": "Stop new dispatches after 5 consecutive invalid or >=5% invalid after 100 completed calls, overall or per model, in completion order. In-flight attempts are retained.", "billing_guard": "Known bills use observed values. Missing bills remain null and retain their reservation, at least $0.01, against the local guard. A local estimate cannot enforce a remote account-level cap.", "resume_policy": "Quality only: identical source, adapter, plan and exact request hashes; dispatch journal prevents every attempted sequence from being retried, including lost outcomes.", "status": "complete", "budget_usd": 5, "calls_completed": 14355, "valid": 14351, "invalid": 4, "charged_usd": null, "known_charge_subtotal_usd": 1.324546401199999, "unknown_billing": 4, "unknown_billing_guard_usd": 0.04, "budget_pending_usd": 0.0, "budget_accounted_usd": 1.364546401199999, "budget_stop_reason": null, "finished_at": "2026-10-02T16:41:46.141386+00:00", "responses_sha256": "9b949a90dcc900c2228ef035987ae306a858c138eab153eb7cf4bb6056357a22", "attempts_sha256": "36eaaed33c2db8cb4f79dc0cb77536d34e382d6e7e3bf17cb08e2d890dd62670" }, "responses_sha256": "9b949a90dcc900c2228ef035987ae306a858c138eab153eb7cf4bb6056357a22", "attempts": 14355, "cost_all_attempts": { "requests": 14355, "known_billed_usd": 1.3245464012, "observed_bills": 14351, "missing_bills": 4, "total_billed_usd": null, "usd_per_1000_attempts": null } } ], "model_provenance": { "Jev": { "source_runs": [ "benchmark/expanded/runs/quality-20261002T093341Z" ], "first_started_at": "2026-10-02T09:33:41.655815+00:00", "last_started_at": "2026-10-02T09:49:23.055621+00:00", "requested_model_ids": [ "typesafe/jev-1.13" ], "returned_model_provider_pairs": [ [ "typesafe/jev-1.13-20260917", "TypeSafe" ] ] }, "Luna": { "source_runs": [ "benchmark/expanded/runs/quality-20261002T093341Z" ], "first_started_at": "2026-10-02T09:33:41.655433+00:00", "last_started_at": "2026-10-02T09:49:22.244067+00:00", "requested_model_ids": [ "openai/gpt-6-luna" ], "returned_model_provider_pairs": [ [ "openai/gpt-6-luna", "OpenAI" ] ] }, "Qwen3.8 Flash": { "source_runs": [ "benchmark/multimodel/runs/quality-20261002T155349573021Z" ], "first_started_at": "2026-10-02T15:53:50.006692+00:00", "last_started_at": "2026-10-02T17:05:21.770721+00:00", "requested_model_ids": [ "qwen/qwen3.8-flash" ], "returned_model_provider_pairs": [ [ "qwen/qwen3.8-flash", "Alibaba" ] ] }, "DeepSeek V4.1 Flash": { "source_runs": [ "benchmark/multimodel/runs/quality-20261002T160544592288Z" ], "first_started_at": "2026-10-02T16:05:45.696738+00:00", "last_started_at": "2026-10-02T16:41:30.921775+00:00", "requested_model_ids": [ "deepseek/deepseek-v4.1-flash" ], "returned_model_provider_pairs": [ [ "deepseek/deepseek-v4.1-flash", "DeepInfra" ] ] }, "MiMo V2.6 Flash": { "source_runs": [ "benchmark/multimodel/runs/quality-20261002T160544592288Z" ], "first_started_at": "2026-10-02T16:05:45.695069+00:00", "last_started_at": "2026-10-02T16:41:33.256370+00:00", "requested_model_ids": [ "xiaomi/mimo-v2.6-flash" ], "returned_model_provider_pairs": [ [ "xiaomi/mimo-v2.6-flash", "Xiaomi" ] ] }, "Hy4 Preview": { "source_runs": [ "benchmark/multimodel/runs/quality-20261002T160544592288Z" ], "first_started_at": "2026-10-02T16:05:45.695640+00:00", "last_started_at": "2026-10-02T16:41:17.958584+00:00", "requested_model_ids": [ "tencent/hy4-preview" ], "returned_model_provider_pairs": [ [ "tencent/hy4-preview", "Tencent" ] ] } }, "quality": { "attempts": 28710, "expected_attempts": 28710, "missing_attempts": 0, "cost_all_quality_sources": { "requests": 28710, "known_billed_usd": 1.8417565032, "observed_bills": 28591, "missing_bills": 119, "total_billed_usd": null, "usd_per_1000_attempts": null }, "cost_quality_phase": { "requests": 28710, "known_billed_usd": 1.8417565032, "observed_bills": 28591, "missing_bills": 119, "total_billed_usd": null, "usd_per_1000_attempts": null }, "bulk_latency_diagnostic_only": { "Jev": { "attempts": 4785, "valid": 4784, "observed_positive_timings": 4785, "median_s": 0.2417347920127213, "p95_s": 0.32409904980449955, "mean_s": 0.25440568755248943 }, "Luna": { "attempts": 4785, "valid": 4785, "observed_positive_timings": 4785, "median_s": 1.0800500409968663, "p95_s": 1.5839749831997325, "mean_s": 1.1784223865195909 }, "Qwen3.8 Flash": { "attempts": 4785, "valid": 4671, "observed_positive_timings": 4785, "median_s": 1.2354236250102986, "p95_s": 3.3684997748001497, "mean_s": 1.5490290967966593 }, "DeepSeek V4.1 Flash": { "attempts": 4785, "valid": 4785, "observed_positive_timings": 4785, "median_s": 0.842129875003593, "p95_s": 2.3143806500011124, "mean_s": 1.1249442134245993 }, "MiMo V2.6 Flash": { "attempts": 4785, "valid": 4785, "observed_positive_timings": 4785, "median_s": 1.6373511250130832, "p95_s": 9.694652474601755, "mean_s": 2.998920932974378 }, "Hy4 Preview": { "attempts": 4785, "valid": 4781, "observed_positive_timings": 4785, "median_s": 2.680595500016352, "p95_s": 3.8576518336078154, "mean_s": 2.9972089135063453 } } }, "tasks": { "boolq": { "expected_cases": 3270, "target_kind": "label", "models": { "Jev": { "attempts": 3270, "valid": 3270, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.998826620388002, 1 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.06289479, "observed_bills": 3270, "missing_bills": 0, "total_billed_usd": 0.06289479, "usd_per_1000_attempts": 0.019233880733944955, "usd_per_correct_answer": 2.095794401866045e-05 }, "quality": { "correct": 3001, "accuracy_all_attempts": 0.917737003058104, "accuracy_all_attempts_wilson95": [ 0.9078221114479283, 0.9266715665031486 ], "balanced_accuracy_all_attempts": 0.9162745579108811, "class_denominators": { "no": { "attempts": 1237, "valid": 1237, "correct": 1126, "recall_all_attempts": 0.910266774454325 }, "yes": { "attempts": 2033, "valid": 2033, "correct": 1875, "recall_all_attempts": 0.9222823413674373 } }, "accuracy_valid_only": 0.917737003058104, "accuracy_valid_only_wilson95": [ 0.9078221114479283, 0.9266715665031486 ], "balanced_accuracy_valid_only": 0.9162745579108811, "brier_valid_only": 0.062442293577981645, "log_loss_valid_only": 0.219908748023455, "proper_score_denominator": 3270 }, "expected_cases": 3270, "missing_attempts": 0 }, "Luna": { "attempts": 3270, "valid": 3270, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.998826620388002, 1 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.1404962, "observed_bills": 3270, "missing_bills": 0, "total_billed_usd": 0.1404962, "usd_per_1000_attempts": 0.042965198776758404, "usd_per_correct_answer": 4.9733168141592913e-05 }, "quality": { "correct": 2825, "accuracy_all_attempts": 0.863914373088685, "accuracy_all_attempts_wilson95": [ 0.8517343947848243, 0.8752403319807552 ], "balanced_accuracy_all_attempts": 0.8577956443023181, "class_denominators": { "no": { "attempts": 1237, "valid": 1237, "correct": 1030, "recall_all_attempts": 0.8326596604688763 }, "yes": { "attempts": 2033, "valid": 2033, "correct": 1795, "recall_all_attempts": 0.88293162813576 } }, "accuracy_valid_only": 0.863914373088685, "accuracy_valid_only_wilson95": [ 0.8517343947848243, 0.8752403319807552 ], "balanced_accuracy_valid_only": 0.8577956443023181, "brier_valid_only": 0.1254407339449541, "log_loss_valid_only": 2.0475586107185118, "proper_score_denominator": 3270 }, "expected_cases": 3270, "missing_attempts": 0 }, "Qwen3.8 Flash": { "attempts": 3270, "valid": 3196, "invalid": 74, "success_rate": 0.9773700305810398, "success_rate_wilson95": [ 0.9716848195435246, 0.9819349690960297 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.16976776, "observed_bills": 3196, "missing_bills": 74, "total_billed_usd": null, "usd_per_1000_attempts": null, "usd_per_correct_answer": null }, "quality": { "correct": 2883, "accuracy_all_attempts": 0.881651376146789, "accuracy_all_attempts_wilson95": [ 0.8701295492173265, 0.8922775591889281 ], "balanced_accuracy_all_attempts": 0.869844613195134, "class_denominators": { "no": { "attempts": 1237, "valid": 1205, "correct": 1016, "recall_all_attempts": 0.8213419563459984 }, "yes": { "attempts": 2033, "valid": 1991, "correct": 1867, "recall_all_attempts": 0.9183472700442695 } }, "accuracy_valid_only": 0.9020650813516896, "accuracy_valid_only_wilson95": [ 0.8912726473426597, 0.9118921447490015 ], "balanced_accuracy_valid_only": 0.8904366328978328, "brier_valid_only": 0.08750831602002504, "log_loss_valid_only": 0.8897137976043077, "proper_score_denominator": 3196 }, "expected_cases": 3270, "missing_attempts": 0 }, "DeepSeek V4.1 Flash": { "attempts": 3270, "valid": 3270, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.998826620388002, 1 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.15220184, "observed_bills": 3270, "missing_bills": 0, "total_billed_usd": 0.15220184, "usd_per_1000_attempts": 0.04654490519877676, "usd_per_correct_answer": 5.3217426573426575e-05 }, "quality": { "correct": 2860, "accuracy_all_attempts": 0.8746177370030581, "accuracy_all_attempts_wilson95": [ 0.8628261706084228, 0.8855301657679089 ], "balanced_accuracy_all_attempts": 0.865295780494914, "class_denominators": { "no": { "attempts": 1237, "valid": 1237, "correct": 1023, "recall_all_attempts": 0.8270008084074374 }, "yes": { "attempts": 2033, "valid": 2033, "correct": 1837, "recall_all_attempts": 0.9035907525823905 } }, "accuracy_valid_only": 0.8746177370030581, "accuracy_valid_only_wilson95": [ 0.8628261706084228, 0.8855301657679089 ], "balanced_accuracy_valid_only": 0.865295780494914, "brier_valid_only": 0.10843817611895688, "log_loss_valid_only": 1.701064681012324, "proper_score_denominator": 3270 }, "expected_cases": 3270, "missing_attempts": 0 }, "MiMo V2.6 Flash": { "attempts": 3270, "valid": 3270, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.998826620388002, 1 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.095622744, "observed_bills": 3270, "missing_bills": 0, "total_billed_usd": 0.095622744, "usd_per_1000_attempts": 0.029242429357798163, "usd_per_correct_answer": 3.2893960784313726e-05 }, "quality": { "correct": 2907, "accuracy_all_attempts": 0.8889908256880734, "accuracy_all_attempts_wilson95": [ 0.8777638470589225, 0.899304936508991 ], "balanced_accuracy_all_attempts": 0.8771715760286716, "class_denominators": { "no": { "attempts": 1237, "valid": 1237, "correct": 1025, "recall_all_attempts": 0.8286176232821342 }, "yes": { "attempts": 2033, "valid": 2033, "correct": 1882, "recall_all_attempts": 0.9257255287752091 } }, "accuracy_valid_only": 0.8889908256880734, "accuracy_valid_only_wilson95": [ 0.8777638470589225, 0.899304936508991 ], "balanced_accuracy_valid_only": 0.8771715760286716, "brier_valid_only": 0.09536354098165138, "log_loss_valid_only": 1.0692482600904558, "proper_score_denominator": 3270 }, "expected_cases": 3270, "missing_attempts": 0 }, "Hy4 Preview": { "attempts": 3270, "valid": 3266, "invalid": 4, "success_rate": 0.998776758409786, "success_rate_wilson95": [ 0.9968588022397419, 0.999524205621317 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.7272430587000001, "observed_bills": 3266, "missing_bills": 4, "total_billed_usd": null, "usd_per_1000_attempts": null, "usd_per_correct_answer": null }, "quality": { "correct": 2909, "accuracy_all_attempts": 0.8896024464831804, "accuracy_all_attempts_wilson95": [ 0.8784006906210592, 0.8998898992103258 ], "balanced_accuracy_all_attempts": 0.8909574478660708, "class_denominators": { "no": { "attempts": 1237, "valid": 1237, "correct": 1109, "recall_all_attempts": 0.8965238480194018 }, "yes": { "attempts": 2033, "valid": 2029, "correct": 1800, "recall_all_attempts": 0.8853910477127398 } }, "accuracy_valid_only": 0.8906919779546846, "accuracy_valid_only_wilson95": [ 0.8795282965301395, 0.9009376777805561 ], "balanced_accuracy_valid_only": 0.8918301842364136, "brier_valid_only": 0.10300349112676056, "log_loss_valid_only": 2.936123009835336, "proper_score_denominator": 3266 }, "expected_cases": 3270, "missing_attempts": 0 } }, "comparisons": { "Luna / Jev": { "direction": "Luna minus Jev", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3270, "invalid_response_pairs": 0, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261002, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.05382262996941896, "ci95": [ -0.06487188927230851, -0.04314538527434187 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.058478913608563, "ci95": [ -0.07053555551408902, -0.04661974990646179 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.06299844036697247, "ci95": [ 0.05413421740104574, 0.07189101253309452 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.8276498626950566, "ci95": [ 1.5795518090371796, 2.08361927992546 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2754, "model_only_correct": 71, "reference_only_correct": 247, "both_incorrect": 198 }, "mcnemar_exact_two_sided_p_exploratory": 6.090829100595564e-24, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Luna", "reference": "Jev" }, "Qwen3.8 Flash / Jev": { "direction": "Qwen3.8 Flash minus Jev", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3196, "invalid_response_pairs": 74, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261003, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.036085626911314984, "ci95": [ -0.04603068438552757, -0.026155469849816006 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.046429944715747164, "ci95": [ -0.057029820300328604, -0.03574148632725449 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.02536732728410513, "ci95": [ 0.018931823481352972, 0.03223035917518614 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.6709340073816001, "ci95": [ 0.5141649550144411, 0.8311340187063352 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2802, "model_only_correct": 81, "reference_only_correct": 199, "both_incorrect": 188 }, "mcnemar_exact_two_sided_p_exploratory": 1.2598593589333536e-12, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Jev" }, "Qwen3.8 Flash / Luna": { "direction": "Qwen3.8 Flash minus Luna", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3196, "invalid_response_pairs": 74, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261004, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.017737003058103974, "ci95": [ 0.005216324025774778, 0.03029407984724555 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.012048968892815832, "ci95": [ -0.0018615755688825212, 0.026099045572692662 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.03844553254067584, "ci95": [ -0.04850481222454987, -0.028405345775200437 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -1.1573943908396733, "ci95": [ -1.4033092768921485, -0.9202395608101466 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2633, "model_only_correct": 250, "reference_only_correct": 192, "both_incorrect": 195 }, "mcnemar_exact_two_sided_p_exploratory": 0.00663801763274788, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Luna" }, "DeepSeek V4.1 Flash / Jev": { "direction": "DeepSeek V4.1 Flash minus Jev", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3270, "invalid_response_pairs": 0, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261005, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.043119266055045874, "ci95": [ -0.053879359549677215, -0.0327115651073721 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.05097877741596718, "ci95": [ -0.06282075022068617, -0.039434909349184664 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.04599588254097523, "ci95": [ 0.038390643576019574, 0.054017700833557156 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.4811559329888688, "ci95": [ 1.2515890410009607, 1.71267232319951 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2770, "model_only_correct": 90, "reference_only_correct": 231, "both_incorrect": 179 }, "mcnemar_exact_two_sided_p_exploratory": 1.93969877002409e-15, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Jev" }, "DeepSeek V4.1 Flash / Luna": { "direction": "DeepSeek V4.1 Flash minus Luna", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3270, "invalid_response_pairs": 0, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261006, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.010703363914373088, "ci95": [ -0.001827820474845092, 0.02321355881293356 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.007500136192595815, "ci95": [ -0.006876287911257969, 0.02186392124097461 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.017002557825997244, "ci95": [ -0.027432445380924438, -0.006427126574292728 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.3464939297061875, "ci95": [ -0.6243277313937058, -0.06930693102207987 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2628, "model_only_correct": 232, "reference_only_correct": 197, "both_incorrect": 213 }, "mcnemar_exact_two_sided_p_exploratory": 0.1005742046978771, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Luna" }, "MiMo V2.6 Flash / Jev": { "direction": "MiMo V2.6 Flash minus Jev", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3270, "invalid_response_pairs": 0, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261007, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.02874617737003058, "ci95": [ -0.03842662825937786, -0.018725319240359688 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.03910298188220951, "ci95": [ -0.04964003253904825, -0.02848620570600139 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.03292124740366973, "ci95": [ 0.026351006507035275, 0.039583275692524594 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.8493395120670008, "ci95": [ 0.6803474198404741, 1.0363984312593697 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2819, "model_only_correct": 88, "reference_only_correct": 182, "both_incorrect": 181 }, "mcnemar_exact_two_sided_p_exploratory": 1.0875055935276827e-08, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Jev" }, "MiMo V2.6 Flash / Luna": { "direction": "MiMo V2.6 Flash minus Luna", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3270, "invalid_response_pairs": 0, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261008, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.02507645259938838, "ci95": [ 0.013190082928380207, 0.0370486608980737 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.019375931726353483, "ci95": [ 0.005698169171641928, 0.03313967894090824 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.030077192963302753, "ci95": [ -0.04001217545269182, -0.02013795772414401 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.9783103506280558, "ci95": [ -1.2307921231092065, -0.7314249293458708 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2665, "model_only_correct": 242, "reference_only_correct": 160, "both_incorrect": 203 }, "mcnemar_exact_two_sided_p_exploratory": 5.057897686177978e-05, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Luna" }, "Hy4 Preview / Jev": { "direction": "Hy4 Preview minus Jev", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3266, "invalid_response_pairs": 4, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261009, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.028134556574923548, "ci95": [ -0.0378051662835933, -0.01888473496974121 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.025317110044810347, "ci95": [ -0.03540255567433637, -0.015449645870049903 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.040493846301285975, "ci95": [ 0.03327109796131445, 0.04820044307773828 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 2.7160174309290226, "ci95": [ 2.4041928802735737, 3.0391821526120597 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2832, "model_only_correct": 77, "reference_only_correct": 169, "both_incorrect": 192 }, "mcnemar_exact_two_sided_p_exploratory": 4.3696015518080695e-09, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Jev" }, "Hy4 Preview / Luna": { "direction": "Hy4 Preview minus Luna", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3266, "invalid_response_pairs": 4, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261010, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.025688073394495414, "ci95": [ 0.014510203990317284, 0.03719512195121951 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.033161803563752645, "ci95": [ 0.020861871793104066, 0.045607160781470694 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.022590844451928962, "ci95": [ -0.032360389435740565, -0.012809885099884533 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.8860597499719922, "ci95": [ 0.5869793905783922, 1.1923514579723544 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2694, "model_only_correct": 215, "reference_only_correct": 131, "both_incorrect": 230 }, "mcnemar_exact_two_sided_p_exploratory": 7.364729817164784e-06, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Luna" } } }, "policy": { "expected_cases": 300, "target_kind": "label", "models": { "Jev": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.006052662, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.006052662, "usd_per_1000_attempts": 0.02017554, "usd_per_correct_answer": 2.017554e-05 }, "quality": { "correct": 300, "accuracy_all_attempts": 1.0, "accuracy_all_attempts_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "balanced_accuracy_all_attempts": 1.0, "class_denominators": { "no": { "attempts": 150, "valid": 150, "correct": 150, "recall_all_attempts": 1.0 }, "yes": { "attempts": 150, "valid": 150, "correct": 150, "recall_all_attempts": 1.0 } }, "accuracy_valid_only": 1.0, "accuracy_valid_only_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "balanced_accuracy_valid_only": 1.0, "brier_valid_only": 0.005199333333333334, "log_loss_valid_only": 0.05776850682166601, "proper_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "Luna": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.0132828, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0132828, "usd_per_1000_attempts": 0.044276, "usd_per_correct_answer": 4.442408026755853e-05 }, "quality": { "correct": 299, "accuracy_all_attempts": 0.9966666666666667, "accuracy_all_attempts_wilson95": [ 0.9813633063054688, 0.9994113422781488 ], "balanced_accuracy_all_attempts": 0.9966666666666666, "class_denominators": { "no": { "attempts": 150, "valid": 150, "correct": 149, "recall_all_attempts": 0.9933333333333333 }, "yes": { "attempts": 150, "valid": 150, "correct": 150, "recall_all_attempts": 1.0 } }, "accuracy_valid_only": 0.9966666666666667, "accuracy_valid_only_wilson95": [ 0.9813633063054688, 0.9994113422781488 ], "balanced_accuracy_valid_only": 0.9966666666666666, "brier_valid_only": 0.0033333333333333335, "log_loss_valid_only": 0.11513191997447059, "proper_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "Qwen3.8 Flash": { "attempts": 300, "valid": 293, "invalid": 7, "success_rate": 0.9766666666666667, "success_rate_wilson95": [ 0.9526280277317246, 0.9886523397008747 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.006091292, "observed_bills": 293, "missing_bills": 7, "total_billed_usd": null, "usd_per_1000_attempts": null, "usd_per_correct_answer": null }, "quality": { "correct": 293, "accuracy_all_attempts": 0.9766666666666667, "accuracy_all_attempts_wilson95": [ 0.9526280277317246, 0.9886523397008747 ], "balanced_accuracy_all_attempts": 0.9766666666666667, "class_denominators": { "no": { "attempts": 150, "valid": 147, "correct": 147, "recall_all_attempts": 0.98 }, "yes": { "attempts": 150, "valid": 146, "correct": 146, "recall_all_attempts": 0.9733333333333334 } }, "accuracy_valid_only": 1.0, "accuracy_valid_only_wilson95": [ 0.9870588871380849, 0.9999999999999999 ], "balanced_accuracy_valid_only": 1.0, "brier_valid_only": 0.0, "log_loss_valid_only": 9.99601725036675e-16, "proper_score_denominator": 293 }, "expected_cases": 300, "missing_attempts": 0 }, "DeepSeek V4.1 Flash": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.0044090536, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0044090536, "usd_per_1000_attempts": 0.014696845333333333, "usd_per_correct_answer": 1.479548187919463e-05 }, "quality": { "correct": 298, "accuracy_all_attempts": 0.9933333333333333, "accuracy_all_attempts_wilson95": [ 0.9760224167496929, 0.998169851642088 ], "balanced_accuracy_all_attempts": 0.9933333333333334, "class_denominators": { "no": { "attempts": 150, "valid": 150, "correct": 150, "recall_all_attempts": 1.0 }, "yes": { "attempts": 150, "valid": 150, "correct": 148, "recall_all_attempts": 0.9866666666666667 } }, "accuracy_valid_only": 0.9933333333333333, "accuracy_valid_only_wilson95": [ 0.9760224167496929, 0.998169851642088 ], "balanced_accuracy_valid_only": 0.9933333333333334, "brier_valid_only": 0.006666666666666667, "log_loss_valid_only": 0.23025850929940556, "proper_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "MiMo V2.6 Flash": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.0051780456, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0051780456, "usd_per_1000_attempts": 0.017260152, "usd_per_correct_answer": 1.74933972972973e-05 }, "quality": { "correct": 296, "accuracy_all_attempts": 0.9866666666666667, "accuracy_all_attempts_wilson95": [ 0.9662244697474675, 0.9948030382606411 ], "balanced_accuracy_all_attempts": 0.9866666666666667, "class_denominators": { "no": { "attempts": 150, "valid": 150, "correct": 148, "recall_all_attempts": 0.9866666666666667 }, "yes": { "attempts": 150, "valid": 150, "correct": 148, "recall_all_attempts": 0.9866666666666667 } }, "accuracy_valid_only": 0.9866666666666667, "accuracy_valid_only_wilson95": [ 0.9662244697474675, 0.9948030382606411 ], "balanced_accuracy_valid_only": 0.9866666666666667, "brier_valid_only": 0.01372642, "log_loss_valid_only": 0.46466073464610685, "proper_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "Hy4 Preview": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.0448371756, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0448371756, "usd_per_1000_attempts": 0.14945725199999998, "usd_per_correct_answer": 0.00015677334125874126 }, "quality": { "correct": 286, "accuracy_all_attempts": 0.9533333333333334, "accuracy_all_attempts_wilson95": [ 0.923202639772079, 0.9720010663176659 ], "balanced_accuracy_all_attempts": 0.9533333333333334, "class_denominators": { "no": { "attempts": 150, "valid": 150, "correct": 138, "recall_all_attempts": 0.92 }, "yes": { "attempts": 150, "valid": 150, "correct": 148, "recall_all_attempts": 0.9866666666666667 } }, "accuracy_valid_only": 0.9533333333333334, "accuracy_valid_only_wilson95": [ 0.923202639772079, 0.9720010663176659 ], "balanced_accuracy_valid_only": 0.9533333333333334, "brier_valid_only": 0.04666666666666667, "log_loss_valid_only": 1.6118415489930407, "proper_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 } }, "comparisons": { "Luna / Jev": { "direction": "Luna minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261102, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.0033333333333333335, "ci95": [ -0.01, 0.0 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0033333333333333335, "ci95": [ -0.01056338028169014, 0.0 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.0018660000000000007, "ci95": [ -0.006145675000000001, 0.005344191666666659 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.05736341315280459, "ci95": [ -0.061878257789966536, 0.2898557157148909 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 299, "model_only_correct": 0, "reference_only_correct": 1, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 1.0, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Luna", "reference": "Jev" }, "Qwen3.8 Flash / Jev": { "direction": "Qwen3.8 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 293, "invalid_response_pairs": 7, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261103, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.023333333333333334, "ci95": [ -0.04, -0.006666666666666667 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.023333333333333334, "ci95": [ -0.0404725544676113, -0.006763886451386454 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.005254607508532423, "ci95": [ -0.006636173877085081, -0.0040879449130480075 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.05802019462434765, "ci95": [ -0.06425874451110178, -0.05214409211784379 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 293, "model_only_correct": 0, "reference_only_correct": 7, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.015625000000000003, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Jev" }, "Qwen3.8 Flash / Luna": { "direction": "Qwen3.8 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 293, "invalid_response_pairs": 7, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261104, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.02, "ci95": [ -0.04, -0.0033333333333333335 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.02, "ci95": [ -0.03913955980163092, -0.0031446540880503146 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.0034129692832764505, "ci95": [ -0.010380622837370242, 0.0 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.11788251191925214, "ci95": [ -0.3585423113391787, 0.0 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 292, "model_only_correct": 1, "reference_only_correct": 7, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.07031250000000003, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Luna" }, "DeepSeek V4.1 Flash / Jev": { "direction": "DeepSeek V4.1 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261105, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.006666666666666667, "ci95": [ -0.016666666666666666, 0.0 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.006666666666666667, "ci95": [ -0.017123287671232876, 0.0 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.0014673333333333326, "ci95": [ -0.005623483333333334, 0.011746774999999992 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.17249000247773955, "ci95": [ -0.05976482274258577, 0.5188561081334732 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 298, "model_only_correct": 0, "reference_only_correct": 2, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.5, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Jev" }, "DeepSeek V4.1 Flash / Luna": { "direction": "DeepSeek V4.1 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261106, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.0033333333333333335, "ci95": [ -0.016666666666666666, 0.006666666666666667 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0033333333333333335, "ci95": [ -0.015625, 0.006993006993006993 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.0033333333333333335, "ci95": [ -0.006666666666666667, 0.016666666666666666 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.11512658932493497, "ci95": [ -0.23026650527370654, 0.5756436079237441 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 297, "model_only_correct": 1, "reference_only_correct": 2, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 1.0, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Luna" }, "MiMo V2.6 Flash / Jev": { "direction": "MiMo V2.6 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261107, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.013333333333333334, "ci95": [ -0.02666666666666667, -0.0033333333333333335 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.013333333333333334, "ci95": [ -0.026763581034581638, -0.003105590062111801 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.008527086666666666, "ci95": [ -0.0015313800833333337, 0.021669585499999998 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.40689222782444084, "ci95": [ 0.06017336988839328, 0.8668725135062876 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 296, "model_only_correct": 0, "reference_only_correct": 4, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.125, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Jev" }, "MiMo V2.6 Flash / Luna": { "direction": "MiMo V2.6 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261108, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.01, "ci95": [ -0.02666666666666667, 0.0033333333333333335 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.01, "ci95": [ -0.02626299004450265, 0.0035565687868319403 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.010393086666666667, "ci95": [ -0.003047595583333333, 0.026847683666666667 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.34952881467163627, "ci95": [ -0.11175104518164916, 0.9236350368121382 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 295, "model_only_correct": 1, "reference_only_correct": 4, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.37500000000000067, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Luna" }, "Hy4 Preview / Jev": { "direction": "Hy4 Preview minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261109, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.04666666666666667, "ci95": [ -0.07333333333333333, -0.023333333333333334 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.04666666666666667, "ci95": [ -0.07159216337882005, -0.024553571428571428 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.04146733333333333, "ci95": [ 0.019387941666666665, 0.06544692499999992 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.5540730421713747, "ci95": [ 0.7545253961250306, 2.4650317680041973 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 286, "model_only_correct": 0, "reference_only_correct": 14, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.00012207031250000005, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Jev" }, "Hy4 Preview / Luna": { "direction": "Hy4 Preview minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261110, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.043333333333333335, "ci95": [ -0.07, -0.02 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.043333333333333335, "ci95": [ -0.06863181312569522, -0.01982574679943101 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.043333333333333335, "ci95": [ 0.02, 0.07 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.49670962901857, "ci95": [ 0.690786189197283, 2.4177569928400247 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 285, "model_only_correct": 1, "reference_only_correct": 14, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.0009765624999999976, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Luna" } } }, "probability": { "expected_cases": 300, "target_kind": "probability", "models": { "Jev": { "attempts": 300, "valid": 299, "invalid": 1, "success_rate": 0.9966666666666667, "success_rate_wilson95": [ 0.9813633063054688, 0.9994113422781488 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.005577474, "observed_bills": 299, "missing_bills": 1, "total_billed_usd": null, "usd_per_1000_attempts": null }, "quality": { "mae_valid_only": 0.07640276740848177, "rmse_valid_only": 0.12256650038782585, "excess_brier_valid_only": 0.015022547017318914, "expected_brier_valid_only": 0.16237160248127255, "error_score_denominator": 299 }, "expected_cases": 300, "missing_attempts": 0 }, "Luna": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.0124422, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0124422, "usd_per_1000_attempts": 0.041474000000000004 }, "quality": { "mae_valid_only": 0.024395889425565125, "rmse_valid_only": 0.06518648458252084, "excess_brier_valid_only": 0.0042492777722272276, "expected_brier_valid_only": 0.1511071697179677, "error_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "Qwen3.8 Flash": { "attempts": 300, "valid": 291, "invalid": 9, "success_rate": 0.97, "success_rate_wilson95": [ 0.9439774608583589, 0.9841381461905676 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.01454715, "observed_bills": 291, "missing_bills": 9, "total_billed_usd": null, "usd_per_1000_attempts": null }, "quality": { "mae_valid_only": 0.061031034066208786, "rmse_valid_only": 0.11986522111015609, "excess_brier_valid_only": 0.01436767123178661, "expected_brier_valid_only": 0.1596741531616886, "error_score_denominator": 291 }, "expected_cases": 300, "missing_attempts": 0 }, "DeepSeek V4.1 Flash": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.01287664, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.01287664, "usd_per_1000_attempts": 0.042922133333333334 }, "quality": { "mae_valid_only": 0.038348388424628684, "rmse_valid_only": 0.08368231372987145, "excess_brier_valid_only": 0.007002729631184632, "expected_brier_valid_only": 0.15386062157692512, "error_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "MiMo V2.6 Flash": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.0079108848, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0079108848, "usd_per_1000_attempts": 0.026369616 }, "quality": { "mae_valid_only": 0.05892329014672775, "rmse_valid_only": 0.12181020223598317, "excess_brier_valid_only": 0.01483772536877112, "expected_brier_valid_only": 0.1616956173145116, "error_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "Hy4 Preview": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.0613542231, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0613542231, "usd_per_1000_attempts": 0.20451407700000002 }, "quality": { "mae_valid_only": 0.04887760465541144, "rmse_valid_only": 0.09360410944665704, "excess_brier_valid_only": 0.008761729305301749, "expected_brier_valid_only": 0.15561962125104223, "error_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 } }, "comparisons": { "Luna / Jev": { "direction": "Luna minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 299, "invalid_response_pairs": 1, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261202, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": -0.05192528637948666, "ci95": [ -0.06378871057453298, -0.04010044712144336 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": -0.010759057613746443, "ci95": [ -0.015567516395841158, -0.006283574001805602 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": -0.05727109929730134, "ci95": [ -0.08223660608430107, -0.03350655734286255 ], "defined_bootstrap_draws": 5000 } }, "model": "Luna", "reference": "Jev" }, "Qwen3.8 Flash / Jev": { "direction": "Qwen3.8 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 290, "invalid_response_pairs": 10, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261203, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": -0.014976150182331546, "ci95": [ -0.026950481402722958, -0.0034219930103654857 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": -0.0007031106813624882, "ci95": [ -0.0065138771293548475, 0.005855484142766468 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": -0.0028930257519377417, "ci95": [ -0.02806378496999935, 0.022083268805977194 ], "defined_bootstrap_draws": 5000 } }, "model": "Qwen3.8 Flash", "reference": "Jev" }, "Qwen3.8 Flash / Luna": { "direction": "Qwen3.8 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 291, "invalid_response_pairs": 9, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261204, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": 0.03683542953222468, "ci95": [ 0.02560280148885307, 0.04936707357444092 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": 0.010060113787372631, "ci95": [ 0.0038378124912402905, 0.018197656516538065 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": 0.05423323619110934, "ci95": [ 0.02185350491030532, 0.08902641155699492 ], "defined_bootstrap_draws": 5000 } }, "model": "Qwen3.8 Flash", "reference": "Luna" }, "DeepSeek V4.1 Flash / Jev": { "direction": "DeepSeek V4.1 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 299, "invalid_response_pairs": 1, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261205, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": -0.03792612350417206, "ci95": [ -0.04964104518529192, -0.026508858239991464 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": -0.007996396885695535, "ci95": [ -0.013843777644533149, -0.001375276566116187 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": -0.03874436648738536, "ci95": [ -0.06980321674077429, -0.006347718074961897 ], "defined_bootstrap_draws": 5000 } }, "model": "DeepSeek V4.1 Flash", "reference": "Jev" }, "DeepSeek V4.1 Flash / Luna": { "direction": "DeepSeek V4.1 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261206, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": 0.013952498999063555, "ci95": [ 0.004774316401553431, 0.024119512424220647 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": 0.0027534518589574044, "ci95": [ -0.0025143462921953423, 0.00981389599931656 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": 0.01849582914735061, "ci95": [ -0.01732676669604676, 0.057594380229559664 ], "defined_bootstrap_draws": 5000 } }, "model": "DeepSeek V4.1 Flash", "reference": "Luna" }, "MiMo V2.6 Flash / Jev": { "direction": "MiMo V2.6 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 299, "invalid_response_pairs": 1, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261207, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": -0.017282409401731515, "ci95": [ -0.028760777793274073, -0.005246101209006715 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": -0.00013519714898668382, "ci95": [ -0.006197130386857073, 0.006228783427978094 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": -0.0005527721916420009, "ci95": [ -0.026592783528214245, 0.024085676490143242 ], "defined_bootstrap_draws": 5000 } }, "model": "MiMo V2.6 Flash", "reference": "Jev" }, "MiMo V2.6 Flash / Luna": { "direction": "MiMo V2.6 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261208, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": 0.03452740072116262, "ci95": [ 0.02407677297049323, 0.04650569845810504 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": 0.010588447596543895, "ci95": [ 0.005107084161945874, 0.017468737178011135 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": 0.05662371765346233, "ci95": [ 0.029026303718582704, 0.08646403575129637 ], "defined_bootstrap_draws": 5000 } }, "model": "MiMo V2.6 Flash", "reference": "Luna" }, "Hy4 Preview / Jev": { "direction": "Hy4 Preview minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 299, "invalid_response_pairs": 1, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261209, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": -0.027361692503386686, "ci95": [ -0.039999336637624644, -0.014391843619790057 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": -0.006231514269524516, "ci95": [ -0.011880492462589964, -0.00015369295690187277 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": -0.0288059929884181, "ci95": [ -0.054549455819958535, -0.0006860256353636531 ], "defined_bootstrap_draws": 5000 } }, "model": "Hy4 Preview", "reference": "Jev" }, "Hy4 Preview / Luna": { "direction": "Hy4 Preview minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261210, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": 0.024481715229846306, "ci95": [ 0.014769578864747, 0.03449687747360977 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": 0.004512451533074521, "ci95": [ 7.071178353467065e-05, 0.009587944189453341 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": 0.028417624864136196, "ci95": [ 0.00040866993314793356, 0.058382976320547564 ], "defined_bootstrap_draws": 5000 } }, "model": "Hy4 Preview", "reference": "Luna" } } }, "rte": { "expected_cases": 277, "target_kind": "label", "models": { "Jev": { "attempts": 277, "valid": 277, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9863216106452903, 1 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.004656624, "observed_bills": 277, "missing_bills": 0, "total_billed_usd": 0.004656624, "usd_per_1000_attempts": 0.016810916967509023, "usd_per_correct_answer": 1.855228685258964e-05 }, "quality": { "correct": 251, "accuracy_all_attempts": 0.9061371841155235, "accuracy_all_attempts_wilson95": [ 0.8660240489486702, 0.9351397142108616 ], "balanced_accuracy_all_attempts": 0.9074296768796403, "class_denominators": { "no": { "attempts": 131, "valid": 131, "correct": 122, "recall_all_attempts": 0.9312977099236641 }, "yes": { "attempts": 146, "valid": 146, "correct": 129, "recall_all_attempts": 0.8835616438356164 } }, "accuracy_valid_only": 0.9061371841155235, "accuracy_valid_only_wilson95": [ 0.8660240489486702, 0.9351397142108616 ], "balanced_accuracy_valid_only": 0.9074296768796403, "brier_valid_only": 0.07178808664259928, "log_loss_valid_only": 0.2483805767638515, "proper_score_denominator": 277 }, "expected_cases": 277, "missing_attempts": 0 }, "Luna": { "attempts": 277, "valid": 277, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9863216106452903, 1 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.0102786, "observed_bills": 277, "missing_bills": 0, "total_billed_usd": 0.0102786, "usd_per_1000_attempts": 0.03710685920577617, "usd_per_correct_answer": 4.264979253112033e-05 }, "quality": { "correct": 241, "accuracy_all_attempts": 0.8700361010830325, "accuracy_all_attempts_wilson95": [ 0.8253226276877503, 0.9046265787464898 ], "balanced_accuracy_all_attempts": 0.8684774652305762, "class_denominators": { "no": { "attempts": 131, "valid": 131, "correct": 110, "recall_all_attempts": 0.8396946564885496 }, "yes": { "attempts": 146, "valid": 146, "correct": 131, "recall_all_attempts": 0.8972602739726028 } }, "accuracy_valid_only": 0.8700361010830325, "accuracy_valid_only_wilson95": [ 0.8253226276877503, 0.9046265787464898 ], "balanced_accuracy_valid_only": 0.8684774652305762, "brier_valid_only": 0.1103158844765343, "log_loss_valid_only": 1.4235066715760918, "proper_score_denominator": 277 }, "expected_cases": 277, "missing_attempts": 0 }, "Qwen3.8 Flash": { "attempts": 277, "valid": 270, "invalid": 7, "success_rate": 0.9747292418772563, "success_rate_wilson95": [ 0.9487653728706874, 0.9877060480668985 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.01191023, "observed_bills": 270, "missing_bills": 7, "total_billed_usd": null, "usd_per_1000_attempts": null, "usd_per_correct_answer": null }, "quality": { "correct": 234, "accuracy_all_attempts": 0.8447653429602888, "accuracy_all_attempts_wilson95": [ 0.7974351903182963, 0.8826638264082396 ], "balanced_accuracy_all_attempts": 0.8374464080309526, "class_denominators": { "no": { "attempts": 131, "valid": 127, "correct": 92, "recall_all_attempts": 0.7022900763358778 }, "yes": { "attempts": 146, "valid": 143, "correct": 142, "recall_all_attempts": 0.9726027397260274 } }, "accuracy_valid_only": 0.8666666666666667, "accuracy_valid_only_wilson95": [ 0.820933956622913, 0.9021121476912128 ], "balanced_accuracy_valid_only": 0.8587082209129453, "brier_valid_only": 0.11132407407407406, "log_loss_valid_only": 0.6062096929230238, "proper_score_denominator": 270 }, "expected_cases": 277, "missing_attempts": 0 }, "DeepSeek V4.1 Flash": { "attempts": 277, "valid": 277, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9863216106452903, 1 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.01062348, "observed_bills": 277, "missing_bills": 0, "total_billed_usd": 0.01062348, "usd_per_1000_attempts": 0.038351913357400715, "usd_per_correct_answer": 4.501474576271186e-05 }, "quality": { "correct": 236, "accuracy_all_attempts": 0.851985559566787, "accuracy_all_attempts_wilson95": [ 0.805360563813589, 0.8889813642580053 ], "balanced_accuracy_all_attempts": 0.8462564048938618, "class_denominators": { "no": { "attempts": 131, "valid": 131, "correct": 97, "recall_all_attempts": 0.7404580152671756 }, "yes": { "attempts": 146, "valid": 146, "correct": 139, "recall_all_attempts": 0.952054794520548 } }, "accuracy_valid_only": 0.851985559566787, "accuracy_valid_only_wilson95": [ 0.805360563813589, 0.8889813642580053 ], "balanced_accuracy_valid_only": 0.8462564048938618, "brier_valid_only": 0.128257761732852, "log_loss_valid_only": 1.8758068422363532, "proper_score_denominator": 277 }, "expected_cases": 277, "missing_attempts": 0 }, "MiMo V2.6 Flash": { "attempts": 277, "valid": 277, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9863216106452903, 1 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.0059315592, "observed_bills": 277, "missing_bills": 0, "total_billed_usd": 0.0059315592, "usd_per_1000_attempts": 0.021413571119133577, "usd_per_correct_answer": 2.556706551724138e-05 }, "quality": { "correct": 232, "accuracy_all_attempts": 0.8375451263537906, "accuracy_all_attempts_wilson95": [ 0.789540911797291, 0.8763151935841865 ], "balanced_accuracy_all_attempts": 0.8302049566035763, "class_denominators": { "no": { "attempts": 131, "valid": 131, "correct": 91, "recall_all_attempts": 0.6946564885496184 }, "yes": { "attempts": 146, "valid": 146, "correct": 141, "recall_all_attempts": 0.9657534246575342 } }, "accuracy_valid_only": 0.8375451263537906, "accuracy_valid_only_wilson95": [ 0.789540911797291, 0.8763151935841865 ], "balanced_accuracy_valid_only": 0.8302049566035763, "brier_valid_only": 0.12324377617328519, "log_loss_valid_only": 1.087372678634681, "proper_score_denominator": 277 }, "expected_cases": 277, "missing_attempts": 0 }, "Hy4 Preview": { "attempts": 277, "valid": 277, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9863216106452903, 1 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.0501196032, "observed_bills": 277, "missing_bills": 0, "total_billed_usd": 0.0501196032, "usd_per_1000_attempts": 0.18093719566787003, "usd_per_correct_answer": 0.00020796515850622407 }, "quality": { "correct": 241, "accuracy_all_attempts": 0.8700361010830325, "accuracy_all_attempts_wilson95": [ 0.8253226276877503, 0.9046265787464898 ], "balanced_accuracy_all_attempts": 0.8673010561539266, "class_denominators": { "no": { "attempts": 131, "valid": 131, "correct": 107, "recall_all_attempts": 0.816793893129771 }, "yes": { "attempts": 146, "valid": 146, "correct": 134, "recall_all_attempts": 0.9178082191780822 } }, "accuracy_valid_only": 0.8700361010830325, "accuracy_valid_only_wilson95": [ 0.8253226276877503, 0.9046265787464898 ], "balanced_accuracy_valid_only": 0.8673010561539266, "brier_valid_only": 0.0944633357400722, "log_loss_valid_only": 2.104598836816039, "proper_score_denominator": 277 }, "expected_cases": 277, "missing_attempts": 0 } }, "comparisons": { "Luna / Jev": { "direction": "Luna minus Jev", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261302, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.036101083032490974, "ci95": [ -0.07581227436823104, 0.0036101083032490976 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0389522116490641, "ci95": [ -0.07927951416787186, -0.0004981124161073831 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.038527797833935014, "ci95": [ 0.009651498194945847, 0.06903178700361003 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.1751260948122404, "ci95": [ 0.5472247818335747, 1.9357642330612883 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 231, "model_only_correct": 10, "reference_only_correct": 20, "both_incorrect": 16 }, "mcnemar_exact_two_sided_p_exploratory": 0.0987371467053885, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Luna", "reference": "Jev" }, "Qwen3.8 Flash / Jev": { "direction": "Qwen3.8 Flash minus Jev", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 270, "invalid_response_pairs": 7, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261303, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.061371841155234655, "ci95": [ -0.11191335740072202, -0.01444043321299639 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.06998326884868765, "ci95": [ -0.11795153266411662, -0.025042511255067106 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.04012814814814815, "ci95": [ 0.007517427318295737, 0.07565096091762853 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.359331507502863, "ci95": [ 0.0687675189238067, 0.752879018453159 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 218, "model_only_correct": 16, "reference_only_correct": 33, "both_incorrect": 10 }, "mcnemar_exact_two_sided_p_exploratory": 0.021294114141386742, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Jev" }, "Qwen3.8 Flash / Luna": { "direction": "Qwen3.8 Flash minus Luna", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 270, "invalid_response_pairs": 7, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261304, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.02527075812274368, "ci95": [ -0.07220216606498195, 0.018050541516245487 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.031031057199623555, "ci95": [ -0.07647132482381076, 0.012519585898936272 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.002715185185185185, "ci95": [ -0.03381912779850745, 0.04155758911420885 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.8342772860490903, "ci95": [ -1.60997529042661, -0.15126021944534754 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 216, "model_only_correct": 18, "reference_only_correct": 25, "both_incorrect": 18 }, "mcnemar_exact_two_sided_p_exploratory": 0.3603776529357701, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Luna" }, "DeepSeek V4.1 Flash / Jev": { "direction": "DeepSeek V4.1 Flash minus Jev", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261305, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.05415162454873646, "ci95": [ -0.09747292418772563, -0.010830324909747292 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.06117327198577853, "ci95": [ -0.10258301200884622, -0.019815112491493644 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.05646967509025271, "ci95": [ 0.024609476534296033, 0.0893348194945848 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.6274262654725018, "ci95": [ 0.8588965251421221, 2.542465012082216 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 224, "model_only_correct": 12, "reference_only_correct": 27, "both_incorrect": 14 }, "mcnemar_exact_two_sided_p_exploratory": 0.023702702470472976, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Jev" }, "DeepSeek V4.1 Flash / Luna": { "direction": "DeepSeek V4.1 Flash minus Luna", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261306, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.018050541516245487, "ci95": [ -0.05776173285198556, 0.021660649819494584 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.02222106033671442, "ci95": [ -0.0633824192394988, 0.01735512380654609 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.017941877256317695, "ci95": [ -0.016791534296028872, 0.05277675992779784 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.45230017066026135, "ci95": [ -0.5331512575605177, 1.484499188545258 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 222, "model_only_correct": 14, "reference_only_correct": 19, "both_incorrect": 22 }, "mcnemar_exact_two_sided_p_exploratory": 0.48685024166479707, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Luna" }, "MiMo V2.6 Flash / Jev": { "direction": "MiMo V2.6 Flash minus Jev", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261307, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.06859205776173286, "ci95": [ -0.11913357400722022, -0.021660649819494584 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.077224720276064, "ci95": [ -0.12302708598864938, -0.03207894736842105 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.05145568953068592, "ci95": [ 0.021786177075812277, 0.08226051028880864 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.8389921018708295, "ci95": [ 0.32327758424814435, 1.45865874242996 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 217, "model_only_correct": 15, "reference_only_correct": 34, "both_incorrect": 11 }, "mcnemar_exact_two_sided_p_exploratory": 0.009399241523872831, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Jev" }, "MiMo V2.6 Flash / Luna": { "direction": "MiMo V2.6 Flash minus Luna", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261308, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.032490974729241874, "ci95": [ -0.07581227436823104, 0.010830324909747292 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.03827250862699989, "ci95": [ -0.07895422792201022, 0.002361516491174732 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.012927891696750904, "ci95": [ -0.017492514350180503, 0.04332573555956676 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.3361339929414109, "ci95": [ -1.140536794957193, 0.41640985087587057 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 219, "model_only_correct": 13, "reference_only_correct": 22, "both_incorrect": 23 }, "mcnemar_exact_two_sided_p_exploratory": 0.1754652489908051, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Luna" }, "Hy4 Preview / Jev": { "direction": "Hy4 Preview minus Jev", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261309, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.036101083032490974, "ci95": [ -0.07581227436823104, 0.0036101083032490976 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.04012862072571369, "ci95": [ -0.08003068825615484, -0.00026753959151520706 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.022675249097472924, "ci95": [ -0.0035023292418772593, 0.04951990974729235 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.8562182600521873, "ci95": [ 1.0000095435879652, 2.8346794674036953 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 229, "model_only_correct": 12, "reference_only_correct": 22, "both_incorrect": 14 }, "mcnemar_exact_two_sided_p_exploratory": 0.12144948495551997, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Jev" }, "Hy4 Preview / Luna": { "direction": "Hy4 Preview minus Luna", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261310, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.0, "ci95": [ -0.039711191335740074, 0.039711191335740074 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0011764090766495875, "ci95": [ -0.042583285824451235, 0.04041807907008709 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.015852548736462094, "ci95": [ -0.049975210830324905, 0.017649202166064942 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.681092165239947, "ci95": [ -0.38785422633747274, 1.7752221023339583 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 224, "model_only_correct": 17, "reference_only_correct": 17, "both_incorrect": 19 }, "mcnemar_exact_two_sided_p_exploratory": 1.0, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Luna" } } }, "wic": { "expected_cases": 638, "target_kind": "label", "models": { "Jev": { "attempts": 638, "valid": 638, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9940149412788755, 1 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.010365599999999999, "observed_bills": 638, "missing_bills": 0, "total_billed_usd": 0.010365599999999999, "usd_per_1000_attempts": 0.016247021943573667, "usd_per_correct_answer": 2.196101694915254e-05 }, "quality": { "correct": 472, "accuracy_all_attempts": 0.7398119122257053, "accuracy_all_attempts_wilson95": [ 0.704404201462688, 0.7723490462353305 ], "balanced_accuracy_all_attempts": 0.7398119122257053, "class_denominators": { "no": { "attempts": 319, "valid": 319, "correct": 221, "recall_all_attempts": 0.6927899686520376 }, "yes": { "attempts": 319, "valid": 319, "correct": 251, "recall_all_attempts": 0.786833855799373 } }, "accuracy_valid_only": 0.7398119122257053, "accuracy_valid_only_wilson95": [ 0.704404201462688, 0.7723490462353305 ], "balanced_accuracy_valid_only": 0.7398119122257053, "brier_valid_only": 0.1717985893416928, "log_loss_valid_only": 0.5200123164526982, "proper_score_denominator": 638 }, "expected_cases": 638, "missing_attempts": 0 }, "Luna": { "attempts": 638, "valid": 638, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9940149412788755, 1 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.0229087, "observed_bills": 638, "missing_bills": 0, "total_billed_usd": 0.0229087, "usd_per_1000_attempts": 0.035907053291536054, "usd_per_correct_answer": 4.926602150537634e-05 }, "quality": { "correct": 465, "accuracy_all_attempts": 0.7288401253918495, "accuracy_all_attempts_wilson95": [ 0.6930507977097585, 0.761890209897501 ], "balanced_accuracy_all_attempts": 0.7288401253918495, "class_denominators": { "no": { "attempts": 319, "valid": 319, "correct": 265, "recall_all_attempts": 0.8307210031347962 }, "yes": { "attempts": 319, "valid": 319, "correct": 200, "recall_all_attempts": 0.6269592476489029 } }, "accuracy_valid_only": 0.7288401253918495, "accuracy_valid_only_wilson95": [ 0.6930507977097585, 0.761890209897501 ], "balanced_accuracy_valid_only": 0.7288401253918495, "brier_valid_only": 0.2600086206896552, "log_loss_valid_only": 6.0042757769812765, "proper_score_denominator": 638 }, "expected_cases": 638, "missing_attempts": 0 }, "Qwen3.8 Flash": { "attempts": 638, "valid": 621, "invalid": 17, "success_rate": 0.9733542319749217, "success_rate_wilson95": [ 0.9577441175672494, 0.9832982406340685 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.02593802, "observed_bills": 621, "missing_bills": 17, "total_billed_usd": null, "usd_per_1000_attempts": null, "usd_per_correct_answer": null }, "quality": { "correct": 467, "accuracy_all_attempts": 0.731974921630094, "accuracy_all_attempts_wilson95": [ 0.6962919010718486, 0.7648811751327708 ], "balanced_accuracy_all_attempts": 0.731974921630094, "class_denominators": { "no": { "attempts": 319, "valid": 307, "correct": 216, "recall_all_attempts": 0.677115987460815 }, "yes": { "attempts": 319, "valid": 314, "correct": 251, "recall_all_attempts": 0.786833855799373 } }, "accuracy_valid_only": 0.7520128824476651, "accuracy_valid_only_wilson95": [ 0.7165678391661234, 0.7843592289428754 ], "balanced_accuracy_valid_only": 0.7514730596070458, "brier_valid_only": 0.223341384863124, "log_loss_valid_only": 3.0872205091441853, "proper_score_denominator": 621 }, "expected_cases": 638, "missing_attempts": 0 }, "DeepSeek V4.1 Flash": { "attempts": 638, "valid": 638, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9940149412788755, 1 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.02429336, "observed_bills": 638, "missing_bills": 0, "total_billed_usd": 0.02429336, "usd_per_1000_attempts": 0.03807736677115987, "usd_per_correct_answer": 5.114391578947368e-05 }, "quality": { "correct": 475, "accuracy_all_attempts": 0.7445141065830722, "accuracy_all_attempts_wilson95": [ 0.7092782498887528, 0.7768231007053056 ], "balanced_accuracy_all_attempts": 0.744514106583072, "class_denominators": { "no": { "attempts": 319, "valid": 319, "correct": 242, "recall_all_attempts": 0.7586206896551724 }, "yes": { "attempts": 319, "valid": 319, "correct": 233, "recall_all_attempts": 0.7304075235109718 } }, "accuracy_valid_only": 0.7445141065830722, "accuracy_valid_only_wilson95": [ 0.7092782498887528, 0.7768231007053056 ], "balanced_accuracy_valid_only": 0.744514106583072, "brier_valid_only": 0.21948714736677116, "log_loss_valid_only": 3.2994352124159394, "proper_score_denominator": 638 }, "expected_cases": 638, "missing_attempts": 0 }, "MiMo V2.6 Flash": { "attempts": 638, "valid": 638, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9940149412788755, 1 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.0120966608, "observed_bills": 638, "missing_bills": 0, "total_billed_usd": 0.0120966608, "usd_per_1000_attempts": 0.018960283385579937, "usd_per_correct_answer": 2.7244731531531533e-05 }, "quality": { "correct": 444, "accuracy_all_attempts": 0.6959247648902821, "accuracy_all_attempts_wilson95": [ 0.6591446418815946, 0.7303596454533879 ], "balanced_accuracy_all_attempts": 0.6959247648902822, "class_denominators": { "no": { "attempts": 319, "valid": 319, "correct": 199, "recall_all_attempts": 0.6238244514106583 }, "yes": { "attempts": 319, "valid": 319, "correct": 245, "recall_all_attempts": 0.768025078369906 } }, "accuracy_valid_only": 0.6959247648902821, "accuracy_valid_only_wilson95": [ 0.6591446418815946, 0.7303596454533879 ], "balanced_accuracy_valid_only": 0.6959247648902822, "brier_valid_only": 0.22654608228840284, "log_loss_valid_only": 1.345381611931422, "proper_score_denominator": 638 }, "expected_cases": 638, "missing_attempts": 0 }, "Hy4 Preview": { "attempts": 638, "valid": 638, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9940149412788755, 1 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.1098480726, "observed_bills": 638, "missing_bills": 0, "total_billed_usd": 0.1098480726, "usd_per_1000_attempts": 0.17217566238244514, "usd_per_correct_answer": 0.00023880015782608695 }, "quality": { "correct": 460, "accuracy_all_attempts": 0.7210031347962382, "accuracy_all_attempts_wilson95": [ 0.6849573750736453, 0.754403461040215 ], "balanced_accuracy_all_attempts": 0.7210031347962382, "class_denominators": { "no": { "attempts": 319, "valid": 319, "correct": 255, "recall_all_attempts": 0.799373040752351 }, "yes": { "attempts": 319, "valid": 319, "correct": 205, "recall_all_attempts": 0.6426332288401254 } }, "accuracy_valid_only": 0.7210031347962382, "accuracy_valid_only_wilson95": [ 0.6849573750736453, 0.754403461040215 ], "balanced_accuracy_valid_only": 0.7210031347962382, "brier_valid_only": 0.2414055608618448, "log_loss_valid_only": 4.593783661446493, "proper_score_denominator": 638 }, "expected_cases": 638, "missing_attempts": 0 } }, "comparisons": { "Luna / Jev": { "direction": "Luna minus Jev", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261402, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.0109717868338558, "ci95": [ -0.047021943573667714, 0.025078369905956112 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.010971786833855801, "ci95": [ -0.04487030647045235, 0.023434599911955314 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.08821003134796238, "ci95": [ 0.06034885579937303, 0.11629181034482759 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 5.484263460528578, "ci95": [ 4.568572972772651, 6.469039919900766 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 400, "model_only_correct": 65, "reference_only_correct": 72, "both_incorrect": 101 }, "mcnemar_exact_two_sided_p_exploratory": 0.6083841068999314, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Luna", "reference": "Jev" }, "Qwen3.8 Flash / Jev": { "direction": "Qwen3.8 Flash minus Jev", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 621, "invalid_response_pairs": 17, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261403, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.007836990595611285, "ci95": [ -0.03918495297805643, 0.023510971786833857 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.007836990595611285, "ci95": [ -0.039720837334376174, 0.023290571735936284 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.0522512077294686, "ci95": [ 0.02992431114179999, 0.07416093850726979 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 2.5682859510830096, "ci95": [ 1.9095417354334503, 3.2641632040210213 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 418, "model_only_correct": 49, "reference_only_correct": 54, "both_incorrect": 117 }, "mcnemar_exact_two_sided_p_exploratory": 0.6936859650244626, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Jev" }, "Qwen3.8 Flash / Luna": { "direction": "Qwen3.8 Flash minus Luna", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 621, "invalid_response_pairs": 17, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261404, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.003134796238244514, "ci95": [ -0.034482758620689655, 0.04075235109717868 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.0031347962382445166, "ci95": [ -0.031609105151947134, 0.03879320656896429 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.037564895330112716, "ci95": [ -0.07272969404186795, -0.003710527041476035 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -3.006150016900281, "ci95": [ -4.141983658073829, -1.9058598387150232 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 394, "model_only_correct": 73, "reference_only_correct": 71, "both_incorrect": 100 }, "mcnemar_exact_two_sided_p_exploratory": 0.9336249535488005, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Luna" }, "DeepSeek V4.1 Flash / Jev": { "direction": "DeepSeek V4.1 Flash minus Jev", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261405, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.004702194357366771, "ci95": [ -0.02821316614420063, 0.03765673981191166 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.004702194357366768, "ci95": [ -0.027718156527495135, 0.03798096640865981 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.047688558025078376, "ci95": [ 0.025477727304467093, 0.06988597181778997 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 2.779422895963241, "ci95": [ 2.084329356295424, 3.488388178541106 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 415, "model_only_correct": 60, "reference_only_correct": 57, "both_incorrect": 106 }, "mcnemar_exact_two_sided_p_exploratory": 0.8534084458956078, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Jev" }, "DeepSeek V4.1 Flash / Luna": { "direction": "DeepSeek V4.1 Flash minus Luna", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261406, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.01567398119122257, "ci95": [ -0.01567398119122257, 0.048589341692789965 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.01567398119122257, "ci95": [ -0.01579985664638888, 0.0472008481060749 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.040521473322884, "ci95": [ -0.06932874605015672, -0.011882911441614435 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -2.704840564565337, "ci95": [ -3.7603025378498365, -1.6559638694224565 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 413, "model_only_correct": 62, "reference_only_correct": 52, "both_incorrect": 111 }, "mcnemar_exact_two_sided_p_exploratory": 0.3993731868805384, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Luna" }, "MiMo V2.6 Flash / Jev": { "direction": "MiMo V2.6 Flash minus Jev", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261407, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.0438871473354232, "ci95": [ -0.07993730407523511, -0.009404388714733543 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0438871473354232, "ci95": [ -0.07993078615003972, -0.009053024430507865 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.05474749294671003, "ci95": [ 0.03450437304075235, 0.07612810348746091 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.8253692954787237, "ci95": [ 0.4858656165315311, 1.2200410864866067 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 393, "model_only_correct": 51, "reference_only_correct": 79, "both_incorrect": 115 }, "mcnemar_exact_two_sided_p_exploratory": 0.01754169530120409, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Jev" }, "MiMo V2.6 Flash / Luna": { "direction": "MiMo V2.6 Flash minus Luna", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261408, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.032915360501567396, "ci95": [ -0.07366771159874608, 0.007836990595611285 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0329153605015674, "ci95": [ -0.07087142103688816, 0.003910317954573282 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.03346253840125235, "ci95": [ -0.06626784719827276, -0.002234893416926358 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -4.658894165049855, "ci95": [ -5.647997786409488, -3.7225494379991053 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 370, "model_only_correct": 74, "reference_only_correct": 95, "both_incorrect": 99 }, "mcnemar_exact_two_sided_p_exploratory": 0.12368263020458219, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Luna" }, "Hy4 Preview / Jev": { "direction": "Hy4 Preview minus Jev", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261409, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.018808777429467086, "ci95": [ -0.05172413793103448, 0.014106583072100314 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.018808777429467086, "ci95": [ -0.05090899012479568, 0.013329031427788586 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.06960697152015204, "ci95": [ 0.04636445808755466, 0.09320833301204748 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 4.073771344993796, "ci95": [ 3.256547483659231, 4.924304217879693 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 406, "model_only_correct": 54, "reference_only_correct": 66, "both_incorrect": 112 }, "mcnemar_exact_two_sided_p_exploratory": 0.3153033008322539, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Jev" }, "Hy4 Preview / Luna": { "direction": "Hy4 Preview minus Luna", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261410, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.007836990595611285, "ci95": [ -0.03761755485893417, 0.023510971786833857 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.007836990595611285, "ci95": [ -0.03761763804108138, 0.022900010134723016 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.018603059827810343, "ci95": [ -0.046815027896628995, 0.0094120726221038 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -1.4104921155347827, "ci95": [ -2.4102871235998773, -0.44297223170598715 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 410, "model_only_correct": 50, "reference_only_correct": 55, "both_incorrect": 123 }, "mcnemar_exact_two_sided_p_exploratory": 0.696467662991644, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Luna" } } } }, "experiments": { "boolq": { "expected_cases": 3270, "target_kind": "label", "models": { "Jev": { "attempts": 3270, "valid": 3270, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.998826620388002, 1 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.06289479, "observed_bills": 3270, "missing_bills": 0, "total_billed_usd": 0.06289479, "usd_per_1000_attempts": 0.019233880733944955, "usd_per_correct_answer": 2.095794401866045e-05 }, "quality": { "correct": 3001, "accuracy_all_attempts": 0.917737003058104, "accuracy_all_attempts_wilson95": [ 0.9078221114479283, 0.9266715665031486 ], "balanced_accuracy_all_attempts": 0.9162745579108811, "class_denominators": { "no": { "attempts": 1237, "valid": 1237, "correct": 1126, "recall_all_attempts": 0.910266774454325 }, "yes": { "attempts": 2033, "valid": 2033, "correct": 1875, "recall_all_attempts": 0.9222823413674373 } }, "accuracy_valid_only": 0.917737003058104, "accuracy_valid_only_wilson95": [ 0.9078221114479283, 0.9266715665031486 ], "balanced_accuracy_valid_only": 0.9162745579108811, "brier_valid_only": 0.062442293577981645, "log_loss_valid_only": 0.219908748023455, "proper_score_denominator": 3270 }, "expected_cases": 3270, "missing_attempts": 0 }, "Luna": { "attempts": 3270, "valid": 3270, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.998826620388002, 1 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.1404962, "observed_bills": 3270, "missing_bills": 0, "total_billed_usd": 0.1404962, "usd_per_1000_attempts": 0.042965198776758404, "usd_per_correct_answer": 4.9733168141592913e-05 }, "quality": { "correct": 2825, "accuracy_all_attempts": 0.863914373088685, "accuracy_all_attempts_wilson95": [ 0.8517343947848243, 0.8752403319807552 ], "balanced_accuracy_all_attempts": 0.8577956443023181, "class_denominators": { "no": { "attempts": 1237, "valid": 1237, "correct": 1030, "recall_all_attempts": 0.8326596604688763 }, "yes": { "attempts": 2033, "valid": 2033, "correct": 1795, "recall_all_attempts": 0.88293162813576 } }, "accuracy_valid_only": 0.863914373088685, "accuracy_valid_only_wilson95": [ 0.8517343947848243, 0.8752403319807552 ], "balanced_accuracy_valid_only": 0.8577956443023181, "brier_valid_only": 0.1254407339449541, "log_loss_valid_only": 2.0475586107185118, "proper_score_denominator": 3270 }, "expected_cases": 3270, "missing_attempts": 0 }, "Qwen3.8 Flash": { "attempts": 3270, "valid": 3196, "invalid": 74, "success_rate": 0.9773700305810398, "success_rate_wilson95": [ 0.9716848195435246, 0.9819349690960297 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.16976776, "observed_bills": 3196, "missing_bills": 74, "total_billed_usd": null, "usd_per_1000_attempts": null, "usd_per_correct_answer": null }, "quality": { "correct": 2883, "accuracy_all_attempts": 0.881651376146789, "accuracy_all_attempts_wilson95": [ 0.8701295492173265, 0.8922775591889281 ], "balanced_accuracy_all_attempts": 0.869844613195134, "class_denominators": { "no": { "attempts": 1237, "valid": 1205, "correct": 1016, "recall_all_attempts": 0.8213419563459984 }, "yes": { "attempts": 2033, "valid": 1991, "correct": 1867, "recall_all_attempts": 0.9183472700442695 } }, "accuracy_valid_only": 0.9020650813516896, "accuracy_valid_only_wilson95": [ 0.8912726473426597, 0.9118921447490015 ], "balanced_accuracy_valid_only": 0.8904366328978328, "brier_valid_only": 0.08750831602002504, "log_loss_valid_only": 0.8897137976043077, "proper_score_denominator": 3196 }, "expected_cases": 3270, "missing_attempts": 0 }, "DeepSeek V4.1 Flash": { "attempts": 3270, "valid": 3270, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.998826620388002, 1 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.15220184, "observed_bills": 3270, "missing_bills": 0, "total_billed_usd": 0.15220184, "usd_per_1000_attempts": 0.04654490519877676, "usd_per_correct_answer": 5.3217426573426575e-05 }, "quality": { "correct": 2860, "accuracy_all_attempts": 0.8746177370030581, "accuracy_all_attempts_wilson95": [ 0.8628261706084228, 0.8855301657679089 ], "balanced_accuracy_all_attempts": 0.865295780494914, "class_denominators": { "no": { "attempts": 1237, "valid": 1237, "correct": 1023, "recall_all_attempts": 0.8270008084074374 }, "yes": { "attempts": 2033, "valid": 2033, "correct": 1837, "recall_all_attempts": 0.9035907525823905 } }, "accuracy_valid_only": 0.8746177370030581, "accuracy_valid_only_wilson95": [ 0.8628261706084228, 0.8855301657679089 ], "balanced_accuracy_valid_only": 0.865295780494914, "brier_valid_only": 0.10843817611895688, "log_loss_valid_only": 1.701064681012324, "proper_score_denominator": 3270 }, "expected_cases": 3270, "missing_attempts": 0 }, "MiMo V2.6 Flash": { "attempts": 3270, "valid": 3270, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.998826620388002, 1 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.095622744, "observed_bills": 3270, "missing_bills": 0, "total_billed_usd": 0.095622744, "usd_per_1000_attempts": 0.029242429357798163, "usd_per_correct_answer": 3.2893960784313726e-05 }, "quality": { "correct": 2907, "accuracy_all_attempts": 0.8889908256880734, "accuracy_all_attempts_wilson95": [ 0.8777638470589225, 0.899304936508991 ], "balanced_accuracy_all_attempts": 0.8771715760286716, "class_denominators": { "no": { "attempts": 1237, "valid": 1237, "correct": 1025, "recall_all_attempts": 0.8286176232821342 }, "yes": { "attempts": 2033, "valid": 2033, "correct": 1882, "recall_all_attempts": 0.9257255287752091 } }, "accuracy_valid_only": 0.8889908256880734, "accuracy_valid_only_wilson95": [ 0.8777638470589225, 0.899304936508991 ], "balanced_accuracy_valid_only": 0.8771715760286716, "brier_valid_only": 0.09536354098165138, "log_loss_valid_only": 1.0692482600904558, "proper_score_denominator": 3270 }, "expected_cases": 3270, "missing_attempts": 0 }, "Hy4 Preview": { "attempts": 3270, "valid": 3266, "invalid": 4, "success_rate": 0.998776758409786, "success_rate_wilson95": [ 0.9968588022397419, 0.999524205621317 ], "unique_cases": 3270, "target_kind": "label", "cost": { "requests": 3270, "known_billed_usd": 0.7272430587000001, "observed_bills": 3266, "missing_bills": 4, "total_billed_usd": null, "usd_per_1000_attempts": null, "usd_per_correct_answer": null }, "quality": { "correct": 2909, "accuracy_all_attempts": 0.8896024464831804, "accuracy_all_attempts_wilson95": [ 0.8784006906210592, 0.8998898992103258 ], "balanced_accuracy_all_attempts": 0.8909574478660708, "class_denominators": { "no": { "attempts": 1237, "valid": 1237, "correct": 1109, "recall_all_attempts": 0.8965238480194018 }, "yes": { "attempts": 2033, "valid": 2029, "correct": 1800, "recall_all_attempts": 0.8853910477127398 } }, "accuracy_valid_only": 0.8906919779546846, "accuracy_valid_only_wilson95": [ 0.8795282965301395, 0.9009376777805561 ], "balanced_accuracy_valid_only": 0.8918301842364136, "brier_valid_only": 0.10300349112676056, "log_loss_valid_only": 2.936123009835336, "proper_score_denominator": 3266 }, "expected_cases": 3270, "missing_attempts": 0 } }, "comparisons": { "Luna / Jev": { "direction": "Luna minus Jev", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3270, "invalid_response_pairs": 0, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261002, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.05382262996941896, "ci95": [ -0.06487188927230851, -0.04314538527434187 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.058478913608563, "ci95": [ -0.07053555551408902, -0.04661974990646179 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.06299844036697247, "ci95": [ 0.05413421740104574, 0.07189101253309452 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.8276498626950566, "ci95": [ 1.5795518090371796, 2.08361927992546 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2754, "model_only_correct": 71, "reference_only_correct": 247, "both_incorrect": 198 }, "mcnemar_exact_two_sided_p_exploratory": 6.090829100595564e-24, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Luna", "reference": "Jev" }, "Qwen3.8 Flash / Jev": { "direction": "Qwen3.8 Flash minus Jev", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3196, "invalid_response_pairs": 74, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261003, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.036085626911314984, "ci95": [ -0.04603068438552757, -0.026155469849816006 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.046429944715747164, "ci95": [ -0.057029820300328604, -0.03574148632725449 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.02536732728410513, "ci95": [ 0.018931823481352972, 0.03223035917518614 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.6709340073816001, "ci95": [ 0.5141649550144411, 0.8311340187063352 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2802, "model_only_correct": 81, "reference_only_correct": 199, "both_incorrect": 188 }, "mcnemar_exact_two_sided_p_exploratory": 1.2598593589333536e-12, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Jev" }, "Qwen3.8 Flash / Luna": { "direction": "Qwen3.8 Flash minus Luna", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3196, "invalid_response_pairs": 74, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261004, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.017737003058103974, "ci95": [ 0.005216324025774778, 0.03029407984724555 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.012048968892815832, "ci95": [ -0.0018615755688825212, 0.026099045572692662 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.03844553254067584, "ci95": [ -0.04850481222454987, -0.028405345775200437 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -1.1573943908396733, "ci95": [ -1.4033092768921485, -0.9202395608101466 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2633, "model_only_correct": 250, "reference_only_correct": 192, "both_incorrect": 195 }, "mcnemar_exact_two_sided_p_exploratory": 0.00663801763274788, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Luna" }, "DeepSeek V4.1 Flash / Jev": { "direction": "DeepSeek V4.1 Flash minus Jev", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3270, "invalid_response_pairs": 0, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261005, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.043119266055045874, "ci95": [ -0.053879359549677215, -0.0327115651073721 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.05097877741596718, "ci95": [ -0.06282075022068617, -0.039434909349184664 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.04599588254097523, "ci95": [ 0.038390643576019574, 0.054017700833557156 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.4811559329888688, "ci95": [ 1.2515890410009607, 1.71267232319951 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2770, "model_only_correct": 90, "reference_only_correct": 231, "both_incorrect": 179 }, "mcnemar_exact_two_sided_p_exploratory": 1.93969877002409e-15, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Jev" }, "DeepSeek V4.1 Flash / Luna": { "direction": "DeepSeek V4.1 Flash minus Luna", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3270, "invalid_response_pairs": 0, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261006, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.010703363914373088, "ci95": [ -0.001827820474845092, 0.02321355881293356 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.007500136192595815, "ci95": [ -0.006876287911257969, 0.02186392124097461 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.017002557825997244, "ci95": [ -0.027432445380924438, -0.006427126574292728 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.3464939297061875, "ci95": [ -0.6243277313937058, -0.06930693102207987 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2628, "model_only_correct": 232, "reference_only_correct": 197, "both_incorrect": 213 }, "mcnemar_exact_two_sided_p_exploratory": 0.1005742046978771, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Luna" }, "MiMo V2.6 Flash / Jev": { "direction": "MiMo V2.6 Flash minus Jev", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3270, "invalid_response_pairs": 0, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261007, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.02874617737003058, "ci95": [ -0.03842662825937786, -0.018725319240359688 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.03910298188220951, "ci95": [ -0.04964003253904825, -0.02848620570600139 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.03292124740366973, "ci95": [ 0.026351006507035275, 0.039583275692524594 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.8493395120670008, "ci95": [ 0.6803474198404741, 1.0363984312593697 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2819, "model_only_correct": 88, "reference_only_correct": 182, "both_incorrect": 181 }, "mcnemar_exact_two_sided_p_exploratory": 1.0875055935276827e-08, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Jev" }, "MiMo V2.6 Flash / Luna": { "direction": "MiMo V2.6 Flash minus Luna", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3270, "invalid_response_pairs": 0, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261008, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.02507645259938838, "ci95": [ 0.013190082928380207, 0.0370486608980737 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.019375931726353483, "ci95": [ 0.005698169171641928, 0.03313967894090824 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.030077192963302753, "ci95": [ -0.04001217545269182, -0.02013795772414401 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.9783103506280558, "ci95": [ -1.2307921231092065, -0.7314249293458708 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2665, "model_only_correct": 242, "reference_only_correct": 160, "both_incorrect": 203 }, "mcnemar_exact_two_sided_p_exploratory": 5.057897686177978e-05, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Luna" }, "Hy4 Preview / Jev": { "direction": "Hy4 Preview minus Jev", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3266, "invalid_response_pairs": 4, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261009, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.028134556574923548, "ci95": [ -0.0378051662835933, -0.01888473496974121 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.025317110044810347, "ci95": [ -0.03540255567433637, -0.015449645870049903 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.040493846301285975, "ci95": [ 0.03327109796131445, 0.04820044307773828 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 2.7160174309290226, "ci95": [ 2.4041928802735737, 3.0391821526120597 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2832, "model_only_correct": 77, "reference_only_correct": 169, "both_incorrect": 192 }, "mcnemar_exact_two_sided_p_exploratory": 4.3696015518080695e-09, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Jev" }, "Hy4 Preview / Luna": { "direction": "Hy4 Preview minus Luna", "attempted_case_union": 3270, "paired_cases": 3270, "unpaired_cases": 0, "valid_response_pairs": 3266, "invalid_response_pairs": 4, "resampling_clusters": 2938, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261010, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.025688073394495414, "ci95": [ 0.014510203990317284, 0.03719512195121951 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.033161803563752645, "ci95": [ 0.020861871793104066, 0.045607160781470694 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.022590844451928962, "ci95": [ -0.032360389435740565, -0.012809885099884533 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.8860597499719922, "ci95": [ 0.5869793905783922, 1.1923514579723544 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 2694, "model_only_correct": 215, "reference_only_correct": 131, "both_incorrect": 230 }, "mcnemar_exact_two_sided_p_exploratory": 7.364729817164784e-06, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Luna" } } }, "policy": { "expected_cases": 300, "target_kind": "label", "models": { "Jev": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.006052662, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.006052662, "usd_per_1000_attempts": 0.02017554, "usd_per_correct_answer": 2.017554e-05 }, "quality": { "correct": 300, "accuracy_all_attempts": 1.0, "accuracy_all_attempts_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "balanced_accuracy_all_attempts": 1.0, "class_denominators": { "no": { "attempts": 150, "valid": 150, "correct": 150, "recall_all_attempts": 1.0 }, "yes": { "attempts": 150, "valid": 150, "correct": 150, "recall_all_attempts": 1.0 } }, "accuracy_valid_only": 1.0, "accuracy_valid_only_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "balanced_accuracy_valid_only": 1.0, "brier_valid_only": 0.005199333333333334, "log_loss_valid_only": 0.05776850682166601, "proper_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "Luna": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.0132828, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0132828, "usd_per_1000_attempts": 0.044276, "usd_per_correct_answer": 4.442408026755853e-05 }, "quality": { "correct": 299, "accuracy_all_attempts": 0.9966666666666667, "accuracy_all_attempts_wilson95": [ 0.9813633063054688, 0.9994113422781488 ], "balanced_accuracy_all_attempts": 0.9966666666666666, "class_denominators": { "no": { "attempts": 150, "valid": 150, "correct": 149, "recall_all_attempts": 0.9933333333333333 }, "yes": { "attempts": 150, "valid": 150, "correct": 150, "recall_all_attempts": 1.0 } }, "accuracy_valid_only": 0.9966666666666667, "accuracy_valid_only_wilson95": [ 0.9813633063054688, 0.9994113422781488 ], "balanced_accuracy_valid_only": 0.9966666666666666, "brier_valid_only": 0.0033333333333333335, "log_loss_valid_only": 0.11513191997447059, "proper_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "Qwen3.8 Flash": { "attempts": 300, "valid": 293, "invalid": 7, "success_rate": 0.9766666666666667, "success_rate_wilson95": [ 0.9526280277317246, 0.9886523397008747 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.006091292, "observed_bills": 293, "missing_bills": 7, "total_billed_usd": null, "usd_per_1000_attempts": null, "usd_per_correct_answer": null }, "quality": { "correct": 293, "accuracy_all_attempts": 0.9766666666666667, "accuracy_all_attempts_wilson95": [ 0.9526280277317246, 0.9886523397008747 ], "balanced_accuracy_all_attempts": 0.9766666666666667, "class_denominators": { "no": { "attempts": 150, "valid": 147, "correct": 147, "recall_all_attempts": 0.98 }, "yes": { "attempts": 150, "valid": 146, "correct": 146, "recall_all_attempts": 0.9733333333333334 } }, "accuracy_valid_only": 1.0, "accuracy_valid_only_wilson95": [ 0.9870588871380849, 0.9999999999999999 ], "balanced_accuracy_valid_only": 1.0, "brier_valid_only": 0.0, "log_loss_valid_only": 9.99601725036675e-16, "proper_score_denominator": 293 }, "expected_cases": 300, "missing_attempts": 0 }, "DeepSeek V4.1 Flash": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.0044090536, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0044090536, "usd_per_1000_attempts": 0.014696845333333333, "usd_per_correct_answer": 1.479548187919463e-05 }, "quality": { "correct": 298, "accuracy_all_attempts": 0.9933333333333333, "accuracy_all_attempts_wilson95": [ 0.9760224167496929, 0.998169851642088 ], "balanced_accuracy_all_attempts": 0.9933333333333334, "class_denominators": { "no": { "attempts": 150, "valid": 150, "correct": 150, "recall_all_attempts": 1.0 }, "yes": { "attempts": 150, "valid": 150, "correct": 148, "recall_all_attempts": 0.9866666666666667 } }, "accuracy_valid_only": 0.9933333333333333, "accuracy_valid_only_wilson95": [ 0.9760224167496929, 0.998169851642088 ], "balanced_accuracy_valid_only": 0.9933333333333334, "brier_valid_only": 0.006666666666666667, "log_loss_valid_only": 0.23025850929940556, "proper_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "MiMo V2.6 Flash": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.0051780456, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0051780456, "usd_per_1000_attempts": 0.017260152, "usd_per_correct_answer": 1.74933972972973e-05 }, "quality": { "correct": 296, "accuracy_all_attempts": 0.9866666666666667, "accuracy_all_attempts_wilson95": [ 0.9662244697474675, 0.9948030382606411 ], "balanced_accuracy_all_attempts": 0.9866666666666667, "class_denominators": { "no": { "attempts": 150, "valid": 150, "correct": 148, "recall_all_attempts": 0.9866666666666667 }, "yes": { "attempts": 150, "valid": 150, "correct": 148, "recall_all_attempts": 0.9866666666666667 } }, "accuracy_valid_only": 0.9866666666666667, "accuracy_valid_only_wilson95": [ 0.9662244697474675, 0.9948030382606411 ], "balanced_accuracy_valid_only": 0.9866666666666667, "brier_valid_only": 0.01372642, "log_loss_valid_only": 0.46466073464610685, "proper_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "Hy4 Preview": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "label", "cost": { "requests": 300, "known_billed_usd": 0.0448371756, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0448371756, "usd_per_1000_attempts": 0.14945725199999998, "usd_per_correct_answer": 0.00015677334125874126 }, "quality": { "correct": 286, "accuracy_all_attempts": 0.9533333333333334, "accuracy_all_attempts_wilson95": [ 0.923202639772079, 0.9720010663176659 ], "balanced_accuracy_all_attempts": 0.9533333333333334, "class_denominators": { "no": { "attempts": 150, "valid": 150, "correct": 138, "recall_all_attempts": 0.92 }, "yes": { "attempts": 150, "valid": 150, "correct": 148, "recall_all_attempts": 0.9866666666666667 } }, "accuracy_valid_only": 0.9533333333333334, "accuracy_valid_only_wilson95": [ 0.923202639772079, 0.9720010663176659 ], "balanced_accuracy_valid_only": 0.9533333333333334, "brier_valid_only": 0.04666666666666667, "log_loss_valid_only": 1.6118415489930407, "proper_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 } }, "comparisons": { "Luna / Jev": { "direction": "Luna minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261102, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.0033333333333333335, "ci95": [ -0.01, 0.0 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0033333333333333335, "ci95": [ -0.01056338028169014, 0.0 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.0018660000000000007, "ci95": [ -0.006145675000000001, 0.005344191666666659 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.05736341315280459, "ci95": [ -0.061878257789966536, 0.2898557157148909 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 299, "model_only_correct": 0, "reference_only_correct": 1, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 1.0, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Luna", "reference": "Jev" }, "Qwen3.8 Flash / Jev": { "direction": "Qwen3.8 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 293, "invalid_response_pairs": 7, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261103, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.023333333333333334, "ci95": [ -0.04, -0.006666666666666667 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.023333333333333334, "ci95": [ -0.0404725544676113, -0.006763886451386454 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.005254607508532423, "ci95": [ -0.006636173877085081, -0.0040879449130480075 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.05802019462434765, "ci95": [ -0.06425874451110178, -0.05214409211784379 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 293, "model_only_correct": 0, "reference_only_correct": 7, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.015625000000000003, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Jev" }, "Qwen3.8 Flash / Luna": { "direction": "Qwen3.8 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 293, "invalid_response_pairs": 7, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261104, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.02, "ci95": [ -0.04, -0.0033333333333333335 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.02, "ci95": [ -0.03913955980163092, -0.0031446540880503146 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.0034129692832764505, "ci95": [ -0.010380622837370242, 0.0 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.11788251191925214, "ci95": [ -0.3585423113391787, 0.0 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 292, "model_only_correct": 1, "reference_only_correct": 7, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.07031250000000003, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Luna" }, "DeepSeek V4.1 Flash / Jev": { "direction": "DeepSeek V4.1 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261105, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.006666666666666667, "ci95": [ -0.016666666666666666, 0.0 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.006666666666666667, "ci95": [ -0.017123287671232876, 0.0 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.0014673333333333326, "ci95": [ -0.005623483333333334, 0.011746774999999992 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.17249000247773955, "ci95": [ -0.05976482274258577, 0.5188561081334732 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 298, "model_only_correct": 0, "reference_only_correct": 2, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.5, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Jev" }, "DeepSeek V4.1 Flash / Luna": { "direction": "DeepSeek V4.1 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261106, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.0033333333333333335, "ci95": [ -0.016666666666666666, 0.006666666666666667 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0033333333333333335, "ci95": [ -0.015625, 0.006993006993006993 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.0033333333333333335, "ci95": [ -0.006666666666666667, 0.016666666666666666 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.11512658932493497, "ci95": [ -0.23026650527370654, 0.5756436079237441 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 297, "model_only_correct": 1, "reference_only_correct": 2, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 1.0, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Luna" }, "MiMo V2.6 Flash / Jev": { "direction": "MiMo V2.6 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261107, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.013333333333333334, "ci95": [ -0.02666666666666667, -0.0033333333333333335 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.013333333333333334, "ci95": [ -0.026763581034581638, -0.003105590062111801 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.008527086666666666, "ci95": [ -0.0015313800833333337, 0.021669585499999998 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.40689222782444084, "ci95": [ 0.06017336988839328, 0.8668725135062876 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 296, "model_only_correct": 0, "reference_only_correct": 4, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.125, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Jev" }, "MiMo V2.6 Flash / Luna": { "direction": "MiMo V2.6 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261108, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.01, "ci95": [ -0.02666666666666667, 0.0033333333333333335 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.01, "ci95": [ -0.02626299004450265, 0.0035565687868319403 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.010393086666666667, "ci95": [ -0.003047595583333333, 0.026847683666666667 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.34952881467163627, "ci95": [ -0.11175104518164916, 0.9236350368121382 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 295, "model_only_correct": 1, "reference_only_correct": 4, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.37500000000000067, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Luna" }, "Hy4 Preview / Jev": { "direction": "Hy4 Preview minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261109, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.04666666666666667, "ci95": [ -0.07333333333333333, -0.023333333333333334 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.04666666666666667, "ci95": [ -0.07159216337882005, -0.024553571428571428 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.04146733333333333, "ci95": [ 0.019387941666666665, 0.06544692499999992 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.5540730421713747, "ci95": [ 0.7545253961250306, 2.4650317680041973 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 286, "model_only_correct": 0, "reference_only_correct": 14, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.00012207031250000005, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Jev" }, "Hy4 Preview / Luna": { "direction": "Hy4 Preview minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261110, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.043333333333333335, "ci95": [ -0.07, -0.02 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.043333333333333335, "ci95": [ -0.06863181312569522, -0.01982574679943101 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.043333333333333335, "ci95": [ 0.02, 0.07 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.49670962901857, "ci95": [ 0.690786189197283, 2.4177569928400247 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 285, "model_only_correct": 1, "reference_only_correct": 14, "both_incorrect": 0 }, "mcnemar_exact_two_sided_p_exploratory": 0.0009765624999999976, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Luna" } } }, "probability": { "expected_cases": 300, "target_kind": "probability", "models": { "Jev": { "attempts": 300, "valid": 299, "invalid": 1, "success_rate": 0.9966666666666667, "success_rate_wilson95": [ 0.9813633063054688, 0.9994113422781488 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.005577474, "observed_bills": 299, "missing_bills": 1, "total_billed_usd": null, "usd_per_1000_attempts": null }, "quality": { "mae_valid_only": 0.07640276740848177, "rmse_valid_only": 0.12256650038782585, "excess_brier_valid_only": 0.015022547017318914, "expected_brier_valid_only": 0.16237160248127255, "error_score_denominator": 299 }, "expected_cases": 300, "missing_attempts": 0 }, "Luna": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.0124422, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0124422, "usd_per_1000_attempts": 0.041474000000000004 }, "quality": { "mae_valid_only": 0.024395889425565125, "rmse_valid_only": 0.06518648458252084, "excess_brier_valid_only": 0.0042492777722272276, "expected_brier_valid_only": 0.1511071697179677, "error_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "Qwen3.8 Flash": { "attempts": 300, "valid": 291, "invalid": 9, "success_rate": 0.97, "success_rate_wilson95": [ 0.9439774608583589, 0.9841381461905676 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.01454715, "observed_bills": 291, "missing_bills": 9, "total_billed_usd": null, "usd_per_1000_attempts": null }, "quality": { "mae_valid_only": 0.061031034066208786, "rmse_valid_only": 0.11986522111015609, "excess_brier_valid_only": 0.01436767123178661, "expected_brier_valid_only": 0.1596741531616886, "error_score_denominator": 291 }, "expected_cases": 300, "missing_attempts": 0 }, "DeepSeek V4.1 Flash": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.01287664, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.01287664, "usd_per_1000_attempts": 0.042922133333333334 }, "quality": { "mae_valid_only": 0.038348388424628684, "rmse_valid_only": 0.08368231372987145, "excess_brier_valid_only": 0.007002729631184632, "expected_brier_valid_only": 0.15386062157692512, "error_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "MiMo V2.6 Flash": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.0079108848, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0079108848, "usd_per_1000_attempts": 0.026369616 }, "quality": { "mae_valid_only": 0.05892329014672775, "rmse_valid_only": 0.12181020223598317, "excess_brier_valid_only": 0.01483772536877112, "expected_brier_valid_only": 0.1616956173145116, "error_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 }, "Hy4 Preview": { "attempts": 300, "valid": 300, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9873570287754538, 0.9999999999999998 ], "unique_cases": 300, "target_kind": "probability", "cost": { "requests": 300, "known_billed_usd": 0.0613542231, "observed_bills": 300, "missing_bills": 0, "total_billed_usd": 0.0613542231, "usd_per_1000_attempts": 0.20451407700000002 }, "quality": { "mae_valid_only": 0.04887760465541144, "rmse_valid_only": 0.09360410944665704, "excess_brier_valid_only": 0.008761729305301749, "expected_brier_valid_only": 0.15561962125104223, "error_score_denominator": 300 }, "expected_cases": 300, "missing_attempts": 0 } }, "comparisons": { "Luna / Jev": { "direction": "Luna minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 299, "invalid_response_pairs": 1, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261202, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": -0.05192528637948666, "ci95": [ -0.06378871057453298, -0.04010044712144336 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": -0.010759057613746443, "ci95": [ -0.015567516395841158, -0.006283574001805602 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": -0.05727109929730134, "ci95": [ -0.08223660608430107, -0.03350655734286255 ], "defined_bootstrap_draws": 5000 } }, "model": "Luna", "reference": "Jev" }, "Qwen3.8 Flash / Jev": { "direction": "Qwen3.8 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 290, "invalid_response_pairs": 10, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261203, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": -0.014976150182331546, "ci95": [ -0.026950481402722958, -0.0034219930103654857 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": -0.0007031106813624882, "ci95": [ -0.0065138771293548475, 0.005855484142766468 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": -0.0028930257519377417, "ci95": [ -0.02806378496999935, 0.022083268805977194 ], "defined_bootstrap_draws": 5000 } }, "model": "Qwen3.8 Flash", "reference": "Jev" }, "Qwen3.8 Flash / Luna": { "direction": "Qwen3.8 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 291, "invalid_response_pairs": 9, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261204, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": 0.03683542953222468, "ci95": [ 0.02560280148885307, 0.04936707357444092 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": 0.010060113787372631, "ci95": [ 0.0038378124912402905, 0.018197656516538065 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": 0.05423323619110934, "ci95": [ 0.02185350491030532, 0.08902641155699492 ], "defined_bootstrap_draws": 5000 } }, "model": "Qwen3.8 Flash", "reference": "Luna" }, "DeepSeek V4.1 Flash / Jev": { "direction": "DeepSeek V4.1 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 299, "invalid_response_pairs": 1, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261205, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": -0.03792612350417206, "ci95": [ -0.04964104518529192, -0.026508858239991464 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": -0.007996396885695535, "ci95": [ -0.013843777644533149, -0.001375276566116187 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": -0.03874436648738536, "ci95": [ -0.06980321674077429, -0.006347718074961897 ], "defined_bootstrap_draws": 5000 } }, "model": "DeepSeek V4.1 Flash", "reference": "Jev" }, "DeepSeek V4.1 Flash / Luna": { "direction": "DeepSeek V4.1 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261206, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": 0.013952498999063555, "ci95": [ 0.004774316401553431, 0.024119512424220647 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": 0.0027534518589574044, "ci95": [ -0.0025143462921953423, 0.00981389599931656 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": 0.01849582914735061, "ci95": [ -0.01732676669604676, 0.057594380229559664 ], "defined_bootstrap_draws": 5000 } }, "model": "DeepSeek V4.1 Flash", "reference": "Luna" }, "MiMo V2.6 Flash / Jev": { "direction": "MiMo V2.6 Flash minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 299, "invalid_response_pairs": 1, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261207, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": -0.017282409401731515, "ci95": [ -0.028760777793274073, -0.005246101209006715 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": -0.00013519714898668382, "ci95": [ -0.006197130386857073, 0.006228783427978094 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": -0.0005527721916420009, "ci95": [ -0.026592783528214245, 0.024085676490143242 ], "defined_bootstrap_draws": 5000 } }, "model": "MiMo V2.6 Flash", "reference": "Jev" }, "MiMo V2.6 Flash / Luna": { "direction": "MiMo V2.6 Flash minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261208, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": 0.03452740072116262, "ci95": [ 0.02407677297049323, 0.04650569845810504 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": 0.010588447596543895, "ci95": [ 0.005107084161945874, 0.017468737178011135 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": 0.05662371765346233, "ci95": [ 0.029026303718582704, 0.08646403575129637 ], "defined_bootstrap_draws": 5000 } }, "model": "MiMo V2.6 Flash", "reference": "Luna" }, "Hy4 Preview / Jev": { "direction": "Hy4 Preview minus Jev", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 299, "invalid_response_pairs": 1, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261209, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": -0.027361692503386686, "ci95": [ -0.039999336637624644, -0.014391843619790057 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": -0.006231514269524516, "ci95": [ -0.011880492462589964, -0.00015369295690187277 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": -0.0288059929884181, "ci95": [ -0.054549455819958535, -0.0006860256353636531 ], "defined_bootstrap_draws": 5000 } }, "model": "Hy4 Preview", "reference": "Jev" }, "Hy4 Preview / Luna": { "direction": "Hy4 Preview minus Luna", "attempted_case_union": 300, "paired_cases": 300, "unpaired_cases": 0, "valid_response_pairs": 300, "invalid_response_pairs": 0, "resampling_clusters": 300, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261210, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "mae_valid_pairs_difference": { "estimate": 0.024481715229846306, "ci95": [ 0.014769578864747, 0.03449687747360977 ], "defined_bootstrap_draws": 5000 }, "excess_brier_valid_pairs_difference": { "estimate": 0.004512451533074521, "ci95": [ 7.071178353467065e-05, 0.009587944189453341 ], "defined_bootstrap_draws": 5000 }, "rmse_valid_pairs_difference": { "estimate": 0.028417624864136196, "ci95": [ 0.00040866993314793356, 0.058382976320547564 ], "defined_bootstrap_draws": 5000 } }, "model": "Hy4 Preview", "reference": "Luna" } } }, "rte": { "expected_cases": 277, "target_kind": "label", "models": { "Jev": { "attempts": 277, "valid": 277, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9863216106452903, 1 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.004656624, "observed_bills": 277, "missing_bills": 0, "total_billed_usd": 0.004656624, "usd_per_1000_attempts": 0.016810916967509023, "usd_per_correct_answer": 1.855228685258964e-05 }, "quality": { "correct": 251, "accuracy_all_attempts": 0.9061371841155235, "accuracy_all_attempts_wilson95": [ 0.8660240489486702, 0.9351397142108616 ], "balanced_accuracy_all_attempts": 0.9074296768796403, "class_denominators": { "no": { "attempts": 131, "valid": 131, "correct": 122, "recall_all_attempts": 0.9312977099236641 }, "yes": { "attempts": 146, "valid": 146, "correct": 129, "recall_all_attempts": 0.8835616438356164 } }, "accuracy_valid_only": 0.9061371841155235, "accuracy_valid_only_wilson95": [ 0.8660240489486702, 0.9351397142108616 ], "balanced_accuracy_valid_only": 0.9074296768796403, "brier_valid_only": 0.07178808664259928, "log_loss_valid_only": 0.2483805767638515, "proper_score_denominator": 277 }, "expected_cases": 277, "missing_attempts": 0 }, "Luna": { "attempts": 277, "valid": 277, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9863216106452903, 1 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.0102786, "observed_bills": 277, "missing_bills": 0, "total_billed_usd": 0.0102786, "usd_per_1000_attempts": 0.03710685920577617, "usd_per_correct_answer": 4.264979253112033e-05 }, "quality": { "correct": 241, "accuracy_all_attempts": 0.8700361010830325, "accuracy_all_attempts_wilson95": [ 0.8253226276877503, 0.9046265787464898 ], "balanced_accuracy_all_attempts": 0.8684774652305762, "class_denominators": { "no": { "attempts": 131, "valid": 131, "correct": 110, "recall_all_attempts": 0.8396946564885496 }, "yes": { "attempts": 146, "valid": 146, "correct": 131, "recall_all_attempts": 0.8972602739726028 } }, "accuracy_valid_only": 0.8700361010830325, "accuracy_valid_only_wilson95": [ 0.8253226276877503, 0.9046265787464898 ], "balanced_accuracy_valid_only": 0.8684774652305762, "brier_valid_only": 0.1103158844765343, "log_loss_valid_only": 1.4235066715760918, "proper_score_denominator": 277 }, "expected_cases": 277, "missing_attempts": 0 }, "Qwen3.8 Flash": { "attempts": 277, "valid": 270, "invalid": 7, "success_rate": 0.9747292418772563, "success_rate_wilson95": [ 0.9487653728706874, 0.9877060480668985 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.01191023, "observed_bills": 270, "missing_bills": 7, "total_billed_usd": null, "usd_per_1000_attempts": null, "usd_per_correct_answer": null }, "quality": { "correct": 234, "accuracy_all_attempts": 0.8447653429602888, "accuracy_all_attempts_wilson95": [ 0.7974351903182963, 0.8826638264082396 ], "balanced_accuracy_all_attempts": 0.8374464080309526, "class_denominators": { "no": { "attempts": 131, "valid": 127, "correct": 92, "recall_all_attempts": 0.7022900763358778 }, "yes": { "attempts": 146, "valid": 143, "correct": 142, "recall_all_attempts": 0.9726027397260274 } }, "accuracy_valid_only": 0.8666666666666667, "accuracy_valid_only_wilson95": [ 0.820933956622913, 0.9021121476912128 ], "balanced_accuracy_valid_only": 0.8587082209129453, "brier_valid_only": 0.11132407407407406, "log_loss_valid_only": 0.6062096929230238, "proper_score_denominator": 270 }, "expected_cases": 277, "missing_attempts": 0 }, "DeepSeek V4.1 Flash": { "attempts": 277, "valid": 277, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9863216106452903, 1 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.01062348, "observed_bills": 277, "missing_bills": 0, "total_billed_usd": 0.01062348, "usd_per_1000_attempts": 0.038351913357400715, "usd_per_correct_answer": 4.501474576271186e-05 }, "quality": { "correct": 236, "accuracy_all_attempts": 0.851985559566787, "accuracy_all_attempts_wilson95": [ 0.805360563813589, 0.8889813642580053 ], "balanced_accuracy_all_attempts": 0.8462564048938618, "class_denominators": { "no": { "attempts": 131, "valid": 131, "correct": 97, "recall_all_attempts": 0.7404580152671756 }, "yes": { "attempts": 146, "valid": 146, "correct": 139, "recall_all_attempts": 0.952054794520548 } }, "accuracy_valid_only": 0.851985559566787, "accuracy_valid_only_wilson95": [ 0.805360563813589, 0.8889813642580053 ], "balanced_accuracy_valid_only": 0.8462564048938618, "brier_valid_only": 0.128257761732852, "log_loss_valid_only": 1.8758068422363532, "proper_score_denominator": 277 }, "expected_cases": 277, "missing_attempts": 0 }, "MiMo V2.6 Flash": { "attempts": 277, "valid": 277, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9863216106452903, 1 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.0059315592, "observed_bills": 277, "missing_bills": 0, "total_billed_usd": 0.0059315592, "usd_per_1000_attempts": 0.021413571119133577, "usd_per_correct_answer": 2.556706551724138e-05 }, "quality": { "correct": 232, "accuracy_all_attempts": 0.8375451263537906, "accuracy_all_attempts_wilson95": [ 0.789540911797291, 0.8763151935841865 ], "balanced_accuracy_all_attempts": 0.8302049566035763, "class_denominators": { "no": { "attempts": 131, "valid": 131, "correct": 91, "recall_all_attempts": 0.6946564885496184 }, "yes": { "attempts": 146, "valid": 146, "correct": 141, "recall_all_attempts": 0.9657534246575342 } }, "accuracy_valid_only": 0.8375451263537906, "accuracy_valid_only_wilson95": [ 0.789540911797291, 0.8763151935841865 ], "balanced_accuracy_valid_only": 0.8302049566035763, "brier_valid_only": 0.12324377617328519, "log_loss_valid_only": 1.087372678634681, "proper_score_denominator": 277 }, "expected_cases": 277, "missing_attempts": 0 }, "Hy4 Preview": { "attempts": 277, "valid": 277, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9863216106452903, 1 ], "unique_cases": 277, "target_kind": "label", "cost": { "requests": 277, "known_billed_usd": 0.0501196032, "observed_bills": 277, "missing_bills": 0, "total_billed_usd": 0.0501196032, "usd_per_1000_attempts": 0.18093719566787003, "usd_per_correct_answer": 0.00020796515850622407 }, "quality": { "correct": 241, "accuracy_all_attempts": 0.8700361010830325, "accuracy_all_attempts_wilson95": [ 0.8253226276877503, 0.9046265787464898 ], "balanced_accuracy_all_attempts": 0.8673010561539266, "class_denominators": { "no": { "attempts": 131, "valid": 131, "correct": 107, "recall_all_attempts": 0.816793893129771 }, "yes": { "attempts": 146, "valid": 146, "correct": 134, "recall_all_attempts": 0.9178082191780822 } }, "accuracy_valid_only": 0.8700361010830325, "accuracy_valid_only_wilson95": [ 0.8253226276877503, 0.9046265787464898 ], "balanced_accuracy_valid_only": 0.8673010561539266, "brier_valid_only": 0.0944633357400722, "log_loss_valid_only": 2.104598836816039, "proper_score_denominator": 277 }, "expected_cases": 277, "missing_attempts": 0 } }, "comparisons": { "Luna / Jev": { "direction": "Luna minus Jev", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261302, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.036101083032490974, "ci95": [ -0.07581227436823104, 0.0036101083032490976 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0389522116490641, "ci95": [ -0.07927951416787186, -0.0004981124161073831 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.038527797833935014, "ci95": [ 0.009651498194945847, 0.06903178700361003 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.1751260948122404, "ci95": [ 0.5472247818335747, 1.9357642330612883 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 231, "model_only_correct": 10, "reference_only_correct": 20, "both_incorrect": 16 }, "mcnemar_exact_two_sided_p_exploratory": 0.0987371467053885, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Luna", "reference": "Jev" }, "Qwen3.8 Flash / Jev": { "direction": "Qwen3.8 Flash minus Jev", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 270, "invalid_response_pairs": 7, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261303, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.061371841155234655, "ci95": [ -0.11191335740072202, -0.01444043321299639 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.06998326884868765, "ci95": [ -0.11795153266411662, -0.025042511255067106 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.04012814814814815, "ci95": [ 0.007517427318295737, 0.07565096091762853 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.359331507502863, "ci95": [ 0.0687675189238067, 0.752879018453159 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 218, "model_only_correct": 16, "reference_only_correct": 33, "both_incorrect": 10 }, "mcnemar_exact_two_sided_p_exploratory": 0.021294114141386742, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Jev" }, "Qwen3.8 Flash / Luna": { "direction": "Qwen3.8 Flash minus Luna", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 270, "invalid_response_pairs": 7, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261304, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.02527075812274368, "ci95": [ -0.07220216606498195, 0.018050541516245487 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.031031057199623555, "ci95": [ -0.07647132482381076, 0.012519585898936272 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.002715185185185185, "ci95": [ -0.03381912779850745, 0.04155758911420885 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.8342772860490903, "ci95": [ -1.60997529042661, -0.15126021944534754 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 216, "model_only_correct": 18, "reference_only_correct": 25, "both_incorrect": 18 }, "mcnemar_exact_two_sided_p_exploratory": 0.3603776529357701, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Luna" }, "DeepSeek V4.1 Flash / Jev": { "direction": "DeepSeek V4.1 Flash minus Jev", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261305, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.05415162454873646, "ci95": [ -0.09747292418772563, -0.010830324909747292 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.06117327198577853, "ci95": [ -0.10258301200884622, -0.019815112491493644 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.05646967509025271, "ci95": [ 0.024609476534296033, 0.0893348194945848 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.6274262654725018, "ci95": [ 0.8588965251421221, 2.542465012082216 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 224, "model_only_correct": 12, "reference_only_correct": 27, "both_incorrect": 14 }, "mcnemar_exact_two_sided_p_exploratory": 0.023702702470472976, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Jev" }, "DeepSeek V4.1 Flash / Luna": { "direction": "DeepSeek V4.1 Flash minus Luna", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261306, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.018050541516245487, "ci95": [ -0.05776173285198556, 0.021660649819494584 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.02222106033671442, "ci95": [ -0.0633824192394988, 0.01735512380654609 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.017941877256317695, "ci95": [ -0.016791534296028872, 0.05277675992779784 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.45230017066026135, "ci95": [ -0.5331512575605177, 1.484499188545258 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 222, "model_only_correct": 14, "reference_only_correct": 19, "both_incorrect": 22 }, "mcnemar_exact_two_sided_p_exploratory": 0.48685024166479707, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Luna" }, "MiMo V2.6 Flash / Jev": { "direction": "MiMo V2.6 Flash minus Jev", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261307, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.06859205776173286, "ci95": [ -0.11913357400722022, -0.021660649819494584 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.077224720276064, "ci95": [ -0.12302708598864938, -0.03207894736842105 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.05145568953068592, "ci95": [ 0.021786177075812277, 0.08226051028880864 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.8389921018708295, "ci95": [ 0.32327758424814435, 1.45865874242996 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 217, "model_only_correct": 15, "reference_only_correct": 34, "both_incorrect": 11 }, "mcnemar_exact_two_sided_p_exploratory": 0.009399241523872831, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Jev" }, "MiMo V2.6 Flash / Luna": { "direction": "MiMo V2.6 Flash minus Luna", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261308, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.032490974729241874, "ci95": [ -0.07581227436823104, 0.010830324909747292 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.03827250862699989, "ci95": [ -0.07895422792201022, 0.002361516491174732 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.012927891696750904, "ci95": [ -0.017492514350180503, 0.04332573555956676 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -0.3361339929414109, "ci95": [ -1.140536794957193, 0.41640985087587057 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 219, "model_only_correct": 13, "reference_only_correct": 22, "both_incorrect": 23 }, "mcnemar_exact_two_sided_p_exploratory": 0.1754652489908051, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Luna" }, "Hy4 Preview / Jev": { "direction": "Hy4 Preview minus Jev", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261309, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.036101083032490974, "ci95": [ -0.07581227436823104, 0.0036101083032490976 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.04012862072571369, "ci95": [ -0.08003068825615484, -0.00026753959151520706 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.022675249097472924, "ci95": [ -0.0035023292418772593, 0.04951990974729235 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 1.8562182600521873, "ci95": [ 1.0000095435879652, 2.8346794674036953 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 229, "model_only_correct": 12, "reference_only_correct": 22, "both_incorrect": 14 }, "mcnemar_exact_two_sided_p_exploratory": 0.12144948495551997, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Jev" }, "Hy4 Preview / Luna": { "direction": "Hy4 Preview minus Luna", "attempted_case_union": 277, "paired_cases": 277, "unpaired_cases": 0, "valid_response_pairs": 277, "invalid_response_pairs": 0, "resampling_clusters": 277, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261310, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.0, "ci95": [ -0.039711191335740074, 0.039711191335740074 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0011764090766495875, "ci95": [ -0.042583285824451235, 0.04041807907008709 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.015852548736462094, "ci95": [ -0.049975210830324905, 0.017649202166064942 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.681092165239947, "ci95": [ -0.38785422633747274, 1.7752221023339583 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 224, "model_only_correct": 17, "reference_only_correct": 17, "both_incorrect": 19 }, "mcnemar_exact_two_sided_p_exploratory": 1.0, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Luna" } } }, "wic": { "expected_cases": 638, "target_kind": "label", "models": { "Jev": { "attempts": 638, "valid": 638, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9940149412788755, 1 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.010365599999999999, "observed_bills": 638, "missing_bills": 0, "total_billed_usd": 0.010365599999999999, "usd_per_1000_attempts": 0.016247021943573667, "usd_per_correct_answer": 2.196101694915254e-05 }, "quality": { "correct": 472, "accuracy_all_attempts": 0.7398119122257053, "accuracy_all_attempts_wilson95": [ 0.704404201462688, 0.7723490462353305 ], "balanced_accuracy_all_attempts": 0.7398119122257053, "class_denominators": { "no": { "attempts": 319, "valid": 319, "correct": 221, "recall_all_attempts": 0.6927899686520376 }, "yes": { "attempts": 319, "valid": 319, "correct": 251, "recall_all_attempts": 0.786833855799373 } }, "accuracy_valid_only": 0.7398119122257053, "accuracy_valid_only_wilson95": [ 0.704404201462688, 0.7723490462353305 ], "balanced_accuracy_valid_only": 0.7398119122257053, "brier_valid_only": 0.1717985893416928, "log_loss_valid_only": 0.5200123164526982, "proper_score_denominator": 638 }, "expected_cases": 638, "missing_attempts": 0 }, "Luna": { "attempts": 638, "valid": 638, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9940149412788755, 1 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.0229087, "observed_bills": 638, "missing_bills": 0, "total_billed_usd": 0.0229087, "usd_per_1000_attempts": 0.035907053291536054, "usd_per_correct_answer": 4.926602150537634e-05 }, "quality": { "correct": 465, "accuracy_all_attempts": 0.7288401253918495, "accuracy_all_attempts_wilson95": [ 0.6930507977097585, 0.761890209897501 ], "balanced_accuracy_all_attempts": 0.7288401253918495, "class_denominators": { "no": { "attempts": 319, "valid": 319, "correct": 265, "recall_all_attempts": 0.8307210031347962 }, "yes": { "attempts": 319, "valid": 319, "correct": 200, "recall_all_attempts": 0.6269592476489029 } }, "accuracy_valid_only": 0.7288401253918495, "accuracy_valid_only_wilson95": [ 0.6930507977097585, 0.761890209897501 ], "balanced_accuracy_valid_only": 0.7288401253918495, "brier_valid_only": 0.2600086206896552, "log_loss_valid_only": 6.0042757769812765, "proper_score_denominator": 638 }, "expected_cases": 638, "missing_attempts": 0 }, "Qwen3.8 Flash": { "attempts": 638, "valid": 621, "invalid": 17, "success_rate": 0.9733542319749217, "success_rate_wilson95": [ 0.9577441175672494, 0.9832982406340685 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.02593802, "observed_bills": 621, "missing_bills": 17, "total_billed_usd": null, "usd_per_1000_attempts": null, "usd_per_correct_answer": null }, "quality": { "correct": 467, "accuracy_all_attempts": 0.731974921630094, "accuracy_all_attempts_wilson95": [ 0.6962919010718486, 0.7648811751327708 ], "balanced_accuracy_all_attempts": 0.731974921630094, "class_denominators": { "no": { "attempts": 319, "valid": 307, "correct": 216, "recall_all_attempts": 0.677115987460815 }, "yes": { "attempts": 319, "valid": 314, "correct": 251, "recall_all_attempts": 0.786833855799373 } }, "accuracy_valid_only": 0.7520128824476651, "accuracy_valid_only_wilson95": [ 0.7165678391661234, 0.7843592289428754 ], "balanced_accuracy_valid_only": 0.7514730596070458, "brier_valid_only": 0.223341384863124, "log_loss_valid_only": 3.0872205091441853, "proper_score_denominator": 621 }, "expected_cases": 638, "missing_attempts": 0 }, "DeepSeek V4.1 Flash": { "attempts": 638, "valid": 638, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9940149412788755, 1 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.02429336, "observed_bills": 638, "missing_bills": 0, "total_billed_usd": 0.02429336, "usd_per_1000_attempts": 0.03807736677115987, "usd_per_correct_answer": 5.114391578947368e-05 }, "quality": { "correct": 475, "accuracy_all_attempts": 0.7445141065830722, "accuracy_all_attempts_wilson95": [ 0.7092782498887528, 0.7768231007053056 ], "balanced_accuracy_all_attempts": 0.744514106583072, "class_denominators": { "no": { "attempts": 319, "valid": 319, "correct": 242, "recall_all_attempts": 0.7586206896551724 }, "yes": { "attempts": 319, "valid": 319, "correct": 233, "recall_all_attempts": 0.7304075235109718 } }, "accuracy_valid_only": 0.7445141065830722, "accuracy_valid_only_wilson95": [ 0.7092782498887528, 0.7768231007053056 ], "balanced_accuracy_valid_only": 0.744514106583072, "brier_valid_only": 0.21948714736677116, "log_loss_valid_only": 3.2994352124159394, "proper_score_denominator": 638 }, "expected_cases": 638, "missing_attempts": 0 }, "MiMo V2.6 Flash": { "attempts": 638, "valid": 638, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9940149412788755, 1 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.0120966608, "observed_bills": 638, "missing_bills": 0, "total_billed_usd": 0.0120966608, "usd_per_1000_attempts": 0.018960283385579937, "usd_per_correct_answer": 2.7244731531531533e-05 }, "quality": { "correct": 444, "accuracy_all_attempts": 0.6959247648902821, "accuracy_all_attempts_wilson95": [ 0.6591446418815946, 0.7303596454533879 ], "balanced_accuracy_all_attempts": 0.6959247648902822, "class_denominators": { "no": { "attempts": 319, "valid": 319, "correct": 199, "recall_all_attempts": 0.6238244514106583 }, "yes": { "attempts": 319, "valid": 319, "correct": 245, "recall_all_attempts": 0.768025078369906 } }, "accuracy_valid_only": 0.6959247648902821, "accuracy_valid_only_wilson95": [ 0.6591446418815946, 0.7303596454533879 ], "balanced_accuracy_valid_only": 0.6959247648902822, "brier_valid_only": 0.22654608228840284, "log_loss_valid_only": 1.345381611931422, "proper_score_denominator": 638 }, "expected_cases": 638, "missing_attempts": 0 }, "Hy4 Preview": { "attempts": 638, "valid": 638, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9940149412788755, 1 ], "unique_cases": 638, "target_kind": "label", "cost": { "requests": 638, "known_billed_usd": 0.1098480726, "observed_bills": 638, "missing_bills": 0, "total_billed_usd": 0.1098480726, "usd_per_1000_attempts": 0.17217566238244514, "usd_per_correct_answer": 0.00023880015782608695 }, "quality": { "correct": 460, "accuracy_all_attempts": 0.7210031347962382, "accuracy_all_attempts_wilson95": [ 0.6849573750736453, 0.754403461040215 ], "balanced_accuracy_all_attempts": 0.7210031347962382, "class_denominators": { "no": { "attempts": 319, "valid": 319, "correct": 255, "recall_all_attempts": 0.799373040752351 }, "yes": { "attempts": 319, "valid": 319, "correct": 205, "recall_all_attempts": 0.6426332288401254 } }, "accuracy_valid_only": 0.7210031347962382, "accuracy_valid_only_wilson95": [ 0.6849573750736453, 0.754403461040215 ], "balanced_accuracy_valid_only": 0.7210031347962382, "brier_valid_only": 0.2414055608618448, "log_loss_valid_only": 4.593783661446493, "proper_score_denominator": 638 }, "expected_cases": 638, "missing_attempts": 0 } }, "comparisons": { "Luna / Jev": { "direction": "Luna minus Jev", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261402, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.0109717868338558, "ci95": [ -0.047021943573667714, 0.025078369905956112 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.010971786833855801, "ci95": [ -0.04487030647045235, 0.023434599911955314 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.08821003134796238, "ci95": [ 0.06034885579937303, 0.11629181034482759 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 5.484263460528578, "ci95": [ 4.568572972772651, 6.469039919900766 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 400, "model_only_correct": 65, "reference_only_correct": 72, "both_incorrect": 101 }, "mcnemar_exact_two_sided_p_exploratory": 0.6083841068999314, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Luna", "reference": "Jev" }, "Qwen3.8 Flash / Jev": { "direction": "Qwen3.8 Flash minus Jev", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 621, "invalid_response_pairs": 17, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261403, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.007836990595611285, "ci95": [ -0.03918495297805643, 0.023510971786833857 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.007836990595611285, "ci95": [ -0.039720837334376174, 0.023290571735936284 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.0522512077294686, "ci95": [ 0.02992431114179999, 0.07416093850726979 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 2.5682859510830096, "ci95": [ 1.9095417354334503, 3.2641632040210213 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 418, "model_only_correct": 49, "reference_only_correct": 54, "both_incorrect": 117 }, "mcnemar_exact_two_sided_p_exploratory": 0.6936859650244626, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Jev" }, "Qwen3.8 Flash / Luna": { "direction": "Qwen3.8 Flash minus Luna", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 621, "invalid_response_pairs": 17, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261404, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.003134796238244514, "ci95": [ -0.034482758620689655, 0.04075235109717868 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.0031347962382445166, "ci95": [ -0.031609105151947134, 0.03879320656896429 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.037564895330112716, "ci95": [ -0.07272969404186795, -0.003710527041476035 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -3.006150016900281, "ci95": [ -4.141983658073829, -1.9058598387150232 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 394, "model_only_correct": 73, "reference_only_correct": 71, "both_incorrect": 100 }, "mcnemar_exact_two_sided_p_exploratory": 0.9336249535488005, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Qwen3.8 Flash", "reference": "Luna" }, "DeepSeek V4.1 Flash / Jev": { "direction": "DeepSeek V4.1 Flash minus Jev", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261405, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.004702194357366771, "ci95": [ -0.02821316614420063, 0.03765673981191166 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.004702194357366768, "ci95": [ -0.027718156527495135, 0.03798096640865981 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.047688558025078376, "ci95": [ 0.025477727304467093, 0.06988597181778997 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 2.779422895963241, "ci95": [ 2.084329356295424, 3.488388178541106 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 415, "model_only_correct": 60, "reference_only_correct": 57, "both_incorrect": 106 }, "mcnemar_exact_two_sided_p_exploratory": 0.8534084458956078, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Jev" }, "DeepSeek V4.1 Flash / Luna": { "direction": "DeepSeek V4.1 Flash minus Luna", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261406, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": 0.01567398119122257, "ci95": [ -0.01567398119122257, 0.048589341692789965 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": 0.01567398119122257, "ci95": [ -0.01579985664638888, 0.0472008481060749 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.040521473322884, "ci95": [ -0.06932874605015672, -0.011882911441614435 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -2.704840564565337, "ci95": [ -3.7603025378498365, -1.6559638694224565 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 413, "model_only_correct": 62, "reference_only_correct": 52, "both_incorrect": 111 }, "mcnemar_exact_two_sided_p_exploratory": 0.3993731868805384, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "DeepSeek V4.1 Flash", "reference": "Luna" }, "MiMo V2.6 Flash / Jev": { "direction": "MiMo V2.6 Flash minus Jev", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261407, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.0438871473354232, "ci95": [ -0.07993730407523511, -0.009404388714733543 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0438871473354232, "ci95": [ -0.07993078615003972, -0.009053024430507865 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.05474749294671003, "ci95": [ 0.03450437304075235, 0.07612810348746091 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 0.8253692954787237, "ci95": [ 0.4858656165315311, 1.2200410864866067 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 393, "model_only_correct": 51, "reference_only_correct": 79, "both_incorrect": 115 }, "mcnemar_exact_two_sided_p_exploratory": 0.01754169530120409, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Jev" }, "MiMo V2.6 Flash / Luna": { "direction": "MiMo V2.6 Flash minus Luna", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261408, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.032915360501567396, "ci95": [ -0.07366771159874608, 0.007836990595611285 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.0329153605015674, "ci95": [ -0.07087142103688816, 0.003910317954573282 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.03346253840125235, "ci95": [ -0.06626784719827276, -0.002234893416926358 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -4.658894165049855, "ci95": [ -5.647997786409488, -3.7225494379991053 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 370, "model_only_correct": 74, "reference_only_correct": 95, "both_incorrect": 99 }, "mcnemar_exact_two_sided_p_exploratory": 0.12368263020458219, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "MiMo V2.6 Flash", "reference": "Luna" }, "Hy4 Preview / Jev": { "direction": "Hy4 Preview minus Jev", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261409, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.018808777429467086, "ci95": [ -0.05172413793103448, 0.014106583072100314 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.018808777429467086, "ci95": [ -0.05090899012479568, 0.013329031427788586 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.06960697152015204, "ci95": [ 0.04636445808755466, 0.09320833301204748 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": 4.073771344993796, "ci95": [ 3.256547483659231, 4.924304217879693 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 406, "model_only_correct": 54, "reference_only_correct": 66, "both_incorrect": 112 }, "mcnemar_exact_two_sided_p_exploratory": 0.3153033008322539, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Jev" }, "Hy4 Preview / Luna": { "direction": "Hy4 Preview minus Luna", "attempted_case_union": 638, "paired_cases": 638, "unpaired_cases": 0, "valid_response_pairs": 638, "invalid_response_pairs": 0, "resampling_clusters": 638, "cluster_rule": "Exact identical BoolQ passage; otherwise case identity", "bootstrap_replicates": 5000, "bootstrap_seed": 20261410, "confidence_level": 0.95, "inference": "exploratory; no multiplicity adjustment", "metrics": { "accuracy_all_attempts_difference": { "estimate": -0.007836990595611285, "ci95": [ -0.03761755485893417, 0.023510971786833857 ], "defined_bootstrap_draws": 5000 }, "balanced_accuracy_all_attempts_difference": { "estimate": -0.007836990595611285, "ci95": [ -0.03761763804108138, 0.022900010134723016 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": -0.018603059827810343, "ci95": [ -0.046815027896628995, 0.0094120726221038 ], "defined_bootstrap_draws": 5000 }, "log_loss_valid_pairs_difference": { "estimate": -1.4104921155347827, "ci95": [ -2.4102871235998773, -0.44297223170598715 ], "defined_bootstrap_draws": 5000 } }, "correctness_pairs": { "both_correct": 410, "model_only_correct": 50, "reference_only_correct": 55, "both_incorrect": 123 }, "mcnemar_exact_two_sided_p_exploratory": 0.696467662991644, "mcnemar_limit": "Treats discordant rows as independent; duplicate passages/shared templates violate that assumption. Do not substitute this test for the passage-cluster intervals.", "model": "Hy4 Preview", "reference": "Luna" } } } }, "timing": { "source": { "run_dir": "benchmark/multimodel/runs/timing-20261002T170524992994Z", "metadata": { "schema_version": 1, "created_at": "2026-10-02T17:05:25.139339+00:00", "models": [ { "label": "Jev", "id": "typesafe/jev-1.13", "provider": "TypeSafe", "provider_name": "TypeSafe", "prices": { "prompt": "0.0000002", "completion": "0.00000075" }, "reasoning": null, "cache_control": "unsupported", "max_tokens": null, "reasoning_mandatory": false, "protocol_difference": null }, { "label": "Luna", "id": "openai/gpt-6-luna", "provider": "OpenAI", "provider_name": "OpenAI", "prices": { "prompt": "0.0000001", "completion": "0.0000005", "web_search": "0.01", "input_cache_read": "0.00000001", "input_cache_write": "0.000000125", "overrides": [ { "min_prompt_tokens": 272000, "prompt": "0.0000002", "completion": "0.00000075", "input_cache_read": "0.00000002", "input_cache_write": "0.00000025" } ] }, "reasoning": { "effort": "none" }, "cache_control": "explicit", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null }, { "label": "Qwen3.8 Flash", "id": "qwen/qwen3.8-flash", "provider": "alibaba", "provider_name": "Alibaba", "prices": { "prompt": "0.00000015", "completion": "0.00000047", "input_cache_read": "0.000000016", "input_cache_write": "0.0000002" }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null }, { "label": "DeepSeek V4.1 Flash", "id": "deepseek/deepseek-v4.1-flash", "provider": "deepinfra/fp8", "provider_name": "DeepInfra", "prices": { "prompt": "0.00000014", "completion": "0.00000042", "input_cache_read": "0.0000000042", "discount": 0.3 }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null, "cache_reason": "No verified matched OpenRouter cache-enable/cache-disable treatment for this provider in this study." }, { "label": "MiMo V2.6 Flash", "id": "xiaomi/mimo-v2.6-flash", "provider": "xiaomi/fp8", "provider_name": "Xiaomi", "prices": { "prompt": "0.00000014", "completion": "0.00000028", "input_cache_read": "0.0000000028", "discount": 0 }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null, "cache_reason": "No verified matched OpenRouter cache-enable/cache-disable treatment for this provider in this study." }, { "label": "Hy4 Preview", "id": "tencent/hy4-preview", "provider": "tencent/fp8", "provider_name": "Tencent", "prices": { "prompt": "0.000000834", "completion": "0.000002501", "input_cache_read": "0.000000042", "discount": 0, "overrides": [ { "utc_start": 0, "utc_end": 1600, "prompt": "0.000000834", "completion": "0.000002501", "input_cache_read": "0.000000042" }, { "utc_start": 1600, "utc_end": 0, "prompt": "0.0000007506", "completion": "0.0000022509", "input_cache_read": "0.0000000378" } ] }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null, "cache_reason": "No verified matched OpenRouter cache-enable/cache-disable treatment for this provider in this study." } ], "phase": "timing", "cases": 60, "planned_calls": 1116, "calls_planned": 1116, "concurrency_pairs": 1, "seed": 20261005, "source_hashes": { "cases_sha256": "e271790b2ee38a1c231e312fa15ece2a56c95f6bf705294f1637d43b2edcf256", "timing_cases_sha256": "aa034a8fd9eedcf2ef2614fd3861d81af9d500eaed94629fb9f72c7d1df504f8" }, "cases_sha256": "aa034a8fd9eedcf2ef2614fd3861d81af9d500eaed94629fb9f72c7d1df504f8", "code_hashes": { "original-runner.py": "1c6d2ea8084da6e671d41a6d89f089edbf13729c75990bf4be0458db5df4242c", "original-adapter.py": "47c300bd57b53f669dd9c707b4317cf47cf2b5888c5233e17b94d1310e6f691b", "original-answer-contract.py": "3a05cb20816fbc5b7c30a065429a7365afa9583f273d683bf69784f4af3e9ad7" }, "plan_sha256": "7298f8c79b0bd68c33c10a08bcfc62b93c2f80dd35e3deb705e0edb7117da0da", "requests_sha256": "93cecde0a629648ce8629eee54004482ba45df6f149c0676f50e2ffc857a3318", "automatic_retries": 0, "repeats": 3, "warmups_per_model_per_block": 2, "unknown_bill_allowance_usd": 0.01, "protocol": "Three serial randomized blocks of the same 60 frozen cases, with two warmups per model per block excluded from measured timing. One persistent adapter, nonstreaming full-response latency. No overlapping quality/timing run through this runner.", "stop_rule": "Stop new dispatches after 5 consecutive invalid or >=5% invalid after 100 completed calls, overall or per model, in completion order. In-flight attempts are retained.", "billing_guard": "Known bills use observed values. Missing bills remain null and retain their reservation, at least $0.01, against the local guard. A local estimate cannot enforce a remote account-level cap.", "resume_policy": "Quality only: identical source, adapter, plan and exact request hashes; dispatch journal prevents every attempted sequence from being retried, including lost outcomes.", "status": "complete", "budget_usd": 1.5, "calls_completed": 1116, "valid": 1116, "invalid": 0, "charged_usd": 0.05492202580000004, "known_charge_subtotal_usd": 0.05492202580000004, "unknown_billing": 0, "unknown_billing_guard_usd": 0.0, "budget_pending_usd": 0.0, "budget_accounted_usd": 0.05492202580000004, "budget_stop_reason": null, "finished_at": "2026-10-02T17:42:25.413958+00:00", "responses_sha256": "7119428ee08d80ac620681f9a9bde1a5da76ffbe003df500b1ca0b968248fd78", "attempts_sha256": "a3139a5882c83a2e865ad8eed7dc73f10c0e1dc7e985fc1c5595855bb656c5c6" }, "responses_sha256": "7119428ee08d80ac620681f9a9bde1a5da76ffbe003df500b1ca0b968248fd78", "attempts": 1116, "cost_all_attempts": { "requests": 1116, "known_billed_usd": 0.0549220258, "observed_bills": 1116, "missing_bills": 0, "total_billed_usd": 0.0549220258, "usd_per_1000_attempts": 0.04921328476702509 } }, "models": [ "Jev", "Luna", "Qwen3.8 Flash", "DeepSeek V4.1 Flash", "MiMo V2.6 Flash", "Hy4 Preview" ], "tasks": { "boolq": { "expected_cases": 12, "case_union": 12, "common_complete_cases": 12, "excluded_from_common_support": 0, "common_case_ids": [ "boolq-dev-0221", "boolq-dev-0519", "boolq-dev-0715", "boolq-dev-0781", "boolq-dev-1124", "boolq-dev-1240", "boolq-dev-1609", "boolq-dev-2142", "boolq-dev-2750", "boolq-dev-2890", "boolq-dev-3044", "boolq-dev-3090" ], "models": { "Jev": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 0.2573744370165514, "p95_s": 0.42748305200075265, "mean_s": 0.28775448031067047 }, "complete_case_count": 12, "primary_case_median_latency_s": 0.2667752709967317, "case_median_distribution": { "n": 12, "median_s": 0.2667752709967317, "p95_s": 0.35314573994546655, "mean_s": 0.27677131608652417 } }, "Luna": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.3402007709955797, "p95_s": 10.395324885503214, "mean_s": 3.950770037028835 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.3766149159928318, "case_median_distribution": { "n": 12, "median_s": 1.3766149159928318, "p95_s": 2.2601582935938493, "mean_s": 1.508123912998902 } }, "Qwen3.8 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.107770562011865, "p95_s": 2.046885468509572, "mean_s": 1.24786807863792 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.1149753329955274, "case_median_distribution": { "n": 12, "median_s": 1.1149753329955274, "p95_s": 1.4513607187385786, "mean_s": 1.1673082951626081 } }, "DeepSeek V4.1 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 0.9197334994969424, "p95_s": 2.065179781006009, "mean_s": 1.1877795938053168 }, "complete_case_count": 12, "primary_case_median_latency_s": 0.9122722080064705, "case_median_distribution": { "n": 12, "median_s": 0.9122722080064705, "p95_s": 1.5250545796501682, "mean_s": 1.055091243002001 } }, "MiMo V2.6 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.5965559169999324, "p95_s": 12.37774844825617, "mean_s": 3.388321368081961 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.6746404794976115, "case_median_distribution": { "n": 12, "median_s": 1.6746404794976115, "p95_s": 6.9148458187541095, "mean_s": 2.643843013999382 } }, "Hy4 Preview": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 2.8128798125108005, "p95_s": 6.792378041245684, "mean_s": 3.638947040470763 }, "complete_case_count": 12, "primary_case_median_latency_s": 2.823063853997155, "case_median_distribution": { "n": 12, "median_s": 2.823063853997155, "p95_s": 3.534944862342672, "mean_s": 2.9659514027443947 } } }, "comparisons": { "Luna / Jev": { "model": "Luna", "reference": "Jev", "paired_cases": 12, "direction": "Luna minus Jev; ratios Luna divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 5.160204357958264, "ci95": [ 4.471644567275055, 6.83462706281865 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.801708079456535, "ci95": [ 4.4308460781531265, 7.260357236037688 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.2313525969123778, "ci95": [ 1.0111039617837376, 1.4850990444758885 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Jev; ratios Qwen3.8 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.17945534767796, "ci95": [ 3.757434249938735, 4.832773738617887 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.256041835523812, "ci95": [ 3.8034854418981574, 5.118248836232557 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.8905369790760839, "ci95": [ 0.7949708012769406, 0.9951263989264891 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna": { "model": "Qwen3.8 Flash", "reference": "Luna", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Luna; ratios Qwen3.8 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.8099398895379492, "ci95": [ 0.6542870263559633, 0.9707629784006117 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.8259200475605883, "ci95": [ 0.6370280004905611, 0.9873906313029338 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.34081561783629394, "ci95": [ -0.6008284537551541, -0.10019538638316614 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Jev; ratios DeepSeek V4.1 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 3.419628080961248, "ci95": [ 3.192572278917972, 4.908669416359866 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 3.648628926093998, "ci95": [ 3.2781612777777775, 4.678789402237193 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.7783199269154769, "ci95": [ 0.6317928726093063, 0.9500766006804041 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna": { "model": "DeepSeek V4.1 Flash", "reference": "Luna", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Luna; ratios DeepSeek V4.1 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.6626923749032084, "ci95": [ 0.5160360207465431, 0.9381806825914548 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.7876209945563536, "ci95": [ 0.47405949978773265, 0.9326531597107353 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.45303266999690095, "ci95": [ -0.7744326968145591, -0.17052387157145873 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Jev; ratios MiMo V2.6 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 6.277345247334142, "ci95": [ 5.62258626112336, 9.594388819956345 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 6.797842934317813, "ci95": [ 5.702242766370249, 9.368404578604135 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.367071697912858, "ci95": [ 1.3968063024753063, 3.66690028120629 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna": { "model": "MiMo V2.6 Flash", "reference": "Luna", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Luna; ratios MiMo V2.6 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.2164915983710956, "ci95": [ 0.9397054809786879, 1.834818861399112 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.3240258038187371, "ci95": [ 0.8629716908359071, 1.8238341758154284 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.1357191010004801, "ci95": [ 0.09978653268050645, 2.514831586039629 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 12, "direction": "Hy4 Preview minus Jev; ratios Hy4 Preview divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 10.582179687980679, "ci95": [ 10.133614298078628, 11.845997493436622 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 10.35179530637304, "ci95": [ 10.158137630215185, 12.211881491412294 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.6891800866578706, "ci95": [ 2.5322840128319513, 2.914688173472011 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna": { "model": "Hy4 Preview", "reference": "Luna", "paired_cases": 12, "direction": "Hy4 Preview minus Luna; ratios Hy4 Preview divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 2.0507287994632297, "ci95": [ 1.6094901022356347, 2.3218962824309197 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.2223080815919083, "ci95": [ 1.7474371472886194, 2.438019927783264 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.4578274897454928, "ci95": [ 1.1637453547138648, 1.7419991995550537 ], "defined_bootstrap_draws": 5000 } } } }, "pairwise_complete_sensitivity": { "Luna / Jev": { "model": "Luna", "reference": "Jev", "paired_cases": 12, "direction": "Luna minus Jev; ratios Luna divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 5.160204357958264, "ci95": [ 4.471644567275055, 6.7762620523091845 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.801708079456535, "ci95": [ 4.4308460781531265, 7.260357236037688 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.2313525969123778, "ci95": [ 1.0029866660562523, 1.4855899012752463 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Jev; ratios Qwen3.8 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.17945534767796, "ci95": [ 3.736339653866685, 4.832744649400371 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.256041835523812, "ci95": [ 3.8034854418981574, 5.118248836232557 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.8905369790760839, "ci95": [ 0.7961898906079418, 0.9917329685765556 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna": { "model": "Qwen3.8 Flash", "reference": "Luna", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Luna; ratios Qwen3.8 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.8099398895379492, "ci95": [ 0.6542870263559633, 0.968364316094414 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.8259200475605883, "ci95": [ 0.6370280004905611, 0.9873906313029338 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.34081561783629394, "ci95": [ -0.5995348571898528, -0.10893401612872076 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Jev; ratios DeepSeek V4.1 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 3.419628080961248, "ci95": [ 3.192572278917972, 4.854102710835852 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 3.648628926093998, "ci95": [ 3.2781612777777775, 4.678789402237193 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.7783199269154769, "ci95": [ 0.6258301780187442, 0.9530921017050786 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna": { "model": "DeepSeek V4.1 Flash", "reference": "Luna", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Luna; ratios DeepSeek V4.1 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.6626923749032084, "ci95": [ 0.5171846062516361, 0.9381806825914548 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.7876209945563536, "ci95": [ 0.47405949978773265, 0.9326531597107353 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.45303266999690095, "ci95": [ -0.7664009867280887, -0.1635858431338421 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Jev; ratios MiMo V2.6 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 6.277345247334142, "ci95": [ 5.62258626112336, 9.351333095982023 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 6.797842934317813, "ci95": [ 5.702242766370249, 9.368404578604135 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.367071697912858, "ci95": [ 1.4072394012368021, 3.599596014705152 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna": { "model": "MiMo V2.6 Flash", "reference": "Luna", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Luna; ratios MiMo V2.6 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.2164915983710956, "ci95": [ 0.9231692384250277, 1.827761397965118 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.3240258038187371, "ci95": [ 0.8629716908359071, 1.8238341758154284 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.1357191010004801, "ci95": [ 0.10015912094513624, 2.4781050431665785 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 12, "direction": "Hy4 Preview minus Jev; ratios Hy4 Preview divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 10.582179687980679, "ci95": [ 10.095064860642607, 11.833955661130377 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 10.35179530637304, "ci95": [ 10.158137630215185, 12.211881491412294 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.6891800866578706, "ci95": [ 2.532375530828055, 2.908650472158721 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna": { "model": "Hy4 Preview", "reference": "Luna", "paired_cases": 12, "direction": "Hy4 Preview minus Luna; ratios Hy4 Preview divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 2.0507287994632297, "ci95": [ 1.609422417233394, 2.3273526699219866 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.2223080815919083, "ci95": [ 1.7474371472886194, 2.438019927783264 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.4578274897454928, "ci95": [ 1.161860339308987, 1.7553691787165007 ], "defined_bootstrap_draws": 5000 } } } }, "repeat_blocks": { "0": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.24990802101092413, "p95_s": 0.4404860648050088, "mean_s": 0.2832866875032778 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.198045937504503, "p95_s": 2.0647660168411672, "mean_s": 1.3452033055800712 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.1114741040073568, "p95_s": 1.9921836687644823, "mean_s": 1.2700824339165895 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.9005816250137286, "p95_s": 1.4137462350525312, "mean_s": 0.9650580172480355 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.3877411459834548, "p95_s": 20.18841410451132, "mean_s": 4.868857086832577 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.8228737710160203, "p95_s": 16.518623897552605, "mean_s": 5.096163899334594 } }, "1": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.2697193954954855, "p95_s": 0.39015207691263637, "mean_s": 0.2918050035077613 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.4798447290086187, "p95_s": 30.752570362502553, "mean_s": 6.932689739505197 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.0682371250004508, "p95_s": 2.167050839941657, "mean_s": 1.2630645451669504 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.1806836875039153, "p95_s": 2.478346308985782, "mean_s": 1.3346214272484456 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.014225229504518, "p95_s": 8.880312593899724, "mean_s": 3.053401409832683 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.893442729502567, "p95_s": 3.534944862342672, "mean_s": 2.9484663611632036 } }, "2": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.25424031251168344, "p95_s": 0.39827140609704653, "mean_s": 0.28817174992097233 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.3764088539901422, "p95_s": 13.474166041996778, "mean_s": 3.5744170660012364 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.1219536869903095, "p95_s": 1.6738198623497733, "mean_s": 1.2104572568302199 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.0426271045143949, "p95_s": 2.138678439405339, "mean_s": 1.2636593369194695 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.8422274999902584, "p95_s": 5.0808568981039555, "mean_s": 2.2427056075806227 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.7190563329932047, "p95_s": 3.4662898251524887, "mean_s": 2.8722108609144925 } } } }, "policy": { "expected_cases": 12, "case_union": 12, "common_complete_cases": 12, "excluded_from_common_support": 0, "common_case_ids": [ "expanded-policy-0049", "expanded-policy-0075", "expanded-policy-0092", "expanded-policy-0107", "expanded-policy-0110", "expanded-policy-0128", "expanded-policy-0134", "expanded-policy-0137", "expanded-policy-0185", "expanded-policy-0202", "expanded-policy-0204", "expanded-policy-0244" ], "models": { "Jev": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 0.2589071249967674, "p95_s": 0.32225854174612323, "mean_s": 0.26642766680500546 }, "complete_case_count": 12, "primary_case_median_latency_s": 0.2589071249967674, "case_median_distribution": { "n": 12, "median_s": 0.2589071249967674, "p95_s": 0.3033180982427438, "mean_s": 0.26394384391460335 } }, "Luna": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.3173393960023532, "p95_s": 13.878983635746408, "mean_s": 3.4383316897801706 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.250741875017411, "case_median_distribution": { "n": 12, "median_s": 1.250741875017411, "p95_s": 4.661991995804415, "mean_s": 1.87474664591961 } }, "Qwen3.8 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.3597179999924265, "p95_s": 4.380932489482802, "mean_s": 1.7001629896096548 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.3677198960067471, "case_median_distribution": { "n": 12, "median_s": 1.3677198960067471, "p95_s": 3.0334949123483947, "mean_s": 1.6584732396683346 } }, "DeepSeek V4.1 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 0.6978027080040192, "p95_s": 1.3263585724926088, "mean_s": 0.8277216689206802 }, "complete_case_count": 12, "primary_case_median_latency_s": 0.7922371245076647, "case_median_distribution": { "n": 12, "median_s": 0.7922371245076647, "p95_s": 0.9856798734152107, "mean_s": 0.7419890382564821 } }, "MiMo V2.6 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.442354583487031, "p95_s": 4.790045343499514, "mean_s": 2.179686965221966 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.477291499992134, "case_median_distribution": { "n": 12, "median_s": 1.477291499992134, "p95_s": 3.927671248107798, "mean_s": 1.8426364375870132 } }, "Hy4 Preview": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 2.949341708503198, "p95_s": 4.263769802499155, "mean_s": 3.0234247800819705 }, "complete_case_count": 12, "primary_case_median_latency_s": 3.0067971455137013, "case_median_distribution": { "n": 12, "median_s": 3.0067971455137013, "p95_s": 3.4898329604562597, "mean_s": 3.005211492997963 } } }, "comparisons": { "Luna / Jev": { "model": "Luna", "reference": "Jev", "paired_cases": 12, "direction": "Luna minus Jev; ratios Luna divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.830851507207565, "ci95": [ 4.472995065872053, 6.058892022821542 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.916009771533298, "ci95": [ 4.36344887330921, 6.123950164844102 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.6108028020050067, "ci95": [ 0.9419422026202786, 2.7901642503598847 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Jev; ratios Qwen3.8 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 5.282666114437075, "ci95": [ 4.718309529564001, 6.32428104037772 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.280217940852246, "ci95": [ 4.738310183030574, 6.489162167666317 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.3945293957537312, "ci95": [ 1.039774207738507, 1.9584364742312266 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna": { "model": "Qwen3.8 Flash", "reference": "Luna", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Luna; ratios Qwen3.8 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.09352690856993, "ci95": [ 0.7992735635264602, 1.329850181421342 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.2192404441381697, "ci95": [ 0.7988483135949247, 1.3580926256309525 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.21627340625127545, "ci95": [ -1.585855036091137, 0.8391003937586592 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Jev; ratios DeepSeek V4.1 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 3.0599278583683485, "ci95": [ 2.088907444500517, 3.2995237855968904 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.995301887760815, "ci95": [ 2.153304001295359, 3.2912649586726905 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.4780451943418787, "ci95": [ 0.38137428844578003, 0.5735736850766746 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna": { "model": "DeepSeek V4.1 Flash", "reference": "Luna", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Luna; ratios DeepSeek V4.1 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.633413768525689, "ci95": [ 0.42968230978733307, 0.7192463728131824 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.5154951245347413, "ci95": [ 0.43959521516187394, 0.7156551030957651 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -1.1327576076631278, "ci95": [ -2.303857139256979, -0.4460666475270046 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Jev; ratios MiMo V2.6 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 5.705874259005344, "ci95": [ 3.724017872846068, 8.838382281039467 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.815140991483925, "ci95": [ 3.883298545223438, 9.255865116464488 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.5786925936724099, "ci95": [ 0.9803268759678152, 2.2961450622749298 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna": { "model": "MiMo V2.6 Flash", "reference": "Luna", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Luna; ratios MiMo V2.6 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.1811321980177318, "ci95": [ 0.7606001179660065, 1.809495037855297 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.2492678880051624, "ci95": [ 0.5214215021224655, 1.803724381725052 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.03211020833259681, "ci95": [ -1.390937968584573, 1.0460596508646025 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 12, "direction": "Hy4 Preview minus Jev; ratios Hy4 Preview divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 11.613419852973312, "ci95": [ 10.982989152605267, 11.927174297020024 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 11.58763033630521, "ci95": [ 10.95599591896677, 11.727723106294869 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.7412676490833596, "ci95": [ 2.5610787900276892, 2.9662701874109185 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna": { "model": "Hy4 Preview", "reference": "Luna", "paired_cases": 12, "direction": "Hy4 Preview minus Luna; ratios Hy4 Preview divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 2.404010935887027, "ci95": [ 1.8861604267802687, 2.5920231435624563 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.3688963188103687, "ci95": [ 1.9664144493613445, 2.58213216697745 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.130464847078353, "ci95": [ -0.14582566472903027, 1.9114393298843422 ], "defined_bootstrap_draws": 5000 } } } }, "pairwise_complete_sensitivity": { "Luna / Jev": { "model": "Luna", "reference": "Jev", "paired_cases": 12, "direction": "Luna minus Jev; ratios Luna divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.830851507207565, "ci95": [ 4.47809619269767, 6.084055377968451 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.916009771533298, "ci95": [ 4.36344887330921, 6.123950164844102 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.6108028020050067, "ci95": [ 0.9496138389301754, 2.7669574142926034 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Jev; ratios Qwen3.8 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 5.282666114437075, "ci95": [ 4.737797956725212, 6.32428104037772 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.280217940852246, "ci95": [ 4.738310183030574, 6.489162167666317 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.3945293957537312, "ci95": [ 1.0397003555464228, 1.9731270472599134 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna": { "model": "Qwen3.8 Flash", "reference": "Luna", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Luna; ratios Qwen3.8 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.09352690856993, "ci95": [ 0.8061455728465883, 1.3417659742886854 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.2192404441381697, "ci95": [ 0.7988483135949247, 1.3580926256309525 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.21627340625127545, "ci95": [ -1.5851728715580369, 0.9095130937326756 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Jev; ratios DeepSeek V4.1 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 3.0599278583683485, "ci95": [ 2.0834855356158424, 3.3031562240224193 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.995301887760815, "ci95": [ 2.153304001295359, 3.2912649586726905 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.4780451943418787, "ci95": [ 0.38166544122893054, 0.5705661887328459 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna": { "model": "DeepSeek V4.1 Flash", "reference": "Luna", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Luna; ratios DeepSeek V4.1 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.633413768525689, "ci95": [ 0.43127112560299197, 0.7171724050351721 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.5154951245347413, "ci95": [ 0.43959521516187394, 0.7156551030957651 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -1.1327576076631278, "ci95": [ -2.2851473901930146, -0.4485879275237193 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Jev; ratios MiMo V2.6 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 5.705874259005344, "ci95": [ 3.724017872846068, 8.94446052834314 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.815140991483925, "ci95": [ 3.883298545223438, 9.255865116464488 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.5786925936724099, "ci95": [ 0.9847619618130921, 2.3047542051531 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna": { "model": "MiMo V2.6 Flash", "reference": "Luna", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Luna; ratios MiMo V2.6 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.1811321980177318, "ci95": [ 0.7664241864991896, 1.8295702668261942 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.2492678880051624, "ci95": [ 0.5214215021224655, 1.803724381725052 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.03211020833259681, "ci95": [ -1.3710617909546878, 1.089040273580819 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 12, "direction": "Hy4 Preview minus Jev; ratios Hy4 Preview divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 11.613419852973312, "ci95": [ 11.017881008282234, 11.927143589201169 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 11.58763033630521, "ci95": [ 10.95599591896677, 11.727723106294869 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.7412676490833596, "ci95": [ 2.5642948993458656, 2.9665092703383076 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna": { "model": "Hy4 Preview", "reference": "Luna", "paired_cases": 12, "direction": "Hy4 Preview minus Luna; ratios Hy4 Preview divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 2.404010935887027, "ci95": [ 1.9045321143068743, 2.6043454510198445 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.3688963188103687, "ci95": [ 1.9664144493613445, 2.58213216697745 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.130464847078353, "ci95": [ -0.0805882469747909, 1.9101845228213885 ], "defined_bootstrap_draws": 5000 } } } }, "repeat_blocks": { "0": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.26014027099881787, "p95_s": 0.3159572666889289, "mean_s": 0.2657650730786069 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.247963896006695, "p95_s": 15.968607681550182, "mean_s": 3.965327041752365 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.522853958493215, "p95_s": 4.418150781086297, "mean_s": 2.0586872117443513 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.6370946039969567, "p95_s": 1.0220559940484235, "mean_s": 0.6694810241654826 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.6892960834957194, "p95_s": 4.325329474991303, "mean_s": 2.118123062498247 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.834959395506303, "p95_s": 3.882259692001389, "mean_s": 2.9454109581686985 } }, "1": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.2686695210140897, "p95_s": 0.32492867514520185, "mean_s": 0.27409294467118644 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.4261476245010272, "p95_s": 17.00745885829818, "mean_s": 4.293307937337279 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.1728145005181432, "p95_s": 3.105983474694947, "mean_s": 1.4967462916659617 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.8166127705189865, "p95_s": 0.9856798734152107, "mean_s": 0.748857791761111 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.310151916986797, "p95_s": 5.866019951750057, "mean_s": 2.73710198616512 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.88555258300039, "p95_s": 3.63260860274604, "mean_s": 2.9067707049956275 } }, "2": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.2504257295076968, "p95_s": 0.3128832581380266, "mean_s": 0.25942498266522307 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.2196106875053374, "p95_s": 5.203323743895449, "mean_s": 2.0563600902508674 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.3923970210016705, "p95_s": 2.391504006255126, "mean_s": 1.5450554654186515 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.7326650830073049, "p95_s": 2.696917128654603, "mean_s": 1.064826190835447 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.1759791665099328, "p95_s": 3.9727873456460654, "mean_s": 1.6838358470025316 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.9841019789892016, "p95_s": 4.478169170647743, "mean_s": 3.218092677081586 } } } }, "probability": { "expected_cases": 12, "case_union": 12, "common_complete_cases": 12, "excluded_from_common_support": 0, "common_case_ids": [ "expanded-probability-0032", "expanded-probability-0038", "expanded-probability-0076", "expanded-probability-0080", "expanded-probability-0098", "expanded-probability-0113", "expanded-probability-0118", "expanded-probability-0127", "expanded-probability-0143", "expanded-probability-0190", "expanded-probability-0216", "expanded-probability-0219" ], "models": { "Jev": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 0.264528812491335, "p95_s": 0.34125712474633474, "mean_s": 0.2947236087234665 }, "complete_case_count": 12, "primary_case_median_latency_s": 0.25876891701773275, "case_median_distribution": { "n": 12, "median_s": 0.25876891701773275, "p95_s": 0.30566662120108956, "mean_s": 0.26373533684090944 } }, "Luna": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.2654068334959447, "p95_s": 2.7522048232567613, "mean_s": 1.5479919872515084 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.2770443120098207, "case_median_distribution": { "n": 12, "median_s": 1.2770443120098207, "p95_s": 1.449510575007298, "mean_s": 1.2927048644196475 } }, "Qwen3.8 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.2959345210110769, "p95_s": 2.5401165829825914, "mean_s": 1.4731943102475877 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.2737293960090028, "case_median_distribution": { "n": 12, "median_s": 1.2737293960090028, "p95_s": 1.5138049060900811, "mean_s": 1.304363538166702 } }, "DeepSeek V4.1 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 0.8107922505005263, "p95_s": 2.2663186142526683, "mean_s": 1.0386834653327242 }, "complete_case_count": 12, "primary_case_median_latency_s": 0.8004174170055194, "case_median_distribution": { "n": 12, "median_s": 0.8004174170055194, "p95_s": 1.2924353350375895, "mean_s": 0.8657877467461125 } }, "MiMo V2.6 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 2.030834062505164, "p95_s": 10.853033604747907, "mean_s": 3.5637626736667394 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.969626062491443, "case_median_distribution": { "n": 12, "median_s": 1.969626062491443, "p95_s": 6.319908743756239, "mean_s": 2.988700204914494 } }, "Hy4 Preview": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 3.083029166489723, "p95_s": 3.8337161562449182, "mean_s": 3.3456124004450327 }, "complete_case_count": 12, "primary_case_median_latency_s": 3.103338145490852, "case_median_distribution": { "n": 12, "median_s": 3.103338145490852, "p95_s": 3.5265933980932456, "mean_s": 3.058740392329734 } } }, "comparisons": { "Luna / Jev": { "model": "Luna", "reference": "Jev", "paired_cases": 12, "direction": "Luna minus Jev; ratios Luna divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.9350761549243884, "ci95": [ 4.62012132699883, 5.381722809126611 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.894458839119427, "ci95": [ 4.552330681473, 5.253390256261255 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.028969527578738, "ci95": [ 0.9774201279143502, 1.0850742148739907 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Jev; ratios Qwen3.8 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.922265821909814, "ci95": [ 4.510535878691225, 5.356820604531621 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.0759886505991325, "ci95": [ 4.47058596102279, 5.342360012993487 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.0406282013257926, "ci95": [ 0.9740334980559662, 1.114049680846559 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna": { "model": "Qwen3.8 Flash", "reference": "Luna", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Luna; ratios Qwen3.8 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.9974042278958974, "ci95": [ 0.915458710396676, 1.0864606227257183 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.0141273929189887, "ci95": [ 0.9364663030630134, 1.0959764615395875 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.011658673747054612, "ci95": [ -0.06326400520162376, 0.08453143501531787 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Jev; ratios DeepSeek V4.1 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 3.0931745057721476, "ci95": [ 2.4735095598455445, 3.7576463545454346 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 3.343426066460834, "ci95": [ 2.5148697263057804, 3.9009234304430054 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.602052409905203, "ci95": [ 0.4819448097314307, 0.7325324881616324 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna": { "model": "DeepSeek V4.1 Flash", "reference": "Luna", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Luna; ratios DeepSeek V4.1 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.6267734090963666, "ci95": [ 0.5080462901543564, 0.7952270065636998 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.6668065757758206, "ci95": [ 0.5023067848522627, 0.7668823238201301 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.4269171176735351, "ci95": [ -0.5715646978669005, -0.2835616367359764 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Jev; ratios MiMo V2.6 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 7.611524928074997, "ci95": [ 5.699888323972072, 17.025581923705374 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 7.723363960926802, "ci95": [ 5.836998271664127, 16.650831339430958 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.7249648680735845, "ci95": [ 1.70284496011609, 3.964920352411536 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna": { "model": "MiMo V2.6 Flash", "reference": "Luna", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Luna; ratios MiMo V2.6 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.5423318078850627, "ci95": [ 1.1410826663043865, 3.460276066663861 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.6183259551801297, "ci95": [ 1.1480834242837705, 3.5068046121005723 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.6959953404948465, "ci95": [ 0.6718771991628576, 2.905745387599501 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 12, "direction": "Hy4 Preview minus Jev; ratios Hy4 Preview divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 11.992700596564264, "ci95": [ 10.682402204094947, 13.060152540587119 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 11.51676067855869, "ci95": [ 10.55774322718743, 12.604457225945904 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.7950050554888244, "ci95": [ 2.6264394305638534, 2.9756116980701943 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna": { "model": "Hy4 Preview", "reference": "Luna", "paired_cases": 12, "direction": "Hy4 Preview minus Luna; ratios Hy4 Preview divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 2.4300943329106555, "ci95": [ 2.0828832218153353, 2.619416666798171 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.4457534829122856, "ci95": [ 2.053498251086894, 2.6881370372365745 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.7660355279100866, "ci95": [ 1.565613936560294, 1.969623296400338 ], "defined_bootstrap_draws": 5000 } } } }, "pairwise_complete_sensitivity": { "Luna / Jev": { "model": "Luna", "reference": "Jev", "paired_cases": 12, "direction": "Luna minus Jev; ratios Luna divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.9350761549243884, "ci95": [ 4.613317762480733, 5.370117418805065 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.894458839119427, "ci95": [ 4.552330681473, 5.253390256261255 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.028969527578738, "ci95": [ 0.9772391112640131, 1.0830680566239606 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Jev; ratios Qwen3.8 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.922265821909814, "ci95": [ 4.476223448147782, 5.459072402902459 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.0759886505991325, "ci95": [ 4.47058596102279, 5.342360012993487 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.0406282013257926, "ci95": [ 0.9732062114633057, 1.1156650204447942 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna": { "model": "Qwen3.8 Flash", "reference": "Luna", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Luna; ratios Qwen3.8 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.9974042278958974, "ci95": [ 0.9151423770656368, 1.0864606227257183 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.0141273929189887, "ci95": [ 0.9364663030630134, 1.0959764615395875 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.011658673747054612, "ci95": [ -0.06344041810328539, 0.08319350515757828 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Jev; ratios DeepSeek V4.1 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 3.0931745057721476, "ci95": [ 2.4735095598455445, 3.8034940600902623 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 3.343426066460834, "ci95": [ 2.5148697263057804, 3.9009234304430054 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.602052409905203, "ci95": [ 0.48262271343737667, 0.7390837577653655 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna": { "model": "DeepSeek V4.1 Flash", "reference": "Luna", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Luna; ratios DeepSeek V4.1 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.6267734090963666, "ci95": [ 0.5115618266143134, 0.7933375257520049 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.6668065757758206, "ci95": [ 0.5023067848522627, 0.7668823238201301 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.4269171176735351, "ci95": [ -0.5688646652390161, -0.2783046715017312 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Jev; ratios MiMo V2.6 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 7.611524928074997, "ci95": [ 5.745544704014323, 17.064558389015804 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 7.723363960926802, "ci95": [ 5.836998271664127, 16.650831339430958 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.7249648680735845, "ci95": [ 1.7148526202551388, 3.9682972698935064 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna": { "model": "MiMo V2.6 Flash", "reference": "Luna", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Luna; ratios MiMo V2.6 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.5423318078850627, "ci95": [ 1.1423500448522386, 3.460276066663861 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.6183259551801297, "ci95": [ 1.1480834242837705, 3.5068046121005723 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.6959953404948465, "ci95": [ 0.6705099229022381, 2.93249069146641 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 12, "direction": "Hy4 Preview minus Jev; ratios Hy4 Preview divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 11.992700596564264, "ci95": [ 10.68129771738377, 13.018105439127169 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 11.51676067855869, "ci95": [ 10.55774322718743, 12.604457225945904 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.7950050554888244, "ci95": [ 2.6214942263399883, 2.968860033380831 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna": { "model": "Hy4 Preview", "reference": "Luna", "paired_cases": 12, "direction": "Hy4 Preview minus Luna; ratios Hy4 Preview divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 2.4300943329106555, "ci95": [ 2.08317651554564, 2.6159798121619104 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.4457534829122856, "ci95": [ 2.053498251086894, 2.6881370372365745 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.7660355279100866, "ci95": [ 1.567085886254366, 1.9666341747784222 ], "defined_bootstrap_draws": 5000 } } } }, "repeat_blocks": { "0": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.2568897289893357, "p95_s": 0.32145203734689853, "mean_s": 0.26999026024956646 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.2496400415111566, "p95_s": 1.389248403806414, "mean_s": 1.2527945797507225 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.2913575830025366, "p95_s": 2.7028866207765527, "mean_s": 1.5711752360730316 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.8004174170055194, "p95_s": 1.2924353350375895, "mean_s": 0.8403472500795033 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.9124056460132124, "p95_s": 9.16632110031205, "mean_s": 3.7143332223349717 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 3.083029166489723, "p95_s": 3.762111406262556, "mean_s": 3.0825391805847175 } }, "1": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.27704885399725754, "p95_s": 0.683388764248229, "mean_s": 0.3438937290823863 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.3082848750054836, "p95_s": 4.0329433921477165, "mean_s": 1.937129913252041 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.301249625001219, "p95_s": 1.9628633587140933, "mean_s": 1.42311563208447 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.7117483124893624, "p95_s": 1.374125008167175, "mean_s": 0.8313041389119462 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.542556145999697, "p95_s": 4.385174448094039, "mean_s": 1.9862754723314235 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 3.0194550624873955, "p95_s": 8.198171981405169, "mean_s": 3.9519022430855935 } }, "2": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.264528812491335, "p95_s": 0.33810693125415126, "mean_s": 0.27028683683844673 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.2741029790049652, "p95_s": 2.1604806477000236, "mean_s": 1.4540514687517618 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.2615324380021775, "p95_s": 2.1679938542016313, "mean_s": 1.4252920625852614 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.889232167013688, "p95_s": 3.7527695828437553, "mean_s": 1.444399007006723 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 3.0398408329929225, "p95_s": 12.562686998098798, "mean_s": 4.990679326333823 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.994746208496508, "p95_s": 3.4015334828611232, "mean_s": 3.0023957776647876 } } } }, "rte": { "expected_cases": 12, "case_union": 12, "common_complete_cases": 12, "excluded_from_common_support": 0, "common_case_ids": [ "rte-validation-0009", "rte-validation-0011", "rte-validation-0064", "rte-validation-0095", "rte-validation-0130", "rte-validation-0143", "rte-validation-0148", "rte-validation-0167", "rte-validation-0170", "rte-validation-0188", "rte-validation-0198", "rte-validation-0220" ], "models": { "Jev": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 0.2602578544901917, "p95_s": 0.3290793235137244, "mean_s": 0.26926005905503797 }, "complete_case_count": 12, "primary_case_median_latency_s": 0.2597315624880139, "case_median_distribution": { "n": 12, "median_s": 0.2597315624880139, "p95_s": 0.3108836395462276, "mean_s": 0.265799843747421 } }, "Luna": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.387714270997094, "p95_s": 8.501623521246074, "mean_s": 2.4899564572778115 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.3477908544882666, "case_median_distribution": { "n": 12, "median_s": 1.3477908544882666, "p95_s": 1.7723492831675682, "mean_s": 1.3689878023336253 } }, "Qwen3.8 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.1657127710059285, "p95_s": 2.240595072747965, "mean_s": 1.377530324110315 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.168272834009258, "case_median_distribution": { "n": 12, "median_s": 1.168272834009258, "p95_s": 1.7134169210490646, "mean_s": 1.2393744306658239 } }, "DeepSeek V4.1 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 0.9557958959921962, "p95_s": 3.2091427500054124, "mean_s": 1.4526896806378824 }, "complete_case_count": 12, "primary_case_median_latency_s": 0.9440041039924836, "case_median_distribution": { "n": 12, "median_s": 0.9440041039924836, "p95_s": 2.3090889105893435, "mean_s": 1.1270616075780708 } }, "MiMo V2.6 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.9666757085069548, "p95_s": 7.185710584009939, "mean_s": 2.597457991973594 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.9624968334974255, "case_median_distribution": { "n": 12, "median_s": 1.9624968334974255, "p95_s": 2.912946939691027, "mean_s": 1.747408045164775 } }, "Hy4 Preview": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 2.964607583009638, "p95_s": 4.15034744800505, "mean_s": 3.0223772499990043 }, "complete_case_count": 12, "primary_case_median_latency_s": 2.9834769580193097, "case_median_distribution": { "n": 12, "median_s": 2.9834769580193097, "p95_s": 3.709168375002627, "mean_s": 2.9943372985823467 } } }, "comparisons": { "Luna / Jev": { "model": "Luna", "reference": "Jev", "paired_cases": 12, "direction": "Luna minus Jev; ratios Luna divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 5.189168546084824, "ci95": [ 4.613254256599394, 5.766885400866188 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.056772339454556, "ci95": [ 4.512417593378769, 5.92688483912877 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.1031879585862043, "ci95": [ 0.9800043326981419, 1.2518481176085816 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Jev; ratios Qwen3.8 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.498001023896245, "ci95": [ 4.281582049179975, 4.736292422008437 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.470584614634566, "ci95": [ 4.181439969528093, 4.655792570718561 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.9735745869184029, "ci95": [ 0.8508805344374802, 1.1687504369376256 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna": { "model": "Qwen3.8 Flash", "reference": "Luna", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Luna; ratios Qwen3.8 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.8668057288850142, "ci95": [ 0.7876863244952111, 0.983242306613461 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.905268863744138, "ci95": [ 0.790281408996576, 1.0361427597445725 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.1296133716678014, "ci95": [ -0.2637466342474606, 0.00165615660062026 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Jev; ratios DeepSeek V4.1 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 3.63453750075541, "ci95": [ 3.2380729715383234, 3.8487693955766122 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 3.5343645876217256, "ci95": [ 3.1618304320544466, 3.8936718913120094 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.8612617638306498, "ci95": [ 0.6030181874604751, 1.2204671608804951 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna": { "model": "DeepSeek V4.1 Flash", "reference": "Luna", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Luna; ratios DeepSeek V4.1 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.700408450501696, "ci95": [ 0.5936178817806, 0.7930334333989925 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.735429189886863, "ci95": [ 0.5661083668361789, 0.8604928159359345 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.24192619475555452, "ci95": [ -0.5219733852813685, 0.08262453060900336 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Jev; ratios MiMo V2.6 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 7.55586581275809, "ci95": [ 3.3055672251723895, 8.71662214935393 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 6.86930845881972, "ci95": [ 3.3377068938311343, 8.35483531578261 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.4816082014173542, "ci95": [ 0.9981009680052012, 1.9839929435990296 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna": { "model": "MiMo V2.6 Flash", "reference": "Luna", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Luna; ratios MiMo V2.6 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.456084100112515, "ci95": [ 0.6613293611441406, 1.7424052411136401 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.1633572668574033, "ci95": [ 0.6713138733020981, 1.7462575460825378 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.3784202428311498, "ci95": [ -0.047990973610649225, 0.8352040714435133 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 12, "direction": "Hy4 Preview minus Jev; ratios Hy4 Preview divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 11.486770916249316, "ci95": [ 10.743459416685262, 12.548298282623456 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 11.18102642512629, "ci95": [ 10.56448348576843, 12.278435476122567 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.728537454834926, "ci95": [ 2.4468313495766782, 3.0103458767416664 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna": { "model": "Hy4 Preview", "reference": "Luna", "paired_cases": 12, "direction": "Hy4 Preview minus Luna; ratios Hy4 Preview divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 2.213605284591492, "ci95": [ 1.994089619126152, 2.5399838740876612 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.2575516853818454, "ci95": [ 1.9934142567858708, 2.582698649668672 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.6253494962487214, "ci95": [ 1.2153708338394305, 1.991214349740888 ], "defined_bootstrap_draws": 5000 } } } }, "pairwise_complete_sensitivity": { "Luna / Jev": { "model": "Luna", "reference": "Jev", "paired_cases": 12, "direction": "Luna minus Jev; ratios Luna divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 5.189168546084824, "ci95": [ 4.61508589556216, 5.734675644238858 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.056772339454556, "ci95": [ 4.512417593378769, 5.92688483912877 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.1031879585862043, "ci95": [ 0.9809393886639858, 1.2457581651866347 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Jev; ratios Qwen3.8 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.498001023896245, "ci95": [ 4.283193681071509, 4.732403515125203 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.470584614634566, "ci95": [ 4.181439969528093, 4.655792570718561 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.9735745869184029, "ci95": [ 0.8511356820941729, 1.1680774983520676 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna": { "model": "Qwen3.8 Flash", "reference": "Luna", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Luna; ratios Qwen3.8 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.8668057288850142, "ci95": [ 0.7878236145113299, 0.9848957483061926 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.905268863744138, "ci95": [ 0.790281408996576, 1.0361427597445725 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.1296133716678014, "ci95": [ -0.2614891990363806, 0.00237162154965808 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Jev; ratios DeepSeek V4.1 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 3.63453750075541, "ci95": [ 3.235785642305783, 3.8467926277067512 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 3.5343645876217256, "ci95": [ 3.1618304320544466, 3.8936718913120094 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.8612617638306498, "ci95": [ 0.6017465422245853, 1.215006755294841 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna": { "model": "DeepSeek V4.1 Flash", "reference": "Luna", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Luna; ratios DeepSeek V4.1 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.700408450501696, "ci95": [ 0.5936178817806, 0.7930593441320385 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.735429189886863, "ci95": [ 0.5661083668361789, 0.8604928159359345 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.24192619475555452, "ci95": [ -0.5286043286776475, 0.08068059247707424 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Jev; ratios MiMo V2.6 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 7.55586581275809, "ci95": [ 3.309089490563295, 8.713125833123454 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 6.86930845881972, "ci95": [ 3.3377068938311343, 8.35483531578261 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.4816082014173542, "ci95": [ 1.0076640678005788, 1.9743382395987885 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna": { "model": "MiMo V2.6 Flash", "reference": "Luna", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Luna; ratios MiMo V2.6 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.456084100112515, "ci95": [ 0.6688040714586677, 1.7577430289563025 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.1633572668574033, "ci95": [ 0.6713138733020981, 1.7462575460825378 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.3784202428311498, "ci95": [ -0.04345288862768938, 0.8460507023273138 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 12, "direction": "Hy4 Preview minus Jev; ratios Hy4 Preview divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 11.486770916249316, "ci95": [ 10.743459416685262, 12.562108577055797 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 11.18102642512629, "ci95": [ 10.56448348576843, 12.278435476122567 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.728537454834926, "ci95": [ 2.4565004724480484, 3.0152825780610626 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna": { "model": "Hy4 Preview", "reference": "Luna", "paired_cases": 12, "direction": "Hy4 Preview minus Luna; ratios Hy4 Preview divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 2.213605284591492, "ci95": [ 1.9970673384627424, 2.5536196910248394 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.2575516853818454, "ci95": [ 1.9934142567858708, 2.582698649668672 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.6253494962487214, "ci95": [ 1.2257174179826205, 1.9973532000158227 ], "defined_bootstrap_draws": 5000 } } } }, "repeat_blocks": { "0": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.24502235450199805, "p95_s": 0.3023945439155796, "mean_s": 0.2551365937542869 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.2642787709919503, "p95_s": 2.556241448257059, "mean_s": 1.4460232294150046 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.168272834009258, "p95_s": 3.0366405357563035, "mean_s": 1.5671484515017557 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.920191145996796, "p95_s": 2.587773335599922, "mean_s": 1.2162858959976195 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.9210889999958454, "p95_s": 7.289592945971525, "mean_s": 2.9445989619222623 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.954948416503612, "p95_s": 4.284012179200363, "mean_s": 3.0773807845810857 } }, "1": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.2602578544901917, "p95_s": 0.3563253688931581, "mean_s": 0.27784449657580507 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.5316849585069576, "p95_s": 14.895532370665745, "mean_s": 4.49509459733963 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.0864620835054666, "p95_s": 1.8004755539528554, "mean_s": 1.2148665486650618 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.9352456664928468, "p95_s": 1.7253596044902217, "mean_s": 1.08478328808269 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.4514642915164586, "p95_s": 7.7470934213488345, "mean_s": 3.21395943066697 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 3.0423250419989927, "p95_s": 3.9559320168526027, "mean_s": 3.1398769271715232 } }, "2": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.2679269165091682, "p95_s": 0.3313384151100763, "mean_s": 0.2747990868350219 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.3178669795015594, "p95_s": 2.5102758518944026, "mean_s": 1.5287515450788003 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.1681147290073568, "p95_s": 2.0163260291883485, "mean_s": 1.3505759721641273 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.8171210005093599, "p95_s": 4.314356681246135, "mean_s": 2.0569998578333375 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.9834761039965088, "p95_s": 2.912946939691027, "mean_s": 1.6338155833315493 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.925392291494063, "p95_s": 3.135248324839631, "mean_s": 2.8498740382444034 } } } }, "wic": { "expected_cases": 12, "case_union": 12, "common_complete_cases": 12, "excluded_from_common_support": 0, "common_case_ids": [ "wic-validation-0031", "wic-validation-0057", "wic-validation-0120", "wic-validation-0134", "wic-validation-0214", "wic-validation-0223", "wic-validation-0297", "wic-validation-0364", "wic-validation-0383", "wic-validation-0455", "wic-validation-0566", "wic-validation-0625" ], "models": { "Jev": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 0.2543736459920183, "p95_s": 0.3234148227420519, "mean_s": 0.2836198645818513 }, "complete_case_count": 12, "primary_case_median_latency_s": 0.25368537550093606, "case_median_distribution": { "n": 12, "median_s": 0.25368537550093606, "p95_s": 0.30710293298761826, "mean_s": 0.262482645829247 } }, "Luna": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.1709933955135057, "p95_s": 6.542056020996824, "mean_s": 2.0536330764718715 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.204547208501026, "case_median_distribution": { "n": 12, "median_s": 1.204547208501026, "p95_s": 4.388623170653589, "mean_s": 1.84934295125034 } }, "Qwen3.8 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 1.232746749999933, "p95_s": 3.1417095627402887, "mean_s": 1.539844539413429 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.232746749999933, "case_median_distribution": { "n": 12, "median_s": 1.232746749999933, "p95_s": 2.3547130063903743, "mean_s": 1.4562526458248612 } }, "DeepSeek V4.1 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 0.8470119164994685, "p95_s": 3.984443520756031, "mean_s": 1.7349614096634065 }, "complete_case_count": 12, "primary_case_median_latency_s": 0.8423815209971508, "case_median_distribution": { "n": 12, "median_s": 0.8423815209971508, "p95_s": 1.8055840479544711, "mean_s": 1.015465128502304 } }, "MiMo V2.6 Flash": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 2.740900333505124, "p95_s": 10.739882124762516, "mean_s": 3.6605685023904595 }, "complete_case_count": 12, "primary_case_median_latency_s": 1.451285146002192, "case_median_distribution": { "n": 12, "median_s": 1.451285146002192, "p95_s": 6.060241012800542, "mean_s": 2.4321024480013875 } }, "Hy4 Preview": { "all_attempts": { "attempts": 36, "valid": 36, "observed_positive_timings": 36, "median_s": 2.783512812500703, "p95_s": 3.4857779057565494, "mean_s": 2.8897121469999547 }, "complete_case_count": 12, "primary_case_median_latency_s": 2.783512812500703, "case_median_distribution": { "n": 12, "median_s": 2.783512812500703, "p95_s": 3.3230860812531313, "mean_s": 2.853456059080296 } } }, "comparisons": { "Luna / Jev": { "model": "Luna", "reference": "Jev", "paired_cases": 12, "direction": "Luna minus Jev; ratios Luna divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.748193332479197, "ci95": [ 4.377493010023091, 7.565170720896542 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.989332717275429, "ci95": [ 4.39652674268386, 6.323282248098764 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.5868603054210932, "ci95": [ 0.9790660782769919, 2.4397731594146537 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Jev; ratios Qwen3.8 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.859352840366568, "ci95": [ 4.370581973443682, 6.333728149015751 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.108367392736511, "ci95": [ 4.413876728160172, 5.693689311870285 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.1937699999956142, "ci95": [ 0.9574922836286229, 1.5192411894999769 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna": { "model": "Qwen3.8 Flash", "reference": "Luna", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Luna; ratios Qwen3.8 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.0234109060233507, "ci95": [ 0.62191193743462, 1.286956102288698 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.0389918711004014, "ci95": [ 0.8627152909940665, 1.2160597183430368 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.3930903054254789, "ci95": [ -1.3527591809803803, 0.3395663422956204 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Jev; ratios DeepSeek V4.1 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 3.3205758090459674, "ci95": [ 2.6551076433263896, 5.175786924454246 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 3.129828883759905, "ci95": [ 2.6664610099910977, 5.023948531124377 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.752982482673057, "ci95": [ 0.5347350098931202, 1.0184345106149217 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna": { "model": "DeepSeek V4.1 Flash", "reference": "Luna", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Luna; ratios DeepSeek V4.1 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.6993345840263373, "ci95": [ 0.4125054713829594, 1.1198261101393623 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.6034436757827928, "ci95": [ 0.5243656928197062, 0.9026211812881626 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.8338778227480361, "ci95": [ -1.8238121761075188, -0.13186858778353794 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Jev; ratios MiMo V2.6 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 5.720807291852884, "ci95": [ 4.78012448160691, 12.420410083865244 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.967073095040021, "ci95": [ 4.648088600286086, 11.652754940362392 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.1696198021721407, "ci95": [ 1.2390599273790333, 3.315330207711425 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna": { "model": "MiMo V2.6 Flash", "reference": "Luna", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Luna; ratios MiMo V2.6 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.2048387441852229, "ci95": [ 0.7101944749351677, 2.791377710652565 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.0863368773917081, "ci95": [ 0.8098045187769742, 1.9565233531282145 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.5827594967510473, "ci95": [ -0.7702426187574626, 2.0244563250729697 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 12, "direction": "Hy4 Preview minus Jev; ratios Hy4 Preview divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 10.972303023003517, "ci95": [ 10.137736641004953, 11.42339004992053 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 10.94094246162917, "ci95": [ 10.045193609375563, 11.634802302653483 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.5909734132510494, "ci95": [ 2.4514543320909676, 2.7931721276863755 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna": { "model": "Hy4 Preview", "reference": "Luna", "paired_cases": 12, "direction": "Hy4 Preview minus Luna; ratios Hy4 Preview divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 2.3108374606293665, "ci95": [ 1.4177671953954247, 2.5446301589067555 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.2578368500571053, "ci95": [ 1.3750773067310569, 2.511751069412824 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.004113107829956, "ci95": [ 0.08068413071802451, 1.6789732037239442 ], "defined_bootstrap_draws": 5000 } } } }, "pairwise_complete_sensitivity": { "Luna / Jev": { "model": "Luna", "reference": "Jev", "paired_cases": 12, "direction": "Luna minus Jev; ratios Luna divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.748193332479197, "ci95": [ 4.377493010023091, 7.565170720896542 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 4.989332717275429, "ci95": [ 4.39652674268386, 6.323282248098764 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.5868603054210932, "ci95": [ 0.9834859385554713, 2.4598617108003347 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Jev; ratios Qwen3.8 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 4.859352840366568, "ci95": [ 4.358582803306023, 6.3331373134677325 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.108367392736511, "ci95": [ 4.413876728160172, 5.693689311870285 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.1937699999956142, "ci95": [ 0.9597039882578732, 1.5131314364264348 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna": { "model": "Qwen3.8 Flash", "reference": "Luna", "paired_cases": 12, "direction": "Qwen3.8 Flash minus Luna; ratios Qwen3.8 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.0234109060233507, "ci95": [ 0.62191193743462, 1.286956102288698 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.0389918711004014, "ci95": [ 0.8627152909940665, 1.2160597183430368 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.3930903054254789, "ci95": [ -1.364856974784925, 0.3394013010621467 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Jev; ratios DeepSeek V4.1 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 3.3205758090459674, "ci95": [ 2.6490080411241688, 5.167621037823705 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 3.129828883759905, "ci95": [ 2.6664610099910977, 5.023948531124377 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.752982482673057, "ci95": [ 0.5232028439483353, 1.0055394262674477 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna": { "model": "DeepSeek V4.1 Flash", "reference": "Luna", "paired_cases": 12, "direction": "DeepSeek V4.1 Flash minus Luna; ratios DeepSeek V4.1 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 0.6993345840263373, "ci95": [ 0.42182893401351246, 1.1248981604442205 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 0.6034436757827928, "ci95": [ 0.5243656928197062, 0.9026211812881626 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": -0.8338778227480361, "ci95": [ -1.8059509335944313, -0.1274757458297256 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Jev; ratios MiMo V2.6 Flash divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 5.720807291852884, "ci95": [ 4.793151272449468, 12.388741437090665 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 5.967073095040021, "ci95": [ 4.648088600286086, 11.652754940362392 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.1696198021721407, "ci95": [ 1.2471369009537132, 3.2536177667401716 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna": { "model": "MiMo V2.6 Flash", "reference": "Luna", "paired_cases": 12, "direction": "MiMo V2.6 Flash minus Luna; ratios MiMo V2.6 Flash divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 1.2048387441852229, "ci95": [ 0.7029850996209521, 2.762723626679212 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 1.0863368773917081, "ci95": [ 0.8098045187769742, 1.9565233531282145 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 0.5827594967510473, "ci95": [ -0.7406468707612779, 1.9410388515504866 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 12, "direction": "Hy4 Preview minus Jev; ratios Hy4 Preview divided by Jev", "metrics": { "ratio_of_case_medians": { "estimate": 10.972303023003517, "ci95": [ 10.138028312962467, 11.42489882370045 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 10.94094246162917, "ci95": [ 10.045193609375563, 11.634802302653483 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 2.5909734132510494, "ci95": [ 2.4524758259192216, 2.7869302192376075 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna": { "model": "Hy4 Preview", "reference": "Luna", "paired_cases": 12, "direction": "Hy4 Preview minus Luna; ratios Hy4 Preview divided by Luna", "metrics": { "ratio_of_case_medians": { "estimate": 2.3108374606293665, "ci95": [ 1.4177671953954247, 2.5522707365359847 ], "defined_bootstrap_draws": 5000 }, "median_of_case_ratios": { "estimate": 2.2578368500571053, "ci95": [ 1.3750773067310569, 2.511751069412824 ], "defined_bootstrap_draws": 5000 }, "mean_case_latency_difference_s": { "estimate": 1.004113107829956, "ci95": [ 0.07840327566476245, 1.6903542375732283 ], "defined_bootstrap_draws": 5000 } } } }, "repeat_blocks": { "0": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.25731156300753355, "p95_s": 0.32575698955479304, "mean_s": 0.26683467025092494 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.2275833955063717, "p95_s": 5.067033527433522, "mean_s": 1.9810116424147661 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.3840435834863456, "p95_s": 4.072065462497994, "mean_s": 1.8049889133277854 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.7062305625004228, "p95_s": 1.2567228268817412, "mean_s": 0.8109250345805776 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.740900333505124, "p95_s": 11.191148414944465, "mean_s": 3.654576204925737 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.746729916500044, "p95_s": 3.8785292288070186, "mean_s": 2.9263959514161493 } }, "1": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.2519675835064845, "p95_s": 0.30317171494243667, "mean_s": 0.2573174618300982 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.1820268540031975, "p95_s": 2.5489638623548663, "mean_s": 1.4633919618330158 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.1696412289893487, "p95_s": 2.5903144438954766, "mean_s": 1.4603826216650002 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.0105602084950078, "p95_s": 9.501481368757947, "mean_s": 2.578840496581203 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.4584443129861029, "p95_s": 12.81054592081927, "mean_s": 4.395566222333097 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.8438163329992676, "p95_s": 3.082248678932956, "mean_s": 2.8294783159168824 } }, "2": { "Jev": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 0.26601133350050077, "p95_s": 0.6229745953431115, "mean_s": 0.32670746166453074 }, "Luna": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.1665710625093197, "p95_s": 9.987310796344534, "mean_s": 2.7164956251678327 }, "Qwen3.8 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.275262000010116, "p95_s": 1.842029168747831, "mean_s": 1.3541620832475019 }, "DeepSeek V4.1 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 1.3654157289856812, "p95_s": 4.856156978629583, "mean_s": 1.8151186978284386 }, "MiMo V2.6 Flash": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 3.5464209584897617, "p95_s": 5.867627081258979, "mean_s": 2.931563079912545 }, "Hy4 Preview": { "attempts": 12, "valid": 12, "observed_positive_timings": 12, "median_s": 2.851818833485595, "p95_s": 3.4457743750055667, "mean_s": 2.913262173666832 } } } } }, "measured_attempts": 1080, "expected_measured_attempts": 1080, "missing_measured_attempts": 0, "warmup_attempts": 36, "cost_all_attempts": { "requests": 1116, "known_billed_usd": 0.0549220258, "observed_bills": 1116, "missing_bills": 0, "total_billed_usd": 0.0549220258, "usd_per_1000_attempts": 0.04921328476702509 }, "cost_measured": { "requests": 1080, "known_billed_usd": 0.0533240682, "observed_bills": 1080, "missing_bills": 0, "total_billed_usd": 0.0533240682, "usd_per_1000_attempts": 0.04937413722222222 }, "method": "New interleaved timing only. Common all-model complete case support; median across 3 repeats then median across cases. Pairwise-complete sensitivity separately.", "comparative_speed_eligible": true }, "cache_sessions": [ { "source": { "run_dir": "benchmark/multimodel/runs/cache-20261002T175446Z", "metadata": { "schema_version": 1, "experiment": "multimodel_cache_policy", "created_at": "2026-10-02T17:54:46.572383+00:00", "models": [ { "label": "Jev", "id": "typesafe/jev-1.13", "provider": "TypeSafe", "provider_name": "TypeSafe", "prices": { "prompt": "0.0000002", "completion": "0.00000075" }, "reasoning": null, "cache_control": "unsupported", "max_tokens": null, "reasoning_mandatory": false, "protocol_difference": null }, { "label": "Luna", "id": "openai/gpt-6-luna", "provider": "OpenAI", "provider_name": "OpenAI", "prices": { "prompt": "0.0000001", "completion": "0.0000005", "web_search": "0.01", "input_cache_read": "0.00000001", "input_cache_write": "0.000000125", "overrides": [ { "min_prompt_tokens": 272000, "prompt": "0.0000002", "completion": "0.00000075", "input_cache_read": "0.00000002", "input_cache_write": "0.00000025" } ] }, "reasoning": { "effort": "none" }, "cache_control": "explicit", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null }, { "label": "Qwen3.8 Flash", "id": "qwen/qwen3.8-flash", "provider": "alibaba", "provider_name": "Alibaba", "prices": { "prompt": "0.00000015", "completion": "0.00000047", "input_cache_read": "0.000000016", "input_cache_write": "0.0000002" }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null }, { "label": "DeepSeek V4.1 Flash", "id": "deepseek/deepseek-v4.1-flash", "provider": "deepinfra/fp8", "provider_name": "DeepInfra", "prices": { "prompt": "0.00000014", "completion": "0.00000042", "input_cache_read": "0.0000000042", "discount": 0.3 }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null, "cache_reason": "No verified matched OpenRouter cache-enable/cache-disable treatment for this provider in this study." }, { "label": "MiMo V2.6 Flash", "id": "xiaomi/mimo-v2.6-flash", "provider": "xiaomi/fp8", "provider_name": "Xiaomi", "prices": { "prompt": "0.00000014", "completion": "0.00000028", "input_cache_read": "0.0000000028", "discount": 0 }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null, "cache_reason": "No verified matched OpenRouter cache-enable/cache-disable treatment for this provider in this study." }, { "label": "Hy4 Preview", "id": "tencent/hy4-preview", "provider": "tencent/fp8", "provider_name": "Tencent", "prices": { "prompt": "0.000000834", "completion": "0.000002501", "input_cache_read": "0.000000042", "discount": 0, "overrides": [ { "utc_start": 0, "utc_end": 1600, "prompt": "0.000000834", "completion": "0.000002501", "input_cache_read": "0.000000042" }, { "utc_start": 1600, "utc_end": 0, "prompt": "0.0000007506", "completion": "0.0000022509", "input_cache_read": "0.0000000378" } ] }, "reasoning": { "effort": "none" }, "cache_control": "unsupported", "max_tokens": 128, "reasoning_mandatory": false, "protocol_difference": null, "cache_reason": "No verified matched OpenRouter cache-enable/cache-disable treatment for this provider in this study." } ], "seed": 20261002, "blocks": 4, "cases": 96, "counterfactual_pairs": 48, "calls_planned": 676, "source_hashes": { "cases_sha256": "73e400bef4a7025d31ab77523f6dcafe9ae46ee2100c1b8e7d4bd444d3a88e0c", "rulebook_sha256": "01006795edc5f9ee13a1e4efc56fc8e19516d74f40cddbce2e72d19b00f0d43b", "prime_source_responses_sha256": "a837d11e4ad92e01c258a5d5e1defe82f6b64b7e4440ec6f383defc30d650e3f" }, "source_dir": "benchmark/expanded/runs/cache-20261002T095722Z", "protocol_sha256": "f3ddc1865467e45265cebd4b5e36c6512505ee1b1d46da11f823085b5283c616", "protocol_file": "cache_protocol_v2.py", "protocol_version": 2, "protocol_amendment": { "id": "administrative-cache-key-length-fix", "original_protocol_file": "cache_protocol.py", "original_protocol_sha256": "65276cccf0f45553045ef30da85c1bb2d7c98046264ac01696a2c58e23cb3374", "reason": "The original 66-character generated key exceeded the provider's 64-character limit and produced HTTP 400 responses.", "change": "Shorten the cache key prefix from model-comparison-cache- to mc-; validate every key before preparing any request. Preserve original HTTP/API errors on already invalid responses and stop after saving a failed priming attempt.", "unchanged": "Frozen cases, reference answers, rulebook, text serialization, schema, model settings, arm order and priming design are unchanged. Fresh per-block identifiers remain required." }, "plan_sha256": "239e42d0dfac906c1ad9af20e3c1c24b66f2d6a58d961f337e356152879a4bda", "automatic_retries": 0, "unknown_bill_allowance_usd": 0.01, "protocol": "Four sequential blocks. Fresh shared prefix nonce per block; one separate prime per controlled model per block. Frozen case order and all-model arm order randomized within each block. One serial injected adapter session. Unsupported cache models receive one baseline arm only.", "resume_policy": "Complete-block boundaries only; no repeated attempts.", "billing_guard": "Reserve max(adapter conservative estimate, $0.01) before each call. Known charges count at actual recorded values; each missing bill uses a $0.01 guard allowance while its reported price remains null. This local guard is not a remote account spending limit.", "status": "complete", "budget_usd": 1.5, "calls_completed": 676, "charged_usd": 0.1568607109, "known_charge_subtotal_usd": 0.1568607109, "cost_missing_n": 0, "unknown_bill_allowance_total_usd": 0.0, "budget_accounted_usd": 0.1568607109, "completed_blocks": [ 0, 1, 2, 3 ], "finished_at": "2026-10-02T18:17:40.202393+00:00", "calls_dispatched": 676, "unresolved_dispatch_n": 0 }, "responses_sha256": "df69dc65a48ec7f87721c187ad5470c51eeec40ae6ab5b928c597d6a0adf125f", "attempts": 676, "cost_all_attempts": { "requests": 676, "known_billed_usd": 0.1568607109, "observed_bills": 676, "missing_bills": 0, "total_billed_usd": 0.1568607109, "usd_per_1000_attempts": 0.23204247174556214 } }, "measured_attempts": 672, "prime_attempts": 4, "arms": { "DeepSeek V4.1 Flash": { "attempts": 96, "valid": 96, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9615244121423243, 1 ], "unique_cases": 96, "target_kind": "label", "cost": { "requests": 96, "known_billed_usd": 0.0076339032, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.0076339032, "usd_per_1000_attempts": 0.079519825, "usd_per_correct_answer": 0.00010602643333333333 }, "quality": { "correct": 72, "accuracy_all_attempts": 0.75, "accuracy_all_attempts_wilson95": [ 0.654901889760503, 0.8258603163106593 ], "balanced_accuracy_all_attempts": 0.75, "class_denominators": { "no": { "attempts": 48, "valid": 48, "correct": 44, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 48, "valid": 48, "correct": 28, "recall_all_attempts": 0.5833333333333334 } }, "accuracy_valid_only": 0.75, "accuracy_valid_only_wilson95": [ 0.654901889760503, 0.8258603163106593 ], "balanced_accuracy_valid_only": 0.75, "brier_valid_only": 0.25, "log_loss_valid_only": 8.634727415287264, "proper_score_denominator": 96 }, "latency": { "attempts": 96, "valid": 96, "observed_positive_timings": 96, "median_s": 1.0940515204856638, "p95_s": 4.6539065412653144, "mean_s": 1.6851937696022408 }, "prime_attempts": 0, "priming_cost": { "requests": 0, "known_billed_usd": 0.0, "observed_bills": 0, "missing_bills": 0, "total_billed_usd": 0.0, "usd_per_1000_attempts": null }, "setup_inclusive_cost": { "requests": 96, "known_billed_usd": 0.0076339032, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.0076339032, "usd_per_1000_attempts": 0.079519825 }, "reported_cache_usage": { "read_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 240896, "total": 240896 }, "write_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 0, "total": 0 }, "positive_read_calls": 92, "all_counters_observed_zero": false }, "setup_inclusive_usd_per_1000_decisions": 0.079519825 }, "Hy4 Preview": { "attempts": 96, "valid": 96, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9615244121423243, 1 ], "unique_cases": 96, "target_kind": "label", "cost": { "requests": 96, "known_billed_usd": 0.0795155877, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.0795155877, "usd_per_1000_attempts": 0.828287371875, "usd_per_correct_answer": 0.0010194306115384616 }, "quality": { "correct": 78, "accuracy_all_attempts": 0.8125, "accuracy_all_attempts_wilson95": [ 0.7229774574820691, 0.8779753001068837 ], "balanced_accuracy_all_attempts": 0.8125, "class_denominators": { "no": { "attempts": 48, "valid": 48, "correct": 40, "recall_all_attempts": 0.8333333333333334 }, "yes": { "attempts": 48, "valid": 48, "correct": 38, "recall_all_attempts": 0.7916666666666666 } }, "accuracy_valid_only": 0.8125, "accuracy_valid_only_wilson95": [ 0.7229774574820691, 0.8779753001068837 ], "balanced_accuracy_valid_only": 0.8125, "brier_valid_only": 0.1875, "log_loss_valid_only": 6.476087207164937, "proper_score_denominator": 96 }, "latency": { "attempts": 96, "valid": 96, "observed_positive_timings": 96, "median_s": 3.3181042079959298, "p95_s": 4.899055228983343, "mean_s": 3.785576113729197 }, "prime_attempts": 0, "priming_cost": { "requests": 0, "known_billed_usd": 0.0, "observed_bills": 0, "missing_bills": 0, "total_billed_usd": 0.0, "usd_per_1000_attempts": null }, "setup_inclusive_cost": { "requests": 96, "known_billed_usd": 0.0795155877, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.0795155877, "usd_per_1000_attempts": 0.828287371875 }, "reported_cache_usage": { "read_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 193920, "total": 193920 }, "write_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 0, "total": 0 }, "positive_read_calls": 76, "all_counters_observed_zero": false }, "setup_inclusive_usd_per_1000_decisions": 0.828287371875 }, "Jev": { "attempts": 96, "valid": 96, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9615244121423243, 1 ], "unique_cases": 96, "target_kind": "label", "cost": { "requests": 96, "known_billed_usd": 0.01293264, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.01293264, "usd_per_1000_attempts": 0.134715, "usd_per_correct_answer": 0.00013906064516129032 }, "quality": { "correct": 93, "accuracy_all_attempts": 0.96875, "accuracy_all_attempts_wilson95": [ 0.9121132067546303, 0.989315929628799 ], "balanced_accuracy_all_attempts": 0.96875, "class_denominators": { "no": { "attempts": 48, "valid": 48, "correct": 45, "recall_all_attempts": 0.9375 }, "yes": { "attempts": 48, "valid": 48, "correct": 48, "recall_all_attempts": 1.0 } }, "accuracy_valid_only": 0.96875, "accuracy_valid_only_wilson95": [ 0.9121132067546303, 0.989315929628799 ], "balanced_accuracy_valid_only": 0.96875, "brier_valid_only": 0.037015625, "log_loss_valid_only": 0.17170647544742138, "proper_score_denominator": 96 }, "latency": { "attempts": 96, "valid": 96, "observed_positive_timings": 96, "median_s": 0.29669247899437323, "p95_s": 0.35924302100465866, "mean_s": 0.303336100270523 }, "prime_attempts": 0, "priming_cost": { "requests": 0, "known_billed_usd": 0.0, "observed_bills": 0, "missing_bills": 0, "total_billed_usd": 0.0, "usd_per_1000_attempts": null }, "setup_inclusive_cost": { "requests": 96, "known_billed_usd": 0.01293264, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.01293264, "usd_per_1000_attempts": 0.134715 }, "reported_cache_usage": { "read_tokens": { "observed_calls": 0, "unknown_calls": 96, "known_subtotal": 0, "total": null }, "write_tokens": { "observed_calls": 0, "unknown_calls": 96, "known_subtotal": 0, "total": null }, "positive_read_calls": 0, "all_counters_observed_zero": false }, "setup_inclusive_usd_per_1000_decisions": 0.134715 }, "Luna cached": { "attempts": 96, "valid": 96, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9615244121423243, 1 ], "unique_cases": 96, "target_kind": "label", "cost": { "requests": 96, "known_billed_usd": 0.0068888000000000005, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.0068888000000000005, "usd_per_1000_attempts": 0.07175833333333334, "usd_per_correct_answer": 8.299759036144578e-05 }, "quality": { "correct": 83, "accuracy_all_attempts": 0.8645833333333334, "accuracy_all_attempts_wilson95": [ 0.7819886149785562, 0.9191229355418891 ], "balanced_accuracy_all_attempts": 0.8645833333333333, "class_denominators": { "no": { "attempts": 48, "valid": 48, "correct": 45, "recall_all_attempts": 0.9375 }, "yes": { "attempts": 48, "valid": 48, "correct": 38, "recall_all_attempts": 0.7916666666666666 } }, "accuracy_valid_only": 0.8645833333333334, "accuracy_valid_only_wilson95": [ 0.7819886149785562, 0.9191229355418891 ], "balanced_accuracy_valid_only": 0.8645833333333333, "brier_valid_only": 0.13541666666666666, "log_loss_valid_only": 4.67715095756385, "proper_score_denominator": 96 }, "latency": { "attempts": 96, "valid": 96, "observed_positive_timings": 96, "median_s": 1.137883604504168, "p95_s": 1.8930704482409055, "mean_s": 1.2779182239073634 }, "prime_attempts": 4, "priming_cost": { "requests": 4, "known_billed_usd": 0.0014666, "observed_bills": 4, "missing_bills": 0, "total_billed_usd": 0.0014666, "usd_per_1000_attempts": 0.36665 }, "setup_inclusive_cost": { "requests": 100, "known_billed_usd": 0.0083554, "observed_bills": 100, "missing_bills": 0, "total_billed_usd": 0.0083554, "usd_per_1000_attempts": 0.083554 }, "reported_cache_usage": { "read_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 246720, "total": 246720 }, "write_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 0, "total": 0 }, "positive_read_calls": 96, "all_counters_observed_zero": false }, "setup_inclusive_usd_per_1000_decisions": 0.08703541666666667 }, "Luna uncached": { "attempts": 96, "valid": 96, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9615244121423243, 1 ], "unique_cases": 96, "target_kind": "label", "cost": { "requests": 96, "known_billed_usd": 0.029084, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.029084, "usd_per_1000_attempts": 0.30295833333333333, "usd_per_correct_answer": 0.00035040963855421687 }, "quality": { "correct": 83, "accuracy_all_attempts": 0.8645833333333334, "accuracy_all_attempts_wilson95": [ 0.7819886149785562, 0.9191229355418891 ], "balanced_accuracy_all_attempts": 0.8645833333333333, "class_denominators": { "no": { "attempts": 48, "valid": 48, "correct": 45, "recall_all_attempts": 0.9375 }, "yes": { "attempts": 48, "valid": 48, "correct": 38, "recall_all_attempts": 0.7916666666666666 } }, "accuracy_valid_only": 0.8645833333333334, "accuracy_valid_only_wilson95": [ 0.7819886149785562, 0.9191229355418891 ], "balanced_accuracy_valid_only": 0.8645833333333333, "brier_valid_only": 0.13541666666666666, "log_loss_valid_only": 4.67715095756385, "proper_score_denominator": 96 }, "latency": { "attempts": 96, "valid": 96, "observed_positive_timings": 96, "median_s": 1.092991729005007, "p95_s": 2.6346865937448456, "mean_s": 1.296985241821858 }, "prime_attempts": 0, "priming_cost": { "requests": 0, "known_billed_usd": 0.0, "observed_bills": 0, "missing_bills": 0, "total_billed_usd": 0.0, "usd_per_1000_attempts": null }, "setup_inclusive_cost": { "requests": 96, "known_billed_usd": 0.029084, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.029084, "usd_per_1000_attempts": 0.30295833333333333 }, "reported_cache_usage": { "read_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 0, "total": 0 }, "write_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 0, "total": 0 }, "positive_read_calls": 0, "all_counters_observed_zero": true }, "setup_inclusive_usd_per_1000_decisions": 0.30295833333333333 }, "MiMo V2.6 Flash": { "attempts": 96, "valid": 96, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9615244121423243, 1 ], "unique_cases": 96, "target_kind": "label", "cost": { "requests": 96, "known_billed_usd": 0.007778512, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.007778512, "usd_per_1000_attempts": 0.08102616666666666, "usd_per_correct_answer": 0.00010101963636363636 }, "quality": { "correct": 77, "accuracy_all_attempts": 0.8020833333333334, "accuracy_all_attempts_wilson95": [ 0.7114483305401257, 0.869472668462529 ], "balanced_accuracy_all_attempts": 0.8020833333333333, "class_denominators": { "no": { "attempts": 48, "valid": 48, "correct": 41, "recall_all_attempts": 0.8541666666666666 }, "yes": { "attempts": 48, "valid": 48, "correct": 36, "recall_all_attempts": 0.75 } }, "accuracy_valid_only": 0.8020833333333334, "accuracy_valid_only_wilson95": [ 0.7114483305401257, 0.869472668462529 ], "balanced_accuracy_valid_only": 0.8020833333333333, "brier_valid_only": 0.19791666666666666, "log_loss_valid_only": 6.8358577988053595, "proper_score_denominator": 96 }, "latency": { "attempts": 96, "valid": 96, "observed_positive_timings": 96, "median_s": 2.0327555624971865, "p95_s": 9.622894812498998, "mean_s": 3.881833391466595 }, "prime_attempts": 0, "priming_cost": { "requests": 0, "known_billed_usd": 0.0, "observed_bills": 0, "missing_bills": 0, "total_billed_usd": 0.0, "usd_per_1000_attempts": null }, "setup_inclusive_cost": { "requests": 96, "known_billed_usd": 0.007778512, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.007778512, "usd_per_1000_attempts": 0.08102616666666666 }, "reported_cache_usage": { "read_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 237440, "total": 237440 }, "write_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 0, "total": 0 }, "positive_read_calls": 90, "all_counters_observed_zero": false }, "setup_inclusive_usd_per_1000_decisions": 0.08102616666666666 }, "Qwen3.8 Flash": { "attempts": 96, "valid": 96, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.9615244121423243, 1 ], "unique_cases": 96, "target_kind": "label", "cost": { "requests": 96, "known_billed_usd": 0.011560668, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.011560668, "usd_per_1000_attempts": 0.12042362499999999, "usd_per_correct_answer": 0.00015836531506849315 }, "quality": { "correct": 73, "accuracy_all_attempts": 0.7604166666666666, "accuracy_all_attempts_wilson95": [ 0.6660761768160148, 0.834717787841446 ], "balanced_accuracy_all_attempts": 0.7604166666666667, "class_denominators": { "no": { "attempts": 48, "valid": 48, "correct": 48, "recall_all_attempts": 1.0 }, "yes": { "attempts": 48, "valid": 48, "correct": 25, "recall_all_attempts": 0.5208333333333334 } }, "accuracy_valid_only": 0.7604166666666666, "accuracy_valid_only_wilson95": [ 0.6660761768160148, 0.834717787841446 ], "balanced_accuracy_valid_only": 0.7604166666666667, "brier_valid_only": 0.23958333333333334, "log_loss_valid_only": 8.274915177947351, "proper_score_denominator": 96 }, "latency": { "attempts": 96, "valid": 96, "observed_positive_timings": 96, "median_s": 1.489837562010507, "p95_s": 4.44317476022843, "mean_s": 1.999996528197092 }, "prime_attempts": 0, "priming_cost": { "requests": 0, "known_billed_usd": 0.0, "observed_bills": 0, "missing_bills": 0, "total_billed_usd": 0.0, "usd_per_1000_attempts": null }, "setup_inclusive_cost": { "requests": 96, "known_billed_usd": 0.011560668, "observed_bills": 96, "missing_bills": 0, "total_billed_usd": 0.011560668, "usd_per_1000_attempts": 0.12042362499999999 }, "reported_cache_usage": { "read_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 238848, "total": 238848 }, "write_tokens": { "observed_calls": 96, "unknown_calls": 0, "known_subtotal": 0, "total": 0 }, "positive_read_calls": 92, "all_counters_observed_zero": false }, "setup_inclusive_usd_per_1000_decisions": 0.12042362499999999 } }, "cost_all_attempts": { "requests": 676, "known_billed_usd": 0.1568607109, "observed_bills": 676, "missing_bills": 0, "total_billed_usd": 0.1568607109, "usd_per_1000_attempts": 0.23204247174556214 }, "comparisons": { "DeepSeek V4.1 Flash / Jev": { "model": "DeepSeek V4.1 Flash", "reference": "Jev", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.21875, "ci95": [ -0.2916666666666667, -0.15625 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.212984375, "ci95": [ 0.142301640625, 0.28785255208333327 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 3.6741954319535193, "ci95": [ 3.3655427095585395, 4.229061788908665 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 1.381857669331718, "ci95": [ 1.0723213421618234, 1.7670511850205837 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 0.5902818914003637, "ci95": [ 0.5030609649459167, 0.7030323727459548 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.7604166666666666, "ci95": [ 0.6875, 0.8333333333333334 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna uncached": { "model": "DeepSeek V4.1 Flash", "reference": "Luna uncached", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.11458333333333333, "ci95": [ -0.19791666666666666, -0.03125 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.11458333333333333, "ci95": [ 0.03125, 0.19791666666666666 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 0.9646863373923716, "ci95": [ 0.8153061726777455, 1.1109767812464924 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 0.3882085277803829, "ci95": [ 0.03827189361168925, 0.7796755161451525 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 0.26247776096822995, "ci95": [ 0.2229870417343145, 0.3120897067329635 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.8020833333333334, "ci95": [ 0.7291666666666666, 0.875 ], "defined_bootstrap_draws": 5000 } } }, "DeepSeek V4.1 Flash / Luna cached": { "model": "DeepSeek V4.1 Flash", "reference": "Luna cached", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.11458333333333333, "ci95": [ -0.19791666666666666, -0.03125 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.11458333333333333, "ci95": [ 0.03125, 0.19791666666666666 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 0.9581496543040593, "ci95": [ 0.8297981522881752, 1.0459228514323586 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 0.4072755456948774, "ci95": [ 0.07020034304964612, 0.8120457593895857 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 1.1081615375682266, "ci95": [ 0.9437523145801486, 1.312388118113394 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.8020833333333334, "ci95": [ 0.7291666666666666, 0.8645833333333334 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Jev": { "model": "Hy4 Preview", "reference": "Jev", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.15625, "ci95": [ -0.23958333333333334, -0.08333333333333333 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.150484375, "ci95": [ 0.07655479166666666, 0.23490916666666659 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 11.249239603022946, "ci95": [ 11.022130257918489, 11.61491231930907 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 3.4822400134586737, "ci95": [ 3.0983575347027683, 3.990597783517955 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 6.14844205823405, "ci95": [ 5.18596326896565, 7.179426570364711 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.8229166666666666, "ci95": [ 0.7395833333333334, 0.8958333333333334 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna uncached": { "model": "Hy4 Preview", "reference": "Luna uncached", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.052083333333333336, "ci95": [ -0.13541666666666666, 0.020833333333333332 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.052083333333333336, "ci95": [ -0.020833333333333332, 0.13541666666666666 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 3.00661246864617, "ci95": [ 2.9371281214864, 3.1222136497789537 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 2.488590871907339, "ci95": [ 2.0706586487800753, 3.0370731452836086 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 2.733997651629762, "ci95": [ 2.300167982819559, 3.1858248544642427 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.84375, "ci95": [ 0.7708333333333334, 0.9065104166666629 ], "defined_bootstrap_draws": 5000 } } }, "Hy4 Preview / Luna cached": { "model": "Hy4 Preview", "reference": "Luna cached", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.052083333333333336, "ci95": [ -0.14583333333333334, 0.041666666666666664 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.052083333333333336, "ci95": [ -0.041666666666666664, 0.14583333333333334 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 2.997661768068612, "ci95": [ 2.810007248515479, 3.0829803603664687 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 2.5076578898218336, "ci95": [ 2.094866117008428, 3.032069402613258 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 11.542734249796771, "ci95": [ 9.692397263679082, 13.422975323688819 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.8020833333333334, "ci95": [ 0.7395833333333334, 0.8645833333333334 ], "defined_bootstrap_draws": 5000 } } }, "Luna cached / Jev": { "model": "Luna cached", "reference": "Jev", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.10416666666666667, "ci95": [ -0.17708333333333334, -0.041666666666666664 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.09840104166666667, "ci95": [ 0.031245546875000002, 0.1690626302083333 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 3.815834886021664, "ci95": [ 3.6349684618805274, 4.037298933977299 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 0.9745821236368405, "ci95": [ 0.8746962963790035, 1.1012339843654368 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 0.5326677306412303, "ci95": [ 0.5324723620775539, 0.5328618530133228 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.8541666666666666, "ci95": [ 0.78125, 0.9166666666666666 ], "defined_bootstrap_draws": 5000 } } }, "Luna cached / Luna uncached": { "model": "Luna cached", "reference": "Luna uncached", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": 0.0, "ci95": [ -0.07291666666666667, 0.0625 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.0, "ci95": [ -0.0625, 0.07291666666666667 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 1.063527991834596, "ci95": [ 1.0121728441603655, 1.0861718216301421 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": -0.0190670179144945, "ci95": [ -0.21037927986230898, 0.1587752964901255 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 0.2368587539540641, "ci95": [ 0.23678002819710464, 0.236937463471654 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.8541666666666666, "ci95": [ 0.7708333333333334, 0.9270833333333334 ], "defined_bootstrap_draws": 5000 } } }, "Luna uncached / Jev": { "model": "Luna uncached", "reference": "Jev", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.10416666666666667, "ci95": [ -0.17708333333333334, -0.041666666666666664 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.09840104166666667, "ci95": [ 0.035059140625, 0.16517528645833332 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 3.795354225874201, "ci95": [ 3.681096142278257, 3.8623096892576787 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 0.993649141551335, "ci95": [ 0.8654671819193481, 1.157535235972735 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 2.248883445298098, "ci95": [ 2.248620529749562, 2.2491360832945917 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.8333333333333334, "ci95": [ 0.7604166666666666, 0.90625 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Jev": { "model": "MiMo V2.6 Flash", "reference": "Jev", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.16666666666666666, "ci95": [ -0.25, -0.08333333333333333 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.16090104166666666, "ci95": [ 0.08314518229166667, 0.24393239583333332 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 7.395829823095138, "ci95": [ 5.590675085688357, 8.318084423614097 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 3.578497291196072, "ci95": [ 2.5264392958795576, 4.896178954997526 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 0.6014635836147917, "ci95": [ 0.4861184188684411, 0.7385644504987149 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.7916666666666666, "ci95": [ 0.7083333333333334, 0.875 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna uncached": { "model": "MiMo V2.6 Flash", "reference": "Luna uncached", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.0625, "ci95": [ -0.16666666666666666, 0.041666666666666664 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.0625, "ci95": [ -0.041666666666666664, 0.16666666666666666 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 1.7810267448584103, "ci95": [ 1.5161925386548538, 2.211345781698135 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 2.584848149644737, "ci95": [ 1.477481876830734, 4.007231633833733 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 0.2674498693439692, "ci95": [ 0.2168023889782943, 0.3274876863250605 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.7708333333333334, "ci95": [ 0.6875, 0.8541666666666666 ], "defined_bootstrap_draws": 5000 } } }, "MiMo V2.6 Flash / Luna cached": { "model": "MiMo V2.6 Flash", "reference": "Luna cached", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.0625, "ci95": [ -0.15625, 0.03125 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.0625, "ci95": [ -0.03125, 0.15625 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 1.9394148586579867, "ci95": [ 1.445387895772096, 2.1926077845819436 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 2.6039151675592316, "ci95": [ 1.5533555976609021, 3.95356937596905 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 1.1291534084310764, "ci95": [ 0.9122090607767188, 1.3910764523324304 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.7708333333333334, "ci95": [ 0.6875, 0.8541666666666666 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Jev": { "model": "Qwen3.8 Flash", "reference": "Jev", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.20833333333333334, "ci95": [ -0.28125, -0.13541666666666666 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.20256770833333335, "ci95": [ 0.1303175260416667, 0.2737657552083333 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 4.960331949047253, "ci95": [ 4.542684407693423, 5.712942476734273 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 1.6966604279265691, "ci95": [ 1.4787098374644454, 1.9466795315784944 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 0.8939140036373083, "ci95": [ 0.8055572984221836, 1.0068333784250318 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.7291666666666666, "ci95": [ 0.65625, 0.8020833333333334 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna uncached": { "model": "Qwen3.8 Flash", "reference": "Luna uncached", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.10416666666666667, "ci95": [ -0.17708333333333334, -0.041666666666666664 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.10416666666666667, "ci95": [ 0.041666666666666664, 0.17708333333333334 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 1.4120695400941914, "ci95": [ 1.2566696687874648, 1.5357134222298363 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 0.7030112863752341, "ci95": [ 0.43743321551673187, 1.0013560350615685 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 0.39749236693714757, "ci95": [ 0.3591618878781386, 0.44604272901331027 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.8125, "ci95": [ 0.7291666666666666, 0.8854166666666666 ], "defined_bootstrap_draws": 5000 } } }, "Qwen3.8 Flash / Luna cached": { "model": "Qwen3.8 Flash", "reference": "Luna cached", "paired_cases": 96, "valid_response_pairs": 96, "clusters": 48, "fixed_blocks": 4, "metrics": { "accuracy_difference": { "estimate": -0.10416666666666667, "ci95": [ -0.17708333333333334, -0.020833333333333332 ], "defined_bootstrap_draws": 5000 }, "brier_valid_pairs_difference": { "estimate": 0.10416666666666667, "ci95": [ 0.020833333333333332, 0.17708333333333334 ], "defined_bootstrap_draws": 5000 }, "median_paired_latency_ratio": { "estimate": 1.3086111128136941, "ci95": [ 1.2378733638523902, 1.4727166066911646 ], "defined_bootstrap_draws": 5000 }, "mean_latency_difference_s": { "estimate": 0.7220783042897286, "ci95": [ 0.46421608751289417, 1.016212921590007 ], "defined_bootstrap_draws": 5000 }, "cost_ratio": { "estimate": 1.6781831378469398, "ci95": [ 1.5118119847353402, 1.884024657857065 ], "defined_bootstrap_draws": 5000 }, "label_agreement_valid_pairs": { "estimate": 0.7916666666666666, "ci95": [ 0.7083333333333334, 0.8645833333333334 ], "defined_bootstrap_draws": 5000 } } } }, "blocks": { "0": { "DeepSeek V4.1 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0019514768, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0019514768, "usd_per_1000_attempts": 0.08131153333333332, "usd_per_correct_answer": 0.00011479275294117647 }, "quality": { "correct": 17, "accuracy_all_attempts": 0.7083333333333334, "accuracy_all_attempts_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_all_attempts": 0.7083333333333334, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 10, "recall_all_attempts": 0.8333333333333334 }, "yes": { "attempts": 12, "valid": 12, "correct": 7, "recall_all_attempts": 0.5833333333333334 } }, "accuracy_valid_only": 0.7083333333333334, "accuracy_valid_only_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_valid_only": 0.7083333333333334, "brier_valid_only": 0.2916666666666667, "log_loss_valid_only": 10.073876414968133, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.452812958494178, "p95_s": 6.41591164184501, "mean_s": 2.2355918508316486 } }, "Hy4 Preview": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.020081179799999998, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.020081179799999998, "usd_per_1000_attempts": 0.8367158249999999, "usd_per_correct_answer": 0.0011812458705882352 }, "quality": { "correct": 17, "accuracy_all_attempts": 0.7083333333333334, "accuracy_all_attempts_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_all_attempts": 0.7083333333333334, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 7, "recall_all_attempts": 0.5833333333333334 }, "yes": { "attempts": 12, "valid": 12, "correct": 10, "recall_all_attempts": 0.8333333333333334 } }, "accuracy_valid_only": 0.7083333333333334, "accuracy_valid_only_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_valid_only": 0.7083333333333334, "brier_valid_only": 0.2916666666666667, "log_loss_valid_only": 10.073976364646908, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 3.3630395629879786, "p95_s": 4.265053635605726, "mean_s": 3.435057916750035 } }, "Jev": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0032315219999999997, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0032315219999999997, "usd_per_1000_attempts": 0.13464675, "usd_per_correct_answer": 0.00014050095652173912 }, "quality": { "correct": 23, "accuracy_all_attempts": 0.9583333333333334, "accuracy_all_attempts_wilson95": [ 0.7975819352134406, 0.9926065438344942 ], "balanced_accuracy_all_attempts": 0.9583333333333333, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 } }, "accuracy_valid_only": 0.9583333333333334, "accuracy_valid_only_wilson95": [ 0.7975819352134406, 0.9926065438344942 ], "balanced_accuracy_valid_only": 0.9583333333333333, "brier_valid_only": 0.05750833333333333, "log_loss_valid_only": 0.23126552318643578, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 0.30488337500719354, "p95_s": 0.41906328925397235, "mean_s": 0.3150322187090448 } }, "Luna cached": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.00172216, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.00172216, "usd_per_1000_attempts": 0.07175666666666666, "usd_per_correct_answer": 9.064e-05 }, "quality": { "correct": 19, "accuracy_all_attempts": 0.7916666666666666, "accuracy_all_attempts_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_all_attempts": 0.7916666666666666, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 8, "recall_all_attempts": 0.6666666666666666 } }, "accuracy_valid_only": 0.7916666666666666, "accuracy_valid_only_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_valid_only": 0.7916666666666666, "brier_valid_only": 0.20833333333333334, "log_loss_valid_only": 7.195611732165985, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.024862916499842, "p95_s": 1.394219979396439, "mean_s": 1.11391420304426 } }, "Luna uncached": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0072688, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0072688, "usd_per_1000_attempts": 0.3028666666666666, "usd_per_correct_answer": 0.0003825684210526316 }, "quality": { "correct": 19, "accuracy_all_attempts": 0.7916666666666666, "accuracy_all_attempts_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_all_attempts": 0.7916666666666667, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 10, "recall_all_attempts": 0.8333333333333334 }, "yes": { "attempts": 12, "valid": 12, "correct": 9, "recall_all_attempts": 0.75 } }, "accuracy_valid_only": 0.7916666666666666, "accuracy_valid_only_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_valid_only": 0.7916666666666667, "brier_valid_only": 0.20833333333333334, "log_loss_valid_only": 7.195645048725576, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 0.967239604506176, "p95_s": 1.1057805766817181, "mean_s": 1.1592397969592032 } }, "MiMo V2.6 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.001783404, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.001783404, "usd_per_1000_attempts": 0.0743085, "usd_per_correct_answer": 0.00010490611764705882 }, "quality": { "correct": 17, "accuracy_all_attempts": 0.7083333333333334, "accuracy_all_attempts_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_all_attempts": 0.7083333333333334, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 10, "recall_all_attempts": 0.8333333333333334 }, "yes": { "attempts": 12, "valid": 12, "correct": 7, "recall_all_attempts": 0.5833333333333334 } }, "accuracy_valid_only": 0.7083333333333334, "accuracy_valid_only_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_valid_only": 0.7083333333333334, "brier_valid_only": 0.2916666666666667, "log_loss_valid_only": 10.073876414968133, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 2.0327555624971865, "p95_s": 4.723352743779833, "mean_s": 2.4133820711249427 } }, "Qwen3.8 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.002931972, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.002931972, "usd_per_1000_attempts": 0.12216550000000001, "usd_per_correct_answer": 0.00016288733333333333 }, "quality": { "correct": 18, "accuracy_all_attempts": 0.75, "accuracy_all_attempts_wilson95": [ 0.55100555994846, 0.8800063377140499 ], "balanced_accuracy_all_attempts": 0.75, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 6, "recall_all_attempts": 0.5 } }, "accuracy_valid_only": 0.75, "accuracy_valid_only_wilson95": [ 0.55100555994846, 0.8800063377140499 ], "balanced_accuracy_valid_only": 0.75, "brier_valid_only": 0.25, "log_loss_valid_only": 8.634694098727673, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.3009495209989836, "p95_s": 2.2365433901519274, "mean_s": 1.4227698454596975 } } }, "1": { "DeepSeek V4.1 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0018881072, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0018881072, "usd_per_1000_attempts": 0.07867113333333334, "usd_per_correct_answer": 9.937406315789474e-05 }, "quality": { "correct": 19, "accuracy_all_attempts": 0.7916666666666666, "accuracy_all_attempts_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_all_attempts": 0.7916666666666666, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 8, "recall_all_attempts": 0.6666666666666666 } }, "accuracy_valid_only": 0.7916666666666666, "accuracy_valid_only_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_valid_only": 0.7916666666666666, "brier_valid_only": 0.20833333333333334, "log_loss_valid_only": 7.195611732165985, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.0346363749995362, "p95_s": 2.262577366898767, "mean_s": 1.1763606805385887 } }, "Hy4 Preview": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0197956233, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0197956233, "usd_per_1000_attempts": 0.8248176375, "usd_per_correct_answer": 0.000899801059090909 }, "quality": { "correct": 22, "accuracy_all_attempts": 0.9166666666666666, "accuracy_all_attempts_wilson95": [ 0.741511978067893, 0.9768411847029567 ], "balanced_accuracy_all_attempts": 0.9166666666666667, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 10, "recall_all_attempts": 0.8333333333333334 } }, "accuracy_valid_only": 0.9166666666666666, "accuracy_valid_only_wilson95": [ 0.741511978067893, 0.9768411847029567 ], "balanced_accuracy_valid_only": 0.9166666666666667, "brier_valid_only": 0.08333333333333333, "log_loss_valid_only": 2.878231366242558, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 3.3566275000193855, "p95_s": 5.135342039348322, "mean_s": 4.194028993042593 } }, "Jev": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.00323379, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.00323379, "usd_per_1000_attempts": 0.13474125, "usd_per_correct_answer": 0.00013474125 }, "quality": { "correct": 24, "accuracy_all_attempts": 1.0, "accuracy_all_attempts_wilson95": [ 0.8620237953250198, 1 ], "balanced_accuracy_all_attempts": 1.0, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 } }, "accuracy_valid_only": 1.0, "accuracy_valid_only_wilson95": [ 0.8620237953250198, 1 ], "balanced_accuracy_valid_only": 1.0, "brier_valid_only": 0.009354166666666665, "log_loss_valid_only": 0.09145425653523921, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 0.28965062501083594, "p95_s": 0.3307093481591437, "mean_s": 0.2984876789572202 } }, "Luna cached": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.00172228, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.00172228, "usd_per_1000_attempts": 0.07176166666666667, "usd_per_correct_answer": 7.828545454545455e-05 }, "quality": { "correct": 22, "accuracy_all_attempts": 0.9166666666666666, "accuracy_all_attempts_wilson95": [ 0.741511978067893, 0.9768411847029567 ], "balanced_accuracy_all_attempts": 0.9166666666666667, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 10, "recall_all_attempts": 0.8333333333333334 } }, "accuracy_valid_only": 0.9166666666666666, "accuracy_valid_only_wilson95": [ 0.741511978067893, 0.9768411847029567 ], "balanced_accuracy_valid_only": 0.9166666666666667, "brier_valid_only": 0.08333333333333333, "log_loss_valid_only": 2.878231366242558, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.1588594999921042, "p95_s": 1.752585599913436, "mean_s": 1.2428047915818752 } }, "Luna uncached": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0072754, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0072754, "usd_per_1000_attempts": 0.30314166666666664, "usd_per_correct_answer": 0.00033069999999999996 }, "quality": { "correct": 22, "accuracy_all_attempts": 0.9166666666666666, "accuracy_all_attempts_wilson95": [ 0.741511978067893, 0.9768411847029567 ], "balanced_accuracy_all_attempts": 0.9166666666666667, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 10, "recall_all_attempts": 0.8333333333333334 } }, "accuracy_valid_only": 0.9166666666666666, "accuracy_valid_only_wilson95": [ 0.741511978067893, 0.9768411847029567 ], "balanced_accuracy_valid_only": 0.9166666666666667, "brier_valid_only": 0.08333333333333333, "log_loss_valid_only": 2.878231366242558, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.033214979499462, "p95_s": 2.471925718741839, "mean_s": 1.2707396840814909 } }, "MiMo V2.6 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0018033848, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0018033848, "usd_per_1000_attempts": 0.07514103333333333, "usd_per_correct_answer": 8.587546666666667e-05 }, "quality": { "correct": 21, "accuracy_all_attempts": 0.875, "accuracy_all_attempts_wilson95": [ 0.6899611872949993, 0.9565566591987655 ], "balanced_accuracy_all_attempts": 0.875, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 9, "recall_all_attempts": 0.75 } }, "accuracy_valid_only": 0.875, "accuracy_valid_only_wilson95": [ 0.6899611872949993, 0.9565566591987655 ], "balanced_accuracy_valid_only": 0.875, "brier_valid_only": 0.125, "log_loss_valid_only": 4.317347049363836, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 2.793738062508055, "p95_s": 9.886241262500697, "mean_s": 4.2030960138314795 } }, "Qwen3.8 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.00281806, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.00281806, "usd_per_1000_attempts": 0.11741916666666666, "usd_per_correct_answer": 0.00013419333333333333 }, "quality": { "correct": 21, "accuracy_all_attempts": 0.875, "accuracy_all_attempts_wilson95": [ 0.6899611872949993, 0.9565566591987655 ], "balanced_accuracy_all_attempts": 0.875, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 9, "recall_all_attempts": 0.75 } }, "accuracy_valid_only": 0.875, "accuracy_valid_only_wilson95": [ 0.6899611872949993, 0.9565566591987655 ], "balanced_accuracy_valid_only": 0.875, "brier_valid_only": 0.125, "log_loss_valid_only": 4.317347049363836, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.7670792919961968, "p95_s": 3.5207512534980197, "mean_s": 2.068007861207055 } } }, "2": { "DeepSeek V4.1 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0018735248, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0018735248, "usd_per_1000_attempts": 0.07806353333333334, "usd_per_correct_answer": 0.00011020734117647059 }, "quality": { "correct": 17, "accuracy_all_attempts": 0.7083333333333334, "accuracy_all_attempts_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_all_attempts": 0.7083333333333333, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 6, "recall_all_attempts": 0.5 } }, "accuracy_valid_only": 0.7083333333333334, "accuracy_valid_only_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_valid_only": 0.7083333333333333, "brier_valid_only": 0.2916666666666667, "log_loss_valid_only": 10.073843098408542, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 0.9835982710064854, "p95_s": 2.390612716099712, "mean_s": 1.1237734444997234 } }, "Hy4 Preview": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0217371465, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0217371465, "usd_per_1000_attempts": 0.9057144374999999, "usd_per_correct_answer": 0.001144060342105263 }, "quality": { "correct": 19, "accuracy_all_attempts": 0.7916666666666666, "accuracy_all_attempts_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_all_attempts": 0.7916666666666667, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 10, "recall_all_attempts": 0.8333333333333334 }, "yes": { "attempts": 12, "valid": 12, "correct": 9, "recall_all_attempts": 0.75 } }, "accuracy_valid_only": 0.7916666666666666, "accuracy_valid_only_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_valid_only": 0.7916666666666667, "brier_valid_only": 0.20833333333333334, "log_loss_valid_only": 7.195645048725576, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 3.2872396250022575, "p95_s": 5.365569293758015, "mean_s": 4.192581486170336 } }, "Jev": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.003231354, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.003231354, "usd_per_1000_attempts": 0.13463975, "usd_per_correct_answer": 0.00014049365217391304 }, "quality": { "correct": 23, "accuracy_all_attempts": 0.9583333333333334, "accuracy_all_attempts_wilson95": [ 0.7975819352134406, 0.9926065438344942 ], "balanced_accuracy_all_attempts": 0.9583333333333333, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 } }, "accuracy_valid_only": 0.9583333333333334, "accuracy_valid_only_wilson95": [ 0.7975819352134406, 0.9926065438344942 ], "balanced_accuracy_valid_only": 0.9583333333333333, "brier_valid_only": 0.027795833333333336, "log_loss_valid_only": 0.147079740899364, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 0.2949892915057717, "p95_s": 0.35291852325754003, "mean_s": 0.3018276301663718 } }, "Luna cached": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.00172272, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.00172272, "usd_per_1000_attempts": 0.07178000000000001, "usd_per_correct_answer": 7.830545454545454e-05 }, "quality": { "correct": 22, "accuracy_all_attempts": 0.9166666666666666, "accuracy_all_attempts_wilson95": [ 0.741511978067893, 0.9768411847029567 ], "balanced_accuracy_all_attempts": 0.9166666666666666, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 } }, "accuracy_valid_only": 0.9166666666666666, "accuracy_valid_only_wilson95": [ 0.741511978067893, 0.9768411847029567 ], "balanced_accuracy_valid_only": 0.9166666666666666, "brier_valid_only": 0.08333333333333333, "log_loss_valid_only": 2.878264682802149, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.0343990420078626, "p95_s": 1.455958373242174, "mean_s": 1.1074061511256634 } }, "Luna uncached": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.007278, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.007278, "usd_per_1000_attempts": 0.30324999999999996, "usd_per_correct_answer": 0.00038305263157894734 }, "quality": { "correct": 19, "accuracy_all_attempts": 0.7916666666666666, "accuracy_all_attempts_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_all_attempts": 0.7916666666666666, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 8, "recall_all_attempts": 0.6666666666666666 } }, "accuracy_valid_only": 0.7916666666666666, "accuracy_valid_only_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_valid_only": 0.7916666666666666, "brier_valid_only": 0.20833333333333334, "log_loss_valid_only": 7.195611732165985, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.1503861465025693, "p95_s": 2.637486370457916, "mean_s": 1.4373784272077803 } }, "MiMo V2.6 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0023895592, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0023895592, "usd_per_1000_attempts": 0.09956496666666666, "usd_per_correct_answer": 0.00011947796 }, "quality": { "correct": 20, "accuracy_all_attempts": 0.8333333333333334, "accuracy_all_attempts_wilson95": [ 0.6414692935030095, 0.9332132367136705 ], "balanced_accuracy_all_attempts": 0.8333333333333333, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 9, "recall_all_attempts": 0.75 } }, "accuracy_valid_only": 0.8333333333333334, "accuracy_valid_only_wilson95": [ 0.6414692935030095, 0.9332132367136705 ], "balanced_accuracy_valid_only": 0.8333333333333333, "brier_valid_only": 0.16666666666666666, "log_loss_valid_only": 5.756496049044706, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.8414089790021535, "p95_s": 6.58044343758083, "mean_s": 3.9579901475347774 } }, "Qwen3.8 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.002865114, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.002865114, "usd_per_1000_attempts": 0.11937975, "usd_per_correct_answer": 0.00016853611764705884 }, "quality": { "correct": 17, "accuracy_all_attempts": 0.7083333333333334, "accuracy_all_attempts_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_all_attempts": 0.7083333333333334, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 5, "recall_all_attempts": 0.4166666666666667 } }, "accuracy_valid_only": 0.7083333333333334, "accuracy_valid_only_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_valid_only": 0.7083333333333334, "brier_valid_only": 0.2916666666666667, "log_loss_valid_only": 10.07380978184895, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.5065319374989485, "p95_s": 2.4560621892873304, "mean_s": 1.913476786328829 } } }, "3": { "DeepSeek V4.1 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0019207944000000001, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0019207944000000001, "usd_per_1000_attempts": 0.08003310000000001, "usd_per_correct_answer": 0.00010109444210526317 }, "quality": { "correct": 19, "accuracy_all_attempts": 0.7916666666666666, "accuracy_all_attempts_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_all_attempts": 0.7916666666666667, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 7, "recall_all_attempts": 0.5833333333333334 } }, "accuracy_valid_only": 0.7916666666666666, "accuracy_valid_only_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_valid_only": 0.7916666666666667, "brier_valid_only": 0.20833333333333334, "log_loss_valid_only": 7.195578415606394, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.1495826665050117, "p95_s": 9.83380924820229, "mean_s": 2.2050491025390024 } }, "Hy4 Preview": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0179016381, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0179016381, "usd_per_1000_attempts": 0.7459015874999999, "usd_per_correct_answer": 0.0008950819049999999 }, "quality": { "correct": 20, "accuracy_all_attempts": 0.8333333333333334, "accuracy_all_attempts_wilson95": [ 0.6414692935030095, 0.9332132367136705 ], "balanced_accuracy_all_attempts": 0.8333333333333333, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 9, "recall_all_attempts": 0.75 } }, "accuracy_valid_only": 0.8333333333333334, "accuracy_valid_only_wilson95": [ 0.6414692935030095, 0.9332132367136705 ], "balanced_accuracy_valid_only": 0.8333333333333333, "brier_valid_only": 0.16666666666666666, "log_loss_valid_only": 5.756496049044706, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 3.259101395495236, "p95_s": 3.9692414562523473, "mean_s": 3.3206360589538235 } }, "Jev": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.003235974, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.003235974, "usd_per_1000_attempts": 0.13483225, "usd_per_correct_answer": 0.00014069452173913043 }, "quality": { "correct": 23, "accuracy_all_attempts": 0.9583333333333334, "accuracy_all_attempts_wilson95": [ 0.7975819352134406, 0.9926065438344942 ], "balanced_accuracy_all_attempts": 0.9583333333333333, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 } }, "accuracy_valid_only": 0.9583333333333334, "accuracy_valid_only_wilson95": [ 0.7975819352134406, 0.9926065438344942 ], "balanced_accuracy_valid_only": 0.9583333333333333, "brier_valid_only": 0.05340416666666667, "log_loss_valid_only": 0.21702638116864661, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 0.29199427099956665, "p95_s": 0.33393304994242495, "mean_s": 0.29799687324945506 } }, "Luna cached": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.00172164, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.00172164, "usd_per_1000_attempts": 0.07173500000000001, "usd_per_correct_answer": 8.6082e-05 }, "quality": { "correct": 20, "accuracy_all_attempts": 0.8333333333333334, "accuracy_all_attempts_wilson95": [ 0.6414692935030095, 0.9332132367136705 ], "balanced_accuracy_all_attempts": 0.8333333333333333, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 9, "recall_all_attempts": 0.75 } }, "accuracy_valid_only": 0.8333333333333334, "accuracy_valid_only_wilson95": [ 0.6414692935030095, 0.9332132367136705 ], "balanced_accuracy_valid_only": 0.8333333333333333, "brier_valid_only": 0.16666666666666666, "log_loss_valid_only": 5.756496049044706, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.276685353994253, "p95_s": 3.1715759705926745, "mean_s": 1.6475477498776552 } }, "Luna uncached": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.0072618, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.0072618, "usd_per_1000_attempts": 0.302575, "usd_per_correct_answer": 0.0003157304347826087 }, "quality": { "correct": 23, "accuracy_all_attempts": 0.9583333333333334, "accuracy_all_attempts_wilson95": [ 0.7975819352134406, 0.9926065438344942 ], "balanced_accuracy_all_attempts": 0.9583333333333333, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 } }, "accuracy_valid_only": 0.9583333333333334, "accuracy_valid_only_wilson95": [ 0.7975819352134406, 0.9926065438344942 ], "balanced_accuracy_valid_only": 0.9583333333333333, "brier_valid_only": 0.041666666666666664, "log_loss_valid_only": 1.4391156831212795, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.1690310834965203, "p95_s": 1.967764762484875, "mean_s": 1.3205830590389571 } }, "MiMo V2.6 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.001802164, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.001802164, "usd_per_1000_attempts": 0.07509016666666667, "usd_per_correct_answer": 9.485073684210526e-05 }, "quality": { "correct": 19, "accuracy_all_attempts": 0.7916666666666666, "accuracy_all_attempts_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_all_attempts": 0.7916666666666666, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 8, "recall_all_attempts": 0.6666666666666666 }, "yes": { "attempts": 12, "valid": 12, "correct": 11, "recall_all_attempts": 0.9166666666666666 } }, "accuracy_valid_only": 0.7916666666666666, "accuracy_valid_only_wilson95": [ 0.5952954674681133, 0.9075517464714814 ], "balanced_accuracy_valid_only": 0.7916666666666666, "brier_valid_only": 0.20833333333333334, "log_loss_valid_only": 7.1957116818447595, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 1.685804166496382, "p95_s": 12.781528658037121, "mean_s": 4.952865333375182 } }, "Qwen3.8 Flash": { "quality": { "attempts": 24, "valid": 24, "invalid": 0, "success_rate": 1.0, "success_rate_wilson95": [ 0.8620237953250198, 1 ], "unique_cases": 24, "target_kind": "label", "cost": { "requests": 24, "known_billed_usd": 0.002945522, "observed_bills": 24, "missing_bills": 0, "total_billed_usd": 0.002945522, "usd_per_1000_attempts": 0.12273008333333332, "usd_per_correct_answer": 0.000173266 }, "quality": { "correct": 17, "accuracy_all_attempts": 0.7083333333333334, "accuracy_all_attempts_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_all_attempts": 0.7083333333333334, "class_denominators": { "no": { "attempts": 12, "valid": 12, "correct": 12, "recall_all_attempts": 1.0 }, "yes": { "attempts": 12, "valid": 12, "correct": 5, "recall_all_attempts": 0.4166666666666667 } }, "accuracy_valid_only": 0.7083333333333334, "accuracy_valid_only_wilson95": [ 0.5083230633550337, 0.8508535180303913 ], "balanced_accuracy_valid_only": 0.7083333333333334, "brier_valid_only": 0.2916666666666667, "log_loss_valid_only": 10.07380978184895, "proper_score_denominator": 24 } }, "latency": { "attempts": 24, "valid": 24, "observed_positive_timings": 24, "median_s": 2.0802034584921785, "p95_s": 5.354588145442539, "mean_s": 2.5957316197927867 } } } }, "inference": "Exploratory paired percentile intervals over pair_id within fixed blocks, conditional on this session. Different sessions are not pooled." } ], "limitations": [ "All intervals are exploratory and unadjusted for multiple comparisons. Wilson marginal intervals assume independent rows.", "Earlier Jev/Luna quality outputs are reused. Model time windows and providers differ; quality comparison is conditional on the saved observations.", "Only the new all-model interleaved timing session supports comparative speed claims. No concurrent bulk timing is pooled into it.", "Invalid outputs count as wrong for accuracy. Proper scores require valid probabilities. Missing attempts and missing bills remain explicit.", "Public labels can be wrong and may overlap training data. Shared synthetic templates and one-session sampling limit generalization.", "Schema equality does not equalize tokenization or internal computation. Reasoning configuration is declared per model; observed counters are audited separately.", "Cache sessions are summarized separately. Cost savings depend on verified cache hits, prefix length, reuse and reported billing." ] }