{ "passed": true, "checks": { "completed": true, "unique_frozen_case_ids": true, "frozen_cases_hash": true, "expected_models_present": true, "frozen_source_code_hashes": true, "plan_count": true, "plan_sha256": true, "seeded_plan_reproduces": true, "unique_attempt_identities": true, "every_record_matches_contract_and_source": true, "complete_quality_cross_product": true, "attempts.jsonl_unique_dispatches": true, "attempts.jsonl_frozen_request_hashes": true, "saved_exact_requests_hash": true, "saved_responses_sha256": true, "saved_attempts_sha256": true, "metadata_billing_complete_or_unknown": true, "named_source_hashes_match": true, "current_code_matches_saved_snapshot": true, "prospective_continuation_evidence_preserved": true }, "issues": [], "issue_count": 0, "run_dir": "benchmark/multimodel/runs/quality-20261002T155349573021Z", "raw_sha256": "a0bc839aaf80fb359b48e808962f21249bde21ef00d058ce163a6bab6ca750b6", "case_sha256": "e271790b2ee38a1c231e312fa15ece2a56c95f6bf705294f1637d43b2edcf256", "attempts": 4785, "valid": 4671, "cost": { "requests": 4785, "known_billed_usd": 0.228254452, "observed_bills": 4671, "missing_bills": 114, "total_billed_usd": null, "usd_per_1000_attempts": null }, "observed_usage": { "Qwen3.8 Flash": { "valid_calls": 4671, "output_tokens": { "observed": 4671, "min": 11, "max": 27, "total": 74005 }, "reasoning_counter_observed": 4671, "reasoning_counter_unknown": 0, "reasoning_tokens_reported_total": 0, "visible_reasoning_calls": 0, "cache_read_counter_observed": 4671, "cache_write_counter_observed": 4671 } }, "exact_payload_factory_supplied": true, "seeded_plan_factory_supplied": true, "limitations": [ "Missing reasoning/cache counters are unobserved, not verified zero. Without factories, quality/timing source content and contracts are checked, but complete payload and seeded plan regeneration are not established." ], "continuation_audit": { "passed": true, "checks": { "amendment_1_metadata_snapshot_hash": true, "amendment_1_safe_continuation_filename": true, "amendment_1_wrapper_hash": true, "amendment_1_saved_amendment_matches_metadata": true, "amendment_1_frozen_configuration": true, "amendment_1_responses.jsonl_preserved_prefix": true, "amendment_1_attempts.jsonl_preserved_prefix": true, "amendment_1_plan.json_preserved_prefix": true, "amendment_1_plan.json_unchanged": true, "amendment_1_requests.jsonl_preserved_prefix": true, "amendment_1_requests.jsonl_unchanged": true, "amendment_1_original_counts": true, "amendment_1_retained_cumulative_invalid_rate": true, "amendment_1_no_original_identity_retried": true, "amendment_1_probe_is_separate": true, "amendment_1_probe_metadata_hash": true, "amendment_1_probe_continuation_file": true, "amendment_1_probe_outcome_hash": true, "amendment_1_probe_dispatches_unique_retained": true, "amendment_1_probe_count_matches": true, "amendment_1_recovery_result_consistent": true, "amendment_1_probe_0_request_journal": true, "amendment_1_probe_0_frozen_identity": true, "amendment_1_probe_0_exact_payload": true, "amendment_1_probe_0_billing_readback": true, "amendment_1_probe_0_valid_observed_zero_reasoning": true, "amendment_1_probe_0_strict_noul_answer": true, "amendment_1_probe_1_request_journal": true, "amendment_1_probe_1_frozen_identity": true, "amendment_1_probe_1_exact_payload": true, "amendment_1_probe_1_billing_readback": true, "amendment_1_probe_1_valid_observed_zero_reasoning": true, "amendment_1_probe_1_strict_noul_answer": true, "amendment_1_probe_billing_complete_or_unknown": true, "amendment_1_probe_budget_retained": true, "amendment_1_earlier_probe_budget_retained": true, "amendment_1_only_historical_circuit_rearmed": true, "amendment_1_original_unknown_guard_retained": true, "amendment_2_metadata_snapshot_hash": true, "amendment_2_safe_continuation_filename": true, "amendment_2_wrapper_hash": true, "amendment_2_saved_amendment_matches_metadata": true, "amendment_2_frozen_configuration": true, "amendment_2_responses.jsonl_preserved_prefix": true, "amendment_2_attempts.jsonl_preserved_prefix": true, "amendment_2_plan.json_preserved_prefix": true, "amendment_2_plan.json_unchanged": true, "amendment_2_requests.jsonl_preserved_prefix": true, "amendment_2_requests.jsonl_unchanged": true, "amendment_2_original_counts": true, "amendment_2_retained_cumulative_invalid_rate": true, "amendment_2_no_original_identity_retried": true, "amendment_2_probe_is_separate": true, "amendment_2_probe_metadata_hash": true, "amendment_2_probe_continuation_file": true, "amendment_2_probe_outcome_hash": true, "amendment_2_probe_dispatches_unique_retained": true, "amendment_2_probe_count_matches": true, "amendment_2_recovery_result_consistent": true, "amendment_2_probe_0_request_journal": true, "amendment_2_probe_0_frozen_identity": true, "amendment_2_probe_0_exact_payload": true, "amendment_2_probe_0_billing_readback": true, "amendment_2_probe_0_valid_observed_zero_reasoning": true, "amendment_2_probe_0_strict_noul_answer": true, "amendment_2_probe_1_request_journal": true, "amendment_2_probe_1_frozen_identity": true, "amendment_2_probe_1_exact_payload": true, "amendment_2_probe_1_billing_readback": true, "amendment_2_probe_1_valid_observed_zero_reasoning": true, "amendment_2_probe_1_strict_noul_answer": true, "amendment_2_probe_billing_complete_or_unknown": true, "amendment_2_probe_budget_retained": true, "amendment_2_earlier_probe_budget_retained": true, "amendment_2_only_historical_circuit_rearmed": true, "amendment_2_original_unknown_guard_retained": true, "amendment_2_paced_first_unattempted_sequence": true, "amendment_2_paced_window_bounds": true, "amendment_2_paced_amendment_fields": true, "amendment_2_paced_run_metadata": true, "amendment_2_paced_new_quality_row_fields": true, "amendment_2_paced_runtime_ceiling": true, "amendment_2_paced_rows_and_dispatches_match": true, "amendment_2_observed_dispatch_spacing_at_least_0_25s": true, "quality_budget_excludes_all_linked_probe_charges": true }, "issues": [], "amendments": [ { "id": "20261002T164150404672Z", "status": "continuation_stopped", "continuation_file": "continuation.py", "original_attempts": 3019, "original_invalid": 47, "recovery_calls": 2, "recovery_budget_accounted_usd": 6.534e-05, "paced_evidence": null }, { "id": "20261002T165108634637Z", "status": "completed", "continuation_file": "continuation_paced.py", "original_attempts": 3651, "original_invalid": 71, "recovery_calls": 2, "recovery_budget_accounted_usd": 6.581e-05, "paced_evidence": { "new_quality_rows": 1134, "observed_minimum_dispatch_gap_s": 0.2501969337463379, "saved_timestamp_tolerance_s": 0.001 } } ], "linked_recovery_budget_accounted_usd": 0.00013115, "limitations": [ "Successful serial probes establish momentary service response; they do not establish sustained provider capacity. Pacing checks compare declared executor limits and recorded dispatch spacing; they do not measure upstream internal concurrency. Original invalid quality outcomes remain in their original denominators." ] }, "generated_at_utc": "2026-10-02T17:48:55.642282+00:00" }