{ "created_utc": "2026-10-31T02:43:01.737185+00:00", "model": { "name": "turboderp/GLM-5.3-Flash-exl3", "revision": "b570edf655a2bb00beca23e18d3b98724bfc2db5", "quantization": "sc_3.10bpw", "engine": "ExLlamaV3 1.6.5-gpuembed1", "GPU": "H100 PCIe80GBx2", "NCCL_P2P_DISABLE": "1", "no_CPU_weight_or_KV_offload": true, "MTP_drafts": 3, "reasoning_effort": "low", "enable_thinking": true, "tools_provided_to_model": false, "primary_input_padding": "Repeated explicitly irrelevant reference log; target question placed last. No dispersed-evidence4K document comprehension claim." }, "primary": { "status": "COMPLETE_INDEPENDENT_AUDIT", "utc": "2026-10-31T02:24:52.410528+00:00", "case": "quality-main-1m", "rows": 3840, "pairs": 1920, "capacity": [ 1048576 ], "GPU_peak_MiB": [ 74003, 74259 ], "residence_snapshots_audited": 21, "frozen_execution_order_verified": true, "summary": { "greedy": { "pairs": 120, "off_correct": 112, "on_correct": 112, "both_correct": 111, "both_wrong": 7, "on_only_correct": 1, "off_only_correct": 1, "full_completion_identical": 96, "answer_identical": 107, "cap_counts": { "False": 0, "True": 0 }, "complete_token_sequence_identical": 96, "format_valid_counts": { "False": 120, "True": 120 }, "condition_metrics": { "False": { "generation_tps_median": 44.69300135762468, "first_answer_seconds_median": 1.5794707329478115, "missing_first_answer": 0, "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "True": { "generation_tps_median": 80.8419816216483, "first_answer_seconds_median": 1.6152822514995933, "missing_first_answer": 0, "accepted_draft_tokens": 1723, "rejected_draft_tokens": 1007 } }, "off_accuracy": 0.9333333333333333, "on_accuracy": 0.9333333333333333, "difference_percentage_points": 0.0 }, "sampling": { "pairs": 1800, "off_correct": 1664, "on_correct": 1664, "both_correct": 1653, "both_wrong": 125, "on_only_correct": 11, "off_only_correct": 11, "full_completion_identical": 1500, "answer_identical": 1644, "cap_counts": { "False": 0, "True": 0 }, "complete_token_sequence_identical": 1500, "format_valid_counts": { "False": 1800, "True": 1799 }, "condition_metrics": { "False": { "generation_tps_median": 45.55469521394798, "first_answer_seconds_median": 1.5852534740697592, "missing_first_answer": 0, "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "True": { "generation_tps_median": 79.71138240494074, "first_answer_seconds_median": 1.6332717120531015, "missing_first_answer": 0, "accepted_draft_tokens": 26247, "rejected_draft_tokens": 17082 } }, "off_accuracy": 0.9244444444444444, "on_accuracy": 0.9244444444444444, "difference_percentage_points": 0.0, "question_clusters": 600, "seeds_per_question": 3, "bootstrap_resamples": 20000, "bootstrap_strata": "30 fixed category/template strata; 3 seeds retained per question", "one_sided_95_lower_difference_pp": -0.33333333333333337, "two_sided_95_difference_pp": [ -0.4444444444444444, 0.44444444444444436 ], "preregistered_noninferiority_margin_pp": 2.0, "noninferiority_pass": true, "questions_with_any_off_only_correct": 11, "conservative95_upper_harmed_question_rate": 0.03016356021374441, "conservative_harm_bound_below_2pp": false, "claim_limit": "Finite fixed Japanese objective task suite, 4K input, tested runtime/model/capacity only; not general answer quality, equality proof, or a1M-history comprehension test." } }, "categories": { "arithmetic": { "False": { "correct": 230, "total": 300 }, "True": { "correct": 231, "total": 300 } }, "code": { "False": { "correct": 261, "total": 300 }, "True": { "correct": 263, "total": 300 } }, "evidence": { "False": { "correct": 299, "total": 300 }, "True": { "correct": 299, "total": 300 } }, "logic": { "False": { "correct": 292, "total": 300 }, "True": { "correct": 290, "total": 300 } }, "metrics": { "False": { "correct": 289, "total": 300 }, "True": { "correct": 289, "total": 300 } }, "records": { "False": { "correct": 293, "total": 300 }, "True": { "correct": 292, "total": 300 } } }, "input_groups": { "greedy-4096": { "pairs": 120, "off_correct": 112, "on_correct": 112, "on_only_correct": 1, "off_only_correct": 1, "caps": { "False": 0, "True": 0 } }, "sampling-4096": { "pairs": 1800, "off_correct": 1664, "on_correct": 1664, "on_only_correct": 11, "off_only_correct": 11, "caps": { "False": 0, "True": 0 } } }, "discordant_pairs": [ { "task": "code-001", "mode": "greedy", "rep": 0, "off": { "answer": "{\"answer\": -2}", "parsed": { "answer": -2 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\": 12}", "parsed": { "answer": 12 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "code-008", "mode": "greedy", "rep": 0, "off": { "answer": "{\"answer\":[18, 12, 11]}", "parsed": { "answer": [ 18, 12, 11 ] }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\":[18, 12, 0]}", "parsed": { "answer": [ 18, 12, 0 ] }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "code-043", "mode": "sampling", "rep": 0, "off": { "answer": "{\"answer\":[11,15,16]}", "parsed": { "answer": [ 11, 15, 16 ] }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\": [11, 2, 4]}", "parsed": { "answer": [ 11, 2, 4 ] }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "arithmetic-023", "mode": "sampling", "rep": 2, "off": { "answer": "{\"answer\":\"95.35%\"}", "parsed": { "answer": "95.35%" }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\":95.35}", "parsed": { "answer": 95.35 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "code-056", "mode": "sampling", "rep": 1, "off": { "answer": "{\"answer\": 10}", "parsed": { "answer": 10 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\": 13}", "parsed": { "answer": 13 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "logic-039", "mode": "sampling", "rep": 2, "off": { "answer": "{\"answer\":1}", "parsed": { "answer": 1 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\": 2}", "parsed": { "answer": 2 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "arithmetic-053", "mode": "sampling", "rep": 2, "off": { "answer": "{\"answer\":93.95}", "parsed": { "answer": 93.95 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\":\"93.95%\"}", "parsed": { "answer": "93.95%" }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "code-037", "mode": "sampling", "rep": 0, "off": { "answer": "{\"answer\":228}", "parsed": { "answer": 228 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\":113}", "parsed": { "answer": 113 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "records-012", "mode": "sampling", "rep": 0, "off": { "answer": "{\"answer\": 542}", "parsed": { "answer": 542 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\":442}", "parsed": { "answer": 442 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "code-071", "mode": "sampling", "rep": 1, "off": { "answer": "{\"answer\": 17}", "parsed": { "answer": 17 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\": 15}", "parsed": { "answer": 15 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "arithmetic-018", "mode": "sampling", "rep": 2, "off": { "answer": "{\"answer\":89.5}", "parsed": { "answer": 89.5 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\":\"89.50%\"}", "parsed": { "answer": "89.50%" }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "code-054", "mode": "sampling", "rep": 1, "off": { "answer": "{\"answer\": 101}", "parsed": { "answer": 101 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\": 87}", "parsed": { "answer": 87 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "code-028", "mode": "sampling", "rep": 1, "off": { "answer": "{\"answer\": [8, -4, -1]}", "parsed": { "answer": [ 8, -4, -1 ] }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\": [8, -4, 0]}", "parsed": { "answer": [ 8, -4, 0 ] }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "logic-027", "mode": "sampling", "rep": 0, "off": { "answer": "{\"answer\":\"断定できない\"}", "parsed": { "answer": "断定できない" }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\": \"断定できる\"}", "parsed": { "answer": "断定できる" }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "arithmetic-092", "mode": "sampling", "rep": 1, "off": { "answer": "{\"answer\":75}", "parsed": { "answer": 75 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\":74}", "parsed": { "answer": 74 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "arithmetic-053", "mode": "sampling", "rep": 1, "off": { "answer": "{\"answer\":\"93.95%\"}", "parsed": { "answer": "93.95%" }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\":93.95}", "parsed": { "answer": 93.95 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "code-056", "mode": "sampling", "rep": 0, "off": { "answer": "{\"answer\": 9}", "parsed": { "answer": 9 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\": 10}", "parsed": { "answer": 10 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "records-072", "mode": "sampling", "rep": 0, "off": { "answer": "{\"answer\":388}", "parsed": { "answer": 388 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\": 408}", "parsed": { "answer": 408 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "arithmetic-033", "mode": "sampling", "rep": 0, "off": { "answer": "{\"answer\":\"53.68%\"}", "parsed": { "answer": "53.68%" }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\": 53.68}", "parsed": { "answer": 53.68 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "records-037", "mode": "sampling", "rep": 1, "off": { "answer": "{\"answer\":371}", "parsed": { "answer": 371 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\": 381}", "parsed": { "answer": 381 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "arithmetic-047", "mode": "sampling", "rep": 1, "off": { "answer": "{\"answer\":20}", "parsed": { "answer": 20 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\":21}", "parsed": { "answer": 21 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "arithmetic-092", "mode": "sampling", "rep": 0, "off": { "answer": "{\"answer\":74}", "parsed": { "answer": 74 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\":75}", "parsed": { "answer": 75 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } }, { "task": "arithmetic-058", "mode": "sampling", "rep": 2, "off": { "answer": "{\"answer\": 87.96}", "parsed": { "answer": 87.96 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "on": { "answer": "{\"answer\":\"87.96%\"}", "parsed": { "answer": "87.96%" }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false } }, { "task": "code-035", "mode": "sampling", "rep": 0, "off": { "answer": "{\"answer\":16}", "parsed": { "answer": 16 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "on": { "answer": "{\"answer\":31}", "parsed": { "answer": 31 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true } } ], "human_reading_completed": false, "analysis_source_sha256": "e3aec503485e9f73c7eee2a49fb316051caf5f3dccc7b6abddda25f40dc12d28", "source_results_sha256": "769ced0a0726ece7a83b4e0f2fc9e8072a7442df00bf7c9865f465b2cddc1f62" }, "wording_sensitivity": { "status": "COMPLETE_POST_HOC_WORDING_SENSITIVITY", "utc": "2026-10-31T02:25:29.859537+00:00", "plan": { "status": "POST_HOC_CONTENT_AMBIGUITY_SENSITIVITY_PLAN", "utc": "2026-10-31T01:46:25.346754+00:00", "discovered_during": "Read-only review of already completed error pairs while main inference continues; some responses were seen. This is not preregistered.", "example": "logic-039 asks six people to be distributed among multiple rooms with capacities[2,6,2], yet asks minimum room count; original oracle is1. Multiple rooms may reasonably imply at least2.", "original_preregistered_primary_unchanged": true, "no_inference_reruns": true, "no_model_or_task_or_scorer_changes": true, "supplementary_exclusion_rule": "Exclude every question in category logic/template_variant4, independent of its answer or MTP direction. Do not change expected answers.", "excluded_ids": [ "logic-004", "logic-009", "logic-014", "logic-019", "logic-024", "logic-029", "logic-034", "logic-039", "logic-044", "logic-049", "logic-054", "logic-059", "logic-064", "logic-069", "logic-074", "logic-079", "logic-084", "logic-089", "logic-094", "logic-099" ], "excluded_questions": 20, "remaining_questions": 580, "remaining_sampling_pairs": 1740, "method": "Same paired question-cluster bootstrap with all3seeds;29 remaining fixed category/template strata,20questions each,20000resamples. Report side by side with original600-question primary, label post-hoc.", "original_oracle_one_room_count": 1, "task_file_sha256": "fcdc18ff49a4ee0321982667046d106a4a58bd50621b6379ea8b66e090fe06d7", "publication_policy": "If sensitivity conflicts with primary, describe result as sensitive/inconclusive; never replace original result with a favorable subset." }, "preregistered": false, "original_primary_replaced": false, "questions": 580, "sampling_pairs": 1740, "bootstrap_resamples": 20000, "strata": 29, "seeds_per_question": 3, "off_correct": 1604, "on_correct": 1605, "off_accuracy": 0.9218390804597701, "on_accuracy": 0.9224137931034483, "difference_percentage_points": 0.05747126436781609, "one_sided_95_lower_difference_pp": -0.28735632183908044, "two_sided_95_difference_pp": [ -0.3448275862068966, 0.45977011494252873 ], "off_only_correct": 10, "on_only_correct": 11, "same_minus_2pp_threshold_pass": true, "agrees_with_original_primary_decision": true, "source_results_sha256": "769ced0a0726ece7a83b4e0f2fc9e8072a7442df00bf7c9865f465b2cddc1f62", "plan_sha256": "90151c914c5a7d29c50e9540724f5de78ab8b1c9946d02e6c9d29f37a95cb306", "analysis_source_sha256": "feb4f206145273e1e967243521d05e8965f31dba5b43b1e3775f04c56515f31c", "interpretation": "Content-based post-hoc exclusion of whole template after some outputs were seen; sensitivity only, not a newly preregistered primary result or general quality guarantee." }, "secondary": { "status": "COMPLETE_INDEPENDENT_AUDIT", "utc": "2026-10-31T02:40:12.532588+00:00", "case": "quality-long-1m", "rows": 48, "pairs": 24, "capacity": [ 1048576 ], "GPU_peak_MiB": [ 74103, 74279 ], "residence_snapshots_audited": 2, "frozen_execution_order_verified": true, "summary": { "sampling": { "pairs": 24, "off_correct": 11, "on_correct": 11, "both_correct": 11, "both_wrong": 13, "on_only_correct": 0, "off_only_correct": 0, "full_completion_identical": 6, "answer_identical": 10, "cap_counts": { "False": 12, "True": 11 }, "complete_token_sequence_identical": 6, "format_valid_counts": { "False": 12, "True": 13 }, "condition_metrics": { "False": { "generation_tps_median": 49.10200405769449, "first_answer_seconds_median": 4.9388292075600475, "missing_first_answer": 12, "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "True": { "generation_tps_median": 85.0156803430397, "first_answer_seconds_median": 3.5588217129698023, "missing_first_answer": 11, "accepted_draft_tokens": 9851, "rejected_draft_tokens": 4459 } }, "off_accuracy": 0.4583333333333333, "on_accuracy": 0.4583333333333333, "difference_percentage_points": 0.0 } }, "categories": { "grounded_long_records": { "False": { "correct": 11, "total": 24 }, "True": { "correct": 11, "total": 24 } } }, "input_groups": { "sampling-4096": { "pairs": 12, "off_correct": 11, "on_correct": 11, "on_only_correct": 0, "off_only_correct": 0, "caps": { "False": 0, "True": 0 }, "field_correct": { "対象部署の件数合計": { "False": { "correct": 11, "total": 12 }, "True": { "correct": 11, "total": 12 } }, "指定IDの待ち秒": { "False": { "correct": 12, "total": 12 }, "True": { "correct": 12, "total": 12 } }, "最大待ち秒": { "False": { "correct": 12, "total": 12 }, "True": { "correct": 12, "total": 12 } }, "最小待ち秒": { "False": { "correct": 12, "total": 12 }, "True": { "correct": 12, "total": 12 } }, "行数": { "False": { "correct": 12, "total": 12 }, "True": { "correct": 12, "total": 12 } }, "遅延原因": { "False": { "correct": 12, "total": 12 }, "True": { "correct": 12, "total": 12 } } } }, "sampling-32768": { "pairs": 12, "off_correct": 0, "on_correct": 0, "on_only_correct": 0, "off_only_correct": 0, "caps": { "False": 12, "True": 11 }, "field_correct": { "対象部署の件数合計": { "False": { "correct": 0, "total": 12 }, "True": { "correct": 0, "total": 12 } }, "指定IDの待ち秒": { "False": { "correct": 0, "total": 12 }, "True": { "correct": 1, "total": 12 } }, "最大待ち秒": { "False": { "correct": 0, "total": 12 }, "True": { "correct": 1, "total": 12 } }, "最小待ち秒": { "False": { "correct": 0, "total": 12 }, "True": { "correct": 1, "total": 12 } }, "行数": { "False": { "correct": 0, "total": 12 }, "True": { "correct": 1, "total": 12 } }, "遅延原因": { "False": { "correct": 0, "total": 12 }, "True": { "correct": 1, "total": 12 } } } } }, "discordant_pairs": [], "human_reading_completed": false, "analysis_source_sha256": "e3aec503485e9f73c7eee2a49fb316051caf5f3dccc7b6abddda25f40dc12d28", "source_results_sha256": "ee818f7e0a72038584ca67705c15a36db029ac6db9c49007ff396aeb545c068d" }, "performance": [ { "status": "COMPLETE_PERFORMANCE_AUDIT_ANSWER_CONTENT_REVIEW_PENDING", "utc": "2026-10-31T00:10:54.836373+00:00", "case": "performance-262144", "groups": [ { "capacity": 262144, "input_target": 4096, "actual_input_tokens": [ 4096 ], "mtp": false, "repetitions": 3, "caps": 1, "generated_tokens": [ 1024, 938, 911 ], "all_trials": [ { "target": 4096, "mtp": false, "seed": 43, "warmup": false, "rep": 1, "case": "main-ctx4096-mtp0-r1", "capacity": 262144, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 1024, "first_generated_seconds": 1.6325488300062716, "first_visible_seconds": 1.6520715119550005, "first_answer_seconds": 1.6520715119550005, "end_to_end_seconds": 22.038368831039406, "generation_tps": 50.085408158082814, "generation_tps_native": 50.13436750134585, "input_seconds_native": 1.610550880432129, "finish_reason": "max_new_tokens", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "22be14553d2a1367d13426145d5aa2765750d901fe88e5d61d074133f5935ebc" }, { "target": 4096, "mtp": false, "seed": 42, "warmup": false, "rep": 0, "case": "main-ctx4096-mtp0-r0", "capacity": 262144, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 938, "first_generated_seconds": 1.6320640180492774, "first_visible_seconds": 1.6320640180492774, "first_answer_seconds": 3.33365441695787, "end_to_end_seconds": 20.358652772032656, "generation_tps": 49.984388052131536, "generation_tps_native": 50.037733183457185, "input_seconds_native": 1.6101512908935547, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "8be938389c8e53c7c63bf5cc04e4dcf22876ce00424965e9b5058ff75e220ccf" }, { "target": 4096, "mtp": false, "seed": 44, "warmup": false, "rep": 2, "case": "main-ctx4096-mtp0-r2", "capacity": 262144, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 911, "first_generated_seconds": 1.6304029129678383, "first_visible_seconds": 1.6499060470378026, "first_answer_seconds": 1.6499060470378026, "end_to_end_seconds": 19.833018483011983, "generation_tps": 49.94013621057814, "generation_tps_native": 49.99501548113921, "input_seconds_native": 1.6086413860321045, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "7edb7e61ff44a76c45a4597203b9a6b39f3827cdb91ff38f5419e6909d44f8d9" } ], "generation_tps_median": 49.984388052131536, "generation_tps_native_median": 50.037733183457185, "first_generated_seconds_median": 1.6320640180492774, "first_visible_seconds_median": 1.6499060470378026, "first_answer_seconds_median": 1.6520715119550005, "end_to_end_seconds_median": 20.358652772032656, "input_seconds_native_median": 1.6101512908935547 }, { "capacity": 262144, "input_target": 4096, "actual_input_tokens": [ 4096 ], "mtp": true, "repetitions": 3, "caps": 0, "generated_tokens": [ 983, 844, 1021 ], "all_trials": [ { "target": 4096, "mtp": true, "seed": 42, "warmup": false, "rep": 0, "case": "main-ctx4096-mtp1-r0", "capacity": 262144, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 983, "first_generated_seconds": 1.6829289799788967, "first_visible_seconds": 1.6829289799788967, "first_answer_seconds": 2.9120616459986195, "end_to_end_seconds": 13.869572410010733, "generation_tps": 80.36714795691806, "generation_tps_native": 80.44898822978662, "input_seconds_native": 1.6473352909088135, "finish_reason": "stop_token", "accepted_draft_tokens": 629, "rejected_draft_tokens": 433, "output_sha256": "3637a4394734d108d72f8b8129d60d610c7874dfafff084c7bd57d737e3fe1ff" }, { "target": 4096, "mtp": true, "seed": 44, "warmup": false, "rep": 2, "case": "main-ctx4096-mtp1-r2", "capacity": 262144, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 844, "first_generated_seconds": 1.6846872990718111, "first_visible_seconds": 1.6847122160252184, "first_answer_seconds": 1.6847122160252184, "end_to_end_seconds": 13.083943187026307, "generation_tps": 73.74067366574269, "generation_tps_native": 73.82814777448023, "input_seconds_native": 1.6487534046173096, "finish_reason": "stop_token", "accepted_draft_tokens": 512, "rejected_draft_tokens": 484, "output_sha256": "c600cfbb8470e6c8a8955635c1e2cd2974dfa3b89c224b68f5336228f0de8109" }, { "target": 4096, "mtp": true, "seed": 43, "warmup": false, "rep": 1, "case": "main-ctx4096-mtp1-r1", "capacity": 262144, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 1021, "first_generated_seconds": 1.6842412049882114, "first_visible_seconds": 1.6842659320682287, "first_answer_seconds": 1.6842659320682287, "end_to_end_seconds": 14.281980014056899, "generation_tps": 80.75674047205295, "generation_tps_native": 80.83591374702556, "input_seconds_native": 1.6483025550842285, "finish_reason": "stop_token", "accepted_draft_tokens": 654, "rejected_draft_tokens": 447, "output_sha256": "f8b62489da2f9cf1cac442f90897a45a5bfbbbb947e3d0f8a2ef7b8020c8927c" } ], "generation_tps_median": 80.36714795691806, "generation_tps_native_median": 80.44898822978662, "first_generated_seconds_median": 1.6842412049882114, "first_visible_seconds_median": 1.6842659320682287, "first_answer_seconds_median": 1.6847122160252184, "end_to_end_seconds_median": 13.869572410010733, "input_seconds_native_median": 1.6483025550842285 }, { "capacity": 262144, "input_target": 32768, "actual_input_tokens": [ 32768 ], "mtp": false, "repetitions": 3, "caps": 1, "generated_tokens": [ 970, 1024, 963 ], "all_trials": [ { "target": 32768, "mtp": false, "seed": 42, "warmup": false, "rep": 0, "case": "main-ctx32768-mtp0-r0", "capacity": 262144, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 970, "first_generated_seconds": 10.520949685014784, "first_visible_seconds": 10.520949685014784, "first_answer_seconds": 12.03945493500214, "end_to_end_seconds": 29.87675105407834, "generation_tps": 50.01302508174486, "generation_tps_native": 50.064638110724985, "input_seconds_native": 10.498457431793213, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "de77fdf6bb57e490d0ff75fa04eade36fb2c420a42d937281d92a8651aa312fb" }, { "target": 32768, "mtp": false, "seed": 43, "warmup": false, "rep": 1, "case": "main-ctx32768-mtp0-r1", "capacity": 262144, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 1024, "first_generated_seconds": 10.520455247955397, "first_visible_seconds": 10.540036577964202, "first_answer_seconds": 10.540036577964202, "end_to_end_seconds": 30.99262812093366, "generation_tps": 49.923378417774046, "generation_tps_native": 49.972179374194155, "input_seconds_native": 10.497770071029663, "finish_reason": "max_new_tokens", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "598ad0b85a68fc5bbb82ba7c05352bc83b156f31589e42286371c27d9da1a4d9" }, { "target": 32768, "mtp": false, "seed": 44, "warmup": false, "rep": 2, "case": "main-ctx32768-mtp0-r2", "capacity": 262144, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 963, "first_generated_seconds": 10.523691480979323, "first_visible_seconds": 10.54324078396894, "first_answer_seconds": 10.54324078396894, "end_to_end_seconds": 29.789736540988088, "generation_tps": 49.88289093507144, "generation_tps_native": 49.93474425205176, "input_seconds_native": 10.50076937675476, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "b78043d889345d99768f4628e20ba589ef2edd42deee981a047ec024a7eefe27" } ], "generation_tps_median": 49.923378417774046, "generation_tps_native_median": 49.972179374194155, "first_generated_seconds_median": 10.520949685014784, "first_visible_seconds_median": 10.540036577964202, "first_answer_seconds_median": 10.54324078396894, "end_to_end_seconds_median": 29.87675105407834, "input_seconds_native_median": 10.498457431793213 }, { "capacity": 262144, "input_target": 32768, "actual_input_tokens": [ 32768 ], "mtp": true, "repetitions": 3, "caps": 0, "generated_tokens": [ 981, 1023, 947 ], "all_trials": [ { "target": 32768, "mtp": true, "seed": 42, "warmup": false, "rep": 0, "case": "main-ctx32768-mtp1-r0", "capacity": 262144, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 981, "first_generated_seconds": 10.837468542042188, "first_visible_seconds": 10.837468542042188, "first_answer_seconds": 12.343281848006882, "end_to_end_seconds": 24.03721951006446, "generation_tps": 74.05937036945149, "generation_tps_native": 74.13494115554276, "input_seconds_native": 10.800851345062256, "finish_reason": "stop_token", "accepted_draft_tokens": 597, "rejected_draft_tokens": 555, "output_sha256": "ba174c5b71b8b5b86ce453908532b8abacb4970fc3d8d6b276cda3a4ae3b898c" }, { "target": 32768, "mtp": true, "seed": 43, "warmup": false, "rep": 1, "case": "main-ctx32768-mtp1-r1", "capacity": 262144, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 1023, "first_generated_seconds": 10.83782317105215, "first_visible_seconds": 10.83784482406918, "first_answer_seconds": 10.83784482406918, "end_to_end_seconds": 23.023671686067246, "generation_tps": 83.64288054977838, "generation_tps_native": 83.72472289865291, "input_seconds_native": 10.800791501998901, "finish_reason": "stop_token", "accepted_draft_tokens": 668, "rejected_draft_tokens": 397, "output_sha256": "c99fce9e82de4388dc736e3c226d9f2ab3a3e1eb40c9bdd3bd53699022f83eb3" }, { "target": 32768, "mtp": true, "seed": 44, "warmup": false, "rep": 2, "case": "main-ctx32768-mtp1-r2", "capacity": 262144, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 947, "first_generated_seconds": 10.835755809908733, "first_visible_seconds": 10.835780425928533, "first_answer_seconds": 10.835780425928533, "end_to_end_seconds": 23.157415264984593, "generation_tps": 76.57060235480449, "generation_tps_native": 76.65154379492586, "input_seconds_native": 10.79870343208313, "finish_reason": "stop_token", "accepted_draft_tokens": 588, "rejected_draft_tokens": 489, "output_sha256": "159336fde6bbc5c448b43fbe8ac24257fd50fe0bf055ddd58a9e3af94bdf24b6" } ], "generation_tps_median": 76.57060235480449, "generation_tps_native_median": 76.65154379492586, "first_generated_seconds_median": 10.837468542042188, "first_visible_seconds_median": 10.837468542042188, "first_answer_seconds_median": 10.83784482406918, "end_to_end_seconds_median": 23.157415264984593, "input_seconds_native_median": 10.800791501998901 } ], "GPU_peak_MiB": [ 66147, 68015 ], "warmup_records_retained": 4, "all_main_records_retained": 12, "MTP_off_draft_allocated": true, "main_model_cache_and_draft_GPU_resident": true, "source_sha256": "b423ce9b0288b0783de58985e90ad093cd00b65fda15ae55d7be2b3c5cf5467a", "limits": [ "Real inputs4K/32K, despite muchlarger allocatedcapacity.", "Measurements usein-process API; remoteclient/network latencyexcluded.", "Loading,warmupandtokenizationexcluded; streaming observationoverhead included.", "Generate speeds arepertrial means, notinstantaneous guarantee.", "MTP-off retainsdraftallocation toholdtargetplacementfixed; thisdoesnotmeasureminimumMTP-offmemory." ] }, { "status": "COMPLETE_PERFORMANCE_AUDIT_ANSWER_CONTENT_REVIEW_PENDING", "utc": "2026-10-31T00:02:22.917182+00:00", "case": "performance-1048576", "groups": [ { "capacity": 1048576, "input_target": 4096, "actual_input_tokens": [ 4096 ], "mtp": false, "repetitions": 3, "caps": 0, "generated_tokens": [ 895, 890, 1013 ], "all_trials": [ { "target": 4096, "mtp": false, "seed": 43, "warmup": false, "rep": 1, "case": "main-ctx4096-mtp0-r1", "capacity": 1048576, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 895, "first_generated_seconds": 1.6267227670177817, "first_visible_seconds": 1.6464293240569532, "first_answer_seconds": 1.6464293240569532, "end_to_end_seconds": 19.616603234084323, "generation_tps": 49.64136778954571, "generation_tps_native": 49.696895046581, "input_seconds_native": 1.6015534400939941, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "d861391e80fe00ef2abe66bb79fc5b85764512c8e37be81a553af42043f16550" }, { "target": 4096, "mtp": false, "seed": 42, "warmup": false, "rep": 0, "case": "main-ctx4096-mtp0-r0", "capacity": 1048576, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 890, "first_generated_seconds": 1.6266138750361279, "first_visible_seconds": 1.6266138750361279, "first_answer_seconds": 2.9724917319836095, "end_to_end_seconds": 19.52084177196957, "generation_tps": 49.627106978748884, "generation_tps_native": 49.6829304961603, "input_seconds_native": 1.601552963256836, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "1fc48a10bf49a3fb66f803393a2d10ef384fae722e307fa0b13b1acfb3d125c5" }, { "target": 4096, "mtp": false, "seed": 44, "warmup": false, "rep": 2, "case": "main-ctx4096-mtp0-r2", "capacity": 1048576, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 1013, "first_generated_seconds": 1.6263057469623163, "first_visible_seconds": 1.6460160090355203, "first_answer_seconds": 1.6460160090355203, "end_to_end_seconds": 21.999215668067336, "generation_tps": 49.62685060614844, "generation_tps_native": 49.67588899607546, "input_seconds_native": 1.6013333797454834, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "0d4938d42e5ad081b2ebbd76f2ca38c7cfa126580e7c521fc38e2b6a451696db" } ], "generation_tps_median": 49.627106978748884, "generation_tps_native_median": 49.6829304961603, "first_generated_seconds_median": 1.6266138750361279, "first_visible_seconds_median": 1.6460160090355203, "first_answer_seconds_median": 1.6464293240569532, "end_to_end_seconds_median": 19.616603234084323, "input_seconds_native_median": 1.601552963256836 }, { "capacity": 1048576, "input_target": 4096, "actual_input_tokens": [ 4096 ], "mtp": true, "repetitions": 3, "caps": 0, "generated_tokens": [ 803, 952, 984 ], "all_trials": [ { "target": 4096, "mtp": true, "seed": 42, "warmup": false, "rep": 0, "case": "main-ctx4096-mtp1-r0", "capacity": 1048576, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 803, "first_generated_seconds": 1.6774975609732792, "first_visible_seconds": 1.6774975609732792, "first_answer_seconds": 2.410736653022468, "end_to_end_seconds": 11.368320718989708, "generation_tps": 82.48480711554818, "generation_tps_native": 82.58765600222593, "input_seconds_native": 1.639216661453247, "finish_reason": "stop_token", "accepted_draft_tokens": 521, "rejected_draft_tokens": 325, "output_sha256": "3a237efc28bd077e71a1fdedcf961690c017dff75fa7664edc48b181cd5bf5ac" }, { "target": 4096, "mtp": true, "seed": 44, "warmup": false, "rep": 2, "case": "main-ctx4096-mtp1-r2", "capacity": 1048576, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 952, "first_generated_seconds": 1.6814543680520728, "first_visible_seconds": 1.68147961597424, "first_answer_seconds": 1.68147961597424, "end_to_end_seconds": 13.598279180005193, "generation_tps": 79.58575931626508, "generation_tps_native": 79.66944570881637, "input_seconds_native": 1.6427600383758545, "finish_reason": "stop_token", "accepted_draft_tokens": 605, "rejected_draft_tokens": 436, "output_sha256": "2c26e99693b2e70230f7ede2f61c7ff4ef3724159f86e4c01749e45dadd0530d" }, { "target": 4096, "mtp": true, "seed": 43, "warmup": false, "rep": 1, "case": "main-ctx4096-mtp1-r1", "capacity": 1048576, "input_tokens": 4096, "prompt_sha256": "10bbc368830cf8bf9228209efade65c08ed5deef90802c6de932dfe40e5004c3", "generated_tokens": 984, "first_generated_seconds": 1.678850447991863, "first_visible_seconds": 1.6788758649490774, "first_answer_seconds": 1.6788758649490774, "end_to_end_seconds": 14.193785553914495, "generation_tps": 78.34179230707696, "generation_tps_native": 78.4214889421808, "input_seconds_native": 1.6399955749511719, "finish_reason": "stop_token", "accepted_draft_tokens": 619, "rejected_draft_tokens": 476, "output_sha256": "6a58c286909b1d11acebae380f9e2af0f2aecbfbb088b2db8000a90dac0607f9" } ], "generation_tps_median": 79.58575931626508, "generation_tps_native_median": 79.66944570881637, "first_generated_seconds_median": 1.678850447991863, "first_visible_seconds_median": 1.6788758649490774, "first_answer_seconds_median": 1.68147961597424, "end_to_end_seconds_median": 13.598279180005193, "input_seconds_native_median": 1.6399955749511719 }, { "capacity": 1048576, "input_target": 32768, "actual_input_tokens": [ 32768 ], "mtp": false, "repetitions": 3, "caps": 0, "generated_tokens": [ 854, 992, 860 ], "all_trials": [ { "target": 32768, "mtp": false, "seed": 42, "warmup": false, "rep": 0, "case": "main-ctx32768-mtp0-r0", "capacity": 1048576, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 854, "first_generated_seconds": 10.511140893097036, "first_visible_seconds": 10.511140893097036, "first_answer_seconds": 12.367131462087855, "end_to_end_seconds": 27.703510682098567, "generation_tps": 49.559445374470776, "generation_tps_native": 49.61754554489806, "input_seconds_native": 10.485308170318604, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "31d430c7588167efa8c11b973e0aa32a412b18f2e5633ec77878650d7c737b8d" }, { "target": 32768, "mtp": false, "seed": 43, "warmup": false, "rep": 1, "case": "main-ctx32768-mtp0-r1", "capacity": 1048576, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 992, "first_generated_seconds": 10.510944269946776, "first_visible_seconds": 10.530611487920396, "first_answer_seconds": 10.530611487920396, "end_to_end_seconds": 30.845779663999565, "generation_tps": 48.68813276599948, "generation_tps_native": 48.737263071515116, "input_seconds_native": 10.485511302947998, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "5fde1e1f13879bd11306b7b35ee6de1d9338b6a98229e842a86e93732b840d70" }, { "target": 32768, "mtp": false, "seed": 44, "warmup": false, "rep": 2, "case": "main-ctx32768-mtp0-r2", "capacity": 1048576, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 860, "first_generated_seconds": 10.518808259046637, "first_visible_seconds": 10.538462918018922, "first_answer_seconds": 10.538462918018922, "end_to_end_seconds": 27.815538955037482, "generation_tps": 49.60730921339835, "generation_tps_native": 49.6650592823313, "input_seconds_native": 10.49313998222351, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "84f4e722a5f552b2cb7e07d2424e27e39c806767536d8fea0468696f97b972e0" } ], "generation_tps_median": 49.559445374470776, "generation_tps_native_median": 49.61754554489806, "first_generated_seconds_median": 10.511140893097036, "first_visible_seconds_median": 10.530611487920396, "first_answer_seconds_median": 10.538462918018922, "end_to_end_seconds_median": 27.815538955037482, "input_seconds_native_median": 10.485511302947998 }, { "capacity": 1048576, "input_target": 32768, "actual_input_tokens": [ 32768 ], "mtp": true, "repetitions": 3, "caps": 0, "generated_tokens": [ 739, 984, 958 ], "all_trials": [ { "target": 32768, "mtp": true, "seed": 42, "warmup": false, "rep": 0, "case": "main-ctx32768-mtp1-r0", "capacity": 1048576, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 739, "first_generated_seconds": 10.829258466954343, "first_visible_seconds": 10.829258466954343, "first_answer_seconds": 12.16216266003903, "end_to_end_seconds": 20.65976138296537, "generation_tps": 74.82916904109065, "generation_tps_native": 74.93056357908671, "input_seconds_native": 10.790263652801514, "finish_reason": "stop_token", "accepted_draft_tokens": 454, "rejected_draft_tokens": 401, "output_sha256": "babfa4fa541bce9a910bef7a682716748265175414adbfa6eb29d180cc33eeb8" }, { "target": 32768, "mtp": true, "seed": 43, "warmup": false, "rep": 1, "case": "main-ctx32768-mtp1-r1", "capacity": 1048576, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 984, "first_generated_seconds": 10.829771901946515, "first_visible_seconds": 10.829793002922088, "first_answer_seconds": 10.829793002922088, "end_to_end_seconds": 23.544281640904956, "generation_tps": 77.11732670362623, "generation_tps_native": 77.19577769722098, "input_seconds_native": 10.79044508934021, "finish_reason": "stop_token", "accepted_draft_tokens": 613, "rejected_draft_tokens": 500, "output_sha256": "d5604908fe28237750c9211cdeb4360211b5d144c44987bee607b05c4430bf04" }, { "target": 32768, "mtp": true, "seed": 44, "warmup": false, "rep": 2, "case": "main-ctx32768-mtp1-r2", "capacity": 1048576, "input_tokens": 32768, "prompt_sha256": "ecbad986389925d0106c42c295bdf4d357266baf5cbf2a86e415d5d590eeed7e", "generated_tokens": 958, "first_generated_seconds": 10.830369401955977, "first_visible_seconds": 10.83039523998741, "first_answer_seconds": 10.83039523998741, "end_to_end_seconds": 23.536025998997502, "generation_tps": 75.1280450290365, "generation_tps_native": 75.20654873335106, "input_seconds_native": 10.791154861450195, "finish_reason": "stop_token", "accepted_draft_tokens": 588, "rejected_draft_tokens": 522, "output_sha256": "771ecfb2189936fc4cf16dee39b0b7880242b4ea175413408111275983fee1c6" } ], "generation_tps_median": 75.1280450290365, "generation_tps_native_median": 75.20654873335106, "first_generated_seconds_median": 10.829771901946515, "first_visible_seconds_median": 10.829793002922088, "first_answer_seconds_median": 10.83039523998741, "end_to_end_seconds_median": 23.536025998997502, "input_seconds_native_median": 10.79044508934021 } ], "GPU_peak_MiB": [ 74123, 74297 ], "warmup_records_retained": 4, "all_main_records_retained": 12, "MTP_off_draft_allocated": true, "main_model_cache_and_draft_GPU_resident": true, "source_sha256": "c15b3e0cca7bfe3a274075e534ec085ca788d10b90ff5e2b9460df6b386fb9f2", "limits": [ "Real inputs4K/32K, despite muchlarger allocatedcapacity.", "Measurements usein-process API; remoteclient/network latencyexcluded.", "Loading,warmupandtokenizationexcluded; streaming observationoverhead included.", "Generate speeds arepertrial means, notinstantaneous guarantee.", "MTP-off retainsdraftallocation toholdtargetplacementfixed; thisdoesnotmeasureminimumMTP-offmemory." ] } ], "performance_answer_reviews": [ { "case": "performance-262144", "reviewer": "Codex", "all_12_full_outputs_read": true, "human_reading_claimed": false, "review_type": "Exploratory non-blinded qualitative reading, separate from preregistered strict objective primary outcome", "results": [ { "case": "main-ctx32768-mtp0-r0", "output_sha256": "de77fdf6bb57e490d0ff75fa04eade36fb2c420a42d937281d92a8651aa312fb", "answer_characters_including_whitespace": 1246, "both_generation_speed_calculations_correct": true, "finish_reason": "stop_token", "replacement_character_count": 0, "notes_ja": [ "各条件1回なのに速度差を「有意」と表現。800/900という生成数があるのに上限1024に到達したか不明と述べる。" ] }, { "case": "main-ctx32768-mtp0-r1", "output_sha256": "598ad0b85a68fc5bbb82ba7c05352bc83b156f31589e42286371c27d9da1a4d9", "answer_characters_including_whitespace": 1411, "both_generation_speed_calculations_correct": true, "finish_reason": "max_new_tokens", "replacement_character_count": 1, "notes_ja": [ "未提示のA/B入力対応を断定し、単発観測を「入力長によらず安定」と一般化。上限到達の切り分け不明という説明も数値記録と整合しない。「比較的�やか」に置換文字。生成上限で終了。" ] }, { "case": "main-ctx32768-mtp0-r2", "output_sha256": "b78043d889345d99768f4628e20ba589ef2edd42deee981a047ec024a7eefe27", "answer_characters_including_whitespace": 1330, "both_generation_speed_calculations_correct": true, "finish_reason": "stop_token", "replacement_character_count": 0, "notes_ja": [ "未提示のBが長入力という対応を断定。「試行Aは生成途中の速度」という意味不明な記述。" ] }, { "case": "main-ctx32768-mtp1-r0", "output_sha256": "ba174c5b71b8b5b86ce453908532b8abacb4970fc3d8d6b276cda3a4ae3b898c", "answer_characters_including_whitespace": 1212, "both_generation_speed_calculations_correct": true, "finish_reason": "stop_token", "replacement_character_count": 0, "notes_ja": [ "計算・時間差・未提示のA/B入力対応に関する留保は妥当。文字数は指定目安より長い。" ] }, { "case": "main-ctx32768-mtp1-r1", "output_sha256": "c99fce9e82de4388dc736e3c226d9f2ab3a3e1eb40c9bdd3bd53699022f83eb3", "answer_characters_including_whitespace": 1423, "both_generation_speed_calculations_correct": true, "finish_reason": "stop_token", "replacement_character_count": 0, "notes_ja": [ "未提示のA=4096/B=32768対応を断定。0.8秒と0.9秒を両方「0.9秒程度」とする丸めも粗い。" ] }, { "case": "main-ctx32768-mtp1-r2", "output_sha256": "159336fde6bbc5c448b43fbe8ac24257fd50fe0bf055ddd58a9e3af94bdf24b6", "answer_characters_including_whitespace": 1310, "both_generation_speed_calculations_correct": true, "finish_reason": "stop_token", "replacement_character_count": 0, "notes_ja": [ "未提示のBが長入力という対応を前提に、長入力で生成速度が低下したと結論。" ] }, { "case": "main-ctx4096-mtp0-r0", "output_sha256": "8be938389c8e53c7c63bf5cc04e4dcf22876ce00424965e9b5058ff75e220ccf", "answer_characters_including_whitespace": 1193, "both_generation_speed_calculations_correct": true, "finish_reason": "stop_token", "replacement_character_count": 0, "notes_ja": [ "未提示のA=4096/B=32768を断定。本文開始前の差分0.8/0.9秒を「本文出力開始後の追加待ち時間」と誤記。" ] }, { "case": "main-ctx4096-mtp0-r1", "output_sha256": "22be14553d2a1367d13426145d5aa2765750d901fe88e5d61d074133f5935ebc", "answer_characters_including_whitespace": 1460, "both_generation_speed_calculations_correct": true, "finish_reason": "max_new_tokens", "replacement_character_count": 0, "notes_ja": [ "未提示のA=4096/B=32768を断定。生成上限で次の評価手順の文中で終了。" ] }, { "case": "main-ctx4096-mtp0-r2", "output_sha256": "7edb7e61ff44a76c45a4597203b9a6b39f3827cdb91ff38f5419e6909d44f8d9", "answer_characters_including_whitespace": 1275, "both_generation_speed_calculations_correct": true, "finish_reason": "stop_token", "replacement_character_count": 0, "notes_ja": [ "未提示の入力対応を推測から確定事項へ移し「長い入力」条件として説明。速度と時間という尺度の異なる変化の大小比較も曖昧。" ] }, { "case": "main-ctx4096-mtp1-r0", "output_sha256": "3637a4394734d108d72f8b8129d60d610c7874dfafff084c7bd57d737e3fe1ff", "answer_characters_including_whitespace": 1238, "both_generation_speed_calculations_correct": true, "finish_reason": "stop_token", "replacement_character_count": 0, "notes_ja": [ "A/B入力対応未記載を正しく留保。本文開始と最初の生成トークンを同じ指標として説明する箇所は区別不足。" ] }, { "case": "main-ctx4096-mtp1-r1", "output_sha256": "f8b62489da2f9cf1cac442f90897a45a5bfbbbb947e3d0f8a2ef7b8020c8927c", "answer_characters_including_whitespace": 1441, "both_generation_speed_calculations_correct": true, "finish_reason": "stop_token", "replacement_character_count": 0, "notes_ja": [ "A/B入力対応を断定しないと書いた後、入力増大の傾向を確定事項として列挙。待ち時間の包含関係への留保自体は追加定義を求める点で合理的。" ] }, { "case": "main-ctx4096-mtp1-r2", "output_sha256": "c600cfbb8470e6c8a8955635c1e2cd2974dfa3b89c224b68f5336228f0de8109", "answer_characters_including_whitespace": 1178, "both_generation_speed_calculations_correct": true, "finish_reason": "stop_token", "replacement_character_count": 0, "notes_ja": [ "未提示のB=32768対応を断定。" ] } ], "summary_ja": "全12回答でA=80、B=60tokens/sの計算は正しい。両MTP条件に未提示の入力対応の断定等が残る。MTPなし2回は生成上限終了し、1回に置換文字を確認。", "utc": "2026-10-31T00:11:39.505609+00:00" }, { "case": "performance-1048576", "reviewer": "Codex", "all_12_full_outputs_read": true, "human_reading_claimed": false, "review_type": "Exploratory non-blinded qualitative reading, separate from preregistered strict objective primary outcome", "results": [ { "case": "main-ctx32768-mtp0-r0", "output_sha256": "31d430c7588167efa8c11b973e0aa32a412b18f2e5633ec77878650d7c737b8d", "answer_characters_including_whitespace": 1056, "both_generation_speed_calculations_correct": true, "notes_ja": [ "試行と入力長の対応は未確定と記した後に「入力トークン数が大きい試行B」と断定し、説明内で矛盾。" ] }, { "case": "main-ctx32768-mtp0-r1", "output_sha256": "5fde1e1f13879bd11306b7b35ee6de1d9338b6a98229e842a86e93732b840d70", "answer_characters_including_whitespace": 1380, "both_generation_speed_calculations_correct": true, "notes_ja": [ "未提示のA=4096/B=32768対応を断定。「観測された事実から直接的」として入力長の増大を原因扱い。" ] }, { "case": "main-ctx32768-mtp0-r2", "output_sha256": "84f4e722a5f552b2cb7e07d2424e27e39c806767536d8fea0468696f97b972e0", "answer_characters_including_whitespace": 1222, "both_generation_speed_calculations_correct": true, "notes_ja": [ "計算・時間差・未提示の入力対応への留保は妥当。文字数は指定目安より長い。" ] }, { "case": "main-ctx32768-mtp1-r0", "output_sha256": "babfa4fa541bce9a910bef7a682716748265175414adbfa6eb29d180cc33eeb8", "answer_characters_including_whitespace": 904, "both_generation_speed_calculations_correct": true, "notes_ja": [ "計算・時間差・原因への留保は妥当。語数ではなく日本語文字数の目安も満たす。速度差20tokens/sと時間差の大小を比較する表現は尺度が曖昧。" ] }, { "case": "main-ctx32768-mtp1-r1", "output_sha256": "d5604908fe28237750c9211cdeb4360211b5d144c44987bee607b05c4430bf04", "answer_characters_including_whitespace": 1419, "both_generation_speed_calculations_correct": true, "notes_ja": [ "未提示のA/B入力長対応を断定。上限1024まで生成すれば「生成速度の上限」を確認できるという提案は根拠不足。" ] }, { "case": "main-ctx32768-mtp1-r2", "output_sha256": "771ecfb2189936fc4cf16dee39b0b7880242b4ea175413408111275983fee1c6", "answer_characters_including_whitespace": 1323, "both_generation_speed_calculations_correct": true, "notes_ja": [ "入力対応の未記載を正しく留保。一回ずつの観測を「安定していた」と呼ぶのは過大表現。" ] }, { "case": "main-ctx4096-mtp0-r0", "output_sha256": "1fc48a10bf49a3fb66f803393a2d10ef384fae722e307fa0b13b1acfb3d125c5", "answer_characters_including_whitespace": 1153, "both_generation_speed_calculations_correct": true, "notes_ja": [ "未提示のA=4096/B=32768対応を断定し、入力長によるメモリ増分と説明。" ] }, { "case": "main-ctx4096-mtp0-r1", "output_sha256": "d861391e80fe00ef2abe66bb79fc5b85764512c8e37be81a553af42043f16550", "answer_characters_including_whitespace": 1285, "both_generation_speed_calculations_correct": true, "notes_ja": [ "未提示のA/B入力対応を断定。入力8倍に対し時間6倍を「比例」と述べるのは厳密には不適切。" ] }, { "case": "main-ctx4096-mtp0-r2", "output_sha256": "0d4938d42e5ad081b2ebbd76f2ca38c7cfa126580e7c521fc38e2b6a451696db", "answer_characters_including_whitespace": 1401, "both_generation_speed_calculations_correct": true, "notes_ja": [ "計算、時間差、原因を断定しない説明は妥当。文字数は指定目安より長い。" ] }, { "case": "main-ctx4096-mtp1-r0", "output_sha256": "3a237efc28bd077e71a1fdedcf961690c017dff75fa7664edc48b181cd5bf5ac", "answer_characters_including_whitespace": 1028, "both_generation_speed_calculations_correct": true, "notes_ja": [ "未提示のBが長入力という対応を断定。本文開始が2.8秒→12.9秒なのに「わずかに延び」と表現。" ] }, { "case": "main-ctx4096-mtp1-r1", "output_sha256": "6a58c286909b1d11acebae380f9e2af0f2aecbfbb088b2db8000a90dac0607f9", "answer_characters_including_whitespace": 1421, "both_generation_speed_calculations_correct": true, "notes_ja": [ "未提示のA/B入力対応を断定。8倍の入力と6倍の時間を「比例」と表現。" ] }, { "case": "main-ctx4096-mtp1-r2", "output_sha256": "2c26e99693b2e70230f7ede2f61c7ff4ef3724159f86e4c01749e45dadd0530d", "answer_characters_including_whitespace": 1348, "both_generation_speed_calculations_correct": true, "notes_ja": [ "入力対応の未記載を留保する点は妥当。生成時間の起算点が資料にないのに本文開始待ち時間へ単純加算し12.8/27.9秒と確定。時間区間の重複を排除できない。" ] } ], "summary_ja": "全12回答でA=80、B=60tokens/sの計算は正しい。両MTP条件に、未記載の入力長対応の断定や指定目安より長い回答などがある。この12回答のみでMTPの品質因果は結論しない。", "utc": "2026-10-31T00:03:34.433488+00:00" } ], "quality_error_review": { "complete": true, "reviewer": "Codex", "human_review_claimed": false, "utc": "2026-10-31T02:42:23.826936+00:00", "review_bundle_sha256": "0928956865a613816cd5068cdc08818182e2a570afd26bf47edb9bf8c47b47fd", "selection_counts": { "quality-main-1m": { "total_responses": 3840, "selected_responses": 312, "wrong_responses": 288, "affected_pairs": 156, "selection": "all pairs with any error, retaining both conditions" }, "quality-long-1m": { "total_responses": 48, "selected_responses": 48, "wrong_responses": 26, "affected_pairs": 13, "selection": "all secondary answers" } }, "primary_early_reviews_reconciled": 312, "secondary_full_completions_read": 48, "total_full_completions_read": 360, "secondary_notes": [ { "case": "long-records-017-sampling-r0-mtp1", "output_sha256": "aadfd3299da50313bf1e042893de63d6213edb55256e17ed94f6f7176b773ce8", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-017-sampling-r0-mtp0", "output_sha256": "4591807af44544e4a5ea68cddb63da29a560b9bd0d5fdb0cbcc542bb2386075d", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-003-sampling-r0-mtp1", "output_sha256": "80f0252c09cd19aca39edfb070c691b853be49a1e17cddca2765887fd61229b4", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-003-sampling-r0-mtp0", "output_sha256": "3b0ab3876d79d22e4894d3baa5d15bd155d85e3d82d8b24b16fcacd93bf1b409", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-004-sampling-r0-mtp1", "output_sha256": "20f37bcde52111530352b0403a11de7270abffbbd7d0c6783c94f164f305944c", "full_completion_reviewed": true, "note": "Both conditions return department sum1375 instead of1273; other five fields correct. Natural stop; not an output cap." }, { "case": "long-records-004-sampling-r0-mtp0", "output_sha256": "9da0a256a0065987099c94f40dc1e411c732032539578428c4114f9f385f0927", "full_completion_reviewed": true, "note": "Both conditions return department sum1375 instead of1273; other five fields correct. Natural stop; not an output cap." }, { "case": "long-records-014-sampling-r0-mtp1", "output_sha256": "cea9875005081b0dedde8022c5a8b5f568539f51c2821e78d830e5f96e27e661", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-014-sampling-r0-mtp0", "output_sha256": "2e0e8aecb3c5730b4952efb080a93c297c1ebc4edc7ad0a86c5dbd572c6c11d0", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-022-sampling-r0-mtp0", "output_sha256": "d56287ad36337bd464db476aab80ec53550454f785b90baccba038070ae580a4", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-022-sampling-r0-mtp1", "output_sha256": "738747ae305b567b272b981364908009f86f9c52b188e25c0c7edc8f64e12329", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-007-sampling-r0-mtp0", "output_sha256": "f832085021c796624b32fd217707c33b0811a3feaf40295a6412438015962a30", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-007-sampling-r0-mtp1", "output_sha256": "92ea66f19d039aa594d16de0f15981563da0ac866b4fb4b9bc9ccca468f72e87", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-000-sampling-r0-mtp0", "output_sha256": "94c1eba4a47411dc62d74b93726bde131b8644ddd94666ffa18303ce565851f4", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-000-sampling-r0-mtp1", "output_sha256": "94c1eba4a47411dc62d74b93726bde131b8644ddd94666ffa18303ce565851f4", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-019-sampling-r0-mtp1", "output_sha256": "211ad91af1a93aabb455c11325da6d00ed2c8ccb232bc4488c522716b128d6fd", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-019-sampling-r0-mtp0", "output_sha256": "ee953c219ad660f11506b375fdb292dae18c2e3c8d2ffdbe1d6facaf6ad233af", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-002-sampling-r0-mtp0", "output_sha256": "eb97f95c5b991e09af2382519bb3e0e225eafa939b1abbf6686ec3ba789a07cf", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-002-sampling-r0-mtp1", "output_sha256": "c1da7564ec56c7f23f8bad7b52f3d2ed674728c3643cfa32127731ca825ccf70", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-006-sampling-r0-mtp1", "output_sha256": "b1ae6dfd7660df38b83057311427a50d037b1dd1e5f0613f2ce4c6c74395c805", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-006-sampling-r0-mtp0", "output_sha256": "b1ae6dfd7660df38b83057311427a50d037b1dd1e5f0613f2ce4c6c74395c805", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-008-sampling-r0-mtp0", "output_sha256": "fe87cfca851ec37f55def3ec61e1003106156cbba2955423d6b4574b1ebf9dfb", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-008-sampling-r0-mtp1", "output_sha256": "c0b093bd73ad4f725de9fde52ed1f0a9df431d16d6108c02b4edd24364bca154", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-011-sampling-r0-mtp0", "output_sha256": "800d64a915b80e5649889906fb9a275139c3f5c4c06b6404c315839edf58e18f", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-011-sampling-r0-mtp1", "output_sha256": "800d64a915b80e5649889906fb9a275139c3f5c4c06b6404c315839edf58e18f", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-018-sampling-r0-mtp1", "output_sha256": "0ec0d68c6191879e46f4e2961335212aa944d787e7afafc0bdcae3f28ccde9ae", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-018-sampling-r0-mtp0", "output_sha256": "ef92a6ced52ea3273544c938fb115482263b6db35cddfef80962f8b2e6751540", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-001-sampling-r0-mtp1", "output_sha256": "25977bf3b227029656473a2941f8cc2f940a82518259ac852f61a623d8eb274e", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-001-sampling-r0-mtp0", "output_sha256": "39861cd1e8d789b1e8ffe94f43e38b07cf934b063e3f6ea360428116ff7cbba4", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-015-sampling-r0-mtp1", "output_sha256": "8c798a707e2ccbcb48c847eaefae76f9d090a3bee398f3d72946852cb0e384dd", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-015-sampling-r0-mtp0", "output_sha256": "573cf25a97ef591be18b9706cf12663f691a1f253f5f60342f36226fb3d8bd9a", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-020-sampling-r0-mtp1", "output_sha256": "d1adb08a422d7213e69acbaf3957ba8a62d00fe340068b07fae460aea1998456", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-020-sampling-r0-mtp0", "output_sha256": "6f3a010648f6b92f9f00f37b2354714a8ba69f150cd73425b07db5a7d321603a", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-010-sampling-r0-mtp0", "output_sha256": "818330edb9c72956bf7d61b2293203139ebb3d09ff4831df8b5666a5714473d1", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle. Reasoning contains an inconsistent initial Sales-row list, although final six-field answer is correct; scoring covers final answer, not reasoning validity." }, { "case": "long-records-010-sampling-r0-mtp1", "output_sha256": "3923307a45f4d1c4147620bd0d0035eabd2a0635dbbf989f168020ac59825898", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-021-sampling-r0-mtp1", "output_sha256": "9ed96a9c95390c52d5314dbd4e48e532263df4938639ac460cb1b01d122a4b13", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-021-sampling-r0-mtp0", "output_sha256": "f9cf72e89dbc8db70ee00000e418965b03759390b4bec7149e908f19352db89a", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-023-sampling-r0-mtp0", "output_sha256": "8b8316f2f2dbbf10acc8ff0b926b48b42b4e08ee9b7cd5b0a8b1b161aefb6ac3", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-023-sampling-r0-mtp1", "output_sha256": "afd8d05b9658058cd3f862cf310035015bd49e05337b6e9f01d29983f62253b7", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-009-sampling-r0-mtp0", "output_sha256": "a7f89e125784c30d257af7dc141627d7e282e951c8680b9e4e6b4689bd071987", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-009-sampling-r0-mtp1", "output_sha256": "a7f89e125784c30d257af7dc141627d7e282e951c8680b9e4e6b4689bd071987", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-005-sampling-r0-mtp0", "output_sha256": "13e6527052963ee8e221fcad86f2bae421300e56109df0c91289ca3c57e8387e", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-005-sampling-r0-mtp1", "output_sha256": "13e6527052963ee8e221fcad86f2bae421300e56109df0c91289ca3c57e8387e", "full_completion_reviewed": true, "note": "Final six-field JSON agrees with independent oracle." }, { "case": "long-records-013-sampling-r0-mtp1", "output_sha256": "8cced956a5c38b020db4d63c0a1b113448934bc0b36a2e294945d236fe34a745", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-013-sampling-r0-mtp0", "output_sha256": "f675d681dbc361da7c4db372620687f6bc50b7e12957c75bb9040c7cb2cd15ce", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-012-sampling-r0-mtp0", "output_sha256": "bfda5cc18058f5c3e54492c644a1e384b8895d56da236ce25955a44e18ed470c", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-012-sampling-r0-mtp1", "output_sha256": "bfda5cc18058f5c3e54492c644a1e384b8895d56da236ce25955a44e18ed470c", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-016-sampling-r0-mtp0", "output_sha256": "23fd0c10395c343d42ee3591bef253957dd7204625773b9df935bb24b5f2d38f", "full_completion_reviewed": true, "note": "Output cap during reasoning/listing or manual aggregation; no completed final answer. Some requested values occur in reasoning, but are not a submitted structured answer; do not reinterpret field scores as independent retrieval accuracy." }, { "case": "long-records-016-sampling-r0-mtp1", "output_sha256": "6ea98fc9e27ee392fe5798f204f1cef914fe54ffc299f9af3762fe8303a4c600", "full_completion_reviewed": true, "note": "Natural stop with five fields correct; substitutes estimated department total9500 for exact9883. Reasoning explicitly abandons exact aggregation; final JSON fails to mark total as approximate." } ], "interpretation": [ "Main fixed objective suite establishes only its preregistered average noninferiority result, not equality or universal quality.", "Secondary4K:11/12 full answers correct in both conditions; sole shared error is department aggregation.", "Secondary32K:0/12 full answers correct in both conditions. Output1024-token budget includes reasoning; off12/12 and on11/12 capped before final answer. One naturally finished MTP-on answer estimates9500 instead of exact9883.", "Secondary field scores concern final structured answers; missing final responses are scored zero. They do not establish that every underlying retrieval operation failed.", "No larger-output-budget or tool-assisted rerun was performed. No quality claim for full1M history." ] }, "preregistered_protocol": { "status": "PREREGISTERED_BEFORE_NEW_MODEL_OUTPUTS", "created_utc": "2026-10-30T23:38:45.219731+00:00", "authority": "Direct human request to design and run H100x2 ExLlamaV3 MTP quality comparison and publish blog/video; later instruction supersedes prior no-further-measurement status for H100 only.", "objective": "Estimate quality effect of MTP on the existing H100x2 deployment without presupposing non-degradation.", "engine": "ExLlamaV3 1.6.5 gpuembed1", "model": "turboderp/GLM-5.3-Flash-exl3", "revision": "b570edf655a2bb00beca23e18d3b98724bfc2db5", "quantization": "sc_3.10bpw", "gpus": 2, "mtp_conditions": [ false, true ], "mtp_draft_tokens": 3, "concurrency": 1, "max_generation_tokens": 1024, "primary": { "unique_questions": 600, "categories": [ "integer_and_decimal_arithmetic", "units_rates_and_statistics", "record_queries_and_aggregation", "python_code_trace", "logic_and_constraints", "grounded_evidence_and_abstention" ], "questions_per_category": 100, "seeds_per_question": 3, "sampler": { "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0 }, "metric": "strict objective answer accuracy, paired by question and seed; natural stop required", "noninferiority_margin_percentage_points": 2.0, "confidence": "one-sided 95% lower bound of MTP-on minus off > -2 percentage points; stratified bootstrap over questions, retaining all 3 seeds as a cluster, 20000 resamples; paired discordance and conservative sensitivity bound also reported", "no_optional_stopping": true, "interpretation": "Bounded claim for this frozen finite task suite and runtime only. Not proof of equality, any-input guarantee, or general Japanese response quality." }, "secondary": { "greedy_pairs": 120, "selection": "20 preselected questions from each category before inference", "long_context_unique_tasks": 24, "long_context_sizes_tokens_approx": [ 4096, 32768 ], "long_context_seeds_per_task": 1, "long_context_rubrics": "grounded numeric summaries, missing-information abstention, association versus causation", "legacy_cases": "retain all prior 12 long-answer quality findings; not retune tests to hide them", "metrics": [ "complete token sequence equality", "answer equality", "answer-format validity", "cap rate", "tokens/s and latency", "actual MTP accepted/rejected token counts" ] }, "operational_limits": { "source_scripts_and_previous_results_preserved": true, "drivers_and_existing_env_unchanged": true, "GPU_peak_limit_GiB_each": 75, "no_CPU_weight_KV_offload": true, "no_B200_access": true, "no_other_jobs_or_data_modified": true, "no_customer_email_or_subscriber_notification": true, "own_containers_only": true }, "4bit_question": { "availability": "same-author sc_4.10bpw exists; metadata archived", "not_assumed_perfect": true, "test_admission": "requires capacity and storage sufficiency while preserving existing data; not a prerequisite for isolating MTP", "quantization_causality": "Cannot attribute existing errors to 3-bit quantization without controlled quantization comparison." }, "publication": "Update existing GLM article with methods, all outcomes and limitations; create Japanese Marin 4K follow-up video preserving prior video. Publish negative or inconclusive results as such.", "updated_utc": "2026-10-30T23:41:08.067784+00:00", "user_preference": "日本語の実務回答を中心にする", "context_amendment": { "authority": "Direct user request to expand 40960 context capacity to at least250K, preferably1M; actual timing inputs4K and32K, report first-answer latency andtokens/s.", "capacities_requested": [ 262144, 1048576 ], "actual_input_tokens_targets": [ 4096, 32768 ], "capacity_qualification": "Separate capacity and resident GPU/KV verification before tests. No claim that short-input tests validate long-history reasoning.", "latency_metrics": [ "first_generated_token", "first_visible_text_including_reasoning", "first_answer_body_after_reasoning", "end_to_end" ], "performance_repeats": 3, "quality_runtime_selection": "Use largest successfully qualified all-GPU capacity; do not change model/runtime weights between MTPon/off. Freeze final task-length allocation before main." } }, "context_allocation": { "utc": "2026-10-30T23:55:51.865626+00:00", "main_quality_capacity": 1048576, "target_allocator_GiB": [ 67, 73 ], "MTP_draft_device": 0, "MTP_drafts": 3, "MTP_off_comparator": "Keep draft model and its cache allocated, but do not pass them to Generator. This fixes target weights/split/memory layout when changing MTP. It does not measure the minimum VRAM of a separately loaded MTP-off deployment.", "primary_inputs": "600 Japanese objective business questions padded to4K, each3independent seeds, MTPoff/on paired;120additional greedy pairs.", "secondary_inputs": "24grounded record questions,12at4K and12at32K, one paired seed each. Different record count acrosslengths; no causal input-length quality claim.", "performance": "Each262144 and1048576capacity: bothMTPconditions,4K/32Ksame supplied records,3seeds;4warmups excluded and retained. Three-repetition medians plus alltrials.", "frozen_source_sha256": { "quality_runner.py": "752cc41e4f52de8233804d2a40302016e3b285ffb0b78fcb287763a01c1d1d21", "performance_runner.py": "03848b6d8f43cd7975f9c3319dbc997eb94b48c7931357fd5701bb5a2add30bb", "score_answers.py": "03866ba67c12766903638f72ca4500b051fa396c3c3063083add691531819536" }, "frozen_tasks_sha256": { "tasks.json": "fcdc18ff49a4ee0321982667046d106a4a58bd50621b6379ea8b66e090fe06d7", "pilot-tasks.json": "831febb3555cb9cff1dfa42cc9c4daab9986ad2d22608989d4b877039386510b", "long-tasks.json": "66923ee4726d24c5d4dc2a21d3030d5326c6987f1ac0379c690a40198d8c99b4" }, "publication_claim_policy": "Report noninferiority only if prespecified lowerCI>-2pp. If inconclusive or worse, publish that outcome. Sampling equality/greedy agreement do not establish all-input equality." }, "capacity_audit": { "status": "PASS_256K_AND_1M_CAPACITY_SHORT_INPUT_QUALIFICATION", "utc": "2026-10-30T23:54:50.526136+00:00", "cases": [ { "case": "capacity-262144-v2", "capacity": 262144, "pass": true, "GPU_peak_MiB": [ 69289, 64837 ], "target_kv_bytes": 4613734400, "draft_kv_bytes": 419430400, "all_model_weights_and_KV_on_GPU": true, "measurements": [ { "case": "ctx4096-mtp0", "capacity": 262144, "input_tokens": 4096, "mtp": false, "generated_tokens": 4, "first_generated_seconds": 2.244705355959013, "first_visible_seconds": 2.2641563219949603, "first_answer_seconds": 2.2641563219949603, "end_to_end_seconds": 2.467787579051219, "generation_tps": 34.40115919840337, "native_tps": 45.868212264537824, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "answer": "確認完了", "cached_tokens": 0, "cached_pages": 0 }, { "case": "ctx4096-mtp1", "capacity": 262144, "input_tokens": 4096, "mtp": true, "generated_tokens": 4, "first_generated_seconds": 2.0138665839331225, "first_visible_seconds": 2.0138976209564134, "first_answer_seconds": 2.0138976209564134, "end_to_end_seconds": 2.0139147859299555, "generation_tps": 10.028214362849681, "native_tps": 13.370952483799575, "finish_reason": "stop_token", "accepted_draft_tokens": 3, "rejected_draft_tokens": 0, "answer": "確認完了", "cached_tokens": 0, "cached_pages": 0 }, { "case": "ctx32768-mtp1", "capacity": 262144, "input_tokens": 32768, "mtp": true, "generated_tokens": 4, "first_generated_seconds": 11.489941479056142, "first_visible_seconds": 11.48996461299248, "first_answer_seconds": 11.48996461299248, "end_to_end_seconds": 11.489979916019365, "generation_tps": 84.0659811997675, "native_tps": 112.08797493302333, "finish_reason": "stop_token", "accepted_draft_tokens": 3, "rejected_draft_tokens": 0, "answer": "確認完了", "cached_tokens": 0, "cached_pages": 0 }, { "case": "ctx32768-mtp0", "capacity": 262144, "input_tokens": 32768, "mtp": false, "generated_tokens": 4, "first_generated_seconds": 10.041666783974506, "first_visible_seconds": 10.060650893952698, "first_answer_seconds": 10.060650893952698, "end_to_end_seconds": 10.099691653973423, "generation_tps": 39.20765275916867, "native_tps": 52.27687034555822, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "answer": "確認完了", "cached_tokens": 0, "cached_pages": 0 } ], "performance_claim": false, "long_history_comprehension_claim": false }, { "case": "capacity-1048576-balanced", "capacity": 1048576, "pass": true, "GPU_peak_MiB": [ 74151, 74385 ], "target_kv_bytes": 18454937600, "draft_kv_bytes": 1677721600, "all_model_weights_and_KV_on_GPU": true, "measurements": [ { "case": "ctx4096-mtp0", "capacity": 1048576, "input_tokens": 4096, "mtp": false, "generated_tokens": 4, "first_generated_seconds": 3.3806308549828827, "first_visible_seconds": 3.400668494985439, "first_answer_seconds": 3.400668494985439, "end_to_end_seconds": 3.6541334650246426, "generation_tps": 33.35112355083412, "native_tps": 44.46816473444549, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "answer": "確認完了", "cached_tokens": 0, "cached_pages": 0 }, { "case": "ctx4096-mtp1", "capacity": 1048576, "input_tokens": 4096, "mtp": true, "generated_tokens": 4, "first_generated_seconds": 2.098727051052265, "first_visible_seconds": 2.098757717059925, "first_answer_seconds": 2.098757717059925, "end_to_end_seconds": 2.098775784019381, "generation_tps": 9.260121222111996, "native_tps": 12.34682829614933, "finish_reason": "stop_token", "accepted_draft_tokens": 3, "rejected_draft_tokens": 0, "answer": "確認完了", "cached_tokens": 0, "cached_pages": 0 }, { "case": "ctx32768-mtp1", "capacity": 1048576, "input_tokens": 32768, "mtp": true, "generated_tokens": 4, "first_generated_seconds": 11.578070224961266, "first_visible_seconds": 11.578093659947626, "first_answer_seconds": 11.578093659947626, "end_to_end_seconds": 11.57810951396823, "generation_tps": 82.60243811174351, "native_tps": 110.13658414899135, "finish_reason": "stop_token", "accepted_draft_tokens": 3, "rejected_draft_tokens": 0, "answer": "確認完了", "cached_tokens": 0, "cached_pages": 0 }, { "case": "ctx32768-mtp0", "capacity": 1048576, "input_tokens": 32768, "mtp": false, "generated_tokens": 4, "first_generated_seconds": 10.119671641034074, "first_visible_seconds": 10.139362593996339, "first_answer_seconds": 10.139362593996339, "end_to_end_seconds": 10.179865972022526, "generation_tps": 37.85199624575843, "native_tps": 50.46932832767791, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "answer": "確認完了", "cached_tokens": 0, "cached_pages": 0 } ], "performance_claim": false, "long_history_comprehension_claim": false } ], "failed_attempts_retained": [ "capacity-262144 instrumentationKeyError elapsed; fixed independently", "capacity-1048576 unbalanced75GiBguard; noOOM; targetallocation revised67/73GiB" ], "quality_capacity_selected": 1048576, "quality_target_allocator_GiB": [ 67, 73 ], "limits": "1M allocated; actual inputs4K/32K only. Short4-token completions are capacity checks, not representative generation speeds or long-history reasoning tests." }, "capacity_placement": { "utc": "2026-10-31T00:26:03.706407+00:00", "comparison": "Descriptive comparison of two qualified capacity configurations. Allocator budgets match, but layer placement changes with KV capacity; not a capacity-only causal estimate.", "capacities": { "262144": { "source_sha256": "2e926605f91699060d134b827eeaeb0525d5085dab3c2c723b26e54663f9edc6", "target": [ { "GPU": 0, "KV_bytes": 838860800, "KV_layers": [ "model.language_model.layers.35.self_attn", "model.language_model.layers.43.self_attn" ] }, { "GPU": 1, "KV_bytes": 3774873600, "KV_layers": [ "model.language_model.layers.11.self_attn", "model.language_model.layers.15.self_attn", "model.language_model.layers.19.self_attn", "model.language_model.layers.23.self_attn", "model.language_model.layers.27.self_attn", "model.language_model.layers.3.self_attn", "model.language_model.layers.31.self_attn", "model.language_model.layers.39.self_attn", "model.language_model.layers.7.self_attn" ] } ], "draft_KV_bytes": 419430400 }, "1048576": { "source_sha256": "6075ac7f04972b764c1b233fafa5f2905f38cbe3c8d6a6c461a4a2381cc668d9", "target": [ { "GPU": 0, "KV_bytes": 6710886400, "KV_layers": [ "model.language_model.layers.19.self_attn", "model.language_model.layers.27.self_attn", "model.language_model.layers.35.self_attn", "model.language_model.layers.43.self_attn" ] }, { "GPU": 1, "KV_bytes": 11744051200, "KV_layers": [ "model.language_model.layers.11.self_attn", "model.language_model.layers.15.self_attn", "model.language_model.layers.23.self_attn", "model.language_model.layers.3.self_attn", "model.language_model.layers.31.self_attn", "model.language_model.layers.39.self_attn", "model.language_model.layers.7.self_attn" ] } ], "draft_KV_bytes": 1677721600 } } }, "model_context_config": { "config_sha256": "d93f855896a545c6221ef54968c678718c185ddc67b53309b363ffd8ed676397", "max_position_embeddings": 1048576, "model_type": "glm5_next_text", "local_remote_config_match": true }, "claim_limits": [ "Fixed 30 templates instantiated as600 questions, not600 distinct task families.", "Primary quality4K;secondary recordreading4K/32K. No250K/1M-history quality test.", "In-process streaming;client/network andcold model loading excluded.", "MTP-off retains draft model/cache to keep target split fixed.", "4.10bit was not benchmarked;quantization qualityeffect was not measured.", "Agent client contextlimit/autocompaction policy was not configured." ], "data_transfer_audits": [ { "status": "ALL_REMOTE_LOCAL_FILES_SHA256_MATCH", "utc": "2026-10-31T02:41:03.979351+00:00", "case": "quality-main-1m", "files": 12752, "bytes": 39713580, "archive_sha256": "40d41a02a0c8142d9d1cb87aacfd2bcddb9f1dc5cb7565e7fb78ea5bd9057daa" }, { "status": "ALL_REMOTE_LOCAL_FILES_SHA256_MATCH", "utc": "2026-10-31T02:41:37.324236+00:00", "case": "quality-long-1m", "files": 205, "bytes": 7807662, "archive_sha256": "0866d7dfd340524f12c22bf7616b8dc1edea72124c2b57a8df0c51f2352898ef" } ], "illustrative_pairs": [ { "task": "code-056", "question": "次のPython 3コードのprintが出力する値を答えてください。コードは実行せず読解してください。\n```python\nxs = [13, -2, 9, 6, 6, -1, 6, 10]\nprint(sorted(set(xs))[-2])\n```", "expected": 10, "mode": "sampling", "rep": 1, "kind": "content_regression", "conditions": { "on": { "seed": 1144997408, "score": { "answer": "{\"answer\": 13}", "parsed": { "answer": 13 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "full_completion": "set sorted: [-2,-1,6,9,10,13] → [-2] = 13.{\"answer\": 13}", "output_sha256": "416cfdc4a6d23a800ebe988eac31ad858ecf4c2eea2039c014b0ba2a4b782f95" }, "off": { "seed": 1144997408, "score": { "answer": "{\"answer\": 10}", "parsed": { "answer": 10 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "full_completion": "set sorted: [-2,-1,6,9,10,13] → [-2] = 13-1... [-2] is 10? sorted ascending: -2,-1,6,9,10,13 → second from end = 10.{\"answer\": 10}", "output_sha256": "7014ecf5584b57f9f1262040edeab8dd01bc939489e8a7feb079bdecc873ed67" } } }, { "task": "code-043", "question": "次のPython 3コードのprintが出力する値を答えてください。コードは実行せず読解してください。\n```python\nxs = [14, 11, 5, 2, 15, 4, 16, -5]\nprint(xs[1:7:2])\n```", "expected": [ 11, 2, 4 ], "mode": "sampling", "rep": 0, "kind": "content_improvement", "conditions": { "on": { "seed": 1335073262, "score": { "answer": "{\"answer\": [11, 2, 4]}", "parsed": { "answer": [ 11, 2, 4 ] }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "full_completion": "[11, 15, 16]? indices 1,3,5 → 11, 2, 4. Wait xs = [14,11,5,2,15,4,16,-5]; indices 1=11,3=2,5=4.{\"answer\": [11, 2, 4]}", "output_sha256": "ac44d741fe530a41c9472912aeb58ad0482eebaa4f710d6e73f508a4beee977e" }, "off": { "seed": 1335073262, "score": { "answer": "{\"answer\":[11,15,16]}", "parsed": { "answer": [ 11, 15, 16 ] }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "full_completion": "[11, 15, 16]{\"answer\":[11,15,16]}", "output_sha256": "941176c6b586162991b54e5d3b7f2263d6f20e42dc1ee8d1d81a9ec9427e77fb" } } }, { "task": "arithmetic-053", "question": "売上43000円、仕入原価2600円です。売上に対する粗利率を百分率で小数第2位まで四捨五入して答えてください。", "expected": 93.95, "mode": "sampling", "rep": 2, "kind": "numeric_unit_representation_regression", "conditions": { "on": { "seed": 1791303408, "score": { "answer": "{\"answer\":\"93.95%\"}", "parsed": { "answer": "93.95%" }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "full_completion": "(43000-2600)/43000 = 40400/43000 = 0.93953... → 93.95%{\"answer\":\"93.95%\"}", "output_sha256": "cc3c1ce21d27eb6666900f8c8d802883e710d9df4b5d933292e3f4425f8ee65e" }, "off": { "seed": 1791303408, "score": { "answer": "{\"answer\":93.95}", "parsed": { "answer": 93.95 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "full_completion": "(43000-2600)/43000=40400/43000≈93.953...→93.95{\"answer\":93.95}", "output_sha256": "078e2015602d4edef674a10063999fcc4954d42a66e51a2704542668c07b21f0" } } }, { "task": "arithmetic-019", "question": "口座残高は18000円。8400円の支払が2件、8500円の入金が1件ありました。入出金の処理後の残高を円で答えてください。", "expected": 9700, "mode": "sampling", "rep": 0, "kind": "shared_arithmetic_error", "conditions": { "off": { "seed": 1686490760, "score": { "answer": "{\"answer\":9500}", "parsed": { "answer": 9500 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "full_completion": "{\"answer\":9500}", "output_sha256": "00403cfe5427bc1ed35ce95fa9e60330d5f3007e5d6bb370bde320eb2f928d85" }, "on": { "seed": 1686490760, "score": { "answer": "{\"answer\":9500}", "parsed": { "answer": 9500 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "full_completion": "{\"answer\":9500}", "output_sha256": "00403cfe5427bc1ed35ce95fa9e60330d5f3007e5d6bb370bde320eb2f928d85" } } }, { "task": "code-056", "question": "次のPython 3コードのprintが出力する値を答えてください。コードは実行せず読解してください。\n```python\nxs = [13, -2, 9, 6, 6, -1, 6, 10]\nprint(sorted(set(xs))[-2])\n```", "expected": 10, "mode": "sampling", "rep": 0, "kind": "content_improvement_same_task", "conditions": { "off": { "seed": 2003929982, "score": { "answer": "{\"answer\": 9}", "parsed": { "answer": 9 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "full_completion": "set sorted: [-2,-1,6,9,10,13], [-2]=9.{\"answer\": 9}", "output_sha256": "944bd2e88ea23ed4165c59694c796e579963323c0bf68e32e578aad2d78475fc" }, "on": { "seed": 2003929982, "score": { "answer": "{\"answer\": 10}", "parsed": { "answer": 10 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "full_completion": "set sorted: [-2,-1,6,9,10,13], [-2] = 10.{\"answer\": 10}", "output_sha256": "a50cb30c8fb9eace34148b39294a44269cb50c2063e30e0efab20fee0e859c77" } } }, { "task": "records-037", "question": "次のJSONだけを根拠に、全行のqtyからreturnedを引いた数の合計を答えてください。\n[{\"id\": \"R01\", \"department\": \"営業\", \"qty\": 42, \"returned\": 4, \"status\": \"確定\"}, {\"id\": \"R02\", \"department\": \"開発\", \"qty\": 58, \"returned\": 2, \"status\": \"確定\"}, {\"id\": \"R03\", \"department\": \"管理\", \"qty\": 27, \"returned\": 4, \"status\": \"保留\"}, {\"id\": \"R04\", \"department\": \"開発\", \"qty\": 13, \"returned\": 0, \"status\": \"保留\"}, {\"id\": \"R05\", \"department\": \"営業\", \"qty\": 12, \"returned\": 2, \"status\": \"保留\"}, {\"id\": \"R06\", \"department\": \"管理\", \"qty\": 30, \"returned\": 0, \"status\": \"保留\"}, {\"id\": \"R07\", \"department\": \"営業\", \"qty\": 9, \"returned\": 2, \"status\": \"確定\"}, {\"id\": \"R08\", \"department\": \"管理\", \"qty\": 61, \"returned\": 4, \"status\": \"確定\"}, {\"id\": \"R09\", \"department\": \"開発\", \"qty\": 38, \"returned\": 1, \"status\": \"確定\"}, {\"id\": \"R10\", \"department\": \"開発\", \"qty\": 31, \"returned\": 4, \"status\": \"保留\"}, {\"id\": \"R11\", \"department\": \"管理\", \"qty\": 14, \"returned\": 0, \"status\": \"保留\"}, {\"id\": \"R12\", \"department\": \"営業\", \"qty\": 62, \"returned\": 3, \"status\": \"保留\"}]", "expected": 371, "mode": "sampling", "rep": 1, "kind": "records_regression", "conditions": { "off": { "seed": 4029129600, "score": { "answer": "{\"answer\":371}", "parsed": { "answer": 371 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "full_completion": "38+56+23+13+10+30+7+57+37+27+14+59 = 371{\"answer\":371}", "output_sha256": "41346347e6945bcf105df86aae0d4f2c4ac50cd47c678bca8c8f0f9bb2658ce2" }, "on": { "seed": 4029129600, "score": { "answer": "{\"answer\": 381}", "parsed": { "answer": 381 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "full_completion": "38+56+23+13+10+30+7+57+37+27+14+59{\"answer\": 381}", "output_sha256": "93c312662937c44ed6d8614fa9f8baaacccfafece03f2d344b69fa3e9c1b71b1" } } }, { "task": "records-072", "question": "次のJSONだけを根拠に、全行のqtyからreturnedを引いた数の合計を答えてください。\n[{\"id\": \"R01\", \"department\": \"開発\", \"qty\": 21, \"returned\": 1, \"status\": \"保留\"}, {\"id\": \"R02\", \"department\": \"営業\", \"qty\": 29, \"returned\": 4, \"status\": \"保留\"}, {\"id\": \"R03\", \"department\": \"開発\", \"qty\": 19, \"returned\": 1, \"status\": \"保留\"}, {\"id\": \"R04\", \"department\": \"管理\", \"qty\": 61, \"returned\": 3, \"status\": \"保留\"}, {\"id\": \"R05\", \"department\": \"管理\", \"qty\": 22, \"returned\": 2, \"status\": \"確定\"}, {\"id\": \"R06\", \"department\": \"営業\", \"qty\": 57, \"returned\": 1, \"status\": \"保留\"}, {\"id\": \"R07\", \"department\": \"営業\", \"qty\": 20, \"returned\": 3, \"status\": \"確定\"}, {\"id\": \"R08\", \"department\": \"営業\", \"qty\": 74, \"returned\": 3, \"status\": \"確定\"}, {\"id\": \"R09\", \"department\": \"営業\", \"qty\": 36, \"returned\": 1, \"status\": \"保留\"}, {\"id\": \"R10\", \"department\": \"営業\", \"qty\": 1, \"returned\": 4, \"status\": \"保留\"}, {\"id\": \"R11\", \"department\": \"開発\", \"qty\": 77, \"returned\": 3, \"status\": \"保留\"}, {\"id\": \"R12\", \"department\": \"開発\", \"qty\": 19, \"returned\": 2, \"status\": \"保留\"}]", "expected": 408, "mode": "sampling", "rep": 0, "kind": "records_improvement", "conditions": { "on": { "seed": 1739021674, "score": { "answer": "{\"answer\": 408}", "parsed": { "answer": 408 }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "full_completion": "20+25+18+58+20+56+17+71+35-3+74+17={\"answer\": 408}", "output_sha256": "e66be6a314a372f8c32034f406422c11f7a41d533067b50a4ef5bba19183fd1d" }, "off": { "seed": 1739021674, "score": { "answer": "{\"answer\":388}", "parsed": { "answer": 388 }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "full_completion": "20+25+18+58+20+56+17+71+35-3+74+17{\"answer\":388}", "output_sha256": "bfbed64379888f3a49b2296b704111e3179f3f63435119ca1b165776615bf8f9" } } } ], "illustrative_selection": "Selected after seeing responses to explain concrete phenomena, not randomly sampled or used to estimate their frequency. All outcomes remain in the full package; original strict scores unchanged." }