{ "created_utc": "2026-10-31T06:46:25.331880+00:00", "purpose": "Post-hoc follow-up to the observed output-cap limitation; original1024 results are retained, not replaced.", "model": { "name": "turboderp/GLM-5.3-Flash-exl3", "revision": "b570edf655a2bb00beca23e18d3b98724bfc2db5", "quantization": "sc_3.10bpw", "engine": "ExLlamaV3 1.6.5-gpuembed1", "GPU": "H100 PCIe80GBx2", "context_capacity": 1048576, "input_targets": [ 4096, 32768 ], "output_caps": [ 4096, 8192 ], "MTP_drafts": 3, "reasoning_effort": "low", "enable_thinking": true, "tools_provided": false, "no_CPU_weight_or_KV_offload": true, "NCCL_P2P_DISABLE": "1" }, "audit": { "status": "COMPLETE_INDEPENDENT_AUDIT", "utc": "2026-10-31T06:46:00.922320+00:00", "case": "quality-output-cap-v1", "rows": 288, "warmup_rows_excluded": 4, "pairs": 144, "all24_prompts_and_token_ids_match_original": true, "all_results_and_timings_recomputed": true, "frozen_order_verified": true, "residence_snapshots_audited": 6, "GPU_observed_peak_MiB": [ 74103, 74279 ], "source_results_sha256": "4a7c0cc27555bbf233e9766c9256c3d271a0284333ff863cfee9ad025e24355e", "source_plan_sha256": "9d984a73c050f52df352980e41904e294c24227a97aae6fd1e6f5f1bfbf0884e", "analysis_source_sha256": "0e2858e6be9909265cbb65e421aabf2aceb14d77f777dc694e4584c8eac3217d", "groups": { "input4096-cap4096-mtp0": { "answers": 36, "unique_tasks": 12, "correct": 30, "semantic_correct": 30, "format_valid": 36, "natural_stops": 36, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 1.6066980255418457, "first_answer_seconds_median_among_observed": 4.775912008539308, "first_answer_observed_n": 36, "request_end_seconds_median_all_including_caps": 5.7706308790366165, "completion_seconds_median_natural_stops_only": 5.7706308790366165, "generation_tps_median_all": 49.30352203357162, "generated_tokens_median": 207.5, "generated_tokens_range": [ 51, 285 ], "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "input4096-cap4096-mtp1": { "answers": 36, "unique_tasks": 12, "correct": 31, "semantic_correct": 31, "format_valid": 36, "natural_stops": 36, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 1.6613684124895371, "first_answer_seconds_median_among_observed": 3.4583046305342577, "first_answer_observed_n": 36, "request_end_seconds_median_all_including_caps": 3.931858022522647, "completion_seconds_median_natural_stops_only": 3.931858022522647, "generation_tps_median_all": 88.33656446493345, "generated_tokens_median": 198.0, "generated_tokens_range": [ 51, 363 ], "accepted_draft_tokens": 4773, "rejected_draft_tokens": 2073 }, "input4096-cap8192-mtp0": { "answers": 36, "unique_tasks": 12, "correct": 30, "semantic_correct": 30, "format_valid": 36, "natural_stops": 36, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 1.6068057339871302, "first_answer_seconds_median_among_observed": 4.774003486440051, "first_answer_observed_n": 36, "request_end_seconds_median_all_including_caps": 5.767837543447968, "completion_seconds_median_natural_stops_only": 5.767837543447968, "generation_tps_median_all": 49.333321687523124, "generated_tokens_median": 207.5, "generated_tokens_range": [ 51, 285 ], "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "input4096-cap8192-mtp1": { "answers": 36, "unique_tasks": 12, "correct": 31, "semantic_correct": 31, "format_valid": 36, "natural_stops": 36, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 1.6610963624552824, "first_answer_seconds_median_among_observed": 3.4707271465449594, "first_answer_observed_n": 36, "request_end_seconds_median_all_including_caps": 3.9477605705033056, "completion_seconds_median_natural_stops_only": 3.9477605705033056, "generation_tps_median_all": 88.41147313277483, "generated_tokens_median": 198.0, "generated_tokens_range": [ 51, 363 ], "accepted_draft_tokens": 4773, "rejected_draft_tokens": 2073 }, "input32768-cap4096-mtp0": { "answers": 36, "unique_tasks": 12, "correct": 1, "semantic_correct": 1, "format_valid": 30, "natural_stops": 31, "output_caps": 5, "missing_first_answer": 6, "first_generated_seconds_median": 11.084140950988512, "first_answer_seconds_median_among_observed": 65.82563911000034, "first_answer_observed_n": 30, "request_end_seconds_median_all_including_caps": 67.83174360648263, "completion_seconds_median_natural_stops_only": 66.6539598229574, "generation_tps_median_all": 48.92624477891208, "generated_tokens_median": 2782.5, "generated_tokens_range": [ 1665, 4096 ], "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "input32768-cap4096-mtp1": { "answers": 36, "unique_tasks": 12, "correct": 3, "semantic_correct": 3, "format_valid": 36, "natural_stops": 36, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 11.413743922545109, "first_answer_seconds_median_among_observed": 44.1056850120076, "first_answer_observed_n": 36, "request_end_seconds_median_all_including_caps": 44.573160738975275, "completion_seconds_median_natural_stops_only": 44.573160738975275, "generation_tps_median_all": 88.4094158966212, "generated_tokens_median": 2841.5, "generated_tokens_range": [ 890, 3913 ], "accepted_draft_tokens": 70298, "rejected_draft_tokens": 26941 }, "input32768-cap8192-mtp0": { "answers": 36, "unique_tasks": 12, "correct": 1, "semantic_correct": 1, "format_valid": 35, "natural_stops": 36, "output_caps": 0, "missing_first_answer": 1, "first_generated_seconds_median": 11.08419430302456, "first_answer_seconds_median_among_observed": 67.25676486094017, "first_answer_observed_n": 35, "request_end_seconds_median_all_including_caps": 67.89382987446152, "completion_seconds_median_natural_stops_only": 67.89382987446152, "generation_tps_median_all": 48.969309167529815, "generated_tokens_median": 2782.5, "generated_tokens_range": [ 1665, 7939 ], "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "input32768-cap8192-mtp1": { "answers": 36, "unique_tasks": 12, "correct": 3, "semantic_correct": 3, "format_valid": 36, "natural_stops": 36, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 11.414829587971326, "first_answer_seconds_median_among_observed": 44.0795789900003, "first_answer_observed_n": 36, "request_end_seconds_median_all_including_caps": 44.540146326005924, "completion_seconds_median_natural_stops_only": 44.540146326005924, "generation_tps_median_all": 88.35862232552009, "generated_tokens_median": 2841.5, "generated_tokens_range": [ 890, 3913 ], "accepted_draft_tokens": 70298, "rejected_draft_tokens": 26941 } }, "paired_MTP_comparisons": { "input4096-cap4096": { "pairs": 36, "both_correct": 30, "off_only_correct": 0, "on_only_correct": 1, "both_wrong": 5, "complete_sequence_identical": 13 }, "input4096-cap8192": { "pairs": 36, "both_correct": 30, "off_only_correct": 0, "on_only_correct": 1, "both_wrong": 5, "complete_sequence_identical": 13 }, "input32768-cap4096": { "pairs": 36, "both_correct": 0, "off_only_correct": 1, "on_only_correct": 3, "both_wrong": 32, "complete_sequence_identical": 0 }, "input32768-cap8192": { "pairs": 36, "both_correct": 0, "off_only_correct": 1, "on_only_correct": 3, "both_wrong": 32, "complete_sequence_identical": 0 } }, "output_budget_paired_comparisons": { "input4096-mtp0": { "pairs": 36, "both_correct": 30, "only4096_correct": 0, "only8192_correct": 0, "cap4096_to_natural8192": 0, "both_natural_below4096_pairs": 36, "both_natural_below4096_identical_full_token_sequence": 36, "note": "Each cap is a newly generated answer under the scheduled same task/seed; not a continuation of the earlier truncated response." }, "input4096-mtp1": { "pairs": 36, "both_correct": 31, "only4096_correct": 0, "only8192_correct": 0, "cap4096_to_natural8192": 0, "both_natural_below4096_pairs": 36, "both_natural_below4096_identical_full_token_sequence": 36, "note": "Each cap is a newly generated answer under the scheduled same task/seed; not a continuation of the earlier truncated response." }, "input32768-mtp0": { "pairs": 36, "both_correct": 1, "only4096_correct": 0, "only8192_correct": 0, "cap4096_to_natural8192": 5, "both_natural_below4096_pairs": 31, "both_natural_below4096_identical_full_token_sequence": 31, "note": "Each cap is a newly generated answer under the scheduled same task/seed; not a continuation of the earlier truncated response." }, "input32768-mtp1": { "pairs": 36, "both_correct": 3, "only4096_correct": 0, "only8192_correct": 0, "cap4096_to_natural8192": 0, "both_natural_below4096_pairs": 36, "both_natural_below4096_identical_full_token_sequence": 36, "note": "Each cap is a newly generated answer under the scheduled same task/seed; not a continuation of the earlier truncated response." } }, "original1024_one_seed_descriptive": { "input4096-cap1024-mtp0": { "answers": 12, "unique_tasks": 12, "correct": 11, "semantic_correct": 11, "format_valid": 12, "natural_stops": 12, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 1.6081986930803396, "first_answer_seconds_median_among_observed": 4.9388292075600475, "first_answer_observed_n": 12, "request_end_seconds_median_all_including_caps": 5.947632939030882, "completion_seconds_median_natural_stops_only": 5.947632939030882, "generation_tps_median_all": 48.77327068278403, "generated_tokens_median": 212.5, "generated_tokens_range": [ 113, 261 ], "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "input4096-cap1024-mtp1": { "answers": 12, "unique_tasks": 12, "correct": 11, "semantic_correct": 11, "format_valid": 12, "natural_stops": 12, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 1.6618528880062513, "first_answer_seconds_median_among_observed": 3.528858389996458, "first_answer_observed_n": 12, "request_end_seconds_median_all_including_caps": 4.002870727039408, "completion_seconds_median_natural_stops_only": 4.002870727039408, "generation_tps_median_all": 87.32381085245257, "generated_tokens_median": 204.5, "generated_tokens_range": [ 89, 261 ], "accepted_draft_tokens": 1665, "rejected_draft_tokens": 741 }, "input32768-cap1024-mtp0": { "answers": 12, "unique_tasks": 12, "correct": 0, "semantic_correct": 0, "format_valid": 0, "natural_stops": 0, "output_caps": 12, "missing_first_answer": 12, "first_generated_seconds_median": 11.093780485447496, "first_answer_seconds_median_among_observed": null, "first_answer_observed_n": 0, "request_end_seconds_median_all_including_caps": 31.90840507345274, "completion_seconds_median_natural_stops_only": null, "generation_tps_median_all": 49.11717145915463, "generated_tokens_median": 1024.0, "generated_tokens_range": [ 1024, 1024 ], "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "input32768-cap1024-mtp1": { "answers": 12, "unique_tasks": 12, "correct": 0, "semantic_correct": 0, "format_valid": 1, "natural_stops": 1, "output_caps": 11, "missing_first_answer": 11, "first_generated_seconds_median": 11.426648918539286, "first_answer_seconds_median_among_observed": 22.372795014060102, "first_answer_observed_n": 1, "request_end_seconds_median_all_including_caps": 23.57838914601598, "completion_seconds_median_natural_stops_only": 22.908072629012167, "generation_tps_median_all": 83.9672601206851, "generated_tokens_median": 1024.0, "generated_tokens_range": [ 890, 1024 ], "accepted_draft_tokens": 8186, "rejected_draft_tokens": 3718 } }, "followup_original_seed_sensitivity": { "input4096-cap4096-mtp0": { "answers": 12, "unique_tasks": 12, "correct": 11, "semantic_correct": 11, "format_valid": 12, "natural_stops": 12, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 1.6066354005597532, "first_answer_seconds_median_among_observed": 4.884383642463945, "first_answer_observed_n": 12, "request_end_seconds_median_all_including_caps": 5.879741168522742, "completion_seconds_median_natural_stops_only": 5.879741168522742, "generation_tps_median_all": 49.35498345118334, "generated_tokens_median": 212.5, "generated_tokens_range": [ 113, 261 ], "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "input4096-cap4096-mtp1": { "answers": 12, "unique_tasks": 12, "correct": 11, "semantic_correct": 11, "format_valid": 12, "natural_stops": 12, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 1.6613386574899778, "first_answer_seconds_median_among_observed": 3.528151090547908, "first_answer_observed_n": 12, "request_end_seconds_median_all_including_caps": 4.000756337016355, "completion_seconds_median_natural_stops_only": 4.000756337016355, "generation_tps_median_all": 87.37227848102677, "generated_tokens_median": 204.5, "generated_tokens_range": [ 89, 261 ], "accepted_draft_tokens": 1665, "rejected_draft_tokens": 741 }, "input4096-cap8192-mtp0": { "answers": 12, "unique_tasks": 12, "correct": 11, "semantic_correct": 11, "format_valid": 12, "natural_stops": 12, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 1.6061230749473907, "first_answer_seconds_median_among_observed": 4.87470141495578, "first_answer_observed_n": 12, "request_end_seconds_median_all_including_caps": 5.876202618004754, "completion_seconds_median_natural_stops_only": 5.876202618004754, "generation_tps_median_all": 49.26752435568173, "generated_tokens_median": 212.5, "generated_tokens_range": [ 113, 261 ], "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "input4096-cap8192-mtp1": { "answers": 12, "unique_tasks": 12, "correct": 11, "semantic_correct": 11, "format_valid": 12, "natural_stops": 12, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 1.6610068754525855, "first_answer_seconds_median_among_observed": 3.534194257983472, "first_answer_observed_n": 12, "request_end_seconds_median_all_including_caps": 4.011071537970565, "completion_seconds_median_natural_stops_only": 4.011071537970565, "generation_tps_median_all": 87.30722843939343, "generated_tokens_median": 204.5, "generated_tokens_range": [ 89, 261 ], "accepted_draft_tokens": 1665, "rejected_draft_tokens": 741 }, "input32768-cap4096-mtp0": { "answers": 12, "unique_tasks": 12, "correct": 0, "semantic_correct": 0, "format_valid": 10, "natural_stops": 10, "output_caps": 2, "missing_first_answer": 2, "first_generated_seconds_median": 11.0827796314843, "first_answer_seconds_median_among_observed": 66.16175173851661, "first_answer_observed_n": 10, "request_end_seconds_median_all_including_caps": 67.83174360648263, "completion_seconds_median_natural_stops_only": 67.29445773904445, "generation_tps_median_all": 48.897612535495945, "generated_tokens_median": 2782.5, "generated_tokens_range": [ 1938, 4096 ], "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "input32768-cap4096-mtp1": { "answers": 12, "unique_tasks": 12, "correct": 0, "semantic_correct": 0, "format_valid": 12, "natural_stops": 12, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 11.413262061076239, "first_answer_seconds_median_among_observed": 40.82581052900059, "first_answer_observed_n": 12, "request_end_seconds_median_all_including_caps": 41.28752931696363, "completion_seconds_median_natural_stops_only": 41.28752931696363, "generation_tps_median_all": 88.9582043808465, "generated_tokens_median": 2617.5, "generated_tokens_range": [ 890, 3820 ], "accepted_draft_tokens": 21929, "rejected_draft_tokens": 8416 }, "input32768-cap8192-mtp0": { "answers": 12, "unique_tasks": 12, "correct": 0, "semantic_correct": 0, "format_valid": 12, "natural_stops": 12, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 11.08361380849965, "first_answer_seconds_median_among_observed": 66.76064704993041, "first_answer_observed_n": 12, "request_end_seconds_median_all_including_caps": 67.89382987446152, "completion_seconds_median_natural_stops_only": 67.89382987446152, "generation_tps_median_all": 48.9124489000205, "generated_tokens_median": 2782.5, "generated_tokens_range": [ 1938, 6109 ], "accepted_draft_tokens": 0, "rejected_draft_tokens": 0 }, "input32768-cap8192-mtp1": { "answers": 12, "unique_tasks": 12, "correct": 0, "semantic_correct": 0, "format_valid": 12, "natural_stops": 12, "output_caps": 0, "missing_first_answer": 0, "first_generated_seconds_median": 11.41357773001073, "first_answer_seconds_median_among_observed": 40.8404664429836, "first_answer_observed_n": 12, "request_end_seconds_median_all_including_caps": 41.30290121096186, "completion_seconds_median_natural_stops_only": 41.30290121096186, "generation_tps_median_all": 88.85240721046337, "generated_tokens_median": 2617.5, "generated_tokens_range": [ 890, 3820 ], "accepted_draft_tokens": 21929, "rejected_draft_tokens": 8416 } }, "limits": [ "24 fixed tasks, 12 per length; 3 seeds per task are correlated, not 36 independent tasks.", "Original1024 used one seed and a different execution session; no pure causal attribution from that baseline alone.", "No general MTP quality-equivalence or noninferiority claim from this small follow-up.", "Timing excludes model load, warmup, tokenization and network; includes prefill and reasoning in request time.", "Generation tokens/s includes thinking/control tokens; completion lengths can differ.", "GPU memory peaks are sampled controller observations." ] }, "all_answer_review": { "utc": "2026-10-31T06:46:14.834606+00:00", "reviewer": "Codex", "human_review_claimed": false, "all288_final_answers_reviewed": true, "scope": "Read all available final-answer fields and malformed final bodies; recorded missing final bodies for output-capped requests. Does not claim human review or reading every internal reasoning token.", "answer_review_sha256": "75446a7b3f3b3d86b14fa6571a77bd0427e6127d7ebead768e3bccb67dfe2aec", "audit_sha256": "7e3860bd61cb3bb88707e58b81c8c0e0b015cd79d38be936f3377d576707ef07", "source_results_sha256": "4a7c0cc27555bbf233e9766c9256c3d271a0284333ff863cfee9ad025e24355e", "all_window_hashes_match_final_collected_outputs": true, "total_answers": 288, "naturally_ended_answers": 283, "output_capped_answers": 5, "missing_first_answer": 7, "natural_answer_errors_by_field": { "対象部署の件数合計": 149, "指定IDの待ち秒": 9, "行数": 11, "最小待ち秒": 2, "最大待ち秒": 2, "遅延原因": 2 }, "windows": [ { "window": "output-cap-early-answer-window-000.json", "window_sha256": "ead7f6c4a4bc24fc19f8a7c04930f2827b7a91341c380f25ac96af51d0656f7b", "review_note": "output-cap-early-answer-window-000-review.json", "review_note_sha256": "ad17be5beed28a2504596c88afb8fc033fb2343b381173284d32b15cf434db9d", "answers": 26, "notes": [ "25natural final JSON answers read in full; index13 capped4096 beforethinkend, no finalanswer.", "Allnatural32Kerrors have department-sum errors; index22 MTPon also reports wrongspecifiedID wait15 versus31.", "32K task015 seed189116 cap4096: MTPon exact11482 sum/all6pass; off11420 sumfails.", "4K task005 seed189106 cap4096: off607 versus631 fails; on631 all6pass.", "Other4K tasks007,010,006,000 in this window all6correct for bothconditions.", "These are partialresults, notfinalfrequencyestimates." ] }, { "window": "output-cap-early-answer-window-026.json", "window_sha256": "d3205e1735b29604201f925d5788e061661870651afd7543f64be1abc2f9f2aa", "review_note": "output-cap-early-answer-window-026-review.json", "review_note_sha256": "831fd7e7524a6041e210f92f8b54355cfcf2a5de4d7465ce3515a91e7caaca76", "answers": 14, "notes": [ "All14 natural stops, allstrict JSON. Only4Ktask006pair all6correct. Otherrows contain incorrectdepartment sum.", "Index36 task017 cap8192 off also wrongID wait10 versus46; correspondingonindex37 IDcorrect, sumwrong.", "Index34 naturallycompletes7939tokens below8192 butsum9859 versus11906; longthinkingdoesnotensurecorrectsum.", "4Ktask004 originalseed reports1375 versus1273 underbothMTPconditions, matchingthehistoricalerror." ] }, { "window": "output-cap-early-answer-window-040.json", "window_sha256": "941d3662b9e727e4ef4ffff808a1dffcd430969871a32b417ba0db99c36ec9c9", "review_note": "output-cap-early-answer-window-040-review.json", "review_note_sha256": "cbe053d9b5e8764a28104b14c22300eb824ec72078516c03f6eb76988cb6012d", "answers": 5, "notes": [ "All five available final bodies are valid six-field JSON and naturally ended. Rows40 and42 have all six values correct.", "Task012 seed289113 cap4096: MTPon correct10994, off wrong9971. Task015 seed289116 cap4096: off correct11482, on wrong11535. Discordance occurs in both directions in these examples.", "Row44 task013 MTPon reports9384 instead of9879; its other five fields match." ] }, { "window": "output-cap-early-answer-window-045.json", "window_sha256": "ee81e1cc01c58dea84b9a1750e00605a36a60edcff28320044b99c9e12770580", "review_note": "output-cap-early-answer-window-045-review.json", "review_note_sha256": "c3e19f316fa6bb0154350c2c147a9d53862ec4b8bd18245ef89ab23aeef43970", "answers": 16, "notes": [ "All16 rows naturally ended with valid strict JSON. All eight4K rows are correct; the eight32K rows have incorrectdepartment sums.", "Index48 task019 cap8192 MTPon also givesIDwait41 ratherthan11; offindex49 hascorrectIDwaitbutwrongsum.", "Index52 task012 cap4096 MTPoff reports215 rows ratherthan600, plusincorrectsum10045 versus10994. Onindex53 correctlyreports600rows buthasthesamewrongsum.", "Task002 originalseed matchesall6fields underbothcapsandMTPconditions; outputsare201tokensonand212offinbothbudgetsettings. Exactsequencecomparisontobeverifiedfromnativeeventlogs." ] }, { "window": "output-cap-early-answer-window-061.json", "window_sha256": "cb9e6a7495437a7eefd950aabd5b08d93d1e6f562aade489908afed9f26eec69", "review_note": "output-cap-early-answer-window-061-review.json", "review_note_sha256": "9d1d21cb2ccbb52ca55fa3ff040ddd024263565a725c01aed809797be30fe59e", "answers": 31, "notes": [ "All31 available final bodies are strict JSON and naturally ended. Nineteenall-fieldpasses.", "4Ktask009seed189110 returns596 instead of834 fordepartment sum underbothMTPconditions; only51tokens andnaturalstop. Largerbudgetalonecannotfixananswerthatendedwrongthisearly.", "32Ktask022seed89123 MTPoff reports500rows insteadof600 andsum9252 insteadof10641. On hascorrectrowcount butwrongsum10815.", "32Ktask022seed189123 MTPon reports214 asdepartment sum insteadof10641; allotherfields correct. Offsum9359alsoincorrect.", "32Ktask012seed289113cap8192: onall6correct, offsum9971wrong. Tokenlengths3876on/2709offmatchthe4096cap pair inrows40/41; nativeeventexactsequencecheckpending.", "Other32Kerrors inthiswindow aredepartment sums; noID/min/max/causeerrors." ] }, { "window": "output-cap-early-answer-window-092.json", "window_sha256": "9546ce45fea34b9d6945ac81b4ee43d5b4dd5fe9660c4df98ab1937e154e9d7f", "review_note": "output-cap-early-answer-window-092-review.json", "review_note_sha256": "759dfc2255d38655292f53ef88f7b9cd0e0cafa52b0e82a607b392a0f8a674d8", "answers": 26, "notes": [ "All26 strictJSON/naturalstop;16all-fieldcorrect.", "4Ktask005seed189106 cap8192 offsum607 versus631, oncorrect; agreeswiththecap4096examplealreadyreviewed. Other4Krowsmatchall6fields.", "32Ktask020seed189121 cap4096 oncorrect10702;off10703 isoneunitwrong andfailsstrictnumericcorrectness. Otherfivefieldsmatch.", "32Ktask022seed89123 cap4096 offrowcount500insteadof600, sum9252insteadof10641;oncorrectrowcountbutwrongsum10815. Samefinalvaluesandlengthsas8192trial,notfixedbylargerbudget.", "Remaining32Ktask014/013/021 failuresareincorrectdepartment sums, notJSONformatfailures oroutputtruncation." ] }, { "window": "output-cap-early-answer-window-118.json", "window_sha256": "a5e099111909f841fe7a64e7bdc6a6edc3d44cf44b4044363bd76628c64d1ac7", "review_note": "output-cap-early-answer-window-118-review.json", "review_note_sha256": "387ea3071bc9b42642d585b00f9692976ff0507a5c08135afb2d60df664da23a", "answers": 23, "notes": [ "Row132 stops at4096 tokens without a final answer. All other22 rows are natural strict JSON.", "4K task004 seed289105 both MTP modes report sum1191 instead of1273; remaining ten4K answers have all six fields correct.", "32K task015 seed289116 cap8192: off correct11482, on wrong11535, matching the4096 pair. Full emitted reasoning and all215 source records separately reviewed; see mechanism review.", "32K task019 seed89120 on also reportsIDwait41 instead of11, plus wrongsum9671; off has correctIDwait butwrongsum9851. Other naturally ended32K failures here are department sums." ] }, { "window": "output-cap-early-answer-window-141.json", "window_sha256": "7513b97fa14d84fcb24deabed52a13967d5d7e25dc0729e6d4af24d4e32d997d", "review_note": "output-cap-early-answer-window-141-review.json", "review_note_sha256": "220c5f34e2da3502ebc72f9bf57e6168e079f76535d02b51d2e65d46e3803354", "answers": 20, "notes": [ "All20 responses naturally ended with strict JSON. All12 4K responses pass.", "All8 32K responses fail department sum. Row143 also gives IDwait10 instead of46.", "Row160 gives department sum214 instead of10641; no output cap occurred. Final output alone does not establish which intermediate operation failed." ] }, { "window": "output-cap-early-answer-window-161.json", "window_sha256": "2b364c0b272b58014ab1b0231f4a7f93e198a665e2f9e3ee2f0993dfbf21b97a", "review_note": "output-cap-early-answer-window-161-review.json", "review_note_sha256": "ed0867565739a16ba41ca4559474c536bdbe1a7765efab4619b9f937ca5a047d", "answers": 15, "notes": [ "Row162 caps at4096 with no final answer. Remaining14 responses naturally end with strict JSON.", "Six4K responses pass. Both4K task010 seed289111 fail department sum after73tokens: off247 andon289, expected324.", "Allsix naturally ended32K responses here fail department sum; their other five fields match. No new off-pass/on-fail pair in this window." ] }, { "window": "output-cap-early-answer-window-176.json", "window_sha256": "a9c150b6eb0860d114a3c318ae5b0e8a699a6e3a8ed0d33f55d8c17dfa2da1c9", "review_note": "output-cap-early-answer-window-176-review.json", "review_note_sha256": "3141af5938effd72f57d3517592da3afa200459924d144fabf5c398363daee0f", "answers": 6, "notes": [ "All six 32K answers ended naturally below4096 tokens. Each failed only the department count sum; other five fields and strict format matched.", "Case0198192: off10083/on9962 vs9971. Case0184096: off9458/on9519 vs9583. Case0154096 seed89116: off11265/on11289 vs11482." ] }, { "window": "output-cap-early-answer-window-182.json", "window_sha256": "f27d8ab15fe12e9cdbe1f82a3ca3d9d3d0c033cb085a2aa3703301ae9341814d", "review_note": "output-cap-early-answer-window-182-review.json", "review_note_sha256": "88334911569693d983d33f959f5dff9b989ad1c4fcecdd8bab145337ac17d12d", "answers": 16, "notes": [ "Index190 had no final body at4096 output cap. Fifteen other answers ended naturally.", "Ten4K answers passed; task009seed189110 both returned596 rather than834,51 outputtokens.", "The three natural32K answers failed department sum only: task014on11562/off10595 vs10639;task017on9903vs9831. All other final fields matched." ] }, { "window": "output-cap-early-answer-window-198.json", "window_sha256": "e185667a4dc7bb98930f977ad18b93328c04ec42e5ce642e0e28c993c908911b", "review_note": "output-cap-early-answer-window-198-review.json", "review_note_sha256": "786329d3313eea5b4c7a5578cedacaefc1ffdc256fb36eed95585579f5075e09", "answers": 12, "notes": [ "All12 ended naturally. Ten4K answers passed all fields.", "Task013cap4096seed189114 failed department sum only: MTPon8364 and off10006 versus9879. Both retained all other five correct fields." ] }, { "window": "output-cap-early-answer-window-210.json", "window_sha256": "d9b7168f4c9d311c7b502686c4205637182547136be1c3ea13de08e70f22ad6e", "review_note": "output-cap-early-answer-window-210-review.json", "review_note_sha256": "d99595f86f6009c94c874a6fc2e6fa495dc4f7f0b30404fe2cd6b060201226bd", "answers": 5, "notes": [ "All five ended naturally. Both4Ktask006 answers passed.", "32Ktask015cap8192seed89116 on11289/off11265 vs11482, and task014on8288vs10639 failed department sum only. Other five fields matched." ] }, { "window": "output-cap-early-answer-window-215.json", "window_sha256": "d8e06e9ac3bec0d3eb5df66bdcb005b4465fbe4d9ebc08b3467bf8d8c5c77a69", "review_note": "output-cap-early-answer-window-215-review.json", "review_note_sha256": "52f64b0d8d00ad858481d5f3bef01054e6a74475bb86e14d7ff4f1d0df29e5d2", "answers": 5, "notes": [ "All five ended naturally. Both4Ktask006seed89107 and task007seed189108 pairs passed.", "32Ktask014cap8192seed89115off returned13019vs10639 for department sum; remaining five fields matched." ] }, { "window": "output-cap-early-answer-window-220.json", "window_sha256": "7961e44f4e3a5407ac3e951afb9366ccdb536a1954e05e69ae7077cd55971a2b", "review_note": "output-cap-early-answer-window-220-review.json", "review_note_sha256": "5496cd6e5b347775e93474497d15a44062cda9524f919b13588598c22954c346", "answers": 2, "notes": [ "Both task016cap4096seed189117 answers ended naturally and failed department sum only: off9761/on9664vs9883. Five other fields correct." ] }, { "window": "output-cap-early-answer-window-222.json", "window_sha256": "a6cef66eaa294ef57e627365c043f5d3f30ae37694b70005945e617247d6d406", "review_note": "output-cap-early-answer-window-222-review.json", "review_note_sha256": "b7591d5a2fe8159c2cbf86cafb1f98b2de4b6964293f56a1687e5bb25d48993f", "answers": 2, "notes": [ "32Ktask020cap8192seed189121: onpassed10702, off10703vs10702 failedsumonly. Bothnatural, allfiveotherfieldsright. This is a reverse discordance (on correct/off wrong), retained." ] }, { "window": "output-cap-early-answer-window-224.json", "window_sha256": "5ebb0b7c62b4f38a5cf9aac42af94378562dcee4dc447e5e2e2657749622f7af", "review_note": "output-cap-early-answer-window-224-review.json", "review_note_sha256": "153bd8682fc5df05f4a8266d424b1f5874fde12b4b655e7bf997031d3756a0d3", "answers": 1, "notes": [ "Task012cap8192seed89113off ended naturally. Row count215vs600 and department sum10045vs10994 were wrong. Other four fields matched." ] }, { "window": "output-cap-early-answer-window-225.json", "window_sha256": "0ba6874ed43445a97efcccda969c219f41948bcc88aaffa169640fe38b8a5e5c", "review_note": "output-cap-early-answer-window-225-review.json", "review_note_sha256": "2882eb36b07fb017edd1308a02d8bce09d187591880b257c925ac9b97d239ea7", "answers": 5, "notes": [ "All five natural; nonepass. Task012on sum10045vs10994, otherfivefieldsright.", "Both4Ktask001seed289102 rowcount21vs50, otherfivefieldsright.", "32Ktask016seed289117: offsum9830vs9883 only; onsum8688vs9883 and IDwait15vs31 wrong. Four other onfieldsright." ] }, { "window": "output-cap-early-answer-window-230.json", "window_sha256": "a9d6f0947959337a3604dbd29077e62aa6de13a5f693d44b633aed6cb39519f2", "review_note": "output-cap-early-answer-window-230-review.json", "review_note_sha256": "e961dd8fb5e15675479a4124346b94259967c3f1e83ee1b54e9c2df09b336edc", "answers": 4, "notes": [ "Allfour natural. Both4Ktask009seed289110passed.", "32Ktask015cap8192seed189116: offsum11420vs11482failedonlysum; onpassedall11482. Reverse discordance retained." ] }, { "window": "output-cap-early-answer-window-234.json", "window_sha256": "7c4634139e888143651abc37889c7d5b47e8ee45ea99aaaa409964509d1769e9", "review_note": "output-cap-early-answer-window-234-review.json", "review_note_sha256": "bf37de7b83cda305fdf2e202444d06a31a2e28e0636b073c4d9021a27aa4c42b", "answers": 3, "notes": [ "Index234task018cap4096seed189119off ended bystop_token at2271tokens without closingthink or finalJSON. Fulloutputread; it ends with uncertain hand-calculated9552vs9583. Naturalstopisnotanswercompletion. Extraevidence output-cap-natural-no-body-234-early.json.", "Index235on natural returned9537vs9583sumonlywrong; fiveotherfieldsright.", "Index236task019cap8192seed189120off natural rowcount189vs600 and sum9879vs9971wrong; remainingfourfieldsright." ] }, { "window": "output-cap-early-answer-window-237.json", "window_sha256": "615ae9107e09e4b8fbe1b9d78bc4ace354f3d821c6363d897bed37102736932d", "review_note": "output-cap-early-answer-window-237-review.json", "review_note_sha256": "a6b5d403b7508ccb0c64d82a9f52c7b95c20e64517fcfb4dace4fa984c2d2dc8", "answers": 10, "notes": [ "Alltennatural. Only task008seed289109twoanswerspassed.", "32Ktask019on sum9929vs9971wrongonly. 4Ktask010both sum247/289vs324wrongonly;task004both1191vs1273wrongonly.", "32Ktask017seed189118 offsum8941vs9831andIDwait10vs46;onsum9028vs9831only. Otherfieldsright.", "4Ktask001offrowcount21vs50only." ] }, { "window": "output-cap-early-answer-window-247.json", "window_sha256": "5d4fa32c7f904fd502cd5fa016ddf6a0e55f7d27a9e4d20b935bb31166dd4340", "review_note": "output-cap-early-answer-window-247-review.json", "review_note_sha256": "e8ec08f5747a333767b09d523abe67dfa4dbb9019c666990efe12bc2304abb6d", "answers": 2, "notes": [ "Bothnatural. task001cap4096seed289102onrow21vs50onlywrong;task023cap8192seed289124offsum9259vs9236onlywrong. Remainingfivefieldsrightinboth." ] }, { "window": "output-cap-early-answer-window-249.json", "window_sha256": "b08388880ca49708789ce496ef152470a70a8cb47cb9359874815d974a8503a8", "review_note": "output-cap-early-answer-window-249-review.json", "review_note_sha256": "fe7a6e987a5f6ed7752d6cc33a671cbb3f1f79d6e666da17ca62bc4cc4a25a6b", "answers": 6, "notes": [ "Allsixnatural. Four4Ktask011seed189112andtask006seed289107answerspassed.", "32Ktask023seed289124on sum9242vs9236wrongonly;task021seed289122on9805vs11906wrongonly. Fiveotherfieldsmatched." ] }, { "window": "output-cap-early-answer-window-255.json", "window_sha256": "37852a810fbf27bb8cf7a155062d54db1eb9377f6785964c312ed07af1ec4056", "review_note": "output-cap-early-answer-window-255-review.json", "review_note_sha256": "cab359005204aeef38cd6b3dfea93ebcd56b4ef408b563402a2fb25cb3d0bb84", "answers": 2, "notes": [ "Bothnatural. Task021cap8192seed289122offsum9637vs11906 and task013cap8192seed289114onsum9384vs9879wrongonly. Remainingfivefieldsright." ] }, { "window": "output-cap-early-answer-window-257.json", "window_sha256": "0920958c270786dfbb7324bff37aeb3a015d9a8d7c0f2b27143538aca3142841", "review_note": "output-cap-early-answer-window-257-review.json", "review_note_sha256": "be5d8d63120fe55f5cedf244621193e57c94726158cf9d99149ecd8a1a4ddfa9", "answers": 5, "notes": [ "Allfive natural. task010cap4096seed189111bothpassed.", "task013cap8192seed289114offsum9663vs9879onlywrong;task004cap4096seed89105bothsum1375vs1273onlywrong. Allotherfivefieldsmatched." ] }, { "window": "output-cap-early-answer-window-262.json", "window_sha256": "ae079d25ca4c792a93b52a2eb8d822afffcfb594f7f0dba668b82622fca7ee22", "review_note": "output-cap-early-answer-window-262-review.json", "review_note_sha256": "6a33cfbabc7dc5cfef7298d845f812c00e99dedbe337ae8ee28fa3d2788d6b5d", "answers": 2, "notes": [ "32Ktask021cap4096seed89122off capped4096 withoutfinalanswer. Onendednaturallywithsum9310vs11906, otherfivefieldsmatched." ] }, { "window": "output-cap-early-answer-window-264.json", "window_sha256": "fdcc7a7e295e75fe3d029f39e16378749512465fd4d112274a5199dd7c1b5bb2", "review_note": "output-cap-early-answer-window-264-review.json", "review_note_sha256": "38c35815633539aa9a62324c9c371a0dcf57f04bfcfad7f9df5dc14d1715dfe9", "answers": 4, "notes": [ "Allfournatural. Task008cap4096seed189109bothpassed.", "Task014cap4096seed289115offsum10595/on11562vs10639onlywrong. Fiveotherfieldsmatched." ] }, { "window": "output-cap-early-answer-window-268.json", "window_sha256": "b522950b0c49428b45684ee20a5d78be4771ae21de3d01d115fa756b727c56d0", "review_note": "output-cap-early-answer-window-268-review.json", "review_note_sha256": "75080d41bd7f40a2905139c2d1cd8146e65683a61fe6dfbac01dd7ac035eb0b1", "answers": 2, "notes": [ "Task018cap8192seed189119: on9537vs9583sumonlywrong, otherfivefieldsright.", "Off269endednaturallyat2271tokensbutnothinkclosureorfinalJSON. FulloutputSHAidenticaltoread234atcap4096. Increasingcapdoesnotchangeit; do not count as finishedanswer." ] }, { "window": "output-cap-early-answer-window-270.json", "window_sha256": "a4f8671cb4c6fa1347a0725d1a2125084f85b3639ff890ed08e44c882f5876bc", "review_note": "output-cap-early-answer-window-270-review.json", "review_note_sha256": "be307289aa0477b7126ba933faafce3acb7f6e0787c7c46d6d96c9212e7e7b31", "answers": 18, "notes": [ "All18natural. Four4Ktask003seed289104andtask007seed89108answerspassed.", "All14other32Kanswersfailedsumonly. Task023r0off9172/on8416vs9236acrossbothcaps;task020r0off10924/on10831vs10702;task014r0off13019/on8288vs10639;task013r0off9844/on9862vs9879;task018r2off9567/on9447vs9583acrossbothcaps. Fiveotherfieldsrightforall." ] } ], "limitations": [ "Review is a verification of this frozen synthetic task and scoring rubric, not proof of arbitrary business-answer quality.", "Natural completion, valid JSON, semantic correctness and output-cap termination are distinct outcomes." ] }, "independent_oracle_check": { "utc": "2026-10-31T04:13:59.904247+00:00", "all24_match": true, "methods": "Parsed the literal JSON record lines from each original question; recomputed count, min, max, department sum and ID lookup without using model outputs. Causal attribution not supported by observational data; instructed unknown answer verified.", "tasks_sha256": "307a64e5621e9b433a0ec65bbf38324e25b79ebc73993609aa7f8b055aea0ea2", "tasks": [ { "task": "long-records-000", "record_count": 50, "target_department": "営業", "target_id": 28, "expected": { "行数": 50, "最小待ち秒": 7, "最大待ち秒": 78, "対象部署の件数合計": 985, "指定IDの待ち秒": 70, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-001", "record_count": 50, "target_department": "管理", "target_id": 22, "expected": { "行数": 50, "最小待ち秒": 4, "最大待ち秒": 79, "対象部署の件数合計": 1098, "指定IDの待ち秒": 5, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-002", "record_count": 50, "target_department": "開発", "target_id": 46, "expected": { "行数": 50, "最小待ち秒": 4, "最大待ち秒": 80, "対象部署の件数合計": 909, "指定IDの待ち秒": 72, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-003", "record_count": 50, "target_department": "営業", "target_id": 19, "expected": { "行数": 50, "最小待ち秒": 2, "最大待ち秒": 77, "対象部署の件数合計": 701, "指定IDの待ち秒": 69, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-004", "record_count": 50, "target_department": "管理", "target_id": 46, "expected": { "行数": 50, "最小待ち秒": 3, "最大待ち秒": 78, "対象部署の件数合計": 1273, "指定IDの待ち秒": 63, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-005", "record_count": 50, "target_department": "営業", "target_id": 41, "expected": { "行数": 50, "最小待ち秒": 1, "最大待ち秒": 79, "対象部署の件数合計": 631, "指定IDの待ち秒": 79, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-006", "record_count": 50, "target_department": "開発", "target_id": 17, "expected": { "行数": 50, "最小待ち秒": 1, "最大待ち秒": 79, "対象部署の件数合計": 565, "指定IDの待ち秒": 21, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-007", "record_count": 50, "target_department": "管理", "target_id": 24, "expected": { "行数": 50, "最小待ち秒": 3, "最大待ち秒": 79, "対象部署の件数合計": 736, "指定IDの待ち秒": 54, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-008", "record_count": 50, "target_department": "営業", "target_id": 26, "expected": { "行数": 50, "最小待ち秒": 3, "最大待ち秒": 78, "対象部署の件数合計": 742, "指定IDの待ち秒": 70, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-009", "record_count": 50, "target_department": "管理", "target_id": 6, "expected": { "行数": 50, "最小待ち秒": 1, "最大待ち秒": 79, "対象部署の件数合計": 834, "指定IDの待ち秒": 17, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-010", "record_count": 50, "target_department": "営業", "target_id": 3, "expected": { "行数": 50, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 324, "指定IDの待ち秒": 54, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-011", "record_count": 50, "target_department": "管理", "target_id": 20, "expected": { "行数": 50, "最小待ち秒": 3, "最大待ち秒": 80, "対象部署の件数合計": 754, "指定IDの待ち秒": 12, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-012", "record_count": 600, "target_department": "開発", "target_id": 559, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 10994, "指定IDの待ち秒": 75, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-013", "record_count": 600, "target_department": "営業", "target_id": 316, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 9879, "指定IDの待ち秒": 69, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-014", "record_count": 600, "target_department": "開発", "target_id": 388, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 10639, "指定IDの待ち秒": 19, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-015", "record_count": 600, "target_department": "管理", "target_id": 512, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 11482, "指定IDの待ち秒": 71, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-016", "record_count": 600, "target_department": "開発", "target_id": 469, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 9883, "指定IDの待ち秒": 31, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-017", "record_count": 600, "target_department": "開発", "target_id": 266, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 9831, "指定IDの待ち秒": 46, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-018", "record_count": 600, "target_department": "営業", "target_id": 33, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 9583, "指定IDの待ち秒": 59, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-019", "record_count": 600, "target_department": "管理", "target_id": 335, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 9971, "指定IDの待ち秒": 11, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-020", "record_count": 600, "target_department": "管理", "target_id": 351, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 10702, "指定IDの待ち秒": 37, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-021", "record_count": 600, "target_department": "管理", "target_id": 483, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 11906, "指定IDの待ち秒": 13, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-022", "record_count": 600, "target_department": "開発", "target_id": 162, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 10641, "指定IDの待ち秒": 2, "遅延原因": "不明" }, "all_six_oracles_match": true }, { "task": "long-records-023", "record_count": 600, "target_department": "営業", "target_id": 479, "expected": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 9236, "指定IDの待ち秒": 4, "遅延原因": "不明" }, "all_six_oracles_match": true } ] }, "transfer_audit": { "status": "ALL_REMOTE_LOCAL_FILES_SHA256_MATCH", "utc": "2026-10-31T06:46:00.969173+00:00", "case": "quality-output-cap-v1", "files": 943, "bytes": 89345091, "archive_sha256": "4e77c3bf1528385266bf4ea1e84eb12237393113c70d2b3e876c8d6c5700935f" }, "protocol": { "status": "FROZEN_BEFORE_OUTPUT_CAP_FOLLOWUP", "created_utc": "2026-10-31T04:06:21.528972+00:00", "question": "How much of the prior 32K failure is attributable to an insufficient output budget, and how do MTP on/off compare with larger budgets?", "study_type": "Post-hoc follow-up motivated by observed 1024-token truncation; not a replacement of the original preregistered study.", "model": "turboderp/GLM-5.3-Flash-exl3", "model_revision": "b570edf655a2bb00beca23e18d3b98724bfc2db5", "quantization": "sc_3.10bpw", "engine": "ExLlamaV3 1.6.5 gpuembed1", "GPUs": "H100 PCIe 80GB x2", "context_capacity": 1048576, "target_allocator_GiB": [ 67, 73 ], "MTP_draft_tokens": 3, "actual_input_targets": [ 4096, 32768 ], "unique_tasks": 24, "seeds_per_task": 3, "max_generation_tokens": [ 4096, 8192 ], "pairs": 144, "main_answers": 288, "original_tasks_sha256": "66923ee4726d24c5d4dc2a21d3030d5326c6987f1ac0379c690a40198d8c99b4", "followup_tasks_sha256": "307a64e5621e9b433a0ec65bbf38324e25b79ebc73993609aa7f8b055aea0ea2", "prompt_rule": "Exactly same system, tasks, padding/rendering, tokenizer and expected answers as original secondary study. Verify all 24 prompt and input-token-ID hashes against collected original run.", "seed_rule": "Original per-task seed plus offsets 100000 and 200000, same across output caps and MTP conditions.", "sampling": { "temperature": 1.0, "top_p": 0.95, "top_k": 0, "min_p": 0 }, "schedule_rng_seed": 202610311304, "order": "Shuffle task/seed/cap pairs and randomize MTP order within each pair; no outcome-dependent retries or exclusions.", "warmup": "Two tasks (first 4K and first 32K) x both MTP conditions, 64 generated tokens, separately retained/excluded. Kernel warmup retained.", "metrics": [ "Strict 6-field full-answer pass requiring natural stop", "Per-field correctness and semantic versus format error", "Output-cap and no-answer rates", "First generated token and first answer-body latency", "End-to-end completion time including prefill and reasoning", "Generation tokens/s excluding first token, includes reasoning/control tokens", "Actual generated tokens and accepted/rejected MTP candidates" ], "analysis": "Summarize separately by input length, output cap and MTP; report all 36 answers per cell and paired MTP disagreements. Same task 3 seeds are correlated. No general noninferiority/equivalence claim from 12 tasks per length. Compare original 1024 descriptively, with matched first-seed sensitivity; original used one seed/task.", "failure_policy": "Preserve incomplete, capped, wrong, and runtime-failed runs. Do not increase beyond 8192 or change prompts/tools/model to chase success in this frozen phase. Any further follow-up must be separately planned and identified.", "hardware_constraints": { "max_GPU_GiB_each": 75, "no_CPU_weights_or_KV_offload": true, "NCCL_P2P_DISABLE": "1", "no_B200_access": true, "no_driver_or_existing_environment_changes": true, "no_other_jobs_or_data_changes": true }, "publication": "Update existing article and create revised Japanese Marin 4K video retaining MTP beginner explanation and original findings; no customer email/newsletter/subscriber notifications." }, "protocol_source_freeze": { "runner_sha256": "6162a462bd1e002841a16065a4ec88f49f9f328d22ba80fe911222c101e7ebc9", "plan_sha256": "9d984a73c050f52df352980e41904e294c24227a97aae6fd1e6f5f1bfbf0884e", "task_sha256": "307a64e5621e9b433a0ec65bbf38324e25b79ebc73993609aa7f8b055aea0ea2", "scorer_sha256": "03866ba67c12766903638f72ca4500b051fa396c3c3063083add691531819536", "original_prompt_and_score_helpers_unchanged": true, "expected_main_answers": 288, "baseline_input_sha256": "63401d0793b564bd2278be14b2ed1a3ad093735c3d51b102f005447671bfad1e", "syntax_and_identical_helpers_verified": true }, "per_answer_fields": [ { "case": "main-long-records-020-cap4096-sampling-r2-mtp0", "task": "long-records-020", "input_target": 32768, "max_generation": 4096, "seed": 289121, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1864, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10767,\"指定IDの待ち秒\":37,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10767, "expected": 10702, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap4096-sampling-r2-mtp1", "task": "long-records-020", "input_target": 32768, "max_generation": 4096, "seed": 289121, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3434, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 10761, \"指定IDの待ち秒\": 37, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10761, "expected": 10702, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap4096-sampling-r1-mtp1", "task": "long-records-015", "input_target": 32768, "max_generation": 4096, "seed": 189116, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2645, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11482,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11482, "expected": 11482, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap4096-sampling-r1-mtp0", "task": "long-records-015", "input_target": 32768, "max_generation": 4096, "seed": 189116, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3450, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 11420, \"指定IDの待ち秒\": 71, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11420, "expected": 11482, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap4096-sampling-r2-mtp0", "task": "long-records-017", "input_target": 32768, "max_generation": 4096, "seed": 289118, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3349, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":8625,\"指定IDの待ち秒\":46,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8625, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 46, "expected": 46, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap4096-sampling-r2-mtp1", "task": "long-records-017", "input_target": 32768, "max_generation": 4096, "seed": 289118, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3047, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":8903,\"指定IDの待ち秒\":46,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8903, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 46, "expected": 46, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap8192-sampling-r1-mtp1", "task": "long-records-013", "input_target": 32768, "max_generation": 8192, "seed": 189114, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3772, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 8364, \"指定IDの待ち秒\": 69, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8364, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap8192-sampling-r1-mtp0", "task": "long-records-013", "input_target": 32768, "max_generation": 8192, "seed": 189114, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3002, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10006,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10006, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap8192-sampling-r2-mtp0", "task": "long-records-007", "input_target": 4096, "max_generation": 8192, "seed": 289108, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 192, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 79, \"対象部署の件数合計\": 736, \"指定IDの待ち秒\": 54, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap8192-sampling-r2-mtp1", "task": "long-records-007", "input_target": 4096, "max_generation": 8192, "seed": 289108, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 192, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 79, \"対象部署の件数合計\": 736, \"指定IDの待ち秒\": 54, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap8192-sampling-r0-mtp0", "task": "long-records-010", "input_target": 4096, "max_generation": 8192, "seed": 89111, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 113, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 324, \"指定IDの待ち秒\": 54, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 324, "expected": 324, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap8192-sampling-r0-mtp1", "task": "long-records-010", "input_target": 4096, "max_generation": 8192, "seed": 89111, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 89, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":324,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 324, "expected": 324, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap4096-sampling-r1-mtp1", "task": "long-records-019", "input_target": 32768, "max_generation": 4096, "seed": 189120, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2649, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9929,\"指定IDの待ち秒\":11,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9929, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 11, "expected": 11, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap4096-sampling-r1-mtp0", "task": "long-records-019", "input_target": 32768, "max_generation": 4096, "seed": 189120, "mtp": false, "finish_reason": "max_new_tokens", "generated_tokens": 4096, "pass": false, "first_answer_missing": true, "answer_text": "管理: count, min/max wait, sum counts, id335 wait.\n\nMin wait in 管理: ids with 1: 101,130,138,208 → 1. Max: 80 (27,231,449). Sum of counts for 管理 rows — need to compute. Let me list 管理 rows with counts:\n\n23,28,60,28,4,9,77,26,32,83,90,14,71,12,32,60,29,96,31,23,58,9,32,58,70,94,25,71,19,27,37,16,3,54,94,5,1,30,74,6,67,2,65,38,6,27,20,48,74,79,1,79,3,27,1,14,15,37,26,55,49,82,16,72,93,49,89,59,39,65,55,12,3,64,57,75,12,49,90,55,45,2,9,59,64,21,68,67,70,67,76,36,82,7,50,69,38,41,14,1,21,73,90,59,76,18,71,23,80,12,6,80,74,63,23,48,71,69,26,56,7,45,81,72,41,56,28,3,7,19,88,13,56,64,53,27,70,52,42,63,100,44,36,49,100,67,30,10,5,41,75,13,66,43,13,51,58,9,94,13,56,63,76,48,29,72,23,13,10,29,48,26,30,8,71,5,7,75,71,28,77,13,14,53,19,61,2,18,30,55,75,54,54,8,85,72,66,14,36,20,41,58,64,29,36,37,7,56,98,3,45,72,29,68,75,97,59,91,53,93,79,73,56,43,26,9,71,77,19,77,9,16,49,58,50,59,61,13,25,75,32,73,38,2,80,57,25,12,25,44,69,73,19,47,27,34,15,7,36,42,19,80,24,49,35,91,11,85,20,3,36,73,46,48,70,25,24,21,53,73,41,12,11,46,16,40,61,98,18,92,22,41,57,89,70,79,38,49,45,19,65,25,13,23,27,41,69,55,14,96,61,60,44,90,12,73,8,19,52,54,42,30,10,96,64,47,60,63.\n\nCount of 管理 rows: let me count. I'll count as I go... easier: total 600 rows, departments roughly equal? Count 管理 entries: I'll count the list above.\n\nGroup by tens:\nRow1: id1管理? ids: 1,3,7,13,17,18,19 → 7 (ids1-19)\n21-40: 27,37,39,40 → wait id20営業,21開発,22-26開発/営業... 管理 in 21-40: 27,37,39,40 → 4? id39管理 yes. So 4.\n41-60: 44,48,49,50,55,57,59 → 7.\n61-80: 62,64,65,67,68,72,75,78 → 8.\n81-100: 84,86,88,89,90,91,92,94,96,100 → 10.\n101-120: 101,103,106,111,113,115,116,117,118 → 9.\n121-140: 122,123,128,129,130,131,134,138,139 → 9.\n141-160: 142,144,146,147,150,156,157 → 7? also 141開発,143営業,145開発,148開発,151開発,153開発,154開発,158開発,159開発 → 7.\n161-180: 162,163,164,168,171,177 → 6.\n181-200: 182,185,187,191,193,195,196,198,200 → 9.\n201-220: 204,206,207,208,214,217,221? (221 is in 201-220) → 204,206,207,208,214,217,221 → 7.\n221-240: 224,225,231,240 → 4.\n241-260: 244,246,247,248,249,256,259,260 → 8.\n261-280: 262,263,265,266,267,271,275,276,277,279,280 → 11.\n281-300: 291,295,298,299,300 → 5.\n301-320: 304,305,310,315,323? no that's 321-340. → 304,305,310,315 → 4.\n321-340: 323,325,332,333,336,337 → 6.\n341-360: 350,351,352,356,359 → 5.\n361-380: 362,364,368,371,372,378 → 6.\n381-400: 388,389,393,397,398,399 → 6.\n401-420: 407,410,418,422? no 422 in 421-440. → 407,410,418 → 3.\n421-440: 422,426,429,430,431,433,439,440 → 8.\n441-460: 441,443,444,446,447,449,450,461? no 461 in 461-480. → 8.\n461-480: 461,464,471,479,480 → 5.\n481-500: 484,497,499,500 → 4.\n501-520: 502,509,516,524? no. → 502,509,516 → 3.\n521-540: 524,525,528,530,532,535,536,540 → 8.\n541-560: 544,545,551,552,555,556,559 → 7.\n561-580: 562,563,566,579 → 4.\n581-600: 583,584,585,590,591,594,596,597,598,600 → 10.\n\nSum counts: 7+4+7+8+10+9+9+7+6+9+7+4+8+11+5+4+6+5+6+6+3+8+8+5+4+3+8+7+4+10 = \n7+4=11;+7=18;+8=26;+10=36;+9=45;+9=54;+7=61;+6=67;+9=76;+7=83;+4=87;+8=95;+11=106;+5=111;+4=115;+6=121;+5=126;+6=132;+6=138;+3=141;+8=149;+8=157;+5=162;+4=166;+3=169;+8=177;+7=184;+4=188;+10=198. 198 rows.\n\nNow sum counts. This is tedious. Let me sum sequentially per groups I already listed... I'll sum from my full list in chunks of 20.\n\nChunk1 (7 values): 23+28+60+28+4+9+77 = 229.\nChunk2 (4): 26+32+83+90 = 231. Wait order: id27:26, id37:90, id39:32, id40:83 → 26+90+32+83=231.\nChunk3 (7): id44:14,48:71,49:12,50:32,55:60,57:29,59:96 = 14+71+12+32+60+29+96=314.\nChunk4 (8): 62:31,64:23,65:58,67:9,68:32,72:58,75:70,78:94 = 31+23+58+9+32+58+70+94=375.\nChunk5 (10): 84:25,86:71,88:19,89:27,90:37,91:16,92:57,94:54,96:94,100:24 = 25+71+19+27+37+16+57+54+94+24=424.\nChunk6 (9): 101:21,103:31,106:93,111:99,113:39,115:57,116:37,117:69,118:73 = 21+31+93+99+39+57+37+69+73=519.\nChunk7 (9): 122:20,123:83,128:47,129:38,130:54,131:56,134:27,138:66,139:4 = 20+83+47+38+54+56+27+66+4=395.\nChunk8 (7): 142:37,144:55,146:82,147:21,150:93,156:89,157:39 = 37+55+82+21+93+89+39=416.\nChunk9 (6): 162:39,163:12,164:52,168:57,171:12,177:90 = 262.\nChunk10 (9): 182:64,185:2,187:59,191:11,193:68,195:70,196:62,198:76,200:82 = 64+2+59+11+68+70+62+76+82=494.\nChunk11 (7): 204:50,206:38,207:14,208:21,214:90,217:76,221:71 = 360.\nChunk12 (4): 224:80,225:6,231:74,240:23 = 183.\nChunk13 (8): 244:71,246:26,247:7,248:81,249:41,256:28,259:7,260:88 = 349.\nChunk14 (11): 262:88,263:56,265:53,266:70,267:42,271:70,275:100,276:36,277:100,279:30,280:5 = 88+56+53+70+42+70+100+36+100+30+5=650.\nChunk15 (5): 291:75,295:66,298:13,299:58,300:94 = 306.\nChunk16 (4): 304:56,305:76,310:29,315:23 = 184.\nChunk17 (6): 323:10,325:48,332:30,333:71,336:7,337:71 = 237.\nChunk18 (5): 350:77,351:14,352:19,356:2,359:30 = 142.\nChunk19 (6): 362:75,364:54,368:85,371:66,372:36,378:41 = 357.\nChunk20 (6): 388:64,389:36,393:7,397:98,398:45,399:51 = 301.\nChunk21 (3): 407:68,410:97,418:91 = 256.\nChunk22 (8): 422:93,426:73,429:43,430:9,431:77,433:77,439:16,440:58 = 446.\nChunk23 (8): 441:59,443:13,444:75,446:73,447:2,449:57,450:12,454? no 454開発. Wait chunk count said 8: 441,443,444,446,447,449,450 — that's 7. Let me recheck 441-460: 管理 ids: 441,443,444,446,447,449,450, and 454開発,455営業... hmm I counted 8 earlier. Also 461? No. Check: id441管理,442営業,443管理,444管理,445開発,446管理,447管理,448開発,449管理,450管理,451営業,...458開発,459営業,460開発. That's 7. Fix chunk counts: previous sum total includes +8 for this chunk but actual 7 → total rows = 197? Also check chunk 441-460. Then chunk 461-480: 461管理,464管理,471管理,479管理,480管理 → 5. Recount earlier segment: after chunk 21 sum 141? Let me redo sums from chunk 21 onward.\n\nRows: 7+4+7+8+10+9+9+7+6+9+7+4+8+11+5+4+6+5+6+6+3+7+5+4+3+8+7+4+10.\nSum: 7+4=11;+7=18;+8=26;+10=36;+9=45;+9=54;+7=61;+6=67;+9=76;+7=83;+4=87;+8=95;+11=106;+5=111;+4=115;+6=121;+5=126;+6=132;+6=138;+3=141;+7=148;+5=153;+4=157;+3=160;+8=168;+7=175;+4=179;+10=189. 189 rows? Hmm, also check chunk 22 (421-440): 422,426,429,430,431,433,439,440 = 8 ✓. chunk23 (441-460)=7. chunk24 (461-480)=5. chunk25 (481-500): 484,497,499,500 =4. chunk26 (501-520): 502,509,516=3. chunk27 (521-540): 524,525,528,530,532,535,536,540=8. chunk28 (541-560): 544,545,551,552,555,556,559=7. chunk29 (561-580): 562,563,566,579=4. chunk30 (581-600)=10.\n\nRecheck some earlier chunks quickly: 81-100 count: 84,86,88,89,90,91,92,94,96,100 =10 ✓.\n\nSum counts chunk23 (7): 441:59,443:13,444:75,446:73,447:2,449:57,450:12 = 291.\nChunk24 (5): 461:44,464:73,471:47,479:34,480:7 = 205.\nChunk25 (4): 484:42,497:80,499:49,500:91 = 262.\nChunk26 (3): 502:85,509:3,516:73 = 161.\nChunk27 (8): 524:48,525:25,528:21,530:73,532:12,535:46,536:40,540:98 = 363.\nChunk28 (7): 544:52,545:66,551:41,552:89,555:79,556:49,559:19 = 395.\nChunk29 (4): 562:25,563:23,566:41,579:55 = 144.\nChunk30 (10): 583:96,584:60,585:90,590:73,591:19,594:54,596:30,597:96,598:47,600:63 = 96+60+90+73+19+54+30+96+47+63=628.\n\nTotal sum: 229+231+314+375+424+519+395+416+262", "fields": { "行数": { "present": false, "actual": null, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": false, "actual": null, "expected": 1, "semantic_correct": false, "pass_with_natural_stop": false }, "最大待ち秒": { "present": false, "actual": null, "expected": 80, "semantic_correct": false, "pass_with_natural_stop": false }, "対象部署の件数合計": { "present": false, "actual": null, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": false, "actual": null, "expected": 11, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": false, "actual": null, "expected": "不明", "semantic_correct": false, "pass_with_natural_stop": false } } }, { "case": "main-long-records-005-cap4096-sampling-r1-mtp0", "task": "long-records-005", "input_target": 4096, "max_generation": 4096, "seed": 189106, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 182, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":607,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 607, "expected": 631, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap4096-sampling-r1-mtp1", "task": "long-records-005", "input_target": 4096, "max_generation": 4096, "seed": 189106, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 167, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":631,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 631, "expected": 631, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap8192-sampling-r2-mtp0", "task": "long-records-022", "input_target": 32768, "max_generation": 8192, "seed": 289123, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 4475, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10578,\"指定IDの待ち秒\":2,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10578, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap8192-sampling-r2-mtp1", "task": "long-records-022", "input_target": 32768, "max_generation": 8192, "seed": 289123, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3500, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9675,\"指定IDの待ち秒\":2,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9675, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap8192-sampling-r1-mtp0", "task": "long-records-006", "input_target": 4096, "max_generation": 8192, "seed": 189107, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 158, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap8192-sampling-r1-mtp1", "task": "long-records-006", "input_target": 4096, "max_generation": 8192, "seed": 189107, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 158, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap8192-sampling-r0-mtp1", "task": "long-records-000", "input_target": 4096, "max_generation": 8192, "seed": 89101, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 261, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":7,\"最大待ち秒\":78,\"対象部署の件数合計\":985,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap8192-sampling-r0-mtp0", "task": "long-records-000", "input_target": 4096, "max_generation": 8192, "seed": 89101, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 261, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":7,\"最大待ち秒\":78,\"対象部署の件数合計\":985,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap4096-sampling-r2-mtp1", "task": "long-records-016", "input_target": 32768, "max_generation": 4096, "seed": 289117, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2796, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 8688, \"指定IDの待ち秒\": 15, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8688, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 15, "expected": 31, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap4096-sampling-r2-mtp0", "task": "long-records-016", "input_target": 32768, "max_generation": 4096, "seed": 289117, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2416, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9830,\"指定IDの待ち秒\":31,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9830, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 31, "expected": 31, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap8192-sampling-r1-mtp1", "task": "long-records-012", "input_target": 32768, "max_generation": 8192, "seed": 189113, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2443, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10025,\"指定IDの待ち秒\":75,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10025, "expected": 10994, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap8192-sampling-r1-mtp0", "task": "long-records-012", "input_target": 32768, "max_generation": 8192, "seed": 189113, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1816, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9254,\"指定IDの待ち秒\":75,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9254, "expected": 10994, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap4096-sampling-r2-mtp0", "task": "long-records-023", "input_target": 32768, "max_generation": 4096, "seed": 289124, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2732, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9259,\"指定IDの待ち秒\":4,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9259, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap4096-sampling-r2-mtp1", "task": "long-records-023", "input_target": 32768, "max_generation": 4096, "seed": 289124, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3573, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9242, \"指定IDの待ち秒\": 4, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9242, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap4096-sampling-r1-mtp0", "task": "long-records-023", "input_target": 32768, "max_generation": 4096, "seed": 189124, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2831, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9191,\"指定IDの待ち秒\":4,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9191, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap4096-sampling-r1-mtp1", "task": "long-records-023", "input_target": 32768, "max_generation": 4096, "seed": 189124, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 1944, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 7168, \"指定IDの待ち秒\": 4, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 7168, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap8192-sampling-r1-mtp1", "task": "long-records-016", "input_target": 32768, "max_generation": 8192, "seed": 189117, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 1926, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9664,\"指定IDの待ち秒\":31,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9664, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 31, "expected": 31, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap8192-sampling-r1-mtp0", "task": "long-records-016", "input_target": 32768, "max_generation": 8192, "seed": 189117, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1665, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9761, \"指定IDの待ち秒\": 31, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9761, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 31, "expected": 31, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap8192-sampling-r0-mtp1", "task": "long-records-006", "input_target": 4096, "max_generation": 8192, "seed": 89107, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 155, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap8192-sampling-r0-mtp0", "task": "long-records-006", "input_target": 4096, "max_generation": 8192, "seed": 89107, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 155, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-021-cap8192-sampling-r1-mtp0", "task": "long-records-021", "input_target": 32768, "max_generation": 8192, "seed": 189122, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 7939, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9859,\"指定IDの待ち秒\":13,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9859, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 13, "expected": 13, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-021-cap8192-sampling-r1-mtp1", "task": "long-records-021", "input_target": 32768, "max_generation": 8192, "seed": 189122, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3913, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9941,\"指定IDの待ち秒\":13,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9941, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 13, "expected": 13, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap8192-sampling-r1-mtp0", "task": "long-records-017", "input_target": 32768, "max_generation": 8192, "seed": 189118, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2357, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 8941, \"指定IDの待ち秒\": 10, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8941, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 10, "expected": 46, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap8192-sampling-r1-mtp1", "task": "long-records-017", "input_target": 32768, "max_generation": 8192, "seed": 189118, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2347, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9028, \"指定IDの待ち秒\": 46, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9028, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 46, "expected": 46, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap8192-sampling-r0-mtp0", "task": "long-records-004", "input_target": 4096, "max_generation": 8192, "seed": 89105, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 254, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1375,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1375, "expected": 1273, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap8192-sampling-r0-mtp1", "task": "long-records-004", "input_target": 4096, "max_generation": 8192, "seed": 89105, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 258, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1375,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1375, "expected": 1273, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap4096-sampling-r2-mtp1", "task": "long-records-012", "input_target": 32768, "max_generation": 4096, "seed": 289113, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3876, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10994,\"指定IDの待ち秒\":75,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10994, "expected": 10994, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap4096-sampling-r2-mtp0", "task": "long-records-012", "input_target": 32768, "max_generation": 4096, "seed": 289113, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2709, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9971,\"指定IDの待ち秒\":75,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9971, "expected": 10994, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap4096-sampling-r2-mtp0", "task": "long-records-015", "input_target": 32768, "max_generation": 4096, "seed": 289116, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3705, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11482,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11482, "expected": 11482, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap4096-sampling-r2-mtp1", "task": "long-records-015", "input_target": 32768, "max_generation": 4096, "seed": 289116, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2887, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11535,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11535, "expected": 11482, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap4096-sampling-r2-mtp1", "task": "long-records-013", "input_target": 32768, "max_generation": 4096, "seed": 289114, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3514, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9384,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9384, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap4096-sampling-r2-mtp0", "task": "long-records-013", "input_target": 32768, "max_generation": 4096, "seed": 289114, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2635, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9663,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9663, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap4096-sampling-r1-mtp1", "task": "long-records-003", "input_target": 4096, "max_generation": 4096, "seed": 189104, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 219, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":2,\"最大待ち秒\":77,\"対象部署の件数合計\":701,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap4096-sampling-r1-mtp0", "task": "long-records-003", "input_target": 4096, "max_generation": 4096, "seed": 189104, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 224, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":2,\"最大待ち秒\":77,\"対象部署の件数合計\":701,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap8192-sampling-r0-mtp1", "task": "long-records-019", "input_target": 32768, "max_generation": 8192, "seed": 89120, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 1807, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9671,\"指定IDの待ち秒\":41,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9671, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 41, "expected": 11, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap8192-sampling-r0-mtp0", "task": "long-records-019", "input_target": 32768, "max_generation": 8192, "seed": 89120, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2608, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9851, \"指定IDの待ち秒\": 11, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9851, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 11, "expected": 11, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap4096-sampling-r0-mtp0", "task": "long-records-000", "input_target": 4096, "max_generation": 4096, "seed": 89101, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 261, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":7,\"最大待ち秒\":78,\"対象部署の件数合計\":985,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap4096-sampling-r0-mtp1", "task": "long-records-000", "input_target": 4096, "max_generation": 4096, "seed": 89101, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 261, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":7,\"最大待ち秒\":78,\"対象部署の件数合計\":985,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap4096-sampling-r0-mtp0", "task": "long-records-012", "input_target": 32768, "max_generation": 4096, "seed": 89113, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3629, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 215, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 10045, \"指定IDの待ち秒\": 75, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 215, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10045, "expected": 10994, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap4096-sampling-r0-mtp1", "task": "long-records-012", "input_target": 32768, "max_generation": 4096, "seed": 89113, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3613, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10045,\"指定IDの待ち秒\":75,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10045, "expected": 10994, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap8192-sampling-r0-mtp1", "task": "long-records-002", "input_target": 4096, "max_generation": 8192, "seed": 89103, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 201, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap8192-sampling-r0-mtp0", "task": "long-records-002", "input_target": 4096, "max_generation": 8192, "seed": 89103, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 212, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap8192-sampling-r0-mtp1", "task": "long-records-018", "input_target": 32768, "max_generation": 8192, "seed": 89119, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2657, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9519, \"指定IDの待ち秒\": 59, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9519, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 59, "expected": 59, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap8192-sampling-r0-mtp0", "task": "long-records-018", "input_target": 32768, "max_generation": 8192, "seed": 89119, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2805, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9458,\"指定IDの待ち秒\":59,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9458, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 59, "expected": 59, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap4096-sampling-r0-mtp0", "task": "long-records-002", "input_target": 4096, "max_generation": 4096, "seed": 89103, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 212, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap4096-sampling-r0-mtp1", "task": "long-records-002", "input_target": 4096, "max_generation": 4096, "seed": 89103, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 201, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap8192-sampling-r0-mtp1", "task": "long-records-020", "input_target": 32768, "max_generation": 8192, "seed": 89121, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2578, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10831,\"指定IDの待ち秒\":37,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10831, "expected": 10702, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap8192-sampling-r0-mtp0", "task": "long-records-020", "input_target": 32768, "max_generation": 8192, "seed": 89121, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2138, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10924,\"指定IDの待ち秒\":37,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10924, "expected": 10702, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap8192-sampling-r2-mtp0", "task": "long-records-012", "input_target": 32768, "max_generation": 8192, "seed": 289113, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2709, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9971,\"指定IDの待ち秒\":75,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9971, "expected": 10994, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap8192-sampling-r2-mtp1", "task": "long-records-012", "input_target": 32768, "max_generation": 8192, "seed": 289113, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3876, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10994,\"指定IDの待ち秒\":75,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10994, "expected": 10994, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap4096-sampling-r0-mtp1", "task": "long-records-016", "input_target": 32768, "max_generation": 4096, "seed": 89117, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 890, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9500, \"指定IDの待ち秒\": 31, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9500, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 31, "expected": 31, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap4096-sampling-r0-mtp0", "task": "long-records-016", "input_target": 32768, "max_generation": 4096, "seed": 89117, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2760, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9004, \"指定IDの待ち秒\": 31, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9004, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 31, "expected": 31, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap8192-sampling-r0-mtp1", "task": "long-records-008", "input_target": 4096, "max_generation": 8192, "seed": 89109, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 208, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":742,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap8192-sampling-r0-mtp0", "task": "long-records-008", "input_target": 4096, "max_generation": 8192, "seed": 89109, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 213, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":742,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap4096-sampling-r1-mtp1", "task": "long-records-009", "input_target": 4096, "max_generation": 4096, "seed": 189110, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 51, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":596,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 596, "expected": 834, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap4096-sampling-r1-mtp0", "task": "long-records-009", "input_target": 4096, "max_generation": 4096, "seed": 189110, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 51, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":596,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 596, "expected": 834, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap8192-sampling-r0-mtp0", "task": "long-records-005", "input_target": 4096, "max_generation": 8192, "seed": 89106, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 195, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":631,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 631, "expected": 631, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap8192-sampling-r0-mtp1", "task": "long-records-005", "input_target": 4096, "max_generation": 8192, "seed": 89106, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 195, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":631,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 631, "expected": 631, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap8192-sampling-r1-mtp0", "task": "long-records-001", "input_target": 4096, "max_generation": 8192, "seed": 189102, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 236, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":79,\"対象部署の件数合計\":1098,\"指定IDの待ち秒\":5,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap8192-sampling-r1-mtp1", "task": "long-records-001", "input_target": 4096, "max_generation": 8192, "seed": 189102, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 238, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":79,\"対象部署の件数合計\":1098,\"指定IDの待ち秒\":5,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap8192-sampling-r0-mtp1", "task": "long-records-022", "input_target": 32768, "max_generation": 8192, "seed": 89123, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3520, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 10815, \"指定IDの待ち秒\": 2, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10815, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap8192-sampling-r0-mtp0", "task": "long-records-022", "input_target": 32768, "max_generation": 8192, "seed": 89123, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1938, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 500, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9252, \"指定IDの待ち秒\": 2, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 500, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9252, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap8192-sampling-r1-mtp1", "task": "long-records-010", "input_target": 4096, "max_generation": 8192, "seed": 189111, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 123, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":324,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 324, "expected": 324, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap8192-sampling-r1-mtp0", "task": "long-records-010", "input_target": 4096, "max_generation": 8192, "seed": 189111, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 106, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":324,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 324, "expected": 324, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap4096-sampling-r0-mtp0", "task": "long-records-001", "input_target": 4096, "max_generation": 4096, "seed": 89102, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 242, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":79,\"対象部署の件数合計\":1098,\"指定IDの待ち秒\":5,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap4096-sampling-r0-mtp1", "task": "long-records-001", "input_target": 4096, "max_generation": 4096, "seed": 89102, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 242, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":79,\"対象部署の件数合計\":1098,\"指定IDの待ち秒\":5,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap8192-sampling-r1-mtp1", "task": "long-records-022", "input_target": 32768, "max_generation": 8192, "seed": 189123, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2132, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":214,\"指定IDの待ち秒\":2,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 214, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap8192-sampling-r1-mtp0", "task": "long-records-022", "input_target": 32768, "max_generation": 8192, "seed": 189123, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3733, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9359,\"指定IDの待ち秒\":2,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9359, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap4096-sampling-r1-mtp1", "task": "long-records-007", "input_target": 4096, "max_generation": 4096, "seed": 189108, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 363, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":79,\"対象部署の件数合計\":736,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap4096-sampling-r1-mtp0", "task": "long-records-007", "input_target": 4096, "max_generation": 4096, "seed": 189108, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 272, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":79,\"対象部署の件数合計\":736,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap8192-sampling-r2-mtp0", "task": "long-records-006", "input_target": 4096, "max_generation": 8192, "seed": 289107, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 167, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap8192-sampling-r2-mtp1", "task": "long-records-006", "input_target": 4096, "max_generation": 8192, "seed": 289107, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 142, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap4096-sampling-r0-mtp0", "task": "long-records-003", "input_target": 4096, "max_generation": 4096, "seed": 89104, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 174, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":2,\"最大待ち秒\":77,\"対象部署の件数合計\":701,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap4096-sampling-r0-mtp1", "task": "long-records-003", "input_target": 4096, "max_generation": 4096, "seed": 89104, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 185, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 2, \"最大待ち秒\": 77, \"対象部署の件数合計\": 701, \"指定IDの待ち秒\": 69, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap8192-sampling-r2-mtp1", "task": "long-records-017", "input_target": 32768, "max_generation": 8192, "seed": 289118, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3047, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":8903,\"指定IDの待ち秒\":46,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8903, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 46, "expected": 46, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap8192-sampling-r2-mtp0", "task": "long-records-017", "input_target": 32768, "max_generation": 8192, "seed": 289118, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3349, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":8625,\"指定IDの待ち秒\":46,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8625, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 46, "expected": 46, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap8192-sampling-r0-mtp1", "task": "long-records-003", "input_target": 4096, "max_generation": 8192, "seed": 89104, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 185, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 2, \"最大待ち秒\": 77, \"対象部署の件数合計\": 701, \"指定IDの待ち秒\": 69, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap8192-sampling-r0-mtp0", "task": "long-records-003", "input_target": 4096, "max_generation": 8192, "seed": 89104, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 174, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":2,\"最大待ち秒\":77,\"対象部署の件数合計\":701,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap4096-sampling-r1-mtp0", "task": "long-records-014", "input_target": 32768, "max_generation": 4096, "seed": 189115, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1830, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9007,\"指定IDの待ち秒\":19,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9007, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap4096-sampling-r1-mtp1", "task": "long-records-014", "input_target": 32768, "max_generation": 4096, "seed": 189115, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2903, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 10989, \"指定IDの待ち秒\": 19, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10989, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap4096-sampling-r0-mtp1", "task": "long-records-010", "input_target": 4096, "max_generation": 4096, "seed": 89111, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 89, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":324,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 324, "expected": 324, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap4096-sampling-r0-mtp0", "task": "long-records-010", "input_target": 4096, "max_generation": 4096, "seed": 89111, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 113, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 324, \"指定IDの待ち秒\": 54, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 324, "expected": 324, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap4096-sampling-r2-mtp0", "task": "long-records-011", "input_target": 4096, "max_generation": 4096, "seed": 289112, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 231, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":80,\"対象部署の件数合計\":754,\"指定IDの待ち秒\":12,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap4096-sampling-r2-mtp1", "task": "long-records-011", "input_target": 4096, "max_generation": 4096, "seed": 289112, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 231, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":80,\"対象部署の件数合計\":754,\"指定IDの待ち秒\":12,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap8192-sampling-r0-mtp1", "task": "long-records-013", "input_target": 32768, "max_generation": 8192, "seed": 89114, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3040, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9862,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9862, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap8192-sampling-r0-mtp0", "task": "long-records-013", "input_target": 32768, "max_generation": 8192, "seed": 89114, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3207, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9844,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9844, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap4096-sampling-r0-mtp1", "task": "long-records-022", "input_target": 32768, "max_generation": 4096, "seed": 89123, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3520, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 10815, \"指定IDの待ち秒\": 2, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10815, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap4096-sampling-r0-mtp0", "task": "long-records-022", "input_target": 32768, "max_generation": 4096, "seed": 89123, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1938, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 500, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9252, \"指定IDの待ち秒\": 2, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 500, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9252, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap8192-sampling-r1-mtp0", "task": "long-records-005", "input_target": 4096, "max_generation": 8192, "seed": 189106, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 182, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":607,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 607, "expected": 631, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap8192-sampling-r1-mtp1", "task": "long-records-005", "input_target": 4096, "max_generation": 8192, "seed": 189106, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 167, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":631,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 631, "expected": 631, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap8192-sampling-r2-mtp1", "task": "long-records-002", "input_target": 4096, "max_generation": 8192, "seed": 289103, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 217, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap8192-sampling-r2-mtp0", "task": "long-records-002", "input_target": 4096, "max_generation": 8192, "seed": 289103, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 217, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap8192-sampling-r2-mtp0", "task": "long-records-008", "input_target": 4096, "max_generation": 8192, "seed": 289109, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 203, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":742,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap8192-sampling-r2-mtp1", "task": "long-records-008", "input_target": 4096, "max_generation": 8192, "seed": 289109, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 138, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":742,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap8192-sampling-r2-mtp0", "task": "long-records-005", "input_target": 4096, "max_generation": 8192, "seed": 289106, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 229, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":631,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 631, "expected": 631, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap8192-sampling-r2-mtp1", "task": "long-records-005", "input_target": 4096, "max_generation": 8192, "seed": 289106, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 233, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":631,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 631, "expected": 631, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-021-cap4096-sampling-r2-mtp1", "task": "long-records-021", "input_target": 32768, "max_generation": 4096, "seed": 289122, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3182, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9805,\"指定IDの待ち秒\":13,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9805, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 13, "expected": 13, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-021-cap4096-sampling-r2-mtp0", "task": "long-records-021", "input_target": 32768, "max_generation": 4096, "seed": 289122, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1867, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9637,\"指定IDの待ち秒\":13,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9637, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 13, "expected": 13, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap4096-sampling-r1-mtp1", "task": "long-records-020", "input_target": 32768, "max_generation": 4096, "seed": 189121, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3759, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 10702, \"指定IDの待ち秒\": 37, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10702, "expected": 10702, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap4096-sampling-r1-mtp0", "task": "long-records-020", "input_target": 32768, "max_generation": 4096, "seed": 189121, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3754, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10703,\"指定IDの待ち秒\":37,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10703, "expected": 10702, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap8192-sampling-r1-mtp0", "task": "long-records-003", "input_target": 4096, "max_generation": 8192, "seed": 189104, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 224, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":2,\"最大待ち秒\":77,\"対象部署の件数合計\":701,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap8192-sampling-r1-mtp1", "task": "long-records-003", "input_target": 4096, "max_generation": 8192, "seed": 189104, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 219, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":2,\"最大待ち秒\":77,\"対象部署の件数合計\":701,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap4096-sampling-r0-mtp1", "task": "long-records-009", "input_target": 4096, "max_generation": 4096, "seed": 89110, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 182, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":834,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 834, "expected": 834, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap4096-sampling-r0-mtp0", "task": "long-records-009", "input_target": 4096, "max_generation": 4096, "seed": 89110, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 182, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":834,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 834, "expected": 834, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap4096-sampling-r2-mtp0", "task": "long-records-019", "input_target": 32768, "max_generation": 4096, "seed": 289120, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3259, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10083,\"指定IDの待ち秒\":11,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10083, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 11, "expected": 11, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap4096-sampling-r2-mtp1", "task": "long-records-019", "input_target": 32768, "max_generation": 4096, "seed": 289120, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2550, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9962,\"指定IDの待ち秒\":11,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9962, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 11, "expected": 11, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap4096-sampling-r2-mtp1", "task": "long-records-002", "input_target": 4096, "max_generation": 4096, "seed": 289103, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 217, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap4096-sampling-r2-mtp0", "task": "long-records-002", "input_target": 4096, "max_generation": 4096, "seed": 289103, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 217, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap8192-sampling-r2-mtp0", "task": "long-records-004", "input_target": 4096, "max_generation": 8192, "seed": 289105, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 51, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1191,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1191, "expected": 1273, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap8192-sampling-r2-mtp1", "task": "long-records-004", "input_target": 4096, "max_generation": 8192, "seed": 289105, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 51, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1191,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1191, "expected": 1273, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap4096-sampling-r0-mtp0", "task": "long-records-019", "input_target": 32768, "max_generation": 4096, "seed": 89120, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2608, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9851, \"指定IDの待ち秒\": 11, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9851, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 11, "expected": 11, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap4096-sampling-r0-mtp1", "task": "long-records-019", "input_target": 32768, "max_generation": 4096, "seed": 89120, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 1807, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9671,\"指定IDの待ち秒\":41,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9671, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 41, "expected": 11, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap8192-sampling-r0-mtp1", "task": "long-records-009", "input_target": 4096, "max_generation": 8192, "seed": 89110, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 182, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":834,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 834, "expected": 834, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap8192-sampling-r0-mtp0", "task": "long-records-009", "input_target": 4096, "max_generation": 8192, "seed": 89110, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 182, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":834,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 834, "expected": 834, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap4096-sampling-r1-mtp0", "task": "long-records-002", "input_target": 4096, "max_generation": 4096, "seed": 189103, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 245, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap4096-sampling-r1-mtp1", "task": "long-records-002", "input_target": 4096, "max_generation": 4096, "seed": 189103, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 245, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap4096-sampling-r1-mtp0", "task": "long-records-012", "input_target": 32768, "max_generation": 4096, "seed": 189113, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1816, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9254,\"指定IDの待ち秒\":75,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9254, "expected": 10994, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap4096-sampling-r1-mtp1", "task": "long-records-012", "input_target": 32768, "max_generation": 4096, "seed": 189113, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2443, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10025,\"指定IDの待ち秒\":75,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10025, "expected": 10994, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap4096-sampling-r2-mtp0", "task": "long-records-022", "input_target": 32768, "max_generation": 4096, "seed": 289123, "mtp": false, "finish_reason": "max_new_tokens", "generated_tokens": 4096, "pass": false, "first_answer_missing": true, "answer_text": "行数600。開発行の件数合計を数える必要がある。開発行を抽出して合計計算。\n\n開発行のリスト:\n44,97,47,47,31,58,69,82,95,14,32,73,74,78,61,18,96,87,92,28,28,87,73,96,55,71,73,96,8,7,49,100,94,5,73,3,38,39,89,64,76,73,56,75,73,78,88,6,31,73,57,78,12,40,47,97,39,31,47,58,24,52,78,32,57,21,32,56,33,47,5,69,75,60,11,48,24,65,89,49,31,77,26,21,24,39,30,62,10,43,50,73,48,27,76,13,60,63,97,3,61,18,28,36,25,39,32,1,15,10,53,54,6,30,55,76,13,74,3,59,61,31,47,31,74,19,24,25,89,57,52,56,62,16,47,58,76,69,33,98,76,35,27,33,68,48,65,77,33,48,65,36,21,49,21,31,44,6,77,20,36,32,99,34,71,34,99,41,89,25,77,77,53,76,18,55,36,18,9,96,71,18,11,14,33,98,18,100,44,73,62,66,14,87,77,55,8,47,53,10,43,44,64,41,76,69,35,27,48,65,19,5,67,25,64,48,73,38,34,71,34,99,59,80,29,2,83,23,30,55,66,94,29,10,95,57,75,10,3,32,1,42,44,64,41,73,64,62,32,94,51,62,64,16,25,71,80,22,36,45,50,18,21,45,4,49,30,54,80,21,3,40,59,10,41,89,25,20,69,19,54,77,53,56,27,76,18,55,36,18,9,96,71,18,11,14,33,98,18,100,44,73,62,66,14,87,77,55,8,47,53,10,43,44,64,41,76,69,35,27,48,65,19,5,67,25,64,48,73,38,34,71,34,99,59 — 待って、後半で繰り返しがある。順に正確に再計算しよう。\n\n開発行をID順に列挙（待ち秒, 件数）:\n1:44, 3:97, 4:47, 7:47, 10:31, 11:58, 12:69, 14:82, 16:95, 26:14, 28:32, 29:5, 30:73, 32:74, 33:78, 34:61, 41:18, 42:96, 45:92, 49:87, 52:28, 57:48, 59:55, 60:71, 61:73, 64:96, 65:8, 68:7, 69:49, 71:100, 73:94, 75:5, 80:38, 86:39, 87:89, 96:64, 97:76, 98:73, 100:56, 105:75, 110:88, 111:69, 113:31, 117:78, 118:12, 122:40, 127:5, 132:97, 133:39, 137:31, 138:74, 139:19, 140:24, 143:52, 147:25, 148:57, 149:52, 150:56, 151:62, 152:16, 156:47, 157:58, 163:75, 165:50, 169:48, 170:36, 172:48, 173:14, 178:80, 182:31, 183:6, 185:20, 186:36, 188:49, 196:24, 200:30, 201:15, 203:10, 205:53, 206:54, 212:6, 217:30, 218:55, 219:76, 224:60, 226:74, 229:3, 230:59, 237:18, 239:25, 248:11, 251:20, 252:56, 255:25, 257:53, 260:11, 261:29, 262:41, 266:16, 271:48, 275:42, 280:65, 281:26, 283:52, 286:27, 289:52, 292:64, 298:43, 300:54, 303:46, 309:43, 311:59, 312:62, 313:54, 319:10, 327:33, 328:59, 330:56, 331:26, 332:46, 333:7, 336:34, 346:74, 349:5, 354:31, 356:57, 357:10, 360:19, 361:46, 366:44, 367:3, 371:42, 376:54, 380:20, 382:1, 383:61, 384:4, 386:18, 391:16, 393:47, 396:76, 397:47, 399:73, 409:49, 410:23, 412:53, 417:53, 421:67, 422:12, 424:45, 428:30, 431:12, 432:61, 433:2, 437:55, 440:55, 443:80, 448:57, 450:10, 456:1, 458:3, 460:11, 461:44, 462:1, 465:63, 466:66, 468:41, 469:64, 470:32, 472:51, 475:3, 478:64, 479:25, 481:80, 482:36, 484:50, 487:21, 488:4, 491:30, 495:80, 496:3, 500:59, 501:41, 502:25, 503:77, 509:77, 511:76, 515:55, 516:18, 519:9, 528:71, 529:11, 531:33, 532:18, 534:44, 536:62, 537:14, 541:77, 542:8, 545:47, 547:53, 548:10, 552:43, 553:64, 563:76, 564:35, 569:3, 574:77, 578:48, 581:19, 583:67, 591:73, 595:34, 598:34, 500行目以降も含む。500:59（id500は開発）、続けてID 501-600。\n\n待ち秒の件数リスト（開発）を合計。グループごとに合計：\n\n100件目まで（id1-100開発）:\n44,97,47,47,31,58,69,82,95,14,32,73,74,78,61,18,96,92,87,28,48,55,71,73,96,8,7,49,100,94,5,38,39,89,64,76,73,56 — 数える。開発は31行？数える:\n1,3,4,7,10,11,12,14,16,26,28,29,30,32,33,34,41,42,45,49,52,57,59,60,61,64,65,68,69,71,73,75,80,86,87,96,97,98,100 → 39行。\n件数: 44+97=141; +47=188; +47=235; +31=266; +58=324; +69=393; +82=475; +95=570; +14=584; +32=616; +73=689; +74=763; +78=841; +61=902; +18=920; +96=1016; +92=1108; +87=1195; +28=1223; +48=1271; +55=1326; +71=1397; +73=1470; +96=1566; +8=1574; +7=1581; +49=1630; +100=1730; +94=1824; +5=1829; +38=1867; +39=1906; +89=1995; +64=2059; +76=2135; +73=2208; +56=2264.\n\nid101-200開発:\n105:75,110:88,111:69,113:31,117:78,118:12,122:40,127:5,132:97,133:39,137:31,138:74,139:19,140:24,143:52,147:25,148:57,149:52,150:56,151:62,152:16,156:47,157:58,163:75,165:50,169:48,170:36,172:48,173:14,178:80,182:31,183:6,185:20,186:36,188:49,196:24,200:30 → 37行。\n75+88=163;+69=232;+31=263;+78=341;+12=353;+40=393;+5=398;+97=495;+39=534;+31=565;+74=639;+19=658;+24=682;+52=734;+25=759;+57=816;+52=868;+56=924;+62=986;+16=1002;+47=1049;+58=1107;+75=1182;+50=1232;+48=1280;+36=1316;+48=1364;+14=1378;+80=1458;+31=1489;+6=1495;+20=1515;+36=1551;+49=1600;+24=1624;+30=1654.\n累計 2264+1654=3918.\n\nid201-300開発:\n201:62,203:43,205:50,206:73,212:79,217:48,218:27,219:76,224:63,226:97,229:2,230:61,237:28,239:36,248:65,251:15,252:31,255:83,257:28,260:93,261:54,262:34,266:41,271:40,275:14,280:68,281:13,283:100,286:7,289:2,292:55,298:78,300:32 → 33行。\n62+43=105;+50=155;+73=228;+79=307;+48=355;+27=382;+76=458;+63=521;+97=618;+2=620;+61=681;+28=709;+36=745;+65=810;+15=825;+31=856;+83=939;+28=967;+93=1060;+54=1114;+34=1148;+41=1189;+40=1229;+14=1243;+68=1311;+13=1324;+100=1424;+7=1431;+2=1433;+55=1488;+78=1566;+32=1598.\n累計 3918+1598=5516.\n\nid301-400開発:\n303:7,309:3,311:23,312:91,313:93,319:47,327:19,328:33,330:56,331:86,332:2,333:66,336:25,346:32,349:34,354:77,356:30,357:74,360:68,361:22,366:7,367:63,371:30,376:58,380:3,382:6,383:54,384:28,386:56,391:53,393:37,396:100,397:87,399:16 → 34行。\n7+3=10;+23=33;+91=124;+93=217;+47=264;+19=283;+33=316;+56=372;+86=458;+2=460;+66=526;+25=551;+32=583;+34=617;+77=694;+30=724;+74=798;+68=866;+22=888;+7=895;+63=958;+30=988;+58=1046;+3=1049;+6=1055;+54=1109;+28=1137;+56=1193;+53=1246;+37=1283;+100=1383;+87=1470;+16=1486.\n累計 5516+1486=7002.\n\nid401-500開発:\n409:80,410:67,412:18,417:74,421:17,422:2,424:71,428:30,431:23,432:33,433:83,437:66,440:94,443:29,448:75,450:95,456:42,458:32,460:71,461:64,462:86,465:48,466:40,468:73,469:62,470:94,472:62,475:32,478:16,479:71,481:22,482:45,484:18,487:45,488:49,491:54,495:21,496:40,500:10 → 39行。\n80+67=147;+18=165;+74=239;+17=256;+2=258;+71=329;+30=359;+23=382;+33=415;+83=498;+66=564;+94=658;+29=687;+75=762;+95=857;+42=899;+32=931;+71=1002;+64=1066;+86=1152;+48=1200;+40=1240;+73=1313;+62=1375;+94=1469;+62=1531;+32=1563;+16=1579;+71=1650;+22=1672;+45=1717;+18=1735;+45=1780;+49=1829;+54=1883;+21=1904;+40=1944;+10=1954.\n累計 7002+1954=8956.\n\nid501-600開発:\n501:89,502:20,503:69,509:53,511:18,515:36,516:67,519:96,528:18,529:14,531:98,532:100,534:73,536:66,537:87,541:55,542:31,545:11,547:29,548:7,552:44,553:41,563:69,564:27,", "fields": { "行数": { "present": false, "actual": null, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": false, "actual": null, "expected": 1, "semantic_correct": false, "pass_with_natural_stop": false }, "最大待ち秒": { "present": false, "actual": null, "expected": 80, "semantic_correct": false, "pass_with_natural_stop": false }, "対象部署の件数合計": { "present": false, "actual": null, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": false, "actual": null, "expected": 2, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": false, "actual": null, "expected": "不明", "semantic_correct": false, "pass_with_natural_stop": false } } }, { "case": "main-long-records-022-cap4096-sampling-r2-mtp1", "task": "long-records-022", "input_target": 32768, "max_generation": 4096, "seed": 289123, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3500, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9675,\"指定IDの待ち秒\":2,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9675, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap8192-sampling-r0-mtp0", "task": "long-records-011", "input_target": 4096, "max_generation": 8192, "seed": 89112, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 242, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 80, \"対象部署の件数合計\": 754, \"指定IDの待ち秒\": 12, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap8192-sampling-r0-mtp1", "task": "long-records-011", "input_target": 4096, "max_generation": 8192, "seed": 89112, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 242, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 80, \"対象部署の件数合計\": 754, \"指定IDの待ち秒\": 12, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap8192-sampling-r2-mtp0", "task": "long-records-015", "input_target": 32768, "max_generation": 8192, "seed": 289116, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3705, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11482,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11482, "expected": 11482, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap8192-sampling-r2-mtp1", "task": "long-records-015", "input_target": 32768, "max_generation": 8192, "seed": 289116, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2887, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11535,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11535, "expected": 11482, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap8192-sampling-r2-mtp0", "task": "long-records-011", "input_target": 4096, "max_generation": 8192, "seed": 289112, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 231, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":80,\"対象部署の件数合計\":754,\"指定IDの待ち秒\":12,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap8192-sampling-r2-mtp1", "task": "long-records-011", "input_target": 4096, "max_generation": 8192, "seed": 289112, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 231, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":80,\"対象部署の件数合計\":754,\"指定IDの待ち秒\":12,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap8192-sampling-r2-mtp1", "task": "long-records-020", "input_target": 32768, "max_generation": 8192, "seed": 289121, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3434, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 10761, \"指定IDの待ち秒\": 37, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10761, "expected": 10702, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap8192-sampling-r2-mtp0", "task": "long-records-020", "input_target": 32768, "max_generation": 8192, "seed": 289121, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1864, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10767,\"指定IDの待ち秒\":37,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10767, "expected": 10702, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap8192-sampling-r0-mtp1", "task": "long-records-017", "input_target": 32768, "max_generation": 8192, "seed": 89118, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3820, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9903,\"指定IDの待ち秒\":46,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9903, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 46, "expected": 46, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap8192-sampling-r0-mtp0", "task": "long-records-017", "input_target": 32768, "max_generation": 8192, "seed": 89118, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 6109, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9736,\"指定IDの待ち秒\":10,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9736, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 10, "expected": 46, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap8192-sampling-r1-mtp1", "task": "long-records-014", "input_target": 32768, "max_generation": 8192, "seed": 189115, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2903, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 10989, \"指定IDの待ち秒\": 19, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10989, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap8192-sampling-r1-mtp0", "task": "long-records-014", "input_target": 32768, "max_generation": 8192, "seed": 189115, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1830, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9007,\"指定IDの待ち秒\":19,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9007, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap4096-sampling-r2-mtp1", "task": "long-records-007", "input_target": 4096, "max_generation": 4096, "seed": 289108, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 192, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 79, \"対象部署の件数合計\": 736, \"指定IDの待ち秒\": 54, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap4096-sampling-r2-mtp0", "task": "long-records-007", "input_target": 4096, "max_generation": 4096, "seed": 289108, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 192, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 79, \"対象部署の件数合計\": 736, \"指定IDの待ち秒\": 54, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap4096-sampling-r1-mtp0", "task": "long-records-001", "input_target": 4096, "max_generation": 4096, "seed": 189102, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 236, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":79,\"対象部署の件数合計\":1098,\"指定IDの待ち秒\":5,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap4096-sampling-r1-mtp1", "task": "long-records-001", "input_target": 4096, "max_generation": 4096, "seed": 189102, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 238, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":79,\"対象部署の件数合計\":1098,\"指定IDの待ち秒\":5,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap4096-sampling-r1-mtp0", "task": "long-records-011", "input_target": 4096, "max_generation": 4096, "seed": 189112, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 192, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":80,\"対象部署の件数合計\":754,\"指定IDの待ち秒\":12,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap4096-sampling-r1-mtp1", "task": "long-records-011", "input_target": 4096, "max_generation": 4096, "seed": 189112, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 192, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":80,\"対象部署の件数合計\":754,\"指定IDの待ち秒\":12,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-021-cap8192-sampling-r0-mtp1", "task": "long-records-021", "input_target": 32768, "max_generation": 8192, "seed": 89122, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 1816, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9310, \"指定IDの待ち秒\": 13, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9310, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 13, "expected": 13, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-021-cap8192-sampling-r0-mtp0", "task": "long-records-021", "input_target": 32768, "max_generation": 8192, "seed": 89122, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 4595, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11716,\"指定IDの待ち秒\":13,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11716, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 13, "expected": 13, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap4096-sampling-r2-mtp1", "task": "long-records-009", "input_target": 4096, "max_generation": 4096, "seed": 289110, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 178, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":834,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 834, "expected": 834, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap4096-sampling-r2-mtp0", "task": "long-records-009", "input_target": 4096, "max_generation": 4096, "seed": 289110, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 134, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":834,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 834, "expected": 834, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap4096-sampling-r0-mtp1", "task": "long-records-011", "input_target": 4096, "max_generation": 4096, "seed": 89112, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 242, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 80, \"対象部署の件数合計\": 754, \"指定IDの待ち秒\": 12, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap4096-sampling-r0-mtp0", "task": "long-records-011", "input_target": 4096, "max_generation": 4096, "seed": 89112, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 242, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 80, \"対象部署の件数合計\": 754, \"指定IDの待ち秒\": 12, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap8192-sampling-r0-mtp1", "task": "long-records-001", "input_target": 4096, "max_generation": 8192, "seed": 89102, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 242, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":79,\"対象部署の件数合計\":1098,\"指定IDの待ち秒\":5,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap8192-sampling-r0-mtp0", "task": "long-records-001", "input_target": 4096, "max_generation": 8192, "seed": 89102, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 242, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":79,\"対象部署の件数合計\":1098,\"指定IDの待ち秒\":5,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap4096-sampling-r1-mtp1", "task": "long-records-022", "input_target": 32768, "max_generation": 4096, "seed": 189123, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2132, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":214,\"指定IDの待ち秒\":2,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 214, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-022-cap4096-sampling-r1-mtp0", "task": "long-records-022", "input_target": 32768, "max_generation": 4096, "seed": 189123, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3733, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9359,\"指定IDの待ち秒\":2,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9359, "expected": 10641, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-021-cap4096-sampling-r1-mtp0", "task": "long-records-021", "input_target": 32768, "max_generation": 4096, "seed": 189122, "mtp": false, "finish_reason": "max_new_tokens", "generated_tokens": 4096, "pass": false, "first_answer_missing": true, "answer_text": "Count rows: 600. Min wait: 1. Max: 80. Management count sum — need to sum counts for department \"management\". That's tedious; approximate? Need accuracy. Let me count management rows and sum. This is heavy but let's try sampling... Actually I must produce a number. Let me attempt to sum management counts.\n\nManagement rows list with counts:\n1:10, 2:41, 4:91, 6:80, 7:26, 9:3, 13:77, 14:71, 21:6, 28:16, 41:24, 43:39, 44:63, 47:86, 49:31, 50:21, 51:40, 52:76, 54:6, 55:66, 57:28, 61:18, 62:13, 66:46, 72:36, 75:7, 78:62, 79:100, 80:53, 81:98, 88:82, 92:33, 94:46, 95:77, 98:70, 105:45, 107:53, 112:64, 116:70, 119:59, 131:41, 134:35, 135:34, 138:72, 139:86, 141:25, 142:39, 143:35, 144:49, 146:42, 149:72, 152:8, 154:81, 155:33, 159:43, 160:95, 162:63, 163:39, 165:97, 171:88, 172:7, 173:67, 177:3, 185:69, 189:4, 191:13, 192:32, 194:42, 195:10, 196:35, 197:23, 198:88, 207:35, 212:57, 215:30, 216:55, 217:94, 222:20, 223:60, 224:34, 228:76, 229:7, 232:55, 239:4, 246:59, 248:67, 249:67, 251:7, 252:33, 254:51, 256:7, 261:18, 268:29, 269:16, 270:44, 271:56, 273:6, 274:11, 275:68, 276:49, 277:27, 278:48, 279:14, 280:51, 282:15, 287:8, 288:69, 289:55, 290:21, 291:60, 292:6, 295:45, 299:11, 301:28, 304:34, 305:49, 306:53, 308:79, 309:60, 310:49, 311:50, 312:7, 319:13, 320:17, 321:6, 322:63, 324:43, 328:5, 333:42, 334:65, 336:51, 337:27, 339:56, 344:72, 346:32, 347:17, 350:21, 356:70, 357:61, 364:35, 365:66, 366:2, 367:71, 371:44, 372:32, 376:16, 377:46, 380:18, 381:30, 389:73, 393:40, 394:21, 396:68, 403:5, 405:51, 406:31, 407:79, 411:78, 413:64, 420:72, 423:77, 426:28, 433:2, 435:43, 440:78, 442:44, 445:53, 446:30, 449:48, 451:39, 452:34, 456:27, 460:23, 463:3, 466:37, 469:58, 470:58, 474:71, 475:53, 476:61, 477:9, 478:39, 480:12, 483:13, 484:39, 488:47, 490:51, 493:28, 497:59, 501:41, 502:76, 506:22, 511:39, 513:73, 514:33, 515:24, 516:2, 517:22, 518:26, 520:76, 525:79, 526:29, 530:69, 531:77, 534:78, 535:34, 537:8, 540:78, 541:77, 542:9, 546:19, 553:3, 554:52, 556:31, 562:42, 566:72, 568:31, 571:48, 576:42, 582:34, 583:35, 584:36, 586:41, 595:72, 596:59, 597:37, 599:8.\n\nCount rows: let me count in chunks of 10:\n1-10: 10,41,91,80,26,3,77,71,6,16 → sum: 10+41=51,+91=142,+80=222,+26=248,+3=251,+77=328,+71=399,+6=405,+16=421. (10 rows)\n11-20: 24,39,63,86,31,21,40,76,6,66 → 24+39=63,+63=126,+86=212,+31=243,+21=264,+40=304,+76=380,+6=386,+66=452. Running: 873.\n21-30: 28,18,13,46,36,7,62,100,53,98 → 28+18=46,+13=59,+46=105,+36=141,+7=148,+62=210,+100=310,+53=363,+98=461. Running: 1334.\n31-40: 82,33,46,77,70,45,53,64,70,59 → 82+33=115,+46=161,+77=238,+70=308,+45=353,+53=406,+64=470,+70=540,+59=599. Running: 1933.\n41-50: 41,35,34,72,86,25,39,35,49,42 → 41+35=76,+34=110,+72=182,+86=268,+25=293,+39=332,+35=367,+49=416,+42=458. Running: 2391.\n51-60: 72,8,81,33,43,95,63,39,97,88 → 72+8=80,+81=161,+33=194,+43=237,+95=332,+63=395,+39=434,+97=531,+88=619. Running: 3010.\n61-70: 7,67,3,69,4,13,32,42,10,35 → 7+67=74,+3=77,+69=146,+4=150,+13=163,+32=195,+42=237,+10=247,+35=282. Running: 3292.\n71-80: 23,88,35,57,30,55,94,20,60,34 → 23+88=111,+35=146,+57=203,+30=233,+55=288,+94=382,+20=402,+60=462,+34=496. Running: 3788.\n81-90: 76,7,55,4,59,67,67,7,33,51 → 76+7=83,+55=138,+4=142,+59=201,+67=268,+67=335,+7=342,+33=375,+51=426. Running: 4214.\n91-100: 7,18,29,16,44,56,6,11,68,49 → 7+18=25,+29=54,+16=70,+44=114,+56=170,+6=176,+11=187,+68=255,+49=304. Running: 4518.\n101-110: 27,48,14,51,15,8,69,55,21,60 → 27+48=75,+14=89,+51=140,+15=155,+8=163,+69=232,+55=287,+21=308,+60=368. Running: 4886.\n111-120: 6,11,28,34,49,53,79,60,49,50 → 6+11=17,+28=45,+34=79,+49=128,+53=181,+79=260,+60=320,+49=369,+50=419. Running: 5305.\n121-130: 7,13,17,6,63,43,5,42,65,51 → 7+13=20,+17=37,+6=43,+63=106,+43=149,+5=154,+42=196,+65=261,+51=312. Running: 5617.\n131-140: 27,56,72,32,17,21,70,61,35,66 → 27+56=83,+72=155,+32=187,+17=204,+21=225,+70=295,+61=356,+35=391,+66=457. Running: 6074.\n141-150: 2,71,44,32,16,46,18,30,73,40 → 2+71=73,+44=117,+32=149,+16=165,+46=211,+18=229,+30=259,+73=332,+40=372. Running: 6446.\n151-160: 21,68,5,68,5,31,79,78,64,72 → 21+68=89,+5=94,+68=162,+5=167,+31=198,+79=277,+78=355,+64=419,+72=491. Running: 6937.\n161-170: 77,28,2,43,12,78,78,44,30,48 → 77+28=105,+2=107,+43=150,+12=162,+78=240,+78=318,+44=362,+30=392,+48=440. Running: 7377.\n171-180: 39,34,27,23,3,37,58,58,71,53 → 39+34=73,+27=100,+23=123,+3=126,+37=163,+58=221,+58=279,+71=350,+53=403. Running: 7780.\n181-190: 61,9,39,12,13,39,47,81,47,51 → 61+9=70,+39=79,+12=91,+13=104,+39=143,+47=190,+81=271,+47=318,+51=369. Running: 8149.\n191-200: 37,28,98,59,41,76,22,39,73,33 → 37+28=65,+98=163,+59=222,+41=263,+76=339,+22=361,+39=400,+73=473,+33=506. Running: 8655.\n201-210: 24,2,39,4,2,22,35,26,76,29 → 24+2=26,+39=65,+4=69,+2=71,+22=93,+35=128,+26=154,+76=230,+29=259. Running: 8914.\n211-220: 69,77,78,34,8,78,77,9,19,3 → 69+77=146,+78=224,+34=258,+8=266,+78=344,+77=421,+9=430,+19=449,+3=452. Running: 9366.\n221-230: 52,31,42,31,48,42,72,64,72,35 → 52+31=83,+42=125,+31=156,+48=204,+42=246,+72=318,+64=382,+72=454,+35=489. Running: 9855.\n231-240: 35,36,41,72,59,37,8,6,8,13 → 35+36=71,+41=112,+72=184,+59=243,+37=280,+8=288,+6=294,+8=302,+13=315. Running: 10170.\n241-250: 3,35,52,72,31,48,39,23,3,89 → 3+35=38,+52=90,+72=162,+31=193,+48=241,+39=280,+23=303,+3=306,+89=395. Running: 10565.\n251-260: 37,47,13,29,24,39,3,89,58,83 → wait id483 is Management, 13 — I already had it? No, id483 appears at chunk 241-250? Let me recheck ordering. After 478:39 comes 480:12, 483:13... Hmm I think my chunk grouping drifted. Let me recount rows count too.\n\nActually I listed management rows sequentially; chunks of 10 in that list. Let me verify my list around id 460-500: 460:23, 463:3, 466:37, 469:58, 470:58, 474:71, 475:53, 476:61, 477:9, 478:39, 480:12, 483:13, 484:39, 488:47, 490:51, 493:28, 497:59. That's 17 items. In my chunks 231-240 I had \"3,35,52,72,31,48,39,23,3,89\" — that maps to 463:3,466:37? mismatch. Hmm, 89 shouldn't be in management there (463:3 yes; 466:37). I wrote \"37,47,13,29,24,39,3,89,58,83\" for chunk 241-250 — that seems wrong. Let me redo the tail carefully.\n\nLet me recount management list tail from id 449 onward:\n449:46? Wait 449 is Management, 46. I had earlier chunks... I think my earlier chunk mapping got misaligned. Let me recheck the list I transcribed:\n\nFrom my original transcription list (management entries in order):\n..., 440:78, 442:44, 445:53, 446:30, 449:46, 451:39, 452:34, 456:27, 460:23, 463:3, 466:37, 469:58, 470:58, 474:71, 475:53, 476:61, 477:9, 478:39, 480:12, 483:13, 484:39, 488:47, 490:51, 493:28, 497:59, 501:41, 502:76, 506:22, 511:39, 513:73, 514:33, 515:24, 516:2, 517:22, 518:26, 520:76, 525:79, 526:29, 530:69, 531:77, 534:78, 535:34, 537:8, 540:78, 541:77, 542:9, 546:19, 553:3, 554:52, 556:31, 562:42, 566:72, 568:31, 571:48, 576:42, 582:34, 583:35, 584:36, 586:41, 595:72, 596:59, 597:37, 599:8.\n\nNow, in my chunked sums, where did alignment break? Chunks 161-170 gave \"77,28,2,43,12,78,78,44,30,48\" — mapping: 525:79? Hmm 79 appears in chunk 151-160 as \"...31,79,78,64,72\" → 525:79, 534:78, 540:", "fields": { "行数": { "present": false, "actual": null, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": false, "actual": null, "expected": 1, "semantic_correct": false, "pass_with_natural_stop": false }, "最大待ち秒": { "present": false, "actual": null, "expected": 80, "semantic_correct": false, "pass_with_natural_stop": false }, "対象部署の件数合計": { "present": false, "actual": null, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": false, "actual": null, "expected": 13, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": false, "actual": null, "expected": "不明", "semantic_correct": false, "pass_with_natural_stop": false } } }, { "case": "main-long-records-021-cap4096-sampling-r1-mtp1", "task": "long-records-021", "input_target": 32768, "max_generation": 4096, "seed": 189122, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3913, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9941,\"指定IDの待ち秒\":13,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9941, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 13, "expected": 13, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap8192-sampling-r1-mtp0", "task": "long-records-004", "input_target": 4096, "max_generation": 8192, "seed": 189105, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 236, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1273,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1273, "expected": 1273, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap8192-sampling-r1-mtp1", "task": "long-records-004", "input_target": 4096, "max_generation": 8192, "seed": 189105, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 235, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1273,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1273, "expected": 1273, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap4096-sampling-r2-mtp1", "task": "long-records-010", "input_target": 4096, "max_generation": 4096, "seed": 289111, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 73, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":289,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 289, "expected": 324, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap4096-sampling-r2-mtp0", "task": "long-records-010", "input_target": 4096, "max_generation": 4096, "seed": 289111, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 73, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":247,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 247, "expected": 324, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap8192-sampling-r1-mtp0", "task": "long-records-000", "input_target": 4096, "max_generation": 8192, "seed": 189101, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 275, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 7, \"最大待ち秒\": 78, \"対象部署の件数合計\": 985, \"指定IDの待ち秒\": 70, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap8192-sampling-r1-mtp1", "task": "long-records-000", "input_target": 4096, "max_generation": 8192, "seed": 189101, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 248, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":7,\"最大待ち秒\":78,\"対象部署の件数合計\":985,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap4096-sampling-r2-mtp0", "task": "long-records-005", "input_target": 4096, "max_generation": 4096, "seed": 289106, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 229, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":631,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 631, "expected": 631, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap4096-sampling-r2-mtp1", "task": "long-records-005", "input_target": 4096, "max_generation": 4096, "seed": 289106, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 233, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":631,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 631, "expected": 631, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap8192-sampling-r1-mtp1", "task": "long-records-023", "input_target": 32768, "max_generation": 8192, "seed": 189124, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 1944, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 7168, \"指定IDの待ち秒\": 4, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 7168, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap8192-sampling-r1-mtp0", "task": "long-records-023", "input_target": 32768, "max_generation": 8192, "seed": 189124, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2831, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9191,\"指定IDの待ち秒\":4,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9191, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap8192-sampling-r0-mtp0", "task": "long-records-016", "input_target": 32768, "max_generation": 8192, "seed": 89117, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2760, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9004, \"指定IDの待ち秒\": 31, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9004, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 31, "expected": 31, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap8192-sampling-r0-mtp1", "task": "long-records-016", "input_target": 32768, "max_generation": 8192, "seed": 89117, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 890, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9500, \"指定IDの待ち秒\": 31, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9500, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 31, "expected": 31, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap8192-sampling-r2-mtp0", "task": "long-records-019", "input_target": 32768, "max_generation": 8192, "seed": 289120, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3259, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10083,\"指定IDの待ち秒\":11,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10083, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 11, "expected": 11, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap8192-sampling-r2-mtp1", "task": "long-records-019", "input_target": 32768, "max_generation": 8192, "seed": 289120, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2550, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9962,\"指定IDの待ち秒\":11,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9962, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 11, "expected": 11, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap4096-sampling-r0-mtp0", "task": "long-records-018", "input_target": 32768, "max_generation": 4096, "seed": 89119, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2805, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9458,\"指定IDの待ち秒\":59,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9458, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 59, "expected": 59, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap4096-sampling-r0-mtp1", "task": "long-records-018", "input_target": 32768, "max_generation": 4096, "seed": 89119, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2657, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9519, \"指定IDの待ち秒\": 59, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9519, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 59, "expected": 59, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap4096-sampling-r0-mtp1", "task": "long-records-015", "input_target": 32768, "max_generation": 4096, "seed": 89116, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3547, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11289,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11289, "expected": 11482, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap4096-sampling-r0-mtp0", "task": "long-records-015", "input_target": 32768, "max_generation": 4096, "seed": 89116, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3624, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 11265, \"指定IDの待ち秒\": 71, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11265, "expected": 11482, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap8192-sampling-r2-mtp1", "task": "long-records-014", "input_target": 32768, "max_generation": 8192, "seed": 289115, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2619, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11562,\"指定IDの待ち秒\":19,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11562, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap8192-sampling-r2-mtp0", "task": "long-records-014", "input_target": 32768, "max_generation": 8192, "seed": 289115, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1977, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10595,\"指定IDの待ち秒\":19,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10595, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap8192-sampling-r0-mtp0", "task": "long-records-007", "input_target": 4096, "max_generation": 8192, "seed": 89108, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 237, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":79,\"対象部署の件数合計\":736,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap8192-sampling-r0-mtp1", "task": "long-records-007", "input_target": 4096, "max_generation": 8192, "seed": 89108, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 249, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 79, \"対象部署の件数合計\": 736, \"指定IDの待ち秒\": 54, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap4096-sampling-r2-mtp1", "task": "long-records-000", "input_target": 4096, "max_generation": 4096, "seed": 289101, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 226, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":7,\"最大待ち秒\":78,\"対象部署の件数合計\":985,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap4096-sampling-r2-mtp0", "task": "long-records-000", "input_target": 4096, "max_generation": 4096, "seed": 289101, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 226, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":7,\"最大待ち秒\":78,\"対象部署の件数合計\":985,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap8192-sampling-r2-mtp1", "task": "long-records-000", "input_target": 4096, "max_generation": 8192, "seed": 289101, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 226, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":7,\"最大待ち秒\":78,\"対象部署の件数合計\":985,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap8192-sampling-r2-mtp0", "task": "long-records-000", "input_target": 4096, "max_generation": 8192, "seed": 289101, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 226, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":7,\"最大待ち秒\":78,\"対象部署の件数合計\":985,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap4096-sampling-r0-mtp0", "task": "long-records-017", "input_target": 32768, "max_generation": 4096, "seed": 89118, "mtp": false, "finish_reason": "max_new_tokens", "generated_tokens": 4096, "pass": false, "first_answer_missing": true, "answer_text": "行数600。最小待ち秒1、最大80。開発部署の件数合計を数える必要がある。\n\n開発の行を数える…手作業で数えるのは大変だが、やるしかない。各idの部署を走査する。\n\n1開,2開,3営,4営,5管,6開,7管,8管,9開,10営,11管,12管,13開,14開,15管,16営,17開,18営,19開,20営,21管,22営,23管,24管,25管,26開,27営,28開,29開,30営,31管,32管,33営,34管,35営,36管,37営,38開,39管,40管,41開,42営,43管,44開,45開,46開,47管,48営,49管,50営,51営,52開,53営,54開,55営,56開,57開,58管,59営,60営,61営,62開,63管,64開,65管,66管,67管,68管,69開,70開,71営,72営,73営,74開,75開,76開,77開,78営,79管,80営,81営,82管,83営,84開,85管,86管,87開,88管,89管,90管,91営,92管,93開,94開,95管,96営,97営,98開,99開,100営\n\n1-100開発: 1,2,6,9,13,14,17,19,26,28,29,38,41,44,45,46,52,54,56,57,62,64,69,70,74,75,76,77,84,87,93,94,98,99 = 数える:1,2(2),6(3),9(4),13(5),14(6),17(7),19(8),26(9),28(10),29(11),38(12),41(13),44(14),45(15),46(16),52(17),54(18),56(19),57(20),62(21),64(22),69(23),70(24),74(25),75(26),76(27),77(28),84(29),87(30),93(31),94(32),98(33),99(34) → 34件\n\n101-200: 101開,102営,103営,104管,105開,106開,107管,108営,109開,110開,111管,112開,113開,114開,115管,116管,117管,118開,119開,120開,121管,122営,123開,124管,125管,126開,127開,128管,129営,130営,131管,132管,133開,134営,135営,136開,137管,138開,139営,140営,141営,142開,143管,144管,145開,146営,147営,148開,149開,150開,151営,152営,153開,154開,155営,156管,157開,158営,159営,160営,161管,162開,163開,164営,165管,166管,167管,168開,169開,170開,171開,172開,173管,174営,175管,176開,177管,178管,179開,180管,181開,182営,183管,184開,185営,186管,187営,188管,189管,190開,191開,192開,193営,194開,195営,196営,197管,198開,199営,200管\n\n開発: 105,106,109,110,112,113,114,118,119,120,123,126,127,133,136,138,142,145,148,149,150,153,154,157,162,163,168,169,170,171,172,176,179,181,184,190,191,192,194,198\n数:105(1),106(2),109(3),110(4),112(5),113(6),114(7),118(8),119(9),120(10),123(11),126(12),127(13),133(14),136(15),138(16),142(17),145(18),148(19),149(20),150(21),153(22),154(23),157(24),162(25),163(26),168(27),169(28),170(29),171(30),172(31),176(32),179(33),181(34),184(35),190(36),191(37),192(38),194(39),198(40) → 40件\n\n201-300: 201開,202開,203営,204開,205営,206営,207開,208営,209開,210営,211営,212営,213開,214開,215管,216開,217開,218管,219開,220開,221営,222管,223営,224営,225管,226開,227開,228管,229開,230管,231開,232営,233管,234開,235管,236管,237開,238開,239開,240管,241管,242営,243管,244営,245管,246開,247営,248営,249開,250営,251営,252開,253開,254管,255開,256営,257営,258管,259営,260開,261開,262営,263営,264管,265営,266営,267開,268営,269管,270営,271営,272開,273管,274管,275営,276管,277管,278営,279開,280開,281営,282管,283開,284開,285開,286開,287営,288営,289開,290管,291営,292管,293営,294開,295営,296開,297管,298管,299営,300営\n\n開発:201,202,204,207,209,213,214,216,217,219,220,226,227,229,231,234,237,238,239,246,249,252,253,255,260,261,267,272,279,280,283,284,285,286,289,294,296\n数:201(1),202(2),204(3),207(4),209(5),213(6),214(7),216(8),217(9),219(10),220(11),226(12),227(13),229(14),231(15),234(16),237(17),238(18),239(19),246(20),249(21),252(22),253(23),255(24),260(25),261(26),267(27),272(28),279(29),280(30),283(31),284(32),285(33),286(34),289(35),294(36),296(37) → 37件\n\n301-400: 301管,302開,303営,304開,305営,306管,307営,308管,309管,310営,311開,312開,313管,314営,315管,316管,317営,318営,319管,320管,321管,322管,323開,324管,325営,326管,327管,328営,329営,330開,331営,332営,333管,334営,335開,336開,337管,338管,339管,340開,341営,342開,343営,344営,345管,346管,347管,348管,349開,350管,351管,352開,353営,354開,355開,356管,357管,358開,359開,360営,361営,362管,363営,364開,365管,366管,367開,368管,369営,370営,371管,372営,373管,374開,375営,376営,377開,378開,379管,380管,381営,382開,383管,384営,385開,386開,387管,388開,389管,390営,391営,392開,393営,394営,395管,396管,397営,398開,399開,400管\n\n開発:302,304,311,312,323,330,335,336,340,342,349,352,354,355,358,359,364,367,374,377,378,382,385,386,388,392,398,399\n数:302(1),304(2),311(3),312(4),323(5),330(6),335(7),336(8),340(9),342(10),349(11),352(12),354(13),355(14),358(15),359(16),364(17),367(18),374(19),377(20),378(21),382(22),385(23),386(24),388(25),392(26),398(27),399(28) → 28件\n\n401-500: 401開,402開,403開,404営,405営,406開,407管,408営,409管,410営,411管,412管,413営,414開,415開,416管,417開,418開,419開,420管,421開,422管,423営,424管,425管,426管,427開,428営,429営,430営,431営,432管,433管,434管,435営,436管,437開,438管,439開,440開,441開,442営,443管,444営,445営,446営,447営,448開,449営,450管,451管,452開,453管,454営,455開,456営,457開,458営,459管,460営,461営,462管,463開,464営,465管,466管,467営,468管,469管,470開,471管,472開,473管,474開,475管,476開,477営,478開,479管,480開,481営,482営,483管,484開,485開,486管,487開,488営,489開,490営,491開,492管,493営,494開,495営,496管,497営,498営,499営,500管\n\n開発:401,402,403,406,414,415,417,418,419,421,427,437,439,440,441,448,452,455,457,463,470,472,474,476,478,480,484,485,487,489,491,494\n数:401(1),402(2),403(3),406(4),414(5),415(6),417(7),418(8),419(9),421(10),427(11),437(12),439(13),440(14),441(15),448(16),452(17),455(18),457(19),463(20),470(21),472(22),474(23),476(24),478(25),480(26),484(27),485(28),487(29),489(30),491(31),494(32) → 32件\n\n501-600: 501開,502開,503開,504開,505管,506開,507開,508管,509開,510管,511開,512管,513管,514営,515営,516営,517営,518営,519営,520管,521営,522管,523管,524営,525営,526営,527管,528開,529営,530管,531営,532開,533開,534開,535営,536管,537開,538営,539営,540管,541営,542管,543管,544開,545開,546開,547管,548管,549営,550管,551管,552営,553管,554開,555管,556管,557開,558営,559営,560管,561開,562営,563管,564管,565開,566管,567営,568営,569営,570営,571管,572営,573開,574営,575管,576開,577開,578開,579管,580開,581開,582管,583営,584管,585開,586開,587開,588管,589管,590営,591管,592管,593管,594管,595管,596開,597営,598営,599営,600管\n\n開発:501,502,503,504,506,507,509,511,528,532,533,534,537,544,545,546,554,557,561,565,573,576,577,578,580,581,585,586,587,596\n数:501(1),502(2),503(3),504(4),506(5),507(6),509(7),511(8),528(9),532(10),533(11),534(12),537(13),544(14),545(15),546(16),554(17),557(18),561(19),565(20),573(21),576(22),577(23),578(24),580(25),581(26),585(27),586(28),587(29),596(30) → 30件\n\n合計件数: 34+40+37+28+32+30 = 201件\n\n開発の件数合計（件数フィールドの合計）を求める必要がある。うーん、「対象部署の件数合計」＝開発部署の「件数」列の合計。これは非常に大変…201行の件数を足す。\n\n開発行とその件数：\n1:40, 2:78, 6:39, 9:45, 13:13, 14:60, 17:9, 19:27, 26:43, 28:44, 29:81, 38:15, 41:39, 44:71, 45:30, 46:99, 52:21, 54:33, 56:82, 57:28, 62:92, 64:53, 69:78, 70:11, 74:94, 75:58, 76:58, 77:3, 84:62, 87:100, 93:3, 94:76, 98:93, 99:9\n\n1-100合計: 40+78=118, +39=157, +45=202, +13=215, +60=275, +9=284, +27=311, +43=354, +44=398, +81=479, +15=494, +39=533, +71=604, +30=634, +99=733, +21=754, +33=787, +82=869, +28=897, +92=989,", "fields": { "行数": { "present": false, "actual": null, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": false, "actual": null, "expected": 1, "semantic_correct": false, "pass_with_natural_stop": false }, "最大待ち秒": { "present": false, "actual": null, "expected": 80, "semantic_correct": false, "pass_with_natural_stop": false }, "対象部署の件数合計": { "present": false, "actual": null, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": false, "actual": null, "expected": 46, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": false, "actual": null, "expected": "不明", "semantic_correct": false, "pass_with_natural_stop": false } } }, { "case": "main-long-records-017-cap4096-sampling-r0-mtp1", "task": "long-records-017", "input_target": 32768, "max_generation": 4096, "seed": 89118, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3820, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9903,\"指定IDの待ち秒\":46,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9903, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 46, "expected": 46, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap8192-sampling-r1-mtp1", "task": "long-records-008", "input_target": 4096, "max_generation": 8192, "seed": 189109, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 181, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 78, \"対象部署の件数合計\": 742, \"指定IDの待ち秒\": 70, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap8192-sampling-r1-mtp0", "task": "long-records-008", "input_target": 4096, "max_generation": 8192, "seed": 189109, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 181, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 78, \"対象部署の件数合計\": 742, \"指定IDの待ち秒\": 70, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap8192-sampling-r1-mtp0", "task": "long-records-009", "input_target": 4096, "max_generation": 8192, "seed": 189110, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 51, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":596,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 596, "expected": 834, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap8192-sampling-r1-mtp1", "task": "long-records-009", "input_target": 4096, "max_generation": 8192, "seed": 189110, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 51, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":596,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 596, "expected": 834, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap4096-sampling-r0-mtp0", "task": "long-records-008", "input_target": 4096, "max_generation": 4096, "seed": 89109, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 213, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":742,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap4096-sampling-r0-mtp1", "task": "long-records-008", "input_target": 4096, "max_generation": 4096, "seed": 89109, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 208, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":742,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap4096-sampling-r0-mtp0", "task": "long-records-005", "input_target": 4096, "max_generation": 4096, "seed": 89106, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 195, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":631,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 631, "expected": 631, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-005-cap4096-sampling-r0-mtp1", "task": "long-records-005", "input_target": 4096, "max_generation": 4096, "seed": 89106, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 195, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":631,\"指定IDの待ち秒\":79,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 631, "expected": 631, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap4096-sampling-r1-mtp1", "task": "long-records-013", "input_target": 32768, "max_generation": 4096, "seed": 189114, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3772, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 8364, \"指定IDの待ち秒\": 69, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8364, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap4096-sampling-r1-mtp0", "task": "long-records-013", "input_target": 32768, "max_generation": 4096, "seed": 189114, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3002, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10006,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10006, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap4096-sampling-r1-mtp0", "task": "long-records-000", "input_target": 4096, "max_generation": 4096, "seed": 189101, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 275, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 7, \"最大待ち秒\": 78, \"対象部署の件数合計\": 985, \"指定IDの待ち秒\": 70, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-000-cap4096-sampling-r1-mtp1", "task": "long-records-000", "input_target": 4096, "max_generation": 4096, "seed": 189101, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 248, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":7,\"最大待ち秒\":78,\"対象部署の件数合計\":985,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 7, "expected": 7, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 985, "expected": 985, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap4096-sampling-r1-mtp1", "task": "long-records-004", "input_target": 4096, "max_generation": 4096, "seed": 189105, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 235, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1273,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1273, "expected": 1273, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap4096-sampling-r1-mtp0", "task": "long-records-004", "input_target": 4096, "max_generation": 4096, "seed": 189105, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 236, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1273,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1273, "expected": 1273, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap8192-sampling-r1-mtp0", "task": "long-records-002", "input_target": 4096, "max_generation": 8192, "seed": 189103, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 245, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-002-cap8192-sampling-r1-mtp1", "task": "long-records-002", "input_target": 4096, "max_generation": 8192, "seed": 189103, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 245, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":4,\"最大待ち秒\":80,\"対象部署の件数合計\":909,\"指定IDの待ち秒\":72,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 909, "expected": 909, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 72, "expected": 72, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap8192-sampling-r2-mtp1", "task": "long-records-003", "input_target": 4096, "max_generation": 8192, "seed": 289104, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 188, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":2,\"最大待ち秒\":77,\"対象部署の件数合計\":701,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap8192-sampling-r2-mtp0", "task": "long-records-003", "input_target": 4096, "max_generation": 8192, "seed": 289104, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 187, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":2,\"最大待ち秒\":77,\"対象部署の件数合計\":701,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap8192-sampling-r0-mtp1", "task": "long-records-015", "input_target": 32768, "max_generation": 8192, "seed": 89116, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3547, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11289,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11289, "expected": 11482, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap8192-sampling-r0-mtp0", "task": "long-records-015", "input_target": 32768, "max_generation": 8192, "seed": 89116, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3624, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 11265, \"指定IDの待ち秒\": 71, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11265, "expected": 11482, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap4096-sampling-r1-mtp1", "task": "long-records-006", "input_target": 4096, "max_generation": 4096, "seed": 189107, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 158, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap4096-sampling-r1-mtp0", "task": "long-records-006", "input_target": 4096, "max_generation": 4096, "seed": 189107, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 158, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap8192-sampling-r0-mtp1", "task": "long-records-014", "input_target": 32768, "max_generation": 8192, "seed": 89115, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2498, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":8288,\"指定IDの待ち秒\":19,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8288, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap8192-sampling-r0-mtp0", "task": "long-records-014", "input_target": 32768, "max_generation": 8192, "seed": 89115, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2068, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":13019,\"指定IDの待ち秒\":19,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 13019, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap4096-sampling-r0-mtp1", "task": "long-records-006", "input_target": 4096, "max_generation": 4096, "seed": 89107, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 155, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap4096-sampling-r0-mtp0", "task": "long-records-006", "input_target": 4096, "max_generation": 4096, "seed": 89107, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 155, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap8192-sampling-r1-mtp1", "task": "long-records-007", "input_target": 4096, "max_generation": 8192, "seed": 189108, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 363, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":79,\"対象部署の件数合計\":736,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap8192-sampling-r1-mtp0", "task": "long-records-007", "input_target": 4096, "max_generation": 8192, "seed": 189108, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 272, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":79,\"対象部署の件数合計\":736,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap4096-sampling-r1-mtp0", "task": "long-records-016", "input_target": 32768, "max_generation": 4096, "seed": 189117, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1665, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9761, \"指定IDの待ち秒\": 31, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9761, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 31, "expected": 31, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap4096-sampling-r1-mtp1", "task": "long-records-016", "input_target": 32768, "max_generation": 4096, "seed": 189117, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 1926, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9664,\"指定IDの待ち秒\":31,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9664, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 31, "expected": 31, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap8192-sampling-r1-mtp1", "task": "long-records-020", "input_target": 32768, "max_generation": 8192, "seed": 189121, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3759, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 10702, \"指定IDの待ち秒\": 37, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10702, "expected": 10702, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap8192-sampling-r1-mtp0", "task": "long-records-020", "input_target": 32768, "max_generation": 8192, "seed": 189121, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3754, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10703,\"指定IDの待ち秒\":37,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10703, "expected": 10702, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap8192-sampling-r0-mtp0", "task": "long-records-012", "input_target": 32768, "max_generation": 8192, "seed": 89113, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3629, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 215, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 10045, \"指定IDの待ち秒\": 75, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 215, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10045, "expected": 10994, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-012-cap8192-sampling-r0-mtp1", "task": "long-records-012", "input_target": 32768, "max_generation": 8192, "seed": 89113, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3613, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10045,\"指定IDの待ち秒\":75,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10045, "expected": 10994, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 75, "expected": 75, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap8192-sampling-r2-mtp1", "task": "long-records-001", "input_target": 4096, "max_generation": 8192, "seed": 289102, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 299, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 21, \"最小待ち秒\": 4, \"最大待ち秒\": 79, \"対象部署の件数合計\": 1098, \"指定IDの待ち秒\": 5, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 21, "expected": 50, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap8192-sampling-r2-mtp0", "task": "long-records-001", "input_target": 4096, "max_generation": 8192, "seed": 289102, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 285, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":21,\"最小待ち秒\":4,\"最大待ち秒\":79,\"対象部署の件数合計\":1098,\"指定IDの待ち秒\":5,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 21, "expected": 50, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap8192-sampling-r2-mtp0", "task": "long-records-016", "input_target": 32768, "max_generation": 8192, "seed": 289117, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2416, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9830,\"指定IDの待ち秒\":31,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9830, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 31, "expected": 31, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-016-cap8192-sampling-r2-mtp1", "task": "long-records-016", "input_target": 32768, "max_generation": 8192, "seed": 289117, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2796, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 8688, \"指定IDの待ち秒\": 15, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8688, "expected": 9883, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 15, "expected": 31, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap8192-sampling-r2-mtp1", "task": "long-records-009", "input_target": 4096, "max_generation": 8192, "seed": 289110, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 178, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":834,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 834, "expected": 834, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-009-cap8192-sampling-r2-mtp0", "task": "long-records-009", "input_target": 4096, "max_generation": 8192, "seed": 289110, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 134, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":834,\"指定IDの待ち秒\":17,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 834, "expected": 834, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 17, "expected": 17, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap8192-sampling-r1-mtp0", "task": "long-records-015", "input_target": 32768, "max_generation": 8192, "seed": 189116, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3450, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 11420, \"指定IDの待ち秒\": 71, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11420, "expected": 11482, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-015-cap8192-sampling-r1-mtp1", "task": "long-records-015", "input_target": 32768, "max_generation": 8192, "seed": 189116, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2645, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11482,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11482, "expected": 11482, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 71, "expected": 71, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap4096-sampling-r1-mtp0", "task": "long-records-018", "input_target": 32768, "max_generation": 4096, "seed": 189119, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2271, "pass": false, "first_answer_missing": true, "answer_text": "行数600、待ち秒最小1、最大80、営業件数合計、ID33の待ち秒=59。\n\n営業件数合計を手計算...多いため概算は不可。真面目に数える必要がある。営業行の件数を列挙する。\n\n営業行を抽出：\n1:12,3:83,7:78,9:95,12:84,13:20,16:23,17:59,25:67,32:69,34:66,42:36,44:33,46:32,47:61,48:86,52:64,53:29,54:36,56:65,58:3,64:4,66:25,68:98,72:74,73:13,75:2,76:26,78:50,81:100,82:89,85:43,89:42,91:36,94:77,95:76,100:84,101:66,105:83,106:23,108:40,112:7,117:85,120:85,124:99,127:54,129:35,133:70,135:8,138:12,142:69,148:7,150:95,154:64,156:71,158:32,159:56,160:79,164:50,166:16,169:11,170:81,172:27,173:91,175:100,176:72,177:13,180:5,181:21,185:12,190:77,195:28,198:79,200:86,204:60,206:28,207:78,215:57,216:18,228:97,229:90,230:59,232:75,233:97,236:86,240:17,245:26,250:28,256:26,263:4,264:92,269:24,270:98,271:74,272:9,274:44,276:83,279:3,280:67,281:95,282:65,283:44,284:80,291:14,293:17,300:10,302:81,306:25,309:78,313:54,318:91,319:28,322:77,325:20,327:67,329:30,332:61,335:25,337:100,348:7,354:35,356:37,358:60,364:83,365:86,366:39,368:80,373:85,375:65,381:15,382:53,384:21,387:60,397:42,398:54,399:74,401:57,402:37,407:20,413:35,414:8,421:39,422:34,423:77,427:9,435:52,449:86,458:74,459:9,462:68,469:70,471:97,485:73,486:14,500:76,501:28,503:90,508:10,509:76,514:20,520:13,521:2,522:88,527:13,528:14,530:52,535:69,538:84,542:39,547:17,549:35,552:24,554:38,559:84,561:23,563:39,566:87,570:41,572:35,575:95,576:41,580:23,581:13,584:19,585:75,586:66,587:36,597:35,598:46\n\n数える：個数を数える。200行あたり？数えてみる：上の列挙の件数＝約160。\n\n合計を計算（段階ごと）：\n12+83=95;+78=173;+95=268;+84=352;+20=372;+23=395;+59=454;+67=521;+69=590;+66=656;+36=692;+33=725;+32=757;+61=818;+86=904;+64=968;+29=997;+36=1033;+65=1098;+3=1101;+4=1105;+25=1130;+98=1228;+74=1302;+13=1315;+2=1317;+26=1343;+50=1393;+100=1493;+89=1582;+43=1625;+42=1667;+36=1703;+77=1780;+76=1856;+84=1940;+66=2006;+83=2089;+23=2112;+40=2152;+7=2159;+85=2244;+85=2329;+99=2428;+54=2482;+35=2517;+70=2587;+8=2595;+12=2607;+69=2676;+7=2683;+95=2778;+64=2842;+71=2913;+32=2945;+56=3001;+79=3080;+50=3130;+16=3146;+11=3157;+81=3238;+27=3265;+91=3356;+100=3456;+72=3528;+13=3541;+5=3546;+21=3567;+12=3579;+77=3656;+28=3684;+79=3763;+86=3849;+60=3909;+28=3937;+78=4015;+57=4072;+18=4090;+97=4187;+90=4277;+59=4336;+75=4411;+97=4508;+86=4594;+17=4611;+26=4637;+28=4665;+26=4691;+4=4695;+92=4787;+24=4811;+98=4909;+74=4983;+9=4992;+44=5036;+83=5119;+3=5122;+67=5189;+95=5284;+65=5349;+44=5393;+80=5473;+14=5487;+17=5504;+10=5514;+81=5595;+25=5620;+78=5698;+54=5752;+91=5843;+28=5871;+77=5948;+20=5968;+67=6035;+30=6065;+61=6126;+25=6151;+100=6251;+7=6258;+35=6293;+37=6330;+60=6390;+83=6473;+86=6559;+39=6598;+80=6678;+85=6763;+65=6828;+15=6843;+53=6896;+21=6917;+60=6977;+42=7019;+54=7073;+74=7147;+57=7204;+37=7241;+20=7261;+35=7296;+8=7304;+39=7343;+34=7377;+77=7454;+9=7463;+52=7515;+86=7601;+74=7675;+9=7684;+68=7752;+70=7822;+97=7919;+73=7992;+14=8006;+76=8082;+28=8110;+90=8200;+10=8210;+76=8286;+20=8306;+13=8319;+2=8321;+88=8409;+13=8422;+14=8436;+52=8488;+69=8557;+84=8641;+39=8680;+17=8697;+35=8732;+24=8756;+38=8794;+84=8878;+23=8901;+39=8940;+87=9027;+41=9068;+35=9103;+95=9198;+41=9239;+23=9262;+13=9275;+19=9294;+75=9369;+66=9435;+36=9471;+35=9506;+46=9552。\n\n営業行数を確認：リスト数＝数え直す。上記160項目か？単純に確信できないが、合計9552。", "fields": { "行数": { "present": false, "actual": null, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": false, "actual": null, "expected": 1, "semantic_correct": false, "pass_with_natural_stop": false }, "最大待ち秒": { "present": false, "actual": null, "expected": 80, "semantic_correct": false, "pass_with_natural_stop": false }, "対象部署の件数合計": { "present": false, "actual": null, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": false, "actual": null, "expected": 59, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": false, "actual": null, "expected": "不明", "semantic_correct": false, "pass_with_natural_stop": false } } }, { "case": "main-long-records-018-cap4096-sampling-r1-mtp1", "task": "long-records-018", "input_target": 32768, "max_generation": 4096, "seed": 189119, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2254, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9537,\"指定IDの待ち秒\":59,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9537, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 59, "expected": 59, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap8192-sampling-r1-mtp0", "task": "long-records-019", "input_target": 32768, "max_generation": 8192, "seed": 189120, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 4453, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":189,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9879,\"指定IDの待ち秒\":11,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 189, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9879, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 11, "expected": 11, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-019-cap8192-sampling-r1-mtp1", "task": "long-records-019", "input_target": 32768, "max_generation": 8192, "seed": 189120, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2649, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9929,\"指定IDの待ち秒\":11,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9929, "expected": 9971, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 11, "expected": 11, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap8192-sampling-r2-mtp0", "task": "long-records-010", "input_target": 4096, "max_generation": 8192, "seed": 289111, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 73, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":247,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 247, "expected": 324, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap8192-sampling-r2-mtp1", "task": "long-records-010", "input_target": 4096, "max_generation": 8192, "seed": 289111, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 73, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":289,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 289, "expected": 324, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap4096-sampling-r2-mtp0", "task": "long-records-008", "input_target": 4096, "max_generation": 4096, "seed": 289109, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 203, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":742,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap4096-sampling-r2-mtp1", "task": "long-records-008", "input_target": 4096, "max_generation": 4096, "seed": 289109, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 138, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":742,\"指定IDの待ち秒\":70,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap4096-sampling-r2-mtp0", "task": "long-records-004", "input_target": 4096, "max_generation": 4096, "seed": 289105, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 51, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1191,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1191, "expected": 1273, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap4096-sampling-r2-mtp1", "task": "long-records-004", "input_target": 4096, "max_generation": 4096, "seed": 289105, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 51, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1191,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1191, "expected": 1273, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap4096-sampling-r1-mtp0", "task": "long-records-017", "input_target": 32768, "max_generation": 4096, "seed": 189118, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2357, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 8941, \"指定IDの待ち秒\": 10, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8941, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 10, "expected": 46, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-017-cap4096-sampling-r1-mtp1", "task": "long-records-017", "input_target": 32768, "max_generation": 4096, "seed": 189118, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2347, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9028, \"指定IDの待ち秒\": 46, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9028, "expected": 9831, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 46, "expected": 46, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap4096-sampling-r2-mtp0", "task": "long-records-001", "input_target": 4096, "max_generation": 4096, "seed": 289102, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 285, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":21,\"最小待ち秒\":4,\"最大待ち秒\":79,\"対象部署の件数合計\":1098,\"指定IDの待ち秒\":5,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 21, "expected": 50, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-001-cap4096-sampling-r2-mtp1", "task": "long-records-001", "input_target": 4096, "max_generation": 4096, "seed": 289102, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 299, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 21, \"最小待ち秒\": 4, \"最大待ち秒\": 79, \"対象部署の件数合計\": 1098, \"指定IDの待ち秒\": 5, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 21, "expected": 50, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1098, "expected": 1098, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 5, "expected": 5, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap8192-sampling-r2-mtp0", "task": "long-records-023", "input_target": 32768, "max_generation": 8192, "seed": 289124, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2732, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9259,\"指定IDの待ち秒\":4,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9259, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap8192-sampling-r2-mtp1", "task": "long-records-023", "input_target": 32768, "max_generation": 8192, "seed": 289124, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3573, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9242, \"指定IDの待ち秒\": 4, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9242, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap8192-sampling-r1-mtp1", "task": "long-records-011", "input_target": 4096, "max_generation": 8192, "seed": 189112, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 192, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":80,\"対象部署の件数合計\":754,\"指定IDの待ち秒\":12,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-011-cap8192-sampling-r1-mtp0", "task": "long-records-011", "input_target": 4096, "max_generation": 8192, "seed": 189112, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 192, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":80,\"対象部署の件数合計\":754,\"指定IDの待ち秒\":12,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 754, "expected": 754, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 12, "expected": 12, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap4096-sampling-r2-mtp0", "task": "long-records-006", "input_target": 4096, "max_generation": 4096, "seed": 289107, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 167, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-006-cap4096-sampling-r2-mtp1", "task": "long-records-006", "input_target": 4096, "max_generation": 4096, "seed": 289107, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 142, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":79,\"対象部署の件数合計\":565,\"指定IDの待ち秒\":21,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 565, "expected": 565, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 21, "expected": 21, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-021-cap8192-sampling-r2-mtp1", "task": "long-records-021", "input_target": 32768, "max_generation": 8192, "seed": 289122, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3182, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9805,\"指定IDの待ち秒\":13,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9805, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 13, "expected": 13, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-021-cap8192-sampling-r2-mtp0", "task": "long-records-021", "input_target": 32768, "max_generation": 8192, "seed": 289122, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1867, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9637,\"指定IDの待ち秒\":13,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9637, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 13, "expected": 13, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap8192-sampling-r2-mtp1", "task": "long-records-013", "input_target": 32768, "max_generation": 8192, "seed": 289114, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3514, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9384,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9384, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap8192-sampling-r2-mtp0", "task": "long-records-013", "input_target": 32768, "max_generation": 8192, "seed": 289114, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2635, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9663,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9663, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap4096-sampling-r0-mtp0", "task": "long-records-004", "input_target": 4096, "max_generation": 4096, "seed": 89105, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 254, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1375,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1375, "expected": 1273, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-004-cap4096-sampling-r0-mtp1", "task": "long-records-004", "input_target": 4096, "max_generation": 4096, "seed": 89105, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 258, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":78,\"対象部署の件数合計\":1375,\"指定IDの待ち秒\":63,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 1375, "expected": 1273, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 63, "expected": 63, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap4096-sampling-r1-mtp0", "task": "long-records-010", "input_target": 4096, "max_generation": 4096, "seed": 189111, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 106, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":324,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 324, "expected": 324, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-010-cap4096-sampling-r1-mtp1", "task": "long-records-010", "input_target": 4096, "max_generation": 4096, "seed": 189111, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 123, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":324,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 324, "expected": 324, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-021-cap4096-sampling-r0-mtp0", "task": "long-records-021", "input_target": 32768, "max_generation": 4096, "seed": 89122, "mtp": false, "finish_reason": "max_new_tokens", "generated_tokens": 4096, "pass": false, "first_answer_missing": true, "answer_text": "行数600。最小1、最大80。管理の件数合計を計算する必要がある。指定ID 483の待ち秒=13。遅延原因不明。\n\n管理の件数を合計する。管理の行を列挙:\n1:10, 2:41, 4:91, 6:80, 7:26, 9:3, 13:77, 14:71, 21:6, 28:16, 41:24, 43:39, 44:63, 47:86, 49:31, 50:21, 51:40, 52:76, 54:6, 55:66, 57:28, 61:18, 62:13, 66:46, 72:36, 75:7, 78:62, 79:100, 80:53, 81:98, 88:82, 92:33, 94:46, 95:77, 96:94, 98:70, 105:45, 107:53, 112:64, 116:70, 119:59, 131:41, 134:35, 135:34, 138:72, 139:86, 141:67, 142:29, 143:35, 144:49, 146:89, 149:72, 152:36, 154:81, 155:33, 159:43, 160:95, 162:63, 163:39, 165:97, 171:88, 172:57, 173:99, 177:32, 185:69, 189:4, 191:13, 192:32, 194:94, 195:10, 196:35, 197:23, 198:88, 207:35, 212:80, 215:30, 216:55, 217:94, 222:67, 223:60, 224:34, 228:76, 229:7, 232:55, 239:77, 246:96, 248:67, 249:67, 251:98, 252:43, 254:51, 256:6, 261:30, 268:92, 269:16, 270:99, 271:9, 273:27, 274:67, 275:45, 276:93, 277:46, 278:42, 279:74, 280:68, 282:38, 287:87, 288:1, 289:53, 290:21, 291:60, 292:6, 295:68, 299:90, 301:32, 304:65, 305:76, 306:37, 308:79, 309:94, 310:49, 311:100, 312:7, 319:72, 320:23, 321:35, 322:85, 324:48, 328:35, 333:71, 334:29, 336:43, 337:4, 339:52, 344:97, 346:60, 347:55, 350:90, 356:33, 357:29, 364:13, 365:95, 366:11, 367:45, 371:25, 372:22, 376:100, 377:49, 380:1, 381:96, 389:69, 393:63, 394:52, 396:11, 403:68, 405:51, 406:98, 407:90, 411:56, 413:28, 420:100, 423:35, 426:70, 433:35, 435:12, 440:7, 442:18, 445:53, 446:30, 449:46, 451:39, 452:34, 456:57, 460:39, 463:89, 466:47, 469:85, 470:83, 474:22, 475:7, 476:37, 477:9, 478:39, 480:86, 483:29, 484:24, 488:81, 490:37, 493:98, 497:66, 501:24, 502:66, 506:2, 511:4, 513:69, 514:66, 515:24, 516:4, 517:35, 518:27, 520:29, 525:50, 526:77, 530:29, 531:10, 534:30, 535:44, 537:74, 540:39, 541:3, 542:38, 546:38, 553:35, 554:72, 556:74, 562:28, 566:64, 568:77, 571:50, 576:14, 582:51, 583:21, 584:1, 586:11, 595:35, 596:38, 597:86, 599:6\n\n管理の行数を数える: 挙げた行を数える。ブロックごと:\n最初から299まで: 1,2,4,6,7,9,13,14 (8), 21,28 (10), 41,43,44,47,49,50,51,52,54,55,57,61,62,66,72,75,78,79,80,81,88 (28), 92,94,95,96,98 (33), 105,107,112,116,119 (38), 131,134,135,138,139,141,142,143,144,146,149,152,154,155,159,160,162,163,165 (57), 171,172,173,177,185,189,191,192,194,195,196,197,198 (70), 207,212,215,216,217,222,223,224,228,229,232,239,246,248,249,251,252,254,256,261,268,269,270,271,273,274,275,276,277,278,279,280,282,287,288,289,290,291,292,295,299 (83).\n\n301以降: 301,304,305,306,308,309,310,311,312,319,320,321,322,324,328,333,334,336,337,339,344,346,347,350,356,357,364,365,366,367,371,372,376,377,380,381,389,393,394,396 (40 → 123), 403,405,406,407,411,413,420,423,426,433,435,440,442,445,446,449,451,452,456,460,463,466,469,470,474,475,476,477,478,480,483,484,488,490,493,497 (36 → 159), 501,502,506,511,513,514,515,516,517,518,520,525,526,530,531,534,535,537,540,541,542,546 (22 → 181), 553,554,556,562,566,568,571,576,582,583,584,586,595,596,597,599 (16 → 197).\n\n管理は197行。合計を計算…大変だがやる。\n\nブロック1(8): 10+41+91+80+26+3+77+71 = 398\nブロック2(2): 6+16=22 → 420\nブロック3(28): 24+39+63+86+31+21+40+76+6+66+28+18+13+46+36+7+62+100+53+98+82+33+46+77+94+70+45+53\n計算: 24+39=63; +63=126; +86=212; +31=243; +21=264; +40=304; +76=380; +6=386; +66=452; +28=480; +18=498; +13=511; +46=557; +36=593; +7=600; +62=662; +100=762; +53=815; +98=913; +82=995; +33=1028; +46=1074; +77=1151; +94=1245; +70=1315; +45=1360; +53=1413.\n→ 420+1413=1833\n\nブロック4(5): 33+46+77+94+70 = 320 → 2153\nブロック5(5): 45+53+64+70+59 = 291 → 2444\nブロック6(19): 41+35+34+72+86+67+29+35+49+89+72+36+81+33+43+95+63+39+97\n41+35=76;+34=110;+72=182;+86=268;+67=335;+29=364;+35=399;+49=448;+89=537;+72=609;+36=645;+81=726;+33=759;+43=802;+95=897;+63=960;+39=999;+97=1096.\n→ 2444+1096=3540\n\nブロック7(13): 88+57+99+32+69+4+13+32+94+10+35+23+88\n88+57=145;+99=244;+32=276;+69=345;+4=349;+13=362;+32=394;+94=488;+10=498;+35=533;+23=556;+88=644.\n→ 3540+644=4184\n\nブロック8(42): 35+80+30+55+94+67+60+34+76+7+55+77+96+67+67+98+43+51+6+30+92+16+99+9+27+67+45+93+46+42+74+68+38+87+1+53+21+60+6+68+90 (41? 数え直し)\nリスト: 207:35,212:80,215:30,216:55,217:94,222:67,223:60,224:34,228:76,229:7,232:55,239:77,246:96,248:67,249:67,251:98,252:43,254:51,256:6,261:30,268:92,269:16,270:99,271:9,273:27,274:67,275:45,276:93,277:46,278:42,279:74,280:68,282:38,287:87,288:1,289:53,290:21,291:60,292:6,295:68,299:90 → 41行。あれ、先ほど83-42=41のはず。ブロック6の後70行、+41=111? でも83と数えた。再確認: 57+13=70、+41=111。でも83と言った—間違い。管理行数を数え直す必要はない厳密には、合計だけ正確ならよい。\n\n合計続行: 35+80=115;+30=145;+55=200;+94=294;+67=361;+60=421;+34=455;+76=531;+7=538;+55=593;+77=670;+96=766;+67=833;+67=900;+98=998;+43=1041;+51=1092;+6=1098;+30=1128;+92=1220;+16=1236;+99=1335;+9=1344;+27=1371;+67=1438;+45=1483;+93=1576;+46=1622;+42=1664;+74=1738;+68=1806;+38=1844;+87=1931;+1=1932;+53=1985;+21=2006;+60=2066;+6=2072;+68=2140;+90=2230.\n→ 4184+2230=6414\n\nブロック9(40): 301:32,304:65,305:76,306:37,308:79,309:94,310:49,311:100,312:7,319:72,320:23,321:35,322:85,324:48,328:35,333:71,334:29,336:43,337:4,339:52,344:97,346:60,347:55,350:90,356:33,357:29,364:13,365:95,366:11,367:45,371:25,372:22,376:100,377:49,380:1,381:96,389:69,393:63,394:52,396:11\n32+65=97;+76=173;+37=210;+79=289;+94=383;+49=432;+100=532;+7=539;+72=611;+23=634;+35=669;+85=754;+48=802;+35=837;+71=908;+29=937;+43=980;+4=984;+52=1036;+97=1133;+60=1193;+55=1248;+90=1338;+33=1371;+29=1400;+13=1413;+95=1508;+11=1519;+45=1564;+25=1589;+22=1611;+100=1711;+49=1760;+1=1761;+96=1857;+69=1926;+63=1989;+52=2041;+11=2052.\n→ 6414+2052=8466\n\nブロック10(36): 403:68,405:51,406:98,407:90,411:56,413:28,420:100,423:35,426:70,433:35,435:12,440:7,442:18,445:53,446:30,449:46,451:39,452:34,456:57,460:39,463:89,466:47,469:85,470:83,474:22,475:7,476:37,477:9,478:39,480:86,483:29,484:24,488:81,490:37,493:98,497:66\n68+51=119;+98=217;+90=307;+56=363;+28=391;+100=491;+35=526;+70=596;+35=631;+12=643;+7=650;+18=668;+53=721;+30=751;+46=797;+39=836;+34=870;+57=927;+39=966;+89=1055;+47=1102;+85=1187;+83=1270;+22=1292;+7=1299;+37=1336;+9=1345;+39=1384;+86=1470;+29=1499;+24=1523;+81=1604;+37=1641;+98=1739;+66=1805.\n→", "fields": { "行数": { "present": false, "actual": null, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": false, "actual": null, "expected": 1, "semantic_correct": false, "pass_with_natural_stop": false }, "最大待ち秒": { "present": false, "actual": null, "expected": 80, "semantic_correct": false, "pass_with_natural_stop": false }, "対象部署の件数合計": { "present": false, "actual": null, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": false, "actual": null, "expected": 13, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": false, "actual": null, "expected": "不明", "semantic_correct": false, "pass_with_natural_stop": false } } }, { "case": "main-long-records-021-cap4096-sampling-r0-mtp1", "task": "long-records-021", "input_target": 32768, "max_generation": 4096, "seed": 89122, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 1816, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9310, \"指定IDの待ち秒\": 13, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9310, "expected": 11906, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 13, "expected": 13, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap4096-sampling-r2-mtp0", "task": "long-records-014", "input_target": 32768, "max_generation": 4096, "seed": 289115, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 1977, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10595,\"指定IDの待ち秒\":19,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10595, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap4096-sampling-r2-mtp1", "task": "long-records-014", "input_target": 32768, "max_generation": 4096, "seed": 289115, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2619, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11562,\"指定IDの待ち秒\":19,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 11562, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap4096-sampling-r1-mtp0", "task": "long-records-008", "input_target": 4096, "max_generation": 4096, "seed": 189109, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 181, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 78, \"対象部署の件数合計\": 742, \"指定IDの待ち秒\": 70, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-008-cap4096-sampling-r1-mtp1", "task": "long-records-008", "input_target": 4096, "max_generation": 4096, "seed": 189109, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 181, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 78, \"対象部署の件数合計\": 742, \"指定IDの待ち秒\": 70, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 78, "expected": 78, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 742, "expected": 742, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 70, "expected": 70, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap8192-sampling-r1-mtp1", "task": "long-records-018", "input_target": 32768, "max_generation": 8192, "seed": 189119, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2254, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9537,\"指定IDの待ち秒\":59,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9537, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 59, "expected": 59, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap8192-sampling-r1-mtp0", "task": "long-records-018", "input_target": 32768, "max_generation": 8192, "seed": 189119, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2271, "pass": false, "first_answer_missing": true, "answer_text": "行数600、待ち秒最小1、最大80、営業件数合計、ID33の待ち秒=59。\n\n営業件数合計を手計算...多いため概算は不可。真面目に数える必要がある。営業行の件数を列挙する。\n\n営業行を抽出：\n1:12,3:83,7:78,9:95,12:84,13:20,16:23,17:59,25:67,32:69,34:66,42:36,44:33,46:32,47:61,48:86,52:64,53:29,54:36,56:65,58:3,64:4,66:25,68:98,72:74,73:13,75:2,76:26,78:50,81:100,82:89,85:43,89:42,91:36,94:77,95:76,100:84,101:66,105:83,106:23,108:40,112:7,117:85,120:85,124:99,127:54,129:35,133:70,135:8,138:12,142:69,148:7,150:95,154:64,156:71,158:32,159:56,160:79,164:50,166:16,169:11,170:81,172:27,173:91,175:100,176:72,177:13,180:5,181:21,185:12,190:77,195:28,198:79,200:86,204:60,206:28,207:78,215:57,216:18,228:97,229:90,230:59,232:75,233:97,236:86,240:17,245:26,250:28,256:26,263:4,264:92,269:24,270:98,271:74,272:9,274:44,276:83,279:3,280:67,281:95,282:65,283:44,284:80,291:14,293:17,300:10,302:81,306:25,309:78,313:54,318:91,319:28,322:77,325:20,327:67,329:30,332:61,335:25,337:100,348:7,354:35,356:37,358:60,364:83,365:86,366:39,368:80,373:85,375:65,381:15,382:53,384:21,387:60,397:42,398:54,399:74,401:57,402:37,407:20,413:35,414:8,421:39,422:34,423:77,427:9,435:52,449:86,458:74,459:9,462:68,469:70,471:97,485:73,486:14,500:76,501:28,503:90,508:10,509:76,514:20,520:13,521:2,522:88,527:13,528:14,530:52,535:69,538:84,542:39,547:17,549:35,552:24,554:38,559:84,561:23,563:39,566:87,570:41,572:35,575:95,576:41,580:23,581:13,584:19,585:75,586:66,587:36,597:35,598:46\n\n数える：個数を数える。200行あたり？数えてみる：上の列挙の件数＝約160。\n\n合計を計算（段階ごと）：\n12+83=95;+78=173;+95=268;+84=352;+20=372;+23=395;+59=454;+67=521;+69=590;+66=656;+36=692;+33=725;+32=757;+61=818;+86=904;+64=968;+29=997;+36=1033;+65=1098;+3=1101;+4=1105;+25=1130;+98=1228;+74=1302;+13=1315;+2=1317;+26=1343;+50=1393;+100=1493;+89=1582;+43=1625;+42=1667;+36=1703;+77=1780;+76=1856;+84=1940;+66=2006;+83=2089;+23=2112;+40=2152;+7=2159;+85=2244;+85=2329;+99=2428;+54=2482;+35=2517;+70=2587;+8=2595;+12=2607;+69=2676;+7=2683;+95=2778;+64=2842;+71=2913;+32=2945;+56=3001;+79=3080;+50=3130;+16=3146;+11=3157;+81=3238;+27=3265;+91=3356;+100=3456;+72=3528;+13=3541;+5=3546;+21=3567;+12=3579;+77=3656;+28=3684;+79=3763;+86=3849;+60=3909;+28=3937;+78=4015;+57=4072;+18=4090;+97=4187;+90=4277;+59=4336;+75=4411;+97=4508;+86=4594;+17=4611;+26=4637;+28=4665;+26=4691;+4=4695;+92=4787;+24=4811;+98=4909;+74=4983;+9=4992;+44=5036;+83=5119;+3=5122;+67=5189;+95=5284;+65=5349;+44=5393;+80=5473;+14=5487;+17=5504;+10=5514;+81=5595;+25=5620;+78=5698;+54=5752;+91=5843;+28=5871;+77=5948;+20=5968;+67=6035;+30=6065;+61=6126;+25=6151;+100=6251;+7=6258;+35=6293;+37=6330;+60=6390;+83=6473;+86=6559;+39=6598;+80=6678;+85=6763;+65=6828;+15=6843;+53=6896;+21=6917;+60=6977;+42=7019;+54=7073;+74=7147;+57=7204;+37=7241;+20=7261;+35=7296;+8=7304;+39=7343;+34=7377;+77=7454;+9=7463;+52=7515;+86=7601;+74=7675;+9=7684;+68=7752;+70=7822;+97=7919;+73=7992;+14=8006;+76=8082;+28=8110;+90=8200;+10=8210;+76=8286;+20=8306;+13=8319;+2=8321;+88=8409;+13=8422;+14=8436;+52=8488;+69=8557;+84=8641;+39=8680;+17=8697;+35=8732;+24=8756;+38=8794;+84=8878;+23=8901;+39=8940;+87=9027;+41=9068;+35=9103;+95=9198;+41=9239;+23=9262;+13=9275;+19=9294;+75=9369;+66=9435;+36=9471;+35=9506;+46=9552。\n\n営業行数を確認：リスト数＝数え直す。上記160項目か？単純に確信できないが、合計9552。", "fields": { "行数": { "present": false, "actual": null, "expected": 600, "semantic_correct": false, "pass_with_natural_stop": false }, "最小待ち秒": { "present": false, "actual": null, "expected": 1, "semantic_correct": false, "pass_with_natural_stop": false }, "最大待ち秒": { "present": false, "actual": null, "expected": 80, "semantic_correct": false, "pass_with_natural_stop": false }, "対象部署の件数合計": { "present": false, "actual": null, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": false, "actual": null, "expected": 59, "semantic_correct": false, "pass_with_natural_stop": false }, "遅延原因": { "present": false, "actual": null, "expected": "不明", "semantic_correct": false, "pass_with_natural_stop": false } } }, { "case": "main-long-records-023-cap4096-sampling-r0-mtp0", "task": "long-records-023", "input_target": 32768, "max_generation": 4096, "seed": 89124, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2744, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9172,\"指定IDの待ち秒\":4,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9172, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap4096-sampling-r0-mtp1", "task": "long-records-023", "input_target": 32768, "max_generation": 4096, "seed": 89124, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2258, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 8416, \"指定IDの待ち秒\": 4, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8416, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap4096-sampling-r0-mtp0", "task": "long-records-020", "input_target": 32768, "max_generation": 4096, "seed": 89121, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2138, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10924,\"指定IDの待ち秒\":37,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10924, "expected": 10702, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-020-cap4096-sampling-r0-mtp1", "task": "long-records-020", "input_target": 32768, "max_generation": 4096, "seed": 89121, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2578, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":10831,\"指定IDの待ち秒\":37,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 10831, "expected": 10702, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 37, "expected": 37, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap4096-sampling-r0-mtp0", "task": "long-records-014", "input_target": 32768, "max_generation": 4096, "seed": 89115, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2068, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":13019,\"指定IDの待ち秒\":19,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 13019, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-014-cap4096-sampling-r0-mtp1", "task": "long-records-014", "input_target": 32768, "max_generation": 4096, "seed": 89115, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2498, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":8288,\"指定IDの待ち秒\":19,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8288, "expected": 10639, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 19, "expected": 19, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap4096-sampling-r2-mtp1", "task": "long-records-003", "input_target": 4096, "max_generation": 4096, "seed": 289104, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 188, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":2,\"最大待ち秒\":77,\"対象部署の件数合計\":701,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-003-cap4096-sampling-r2-mtp0", "task": "long-records-003", "input_target": 4096, "max_generation": 4096, "seed": 289104, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 187, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":2,\"最大待ち秒\":77,\"対象部署の件数合計\":701,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 2, "expected": 2, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 77, "expected": 77, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 701, "expected": 701, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap4096-sampling-r0-mtp0", "task": "long-records-013", "input_target": 32768, "max_generation": 4096, "seed": 89114, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3207, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9844,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9844, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-013-cap4096-sampling-r0-mtp1", "task": "long-records-013", "input_target": 32768, "max_generation": 4096, "seed": 89114, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3040, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9862,\"指定IDの待ち秒\":69,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9862, "expected": 9879, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 69, "expected": 69, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap4096-sampling-r2-mtp0", "task": "long-records-018", "input_target": 32768, "max_generation": 4096, "seed": 289119, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3040, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9567, \"指定IDの待ち秒\": 59, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9567, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 59, "expected": 59, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap4096-sampling-r2-mtp1", "task": "long-records-018", "input_target": 32768, "max_generation": 4096, "seed": 289119, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3002, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9447,\"指定IDの待ち秒\":59,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9447, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 59, "expected": 59, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap8192-sampling-r2-mtp1", "task": "long-records-018", "input_target": 32768, "max_generation": 8192, "seed": 289119, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 3002, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9447,\"指定IDの待ち秒\":59,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9447, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 59, "expected": 59, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-018-cap8192-sampling-r2-mtp0", "task": "long-records-018", "input_target": 32768, "max_generation": 8192, "seed": 289119, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 3040, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 9567, \"指定IDの待ち秒\": 59, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9567, "expected": 9583, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 59, "expected": 59, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap8192-sampling-r0-mtp0", "task": "long-records-023", "input_target": 32768, "max_generation": 8192, "seed": 89124, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 2744, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":9172,\"指定IDの待ち秒\":4,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 9172, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-023-cap8192-sampling-r0-mtp1", "task": "long-records-023", "input_target": 32768, "max_generation": 8192, "seed": 89124, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 2258, "pass": false, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 600, \"最小待ち秒\": 1, \"最大待ち秒\": 80, \"対象部署の件数合計\": 8416, \"指定IDの待ち秒\": 4, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 600, "expected": 600, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 1, "expected": 1, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 80, "expected": 80, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 8416, "expected": 9236, "semantic_correct": false, "pass_with_natural_stop": false }, "指定IDの待ち秒": { "present": true, "actual": 4, "expected": 4, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap4096-sampling-r0-mtp0", "task": "long-records-007", "input_target": 4096, "max_generation": 4096, "seed": 89108, "mtp": false, "finish_reason": "stop_token", "generated_tokens": 237, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\":{\"行数\":50,\"最小待ち秒\":3,\"最大待ち秒\":79,\"対象部署の件数合計\":736,\"指定IDの待ち秒\":54,\"遅延原因\":\"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } }, { "case": "main-long-records-007-cap4096-sampling-r0-mtp1", "task": "long-records-007", "input_target": 4096, "max_generation": 4096, "seed": 89108, "mtp": true, "finish_reason": "stop_token", "generated_tokens": 249, "pass": true, "first_answer_missing": false, "answer_text": "{\"answer\": {\"行数\": 50, \"最小待ち秒\": 3, \"最大待ち秒\": 79, \"対象部署の件数合計\": 736, \"指定IDの待ち秒\": 54, \"遅延原因\": \"不明\"}}", "fields": { "行数": { "present": true, "actual": 50, "expected": 50, "semantic_correct": true, "pass_with_natural_stop": true }, "最小待ち秒": { "present": true, "actual": 3, "expected": 3, "semantic_correct": true, "pass_with_natural_stop": true }, "最大待ち秒": { "present": true, "actual": 79, "expected": 79, "semantic_correct": true, "pass_with_natural_stop": true }, "対象部署の件数合計": { "present": true, "actual": 736, "expected": 736, "semantic_correct": true, "pass_with_natural_stop": true }, "指定IDの待ち秒": { "present": true, "actual": 54, "expected": 54, "semantic_correct": true, "pass_with_natural_stop": true }, "遅延原因": { "present": true, "actual": "不明", "expected": "不明", "semantic_correct": true, "pass_with_natural_stop": true } } } ], "paired_regression_review": { "utc": "2026-10-31T06:46:14.794968+00:00", "reviewer": "Codex", "human_review_claimed": false, "source_audit_sha256": "7e3860bd61cb3bb88707e58b81c8c0e0b015cd79d38be936f3377d576707ef07", "source_results_sha256": "4a7c0cc27555bbf233e9766c9256c3d271a0284333ff863cfee9ad025e24355e", "all_off_correct_on_wrong_pairs_reviewed": true, "all_evidence_reconciled": true, "case015_verified": true, "off_correct_on_wrong_pairs": 2, "unique_task_seed_combinations": 1, "all_paired_outcomes": { "input4096-cap4096": { "pairs": 36, "both_correct": 30, "off_only_correct": 0, "on_only_correct": 1, "both_wrong": 5, "complete_sequence_identical": 13 }, "input4096-cap8192": { "pairs": 36, "both_correct": 30, "off_only_correct": 0, "on_only_correct": 1, "both_wrong": 5, "complete_sequence_identical": 13 }, "input32768-cap4096": { "pairs": 36, "both_correct": 0, "off_only_correct": 1, "on_only_correct": 3, "both_wrong": 32, "complete_sequence_identical": 0 }, "input32768-cap8192": { "pairs": 36, "both_correct": 0, "off_only_correct": 1, "on_only_correct": 3, "both_wrong": 32, "complete_sequence_identical": 0 } }, "pairs": [ { "task": "long-records-015", "seed": 289116, "max_generation": 4096, "review": { "category": "wrong_column_copy", "finding_ja": "対象215行のIDは両方一度ずつ列挙。ありは38行で件数ではなく同じ行の待ち秒を転記し、そのリストの合計11535を回答。なしの11482が正解。両上限とも自然終了。", "review_receipt": "output-cap-discordant-case015-r2-mechanism-review.json", "review_receipt_sha256": "33f8ea1aa773156e69cd6c20f7264d1f4949581facd748f0149d63bc88616ac6", "evidence_strength": "emitted_list_reconciled_against_all_target_source_rows" }, "off": { "case": "main-long-records-015-cap4096-sampling-r2-mtp0", "phase": "main", "max_generation": 4096, "task": "long-records-015", "category": "grounded_long_records", "template_variant": 3, "mode": "sampling", "seed": 289116, "rep": 2, "mtp": false, "capacity": 1048576, "input_tokens": 32768, "generated_tokens": 3705, "first_generated_seconds": 11.084076575003564, "first_visible_seconds": 11.084076575003564, "first_answer_seconds": 85.80515068001114, "end_to_end_seconds": 86.81465896905866, "generation_tps": 48.90038906079581, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "dd8a06b8f77f009c8c4fe9956e66e7ac0b12cb14c1f5aa7dd6a9832456c35afa", "score": { "answer": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11482,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "parsed": { "answer": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 11482, "指定IDの待ち秒": 71, "遅延原因": "不明" } }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "complete_token_sequence_sha256": "be70e6aff32bd50477fc1a6282788e5f321ef0a31c528cf37d4545ffce87f9e2", "input_target": 32768, "full_completion": "Count lines: 600. Min wait: 1, max: 80. 管理件数合計: need count. ID512: 71秒. That's heavy—need sum of 管理 counts. Let me count 管理 entries and sums.\n\n管理 ids: 1,9,11,17,20,23,30,35,36,37,38,40,43,44,45,47,52,54,55,60,61,66,69,74,79,80,82,83,84,87,91,94,97,98,109,110,116,117,120,126,127,130,131,132,133,136,140,141,143,145,146,150,153,156,160,161,163,164,167,171,174,176,178,179,182,183,187,205,207,210,214,216,217,219,221,222,224,225,231,235,236,239,242,243,245,246,251,253,260,261,264,266,275,276,284,286,288,290,296,297,301,302,304,318,320,322,323,324,330,331,335,339,341,343,346,348,354,355,356,358,359,360,361,366,369,372,382,383,385,388,389,390,391,400,402,403,405,409,410,411,415,416,422,423,427,430,435,436,437,444,448,450,452,453,457,459,465,469,472,473,480,484,485,487,488,501,502,504,505,507,510,512,513,516,517,518,521,523,526,528,532,533,536,537,540,542,544,546,548,554,557,561,562,563,565,568,569,570,577,579,580,581,582,583,584,585,586,588,589,590,593,595,596,597,599.\n\nCount them: let me count in groups.\nGroup 1-100: ids 1,9,11,17,20,23,30,35,36,37,38,40,43,44,45,47,52,54,55,60,61,66,69,74,79,80,82,83,84,87,91,94,97,98 → 34.\n101-200: 109,110,116,117,120,126,127,130,131,132,133,136,140,141,143,145,146,150,153,156,160,161,163,164,167,171,174,176,178,179,182,183,187 → 33.\n201-300: 205,207,210,214,216,217,219,221,222,224,225,231,235,236,239,242,243,245,246,251,253,260,261,264,266,275,276,284,286,288,290,296,297 → 33.\n301-400: 301,302,304,318,320,322,323,324,330,331,335,339,341,343,346,348,354,355,356,358,359,360,361,366,369,372,382,383,385,388,389,390,391,400 → 34.\n401-500: 402,403,405,409,410,411,415,416,422,423,427,430,435,436,437,444,448,450,452,453,457,459,465,469,472,473,480,484,485,487,488 → 31.\n501-600: 501,502,504,505,507,510,512,513,516,517,518,521,523,526,528,532,533,536,537,540,542,544,546,548,554,557,561,562,563,565,568,569,570,577,579,580,581,582,583,584,585,586,588,589,590,593,595,596,597,599 → 50.\nTotal = 34+33+33+34+31+50 = 215.\n\nNow sum counts for each 管理 line.\n\n1-100 group:\n1:83\n9:72\n11:40\n17:94\n20:61\n23:60\n30:100\n35:95\n36:57\n37:25\n38:18\n40:93\n43:25\n44:99\n45:58\n47:87\n52:31\n54:70\n55:80\n60:80\n61:10\n66:29\n69:78\n74:8\n79:93\n80:57\n82:34\n83:66\n84:37\n87:43\n91:69\n94:100\n97:65\n98:64\n\nSum: 83+72=155; +40=195; +94=289; +61=350; +60=410; +100=510; +95=605; +57=662; +25=687; +18=705; +93=798; +25=823; +99=922; +58=980; +87=1067; +31=1098; +70=1168; +80=1248; +80=1328; +10=1338; +29=1367; +78=1445; +8=1453; +93=1546; +57=1603; +34=1637; +66=1703; +37=1740; +43=1783; +69=1852; +100=1952; +65=2017; +64=2081.\n\n101-200:\n109:16\n110:69\n116:83\n117:98\n120:98\n126:22\n127:70\n130:7\n131:27\n132:79\n133:60\n136:77\n140:6\n141:3\n143:42\n145:80\n146:21\n150:88\n153:50\n156:7\n160:83\n161:70\n163:87\n164:44\n167:47\n171:21\n174:19\n176:50\n178:18\n179:83\n182:54\n183:84\n187:74\n\nSum: 16+69=85;+83=168;+98=266;+98=364;+22=386;+70=456;+7=463;+27=490;+79=569;+60=629;+77=706;+6=712;+3=715;+42=757;+80=837;+21=858;+88=946;+50=996;+7=1003;+83=1086;+70=1156;+87=1243;+44=1287;+47=1334;+21=1355;+19=1374;+50=1424;+18=1442;+83=1525;+54=1579;+84=1663;+74=1737.\n\nRunning total: 2081+1737=3818.\n\n201-300:\n205:83\n207:48\n210:22\n214:34\n216:63\n217:86\n219:71\n221:31\n222:55\n224:74\n225:16\n231:38\n235:42\n236:22\n239:40\n242:12\n243:39\n245:34\n246:86\n251:66\n253:82\n260:41\n261:53\n264:1\n266:32\n275:34\n276:71\n284:95\n286:40\n288:78\n290:60\n296:82\n297:50\n\nSum: 83+48=131;+22=153;+34=187;+63=250;+86=336;+71=407;+31=438;+55=493;+74=567;+16=583;+38=621;+42=663;+22=685;+40=725;+12=737;+39=776;+34=810;+86=896;+66=962;+82=1044;+41=1085;+53=1138;+1=1139;+32=1171;+34=1205;+71=1276;+95=1371;+40=1411;+78=1489;+60=1549;+82=1631;+50=1681.\n\nTotal: 3818+1681=5499.\n\n301-400:\n301:51\n302:48\n304:98\n318:55\n320:3\n322:42\n323:7\n324:51\n330:62\n331:44\n335:47\n339:74\n341:21\n343:59\n346:90\n348:96\n354:75\n355:53\n356:72\n358:33\n359:3\n360:5\n361:59\n366:28\n369:66\n372:79\n382:46\n383:9\n385:23\n388:84\n389:42\n390:60\n391:6\n400:24\n\nSum: 51+48=99;+98=197;+55=252;+3=255;+42=297;+7=304;+51=355;+62=417;+44=461;+47=508;+74=582;+21=603;+59=662;+90=752;+96=848;+75=923;+53=976;+72=1048;+33=1081;+3=1084;+5=1089;+59=1148;+28=1176;+66=1242;+79=1321;+46=1367;+9=1376;+23=1399;+84=1483;+42=1525;+60=1585;+6=1591;+24=1615.\n\nTotal: 5499+1615=7114.\n\n401-500:\n402:49\n403:3\n405:39\n409:71\n410:52\n411:25\n415:21\n416:83\n422:74\n423:66\n427:5\n430:98\n435:100\n436:88\n437:52\n444:86\n448:48\n450:61\n452:44\n453:76\n457:24\n459:59\n465:45\n469:78\n472:19\n473:42\n480:37\n484:65\n485:85\n487:92\n488:44\n\nSum: 49+3=52;+39=91;+71=162;+52=214;+25=239;+21=260;+83=343;+74=417;+66=483;+5=488;+98=586;+100=686;+88=774;+52=826;+86=912;+48=960;+61=1021;+44=1065;+76=1141;+24=1165;+59=1224;+45=1269;+78=1347;+19=1366;+42=1408;+37=1445;+65=1510;+85=1595;+92=1687;+44=1731.\n\nTotal: 7114+1731=8845.\n\n501-600:\n501:21\n502:81\n504:69\n505:26\n507:90\n510:80\n512:84\n513:20\n516:8\n517:37\n518:41\n521:66\n523:74\n526:22\n528:43\n532:62\n533:69\n536:8\n537:79\n540:58\n542:58\n544:83\n546:8\n548:47\n554:36\n557:40\n561:63\n562:43\n563:6\n565:35\n568:80\n569:4\n570:24\n577:44\n579:60\n580:44\n581:84\n582:67\n583:77\n584:71\n585:89\n586:66\n588:95\n589:87\n590:34\n593:84\n595:64\n596:20\n597:44\n599:42\n\nSum: 21+81=102;+69=171;+26=197;+90=287;+80=367;+84=451;+20=471;+8=479;+37=516;+41=557;+66=623;+74=697;+22=719;+43=762;+62=824;+69=893;+8=901;+79=980;+58=1038;+58=1096;+83=1179;+8=1187;+47=1234;+36=1270;+40=1310;+63=1373;+43=1416;+6=1422;+35=1457;+80=1537;+4=1541;+24=1565;+44=1609;+60=1669;+44=1713;+84=1797;+67=1864;+77=1941;+71=2012;+89=2101;+66=2167;+95=2262;+87=2349;+34=2383;+84=2467;+64=2531;+20=2551;+44=2595;+42=2637.\n\nTotal: 8845+2637=11482.\n\nAnswer: {\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11482,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11482,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}" }, "on": { "case": "main-long-records-015-cap4096-sampling-r2-mtp1", "phase": "main", "max_generation": 4096, "task": "long-records-015", "category": "grounded_long_records", "template_variant": 3, "mode": "sampling", "seed": 289116, "rep": 2, "mtp": true, "capacity": 1048576, "input_tokens": 32768, "generated_tokens": 2887, "first_generated_seconds": 11.418474373989739, "first_visible_seconds": 11.418474373989739, "first_answer_seconds": 44.89141746493988, "end_to_end_seconds": 45.355748873902485, "generation_tps": 84.96200217295043, "finish_reason": "stop_token", "accepted_draft_tokens": 1940, "rejected_draft_tokens": 901, "output_sha256": "6c24bb6968f61c3e518c0968ed5e8c9daecf841c5c7f21a08fc806305e32184e", "score": { "answer": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11535,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "parsed": { "answer": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 11535, "指定IDの待ち秒": 71, "遅延原因": "不明" } }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "complete_token_sequence_sha256": "73d32dfc9ea8f28ffb623ea3f7c55a71077835944c95c640943c4f65ad7e4f45", "input_target": 32768, "full_completion": "Count lines: 600. Min wait: 1, max: 80. 管理件数合計 — need to sum 管理件数. ID512: 71. 遅延原因: 不明.\n\nSum 管理件数: let's list 管理 ids and 件数:\n1:83,9:72,11:40,17:94,20:61,23:60,30:100,35:95,36:57,37:25,38:18,40:93,43:25,44:99,45:58,47:87,52:31,54:70,55:80,60:80,61:10,66:29,69:78,74:8,79:93,80:57,82:34,83:66,84:33,87:32,91:64,94:100,97:65,98:64,109:16,110:69,116:83,117:98,120:98,126:22,127:70,130:7,131:27,132:79,133:55,136:77,140:78,141:3,143:55,145:80,146:21,150:88,153:50,156:7,160:83,161:70,163:87,164:44,167:47,171:67,174:59,176:50,178:3,179:83,182:54,183:84,187:74,205:83,207:47,210:34,214:34,216:63,217:86,219:71,221:39,222:55,224:10,225:19,231:72,235:42,236:22,239:47,242:10,243:39,245:61,246:86,251:66,253:82,260:65,261:39,264:1,266:30,275:34,276:71,284:95,286:40,288:78,290:60,296:82,297:50,301:43,302:79,304:98,318:55,320:3,322:42,323:7,324:51,330:62,331:44,335:47,339:74,341:21,343:59,346:90,348:96,354:75,355:53,356:72,358:42,359:3,360:5,361:59,366:28,369:66,372:79,382:46,383:9,385:23,388:84,389:42,390:60,391:6,400:24,402:49,403:3,405:39,409:71,410:52,411:25,415:21,416:83,422:74,423:66,427:5,430:98,435:100,436:88,437:52,444:86,448:48,450:61,452:44,453:76,457:24,459:59,465:45,469:78,472:19,473:42,480:37,484:65,485:85,487:92,488:44,501:21,502:81,504:69,505:26,507:90,510:79,512:71,513:55,516:8,517:12,518:41,521:69,523:74,526:22,528:61,532:62,533:69,536:8,537:79,540:73,542:27,544:83,546:8,548:47,554:36,557:40,561:2,562:43,563:6,565:35,568:80,569:4,570:24,577:44,579:60,580:44,581:84,582:67,583:69,584:53,585:89,586:2,588:95,589:87,590:72,593:84,595:64,596:20,597:14,599:42.\n\nCount 管理 entries: roughly let's count them: I'll count list length. Groups of 10:\n83,72,40,94,61,60,100,95,57,25 →10\n18,93,25,99,58,87,31,70,80,80 →20\n10,29,78,8,93,57,34,66,33,32 →30\n64,100,65,64,16,69,83,98,98,22 →40\n70,7,27,79,55,77,78,3,55,80 →50\n21,88,50,7,83,70,87,44,47,67 →60\n59,50,3,83,54,84,74,83,47,34 →70\n34,63,86,71,39,55,10,19,72,42 →80\n22,47,10,39,61,86,66,82,65,39 →90\n1,30,34,71,95,40,78,60,82,50 →100\n43,79,98,55,3,42,7,51,62,44 →110\n47,74,21,59,90,96,75,53,72,42 →120\n3,5,59,28,66,79,46,9,23,84 →130\n42,60,6,24,49,3,39,71,52,25 →140\n21,83,74,66,5,98,100,88,52,86 →150\n48,61,44,76,24,59,45,78,19,42 →160\n37,65,85,92,44,21,81,69,26,90 →170\n79,71,55,8,12,41,69,74,22,61 →180\n62,69,8,79,73,27,83,8,47,36 →190\n40,2,43,6,35,80,4,24,44,60 →200\n44,84,67,69,53,89,2,95,87,72 →210\n84,64,20,14,42 →215\n\nSum per group:\nG1:83+72=155,+40=195,+94=289,+61=350,+60=410,+100=510,+95=605,+57=662,+25=687\nG2:687+18=705,+93=798,+25=823,+99=922,+58=980,+87=1067,+31=1098,+70=1168,+80=1248,+80=1328\nG3:+10=1338,+29=1367,+78=1445,+8=1453,+93=1546,+57=1603,+34=1637,+66=1703,+33=1736,+32=1768\nG4:+64=1832,+100=1932,+65=1997,+64=2061,+16=2077,+69=2146,+83=2229,+98=2327,+98=2425,+22=2447\nG5:+70=2517,+7=2524,+27=2551,+79=2630,+55=2685,+77=2762,+78=2840,+3=2843,+55=2898,+80=2978\nG6:+21=2999,+88=3087,+50=3137,+7=3144,+83=3227,+70=3297,+87=3384,+44=3428,+47=3475,+67=3542\nG7:+59=3601,+50=3651,+3=3654,+83=3737,+54=3791,+84=3875,+74=3949,+83=4032,+47=4079,+34=4113\nG8:+34=4147,+63=4210,+86=4296,+71=4367,+39=4406,+55=4461,+10=4471,+19=4490,+72=4562,+42=4604\nG9:+22=4626,+47=4673,+10=4683,+39=4722,+61=4783,+86=4869,+66=4935,+82=5017,+65=5082,+39=5121\nG10:+1=5122,+30=5152,+34=5186,+71=5257,+95=5352,+40=5392,+78=5470,+60=5530,+82=5612,+50=5662\nG11:+43=5705,+79=5784,+98=5882,+55=5937,+3=5940,+42=5982,+7=5989,+51=6040,+62=6102,+44=6146\nG12:+47=6193,+74=6267,+21=6288,+59=6347,+90=6437,+96=6533,+75=6608,+53=6661,+72=6733,+42=6775\nG13:+3=6778,+5=6783,+59=6842,+28=6870,+66=6936,+79=7015,+46=7061,+9=7070,+23=7093,+84=7177\nG14:+42=7219,+60=7279,+6=7285,+24=7309,+49=7358,+3=7361,+39=7400,+71=7471,+52=7523,+25=7548\nG15:+21=7569,+83=7652,+74=7726,+66=7792,+5=7797,+98=7895,+100=7995,+88=8083,+52=8135,+86=8221\nG16:+48=8269,+61=8330,+44=8374,+76=8450,+24=8474,+59=8533,+45=8578,+78=8656,+19=8675,+42=8717\nG17:+37=8754,+65=8819,+85=8904,+92=8996,+44=9040,+21=9061,+81=9142,+69=9211,+26=9237,+90=9327\nG18:+79=9406,+71=9477,+55=9532,+8=9540,+12=9552,+41=9593,+69=9662,+74=9736,+22=9758,+61=9819\nG19:+62=9881,+69=9950,+8=9958,+79=10037,+73=10110,+27=10137,+83=10220,+8=10228,+47=10275,+36=10311\nG20:+40=10351,+2=10353,+43=10396,+6=10402,+35=10437,+80=10517,+4=10521,+24=10545,+44=10589,+60=10649\nG21:+44=10693,+84=10777,+67=10844,+69=10913,+53=10966,+89=11055,+2=11057,+95=11152,+87=11239,+72=11311\nG22:+84=11395,+64=11459,+20=11479,+14=11493,+42=11535\n\n合計 11535.{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11535,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}" } }, { "task": "long-records-015", "seed": 289116, "max_generation": 8192, "review": { "category": "wrong_column_copy", "finding_ja": "対象215行のIDは両方一度ずつ列挙。ありは38行で件数ではなく同じ行の待ち秒を転記し、そのリストの合計11535を回答。なしの11482が正解。両上限とも自然終了。", "review_receipt": "output-cap-discordant-case015-r2-mechanism-review.json", "review_receipt_sha256": "33f8ea1aa773156e69cd6c20f7264d1f4949581facd748f0149d63bc88616ac6", "evidence_strength": "emitted_list_reconciled_against_all_target_source_rows" }, "off": { "case": "main-long-records-015-cap8192-sampling-r2-mtp0", "phase": "main", "max_generation": 8192, "task": "long-records-015", "category": "grounded_long_records", "template_variant": 3, "mode": "sampling", "seed": 289116, "rep": 2, "mtp": false, "capacity": 1048576, "input_tokens": 32768, "generated_tokens": 3705, "first_generated_seconds": 11.084528874023817, "first_visible_seconds": 11.084528874023817, "first_answer_seconds": 85.42091396695469, "end_to_end_seconds": 86.40231840591878, "generation_tps": 49.16640105614594, "finish_reason": "stop_token", "accepted_draft_tokens": 0, "rejected_draft_tokens": 0, "output_sha256": "dd8a06b8f77f009c8c4fe9956e66e7ac0b12cb14c1f5aa7dd6a9832456c35afa", "score": { "answer": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11482,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "parsed": { "answer": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 11482, "指定IDの待ち秒": 71, "遅延原因": "不明" } }, "format_valid": true, "semantic_correct": true, "natural_stop": true, "pass": true }, "complete_token_sequence_sha256": "be70e6aff32bd50477fc1a6282788e5f321ef0a31c528cf37d4545ffce87f9e2", "input_target": 32768, "full_completion": "Count lines: 600. Min wait: 1, max: 80. 管理件数合計: need count. ID512: 71秒. That's heavy—need sum of 管理 counts. Let me count 管理 entries and sums.\n\n管理 ids: 1,9,11,17,20,23,30,35,36,37,38,40,43,44,45,47,52,54,55,60,61,66,69,74,79,80,82,83,84,87,91,94,97,98,109,110,116,117,120,126,127,130,131,132,133,136,140,141,143,145,146,150,153,156,160,161,163,164,167,171,174,176,178,179,182,183,187,205,207,210,214,216,217,219,221,222,224,225,231,235,236,239,242,243,245,246,251,253,260,261,264,266,275,276,284,286,288,290,296,297,301,302,304,318,320,322,323,324,330,331,335,339,341,343,346,348,354,355,356,358,359,360,361,366,369,372,382,383,385,388,389,390,391,400,402,403,405,409,410,411,415,416,422,423,427,430,435,436,437,444,448,450,452,453,457,459,465,469,472,473,480,484,485,487,488,501,502,504,505,507,510,512,513,516,517,518,521,523,526,528,532,533,536,537,540,542,544,546,548,554,557,561,562,563,565,568,569,570,577,579,580,581,582,583,584,585,586,588,589,590,593,595,596,597,599.\n\nCount them: let me count in groups.\nGroup 1-100: ids 1,9,11,17,20,23,30,35,36,37,38,40,43,44,45,47,52,54,55,60,61,66,69,74,79,80,82,83,84,87,91,94,97,98 → 34.\n101-200: 109,110,116,117,120,126,127,130,131,132,133,136,140,141,143,145,146,150,153,156,160,161,163,164,167,171,174,176,178,179,182,183,187 → 33.\n201-300: 205,207,210,214,216,217,219,221,222,224,225,231,235,236,239,242,243,245,246,251,253,260,261,264,266,275,276,284,286,288,290,296,297 → 33.\n301-400: 301,302,304,318,320,322,323,324,330,331,335,339,341,343,346,348,354,355,356,358,359,360,361,366,369,372,382,383,385,388,389,390,391,400 → 34.\n401-500: 402,403,405,409,410,411,415,416,422,423,427,430,435,436,437,444,448,450,452,453,457,459,465,469,472,473,480,484,485,487,488 → 31.\n501-600: 501,502,504,505,507,510,512,513,516,517,518,521,523,526,528,532,533,536,537,540,542,544,546,548,554,557,561,562,563,565,568,569,570,577,579,580,581,582,583,584,585,586,588,589,590,593,595,596,597,599 → 50.\nTotal = 34+33+33+34+31+50 = 215.\n\nNow sum counts for each 管理 line.\n\n1-100 group:\n1:83\n9:72\n11:40\n17:94\n20:61\n23:60\n30:100\n35:95\n36:57\n37:25\n38:18\n40:93\n43:25\n44:99\n45:58\n47:87\n52:31\n54:70\n55:80\n60:80\n61:10\n66:29\n69:78\n74:8\n79:93\n80:57\n82:34\n83:66\n84:37\n87:43\n91:69\n94:100\n97:65\n98:64\n\nSum: 83+72=155; +40=195; +94=289; +61=350; +60=410; +100=510; +95=605; +57=662; +25=687; +18=705; +93=798; +25=823; +99=922; +58=980; +87=1067; +31=1098; +70=1168; +80=1248; +80=1328; +10=1338; +29=1367; +78=1445; +8=1453; +93=1546; +57=1603; +34=1637; +66=1703; +37=1740; +43=1783; +69=1852; +100=1952; +65=2017; +64=2081.\n\n101-200:\n109:16\n110:69\n116:83\n117:98\n120:98\n126:22\n127:70\n130:7\n131:27\n132:79\n133:60\n136:77\n140:6\n141:3\n143:42\n145:80\n146:21\n150:88\n153:50\n156:7\n160:83\n161:70\n163:87\n164:44\n167:47\n171:21\n174:19\n176:50\n178:18\n179:83\n182:54\n183:84\n187:74\n\nSum: 16+69=85;+83=168;+98=266;+98=364;+22=386;+70=456;+7=463;+27=490;+79=569;+60=629;+77=706;+6=712;+3=715;+42=757;+80=837;+21=858;+88=946;+50=996;+7=1003;+83=1086;+70=1156;+87=1243;+44=1287;+47=1334;+21=1355;+19=1374;+50=1424;+18=1442;+83=1525;+54=1579;+84=1663;+74=1737.\n\nRunning total: 2081+1737=3818.\n\n201-300:\n205:83\n207:48\n210:22\n214:34\n216:63\n217:86\n219:71\n221:31\n222:55\n224:74\n225:16\n231:38\n235:42\n236:22\n239:40\n242:12\n243:39\n245:34\n246:86\n251:66\n253:82\n260:41\n261:53\n264:1\n266:32\n275:34\n276:71\n284:95\n286:40\n288:78\n290:60\n296:82\n297:50\n\nSum: 83+48=131;+22=153;+34=187;+63=250;+86=336;+71=407;+31=438;+55=493;+74=567;+16=583;+38=621;+42=663;+22=685;+40=725;+12=737;+39=776;+34=810;+86=896;+66=962;+82=1044;+41=1085;+53=1138;+1=1139;+32=1171;+34=1205;+71=1276;+95=1371;+40=1411;+78=1489;+60=1549;+82=1631;+50=1681.\n\nTotal: 3818+1681=5499.\n\n301-400:\n301:51\n302:48\n304:98\n318:55\n320:3\n322:42\n323:7\n324:51\n330:62\n331:44\n335:47\n339:74\n341:21\n343:59\n346:90\n348:96\n354:75\n355:53\n356:72\n358:33\n359:3\n360:5\n361:59\n366:28\n369:66\n372:79\n382:46\n383:9\n385:23\n388:84\n389:42\n390:60\n391:6\n400:24\n\nSum: 51+48=99;+98=197;+55=252;+3=255;+42=297;+7=304;+51=355;+62=417;+44=461;+47=508;+74=582;+21=603;+59=662;+90=752;+96=848;+75=923;+53=976;+72=1048;+33=1081;+3=1084;+5=1089;+59=1148;+28=1176;+66=1242;+79=1321;+46=1367;+9=1376;+23=1399;+84=1483;+42=1525;+60=1585;+6=1591;+24=1615.\n\nTotal: 5499+1615=7114.\n\n401-500:\n402:49\n403:3\n405:39\n409:71\n410:52\n411:25\n415:21\n416:83\n422:74\n423:66\n427:5\n430:98\n435:100\n436:88\n437:52\n444:86\n448:48\n450:61\n452:44\n453:76\n457:24\n459:59\n465:45\n469:78\n472:19\n473:42\n480:37\n484:65\n485:85\n487:92\n488:44\n\nSum: 49+3=52;+39=91;+71=162;+52=214;+25=239;+21=260;+83=343;+74=417;+66=483;+5=488;+98=586;+100=686;+88=774;+52=826;+86=912;+48=960;+61=1021;+44=1065;+76=1141;+24=1165;+59=1224;+45=1269;+78=1347;+19=1366;+42=1408;+37=1445;+65=1510;+85=1595;+92=1687;+44=1731.\n\nTotal: 7114+1731=8845.\n\n501-600:\n501:21\n502:81\n504:69\n505:26\n507:90\n510:80\n512:84\n513:20\n516:8\n517:37\n518:41\n521:66\n523:74\n526:22\n528:43\n532:62\n533:69\n536:8\n537:79\n540:58\n542:58\n544:83\n546:8\n548:47\n554:36\n557:40\n561:63\n562:43\n563:6\n565:35\n568:80\n569:4\n570:24\n577:44\n579:60\n580:44\n581:84\n582:67\n583:77\n584:71\n585:89\n586:66\n588:95\n589:87\n590:34\n593:84\n595:64\n596:20\n597:44\n599:42\n\nSum: 21+81=102;+69=171;+26=197;+90=287;+80=367;+84=451;+20=471;+8=479;+37=516;+41=557;+66=623;+74=697;+22=719;+43=762;+62=824;+69=893;+8=901;+79=980;+58=1038;+58=1096;+83=1179;+8=1187;+47=1234;+36=1270;+40=1310;+63=1373;+43=1416;+6=1422;+35=1457;+80=1537;+4=1541;+24=1565;+44=1609;+60=1669;+44=1713;+84=1797;+67=1864;+77=1941;+71=2012;+89=2101;+66=2167;+95=2262;+87=2349;+34=2383;+84=2467;+64=2531;+20=2551;+44=2595;+42=2637.\n\nTotal: 8845+2637=11482.\n\nAnswer: {\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11482,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11482,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}" }, "on": { "case": "main-long-records-015-cap8192-sampling-r2-mtp1", "phase": "main", "max_generation": 8192, "task": "long-records-015", "category": "grounded_long_records", "template_variant": 3, "mode": "sampling", "seed": 289116, "rep": 2, "mtp": true, "capacity": 1048576, "input_tokens": 32768, "generated_tokens": 2887, "first_generated_seconds": 11.42075869999826, "first_visible_seconds": 11.42075869999826, "first_answer_seconds": 45.21649359294679, "end_to_end_seconds": 45.693056011921726, "generation_tps": 84.16687832104591, "finish_reason": "stop_token", "accepted_draft_tokens": 1940, "rejected_draft_tokens": 901, "output_sha256": "6c24bb6968f61c3e518c0968ed5e8c9daecf841c5c7f21a08fc806305e32184e", "score": { "answer": "{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11535,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}", "parsed": { "answer": { "行数": 600, "最小待ち秒": 1, "最大待ち秒": 80, "対象部署の件数合計": 11535, "指定IDの待ち秒": 71, "遅延原因": "不明" } }, "format_valid": true, "semantic_correct": false, "natural_stop": true, "pass": false }, "complete_token_sequence_sha256": "73d32dfc9ea8f28ffb623ea3f7c55a71077835944c95c640943c4f65ad7e4f45", "input_target": 32768, "full_completion": "Count lines: 600. Min wait: 1, max: 80. 管理件数合計 — need to sum 管理件数. ID512: 71. 遅延原因: 不明.\n\nSum 管理件数: let's list 管理 ids and 件数:\n1:83,9:72,11:40,17:94,20:61,23:60,30:100,35:95,36:57,37:25,38:18,40:93,43:25,44:99,45:58,47:87,52:31,54:70,55:80,60:80,61:10,66:29,69:78,74:8,79:93,80:57,82:34,83:66,84:33,87:32,91:64,94:100,97:65,98:64,109:16,110:69,116:83,117:98,120:98,126:22,127:70,130:7,131:27,132:79,133:55,136:77,140:78,141:3,143:55,145:80,146:21,150:88,153:50,156:7,160:83,161:70,163:87,164:44,167:47,171:67,174:59,176:50,178:3,179:83,182:54,183:84,187:74,205:83,207:47,210:34,214:34,216:63,217:86,219:71,221:39,222:55,224:10,225:19,231:72,235:42,236:22,239:47,242:10,243:39,245:61,246:86,251:66,253:82,260:65,261:39,264:1,266:30,275:34,276:71,284:95,286:40,288:78,290:60,296:82,297:50,301:43,302:79,304:98,318:55,320:3,322:42,323:7,324:51,330:62,331:44,335:47,339:74,341:21,343:59,346:90,348:96,354:75,355:53,356:72,358:42,359:3,360:5,361:59,366:28,369:66,372:79,382:46,383:9,385:23,388:84,389:42,390:60,391:6,400:24,402:49,403:3,405:39,409:71,410:52,411:25,415:21,416:83,422:74,423:66,427:5,430:98,435:100,436:88,437:52,444:86,448:48,450:61,452:44,453:76,457:24,459:59,465:45,469:78,472:19,473:42,480:37,484:65,485:85,487:92,488:44,501:21,502:81,504:69,505:26,507:90,510:79,512:71,513:55,516:8,517:12,518:41,521:69,523:74,526:22,528:61,532:62,533:69,536:8,537:79,540:73,542:27,544:83,546:8,548:47,554:36,557:40,561:2,562:43,563:6,565:35,568:80,569:4,570:24,577:44,579:60,580:44,581:84,582:67,583:69,584:53,585:89,586:2,588:95,589:87,590:72,593:84,595:64,596:20,597:14,599:42.\n\nCount 管理 entries: roughly let's count them: I'll count list length. Groups of 10:\n83,72,40,94,61,60,100,95,57,25 →10\n18,93,25,99,58,87,31,70,80,80 →20\n10,29,78,8,93,57,34,66,33,32 →30\n64,100,65,64,16,69,83,98,98,22 →40\n70,7,27,79,55,77,78,3,55,80 →50\n21,88,50,7,83,70,87,44,47,67 →60\n59,50,3,83,54,84,74,83,47,34 →70\n34,63,86,71,39,55,10,19,72,42 →80\n22,47,10,39,61,86,66,82,65,39 →90\n1,30,34,71,95,40,78,60,82,50 →100\n43,79,98,55,3,42,7,51,62,44 →110\n47,74,21,59,90,96,75,53,72,42 →120\n3,5,59,28,66,79,46,9,23,84 →130\n42,60,6,24,49,3,39,71,52,25 →140\n21,83,74,66,5,98,100,88,52,86 →150\n48,61,44,76,24,59,45,78,19,42 →160\n37,65,85,92,44,21,81,69,26,90 →170\n79,71,55,8,12,41,69,74,22,61 →180\n62,69,8,79,73,27,83,8,47,36 →190\n40,2,43,6,35,80,4,24,44,60 →200\n44,84,67,69,53,89,2,95,87,72 →210\n84,64,20,14,42 →215\n\nSum per group:\nG1:83+72=155,+40=195,+94=289,+61=350,+60=410,+100=510,+95=605,+57=662,+25=687\nG2:687+18=705,+93=798,+25=823,+99=922,+58=980,+87=1067,+31=1098,+70=1168,+80=1248,+80=1328\nG3:+10=1338,+29=1367,+78=1445,+8=1453,+93=1546,+57=1603,+34=1637,+66=1703,+33=1736,+32=1768\nG4:+64=1832,+100=1932,+65=1997,+64=2061,+16=2077,+69=2146,+83=2229,+98=2327,+98=2425,+22=2447\nG5:+70=2517,+7=2524,+27=2551,+79=2630,+55=2685,+77=2762,+78=2840,+3=2843,+55=2898,+80=2978\nG6:+21=2999,+88=3087,+50=3137,+7=3144,+83=3227,+70=3297,+87=3384,+44=3428,+47=3475,+67=3542\nG7:+59=3601,+50=3651,+3=3654,+83=3737,+54=3791,+84=3875,+74=3949,+83=4032,+47=4079,+34=4113\nG8:+34=4147,+63=4210,+86=4296,+71=4367,+39=4406,+55=4461,+10=4471,+19=4490,+72=4562,+42=4604\nG9:+22=4626,+47=4673,+10=4683,+39=4722,+61=4783,+86=4869,+66=4935,+82=5017,+65=5082,+39=5121\nG10:+1=5122,+30=5152,+34=5186,+71=5257,+95=5352,+40=5392,+78=5470,+60=5530,+82=5612,+50=5662\nG11:+43=5705,+79=5784,+98=5882,+55=5937,+3=5940,+42=5982,+7=5989,+51=6040,+62=6102,+44=6146\nG12:+47=6193,+74=6267,+21=6288,+59=6347,+90=6437,+96=6533,+75=6608,+53=6661,+72=6733,+42=6775\nG13:+3=6778,+5=6783,+59=6842,+28=6870,+66=6936,+79=7015,+46=7061,+9=7070,+23=7093,+84=7177\nG14:+42=7219,+60=7279,+6=7285,+24=7309,+49=7358,+3=7361,+39=7400,+71=7471,+52=7523,+25=7548\nG15:+21=7569,+83=7652,+74=7726,+66=7792,+5=7797,+98=7895,+100=7995,+88=8083,+52=8135,+86=8221\nG16:+48=8269,+61=8330,+44=8374,+76=8450,+24=8474,+59=8533,+45=8578,+78=8656,+19=8675,+42=8717\nG17:+37=8754,+65=8819,+85=8904,+92=8996,+44=9040,+21=9061,+81=9142,+69=9211,+26=9237,+90=9327\nG18:+79=9406,+71=9477,+55=9532,+8=9540,+12=9552,+41=9593,+69=9662,+74=9736,+22=9758,+61=9819\nG19:+62=9881,+69=9950,+8=9958,+79=10037,+73=10110,+27=10137,+83=10220,+8=10228,+47=10275,+36=10311\nG20:+40=10351,+2=10353,+43=10396,+6=10402,+35=10437,+80=10517,+4=10521,+24=10545,+44=10589,+60=10649\nG21:+44=10693,+84=10777,+67=10844,+69=10913,+53=10966,+89=11055,+2=11057,+95=11152,+87=11239,+72=11311\nG22:+84=11395,+64=11459,+20=11479,+14=11493,+42=11535\n\n合計 11535.{\"answer\":{\"行数\":600,\"最小待ち秒\":1,\"最大待ち秒\":80,\"対象部署の件数合計\":11535,\"指定IDの待ち秒\":71,\"遅延原因\":\"不明\"}}" } } ], "prior600_review_sha256": "1dc076be28cee27346d9bcf5d634ccf6f5d75ee598eae8a90cd82913deebe945", "limits": [ "The generated reasoning is observable text, not a direct measurement of hidden computation.", "No causal attribution to quantization, kernel arithmetic, RNG, draft acceptance or cache handling is established.", "The same task and seed under two caps are not independent tasks." ] }, "prior600_paired_regression_review": { "utc": "2026-10-31T05:29:41.250859+00:00", "reviewer": "Codex", "human_review_claimed": false, "all_original_off_correct_on_wrong_pairs_read": true, "source_input_sha256": "ac51ba6a09b76d0d7860dc705cca40e7566dadebb3cf2af0642217a7fdc2eb2d", "source_audit_sha256": "95cece7e5a3827a19aa0b6a5fa03aaeb9ddb00436394c55b40003f91fe08cd67", "sampling_pairs": 11, "greedy_pairs_separate": 1, "original_sampling_off_correct_denominator": 1664, "regressions_among_off_passes_percentage": 0.6610576923076923, "sampling_classification": { "clear_content_error": 7, "output_contract": 3, "wording_ambiguity": 1 }, "pairs": [ { "task": "code-008", "mode": "greedy", "rep": 0, "seed": 3571, "expected": [ 18, 12, 11 ], "off_answer": [ 18, 12, 11 ], "on_answer": [ 18, 12, 0 ], "off_output_sha256": "44edb398a82aabb193a47db3648214e4ddd2d12b61c9e8bf5c45aa233cbdf864", "on_output_sha256": "fd91417dd271c0328891d2dbfa019acaccf280f60a3e2771c33e0dde212bba4c", "category": "slice_selection", "label_ja": "スライスの位置違い", "finding_ja": "正しい添字は1,3,5。ありの末尾0は添字6の値。思考文はなく、内部でどの計算をしたかは不明。", "evidence_strength": "final_answer_only", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "code-056", "mode": "sampling", "rep": 1, "seed": 1144997408, "expected": 10, "off_answer": 10, "on_answer": 13, "off_output_sha256": "7014ecf5584b57f9f1262040edeab8dd01bc939489e8a7feb079bdecc873ed67", "on_output_sha256": "416cfdc4a6d23a800ebe988eac31ad858ecf4c2eea2039c014b0ba2a4b782f95", "category": "index_interpretation", "label_ja": "末尾から2番目の取り違え", "finding_ja": "両方とも昇順リストは正しく列挙。ありは[-2]を最後の値13としている。正解は10。", "evidence_strength": "emitted_steps_show_error", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "logic-039", "mode": "sampling", "rep": 2, "seed": 1427608789, "expected": 1, "off_answer": 1, "on_answer": 2, "off_output_sha256": "31dfd8660625903f4699650ed7010875db068b3e66f676011f99e092edb085d7", "on_output_sha256": "6de1b659b5791e1538780ee07b5f5c6be971dcc606d56f1c48513c70621648e6", "category": "wording_ambiguity", "label_ja": "設問の「複数」と最小値が曖昧", "finding_ja": "6人は定員6のBだけに入るため元の正解は1。ただし「複数の会議室」を文字どおり読むと2も合理的。明確なMTP品質悪化とは扱わない。既存の20問一括除外感度分析も保持。", "evidence_strength": "ambiguous_question", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "arithmetic-053", "mode": "sampling", "rep": 2, "seed": 1791303408, "expected": 93.95, "off_answer": 93.95, "on_answer": "93.95%", "off_output_sha256": "078e2015602d4edef674a10063999fcc4954d42a66e51a2704542668c07b21f0", "on_output_sha256": "cc3c1ce21d27eb6666900f8c8d802883e710d9df4b5d933292e3f4425f8ee65e", "category": "output_contract", "label_ja": "数値型・単位の指定違反", "finding_ja": "数値93.95は両方同じ。ありは単位付き文字列93.95%を返し、数値に単位を付けない出力仕様に違反。JSON構文自体は妥当。", "evidence_strength": "direct_output_contract", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "code-037", "mode": "sampling", "rep": 0, "seed": 501238494, "expected": 228, "off_answer": 228, "on_answer": 113, "off_output_sha256": "c81bd0d630230da9424508ee05b4b82007bf3d712ffb45210fa54459a51dc71b", "on_output_sha256": "589360f95bd9604828bbbb57db7d844b1a2a1322d2d6604297eb134428de38dd", "category": "loop_state", "label_ja": "ループ状態更新の誤り", "finding_ja": "正しい状態は29→58→115→228。ありの出力過程は29→58→57→113で、2回目の更新から異なる。", "evidence_strength": "emitted_steps_show_error", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "records-012", "mode": "sampling", "rep": 0, "seed": 3204126560, "expected": 542, "off_answer": 542, "on_answer": 442, "off_output_sha256": "e542dda9c47c4b5dca6feade7e667dd970293aa6dbb254ca3cf9a94faf25939f", "on_output_sha256": "8e8dead8064ee5ebcf6a50c8866a280657600643d17d43f4a076c08996d58812", "category": "addition", "label_ja": "正しい加数の合計を誤る", "finding_ja": "両方とも同じ12個の正しい加数を列挙。合計は542だが、ありは442と出力した。", "evidence_strength": "emitted_steps_show_error", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "arithmetic-018", "mode": "sampling", "rep": 2, "seed": 1902073540, "expected": 89.5, "off_answer": 89.5, "on_answer": "89.50%", "off_output_sha256": "4d6b5ce4ce7798b1f41e32b5a9100717d1e0ddc2cd7e86febc007890d793ed8b", "on_output_sha256": "956b6b09dde0ede9bc3d367731bdbb14a0b27131acaa887d8305f7eb952c49ee", "category": "output_contract", "label_ja": "数値型・単位の指定違反", "finding_ja": "89.5と89.50%は数量として同じだが、ありは単位付き文字列を返す。計算の間違いと区別する。", "evidence_strength": "direct_output_contract", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "code-054", "mode": "sampling", "rep": 1, "seed": 332336680, "expected": 101, "off_answer": 101, "on_answer": 87, "off_output_sha256": "6cc824f3be5b67feda66dd7454d4502b9d71d3eaea78d1a307bb027f1822cba2", "on_output_sha256": "df2599fd7f7b78a28e7cfea9c213c13b5e32ebeea98459c959b4b767e6c800d5", "category": "weighted_terms", "label_ja": "添字と値の積の取り違え", "finding_ja": "正しい項は0,-5,18,18,4,45,0,21。ありは2項目を-25、4項目を24としている。その誤った項の合計は出力87と一致する。", "evidence_strength": "emitted_steps_show_error", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "logic-027", "mode": "sampling", "rep": 0, "seed": 3179642043, "expected": "断定できない", "off_answer": "断定できない", "on_answer": "断定できる", "off_output_sha256": "5ecb23cbfd41d73890c3dd1add61e5b195d3d416badcfcbc1523a48da5af338b", "on_output_sha256": "c2942c2ea84287c71f36afc2204ea14610789442b4bdeebc770b446dc3710a9b", "category": "implication", "label_ja": "必要条件から原因を断定", "finding_ja": "障害なら通知という規則と通知の存在だけでは障害を断定できない。ありの断定できるは誤り。思考文がないので内部の誤推論の経路は不明。", "evidence_strength": "final_answer_only", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "arithmetic-092", "mode": "sampling", "rep": 1, "seed": 4096328267, "expected": 75, "off_answer": 75, "on_answer": 74, "off_output_sha256": "d45a5571ecfb6a7ffe593c58607a5076de3b56c2820916e5a6c5597848b45f44", "on_output_sha256": "7d15928f4cc1066209814e39f636c66ac50e6dfed244c2758a5700f8ab2cfa74", "category": "ceiling", "label_ja": "端数の切り上げを落とす", "finding_ja": "両方とも商74.01…を認識。74箱では5402個しか入らず、残り1個のため75箱必要。ありは74箱と回答した。", "evidence_strength": "emitted_steps_show_error", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "records-037", "mode": "sampling", "rep": 1, "seed": 4029129600, "expected": 371, "off_answer": 371, "on_answer": 381, "off_output_sha256": "41346347e6945bcf105df86aae0d4f2c4ac50cd47c678bca8c8f0f9bb2658ce2", "on_output_sha256": "93c312662937c44ed6d8614fa9f8baaacccfafece03f2d344b69fa3e9c1b71b1", "category": "addition", "label_ja": "正しい加数と異なる合計を返す", "finding_ja": "両方とも同じ12個の正しい加数を列挙。合計は371だが、ありは381と出力した。", "evidence_strength": "emitted_steps_show_error", "natural_stop_both": true, "max_output_tokens": 1024 }, { "task": "arithmetic-058", "mode": "sampling", "rep": 2, "seed": 3730798878, "expected": 87.96, "off_answer": 87.96, "on_answer": "87.96%", "off_output_sha256": "97f618f32248854e705c48229b346be10dc3f397e722464f888d6ecae2af90e9", "on_output_sha256": "474901e99f879599af9a2f29e7aeed796e681f9244f36b15d7eb493706cf2b5b", "category": "output_contract", "label_ja": "数値型・単位の指定違反", "finding_ja": "数値87.96は両方同じ。ありだけ単位付き文字列87.96%を返す出力仕様違反。", "evidence_strength": "direct_output_contract", "natural_stop_both": true, "max_output_tokens": 1024 } ], "original_scores_unchanged": true, "claim_limit": "Observed differences and visible error steps; not a proof that quantization, draft acceptance, RNG handling, KV state or a particular kernel caused the divergence. No missing reasoning is invented." }, "literature_background": { "reviewed_utc": "2026-10-31T05:34:53.682278+00:00", "reviewer": "Codex", "primary_sources_only": true, "sources": [ { "id": "leviathan2023", "title": "Fast Inference from Transformers via Speculative Decoding", "year": 2023, "status": "ICML 2023", "url": "https://proceedings.mlr.press/v202/leviathan23a.html", "finding_ja": "正しい検証・サンプリングで対象モデルの出力確率分布を保つ方式を示し、T5-XXLで2〜3倍の高速化を報告。", "limit_ja": "GLM・ExLlamaV3・3.10bpwでの個別回答の一致や品質を直接検証した研究ではない。" }, { "id": "chen2023", "title": "Accelerating Large Language Model Decoding with Speculative Sampling", "year": 2023, "status": "arXiv 2302.01318", "url": "https://arxiv.org/abs/2302.01318", "finding_ja": "Chinchilla70Bで2〜2.5倍のデコード高速化。出力分布の保存にはhardware numericsという留保がある。", "limit_ja": "検証は事実の正誤を保証する処理ではない。元のモデルが持つ誤答も分布に含まれる。" }, { "id": "deepseekv3", "title": "DeepSeek-V3 Technical Report, section5.4.3", "year": 2024, "status": "Technical report v2", "url": "https://arxiv.org/html/2412.19437v2#S5.SS4.SSS3", "finding_ja": "MTPの2番目トークンの採用率85〜90%、生成tokens/s1.8倍を報告。", "limit_ja": "採用率は回答正解率ではない。この節はなし正解・あり誤答の全例分類を提供していない。MTP学習のアブレーションと推論時ON/OFFは別の比較。" }, { "id": "liu2026", "title": "Speculative Decoding: Performance or Illusion?", "year": 2026, "status": "arXiv2601.11580v2, revised2026-03-18", "url": "https://arxiv.org/html/2601.11580v2", "finding_ja": "H100・vLLMでMTPを含む方式を評価。候補生成・検証の時間、採用率、バッチ数によって効果が変化。GLM-4.5-AirのMTPも測定。greedyでも出力長の差を観測。", "limit_ja": "メモリ表はモデル仕様に基づく重みとKVの計算値で、実行中activationを含まない。GLM-5.3-Flashの3bit品質比較ではない。著者の数値非決定性という説明だけで今回の原因を確定しない。" }, { "id": "yuan2025", "title": "Understanding and Mitigating Numerical Sources of Nondeterminism in LLM Inference", "year": 2025, "status": "NeurIPS2025", "url": "https://proceedings.neurips.cc/paper_files/paper/2025/file/f80094a824ba5912d4a2de169c404a40-Paper-Conference.pdf", "finding_ja": "バッチ数・GPU構成などによる浮動小数点の丸め差が、推論モデルの回答列や精度へ波及することを実測。", "limit_ja": "MTP専用研究ではなく、今回の差の原因を証明しない。重みを3bitから4bitにする変更と、演算精度をFP32へ変える変更は同じでない。" }, { "id": "yin2024", "title": "A Theoretical Perspective for Speculative Decoding Algorithm", "year": 2024, "status": "NeurIPS2024", "url": "https://proceedings.neurips.cc/paper_files/paper/2024/hash/e7349e785900b93d8b4971a3f2c1cefe-Abstract-Conference.html", "finding_ja": "分布保存方式を解析した上で、分布の偏りを許す拡張について推論コストと品質指標のトレードオフを理論解析。", "limit_ja": "対象は定義されたアルゴリズム族と分布のずれであり、一般的な日本語実務正解率の低下量ではない。今回ExLlamaV3がその緩い採用方式を使う証拠として扱わない。" }, { "id": "windowed2026", "title": "Windowed-MTP: Removing the Full-Context Draft-KV Tax at Million-Token Context", "year": 2026, "status": "arXiv2607.21535v1 preprint", "url": "https://arxiv.org/html/2607.21535v1", "finding_ja": "実1M入力ではMTP候補側の全履歴KV参照が負担になる。SGLangの別モデル群で、深い先読みがMTPなしより遅くなる条件、候補KVが全KVの7.7〜11%を占める測定を報告。", "limit_ja": "GLM/ExLlamaV3の実測ではない。当試験は1M容量だが実入力は4K/32Kであり、1Mを埋めた結果へ転用しない。" } ], "scope": "Literature background for the ongoing output-budget and paired-regression revision. No new GPU campaign authorized or launched by this review.", "conclusions_ja": [ "MTPの候補生成と、候補を本体で検証・採用する手続きは区別する。", "分布保存と、同じseedで同じ文字列になることと、回答が正解であることは別。", "追加計算・重みやKVメモリ・待ち時間・同時処理性能を測る。省エネ効果は実測なしに主張しない。", "今回のON/OFFでは補助モデルとKVを両方に保持しており、MTPを全くロードしない場合に対する追加VRAM差は測っていない。", "内部原因の確定には同じprefixでのlogits・採用判定・RNG状態等の別検証が必要。現在の最終回答・出力された思考だけで演算原因を断定しない。" ] }, "claim_limits": [ "Each cell has12 unique tasks repeated3seeds;36 answers are not36 independent tasks.", "This24-task follow-up does not establish general MTP noninferiority or quality equivalence.", "The older600-question primary study and its predeclared criterion are separate evidence.", "4K uses50 records and32K uses600 records; input-length differences also change task difficulty.", "Synthetic records are placed after repeated irrelevant reference-log padding; input length includes that padding and chat control tokens. No positional retrieval sweep was performed.", "Original1024 budget used1seed and a different session; matched-original-seed results are also shown.", "No calculator, retrieval or code-execution tools were supplied to the model.", "First-answer medians exclude unanswered requests; count and cap rate must be read together.", "Generation tokens/s includes reasoning/control tokens; not a completion-time ratio.", "No actual250K/1M-filled input, client compression-policy or4bit comparison was tested.", "Output caps above8192 were not tested in this frozen follow-up." ] }