{ "model": "GLM-5.3-Flash", "host": "HGX B200", "concurrency": 1, "context_capacity": 40960, "generation_limit": 1024, "rows": [ { "input_tokens": 8193, "repetitions_each": 3, "llama_cpp_g1": { "generation_tps": { "values": [ 63.577998996895296, 63.5300878761998, 63.56585265865646 ], "median": 63.56585265865646, "min": 63.5300878761998, "max": 63.577998996895296 }, "first_generated_seconds": { "values": [ 22.320389157976024, 22.33184133504983, 22.37715708394535 ], "median": 22.33184133504983, "min": 22.320389157976024, "max": 22.37715708394535 }, "input_tps": { "values": [ 367.37935032102985, 367.31445609910793, 366.65549947894914 ], "median": 367.31445609910793, "min": 366.65549947894914, "max": 367.37935032102985 } }, "exl3_g2_off": { "actual_input_tokens": 8193, "n": 3, "all_conservative_intertoken_tps_above50": true, "generation_tps_native": { "values": [ 60.09055744850449, 59.4433888937625, 59.524057443092495 ], "median": 59.524057443092495, "min": 59.4433888937625, "max": 60.09055744850449 }, "generation_tps_intertoken": { "values": [ 60.015066798443556, 59.377778089243996, 59.456029948871816 ], "median": 59.456029948871816, "min": 59.377778089243996, "max": 60.015066798443556 }, "input_seconds_native": { "values": [ 2.478013515472412, 2.092078685760498, 2.0884130001068115 ], "median": 2.092078685760498, "min": 2.0884130001068115, "max": 2.478013515472412 }, "input_tps_native": { "values": [ 3306.2773664646757, 3916.200693484785, 3923.074602380358 ], "median": 3916.200693484785, "min": 3306.2773664646757, "max": 3923.074602380358 }, "first_generated_seconds": { "values": [ 2.511534914956428, 2.1108196099521592, 2.1073353299871087 ], "median": 2.1108196099521592, "min": 2.1073353299871087, "max": 2.511534914956428 }, "first_visible_seconds": { "values": [ 2.511534914956428, 2.1271576220169663, 2.12368782190606 ], "median": 2.1271576220169663, "min": 2.12368782190606, "max": 2.511534914956428 }, "first_answer_visible_seconds": { "values": [ 2.6757730098906904, 2.1271576220169663, 2.12368782190606 ], "median": 2.1271576220169663, "min": 2.12368782190606, "max": 2.6757730098906904 }, "end_to_end_seconds": { "values": [ 15.784663100959733, 17.33802134799771, 16.793299278942868 ], "median": 16.793299278942868, "min": 15.784663100959733, "max": 17.33802134799771 } }, "exl3_g2_mtp3": { "actual_input_tokens": 8193, "n": 3, "all_conservative_intertoken_tps_above50": true, "generation_tps_native": { "values": [ 82.89585427095253, 95.55501805046923, 100.92978586160334 ], "median": 95.55501805046923, "min": 82.89585427095253, "max": 100.92978586160334 }, "generation_tps_intertoken": { "values": [ 82.79513026819316, 95.46170260315432, 100.81390780665316 ], "median": 95.46170260315432, "min": 82.79513026819316, "max": 100.81390780665316 }, "input_seconds_native": { "values": [ 2.5195040702819824, 2.134533405303955, 2.135387897491455 ], "median": 2.135387897491455, "min": 2.134533405303955, "max": 2.5195040702819824 }, "input_tps_native": { "values": [ 3251.830428312442, 3838.3095713760104, 3836.7736417466444 ], "median": 3836.7736417466444, "min": 3251.830428312442, "max": 3838.3095713760104 }, "first_generated_seconds": { "values": [ 3.1026840769918635, 2.1630470579257235, 2.1635261320043355 ], "median": 2.1635261320043355, "min": 2.1630470579257235, "max": 3.1026840769918635 }, "first_visible_seconds": { "values": [ 3.1026840769918635, 2.1630665080156177, 2.16354475193657 ], "median": 2.16354475193657, "min": 2.1630665080156177, "max": 3.1026840769918635 }, "first_answer_visible_seconds": { "values": [ 3.919887642026879, 2.1630665080156177, 2.16354475193657 ], "median": 2.16354475193657, "min": 2.1630665080156177, "max": 3.919887642026879 }, "end_to_end_seconds": { "values": [ 12.449465634999797, 12.85473516897764, 10.768680646899156 ], "median": 12.449465634999797, "min": 10.768680646899156, "max": 12.85473516897764 }, "draft_acceptance_ratio": { "values": [ 0.4929718875502008, 0.49514563106796117, 0.5411646586345381 ], "median": 0.49514563106796117, "min": 0.4929718875502008, "max": 0.5411646586345381 } }, "mtp_speedup": 1.6055848781905715 }, { "input_tokens": 32768, "repetitions_each": 3, "llama_cpp_g1": { "generation_tps": { "values": [ 61.71993843885557, 61.65063518672581, 61.73078791007911 ], "median": 61.71993843885557, "min": 61.65063518672581, "max": 61.73078791007911 }, "first_generated_seconds": { "values": [ 84.18414412299171, 84.15482447296381, 83.95840363309253 ], "median": 84.15482447296381, "min": 83.95840363309253, "max": 84.18414412299171 }, "input_tps": { "values": [ 389.4147497277585, 389.5557868067508, 390.57357913723 ], "median": 389.5557868067508, "min": 389.4147497277585, "max": 390.57357913723 } }, "exl3_g2_off": { "actual_input_tokens": 32768, "n": 3, "all_conservative_intertoken_tps_above50": true, "generation_tps_native": { "values": [ 59.67992378462389, 60.15170206566067, 58.888886431651 ], "median": 59.67992378462389, "min": 58.888886431651, "max": 60.15170206566067 }, "generation_tps_intertoken": { "values": [ 59.614699824203534, 60.08988120022834, 58.82786167887209 ], "median": 59.614699824203534, "min": 58.82786167887209, "max": 60.08988120022834 }, "input_seconds_native": { "values": [ 8.95195460319519, 8.263920545578003, 8.264732360839844 ], "median": 8.264732360839844, "min": 8.263920545578003, "max": 8.95195460319519 }, "input_tps_native": { "values": [ 3660.4296438572455, 3965.188171797471, 3964.798685467679 ], "median": 3964.798685467679, "min": 3660.4296438572455, "max": 3965.188171797471 }, "first_generated_seconds": { "values": [ 8.971125897020102, 8.28313649003394, 8.28419387107715 ], "median": 8.28419387107715, "min": 8.28313649003394, "max": 8.971125897020102 }, "first_visible_seconds": { "values": [ 8.971125897020102, 8.299386703991331, 8.30083858803846 ], "median": 8.30083858803846, "min": 8.299386703991331, "max": 8.971125897020102 }, "first_answer_visible_seconds": { "values": [ 9.810961136012338, 8.299386703991331, 8.30083858803846 ], "median": 8.30083858803846, "min": 8.299386703991331, "max": 9.810961136012338 }, "end_to_end_seconds": { "values": [ 24.287577643990517, 24.445283739012666, 24.656483210041188 ], "median": 24.445283739012666, "min": 24.287577643990517, "max": 24.656483210041188 } }, "exl3_g2_mtp3": { "actual_input_tokens": 32768, "n": 3, "all_conservative_intertoken_tps_above50": true, "generation_tps_native": { "values": [ 93.06145845088606, 93.02216469748399, 92.72427426220128 ], "median": 93.02216469748399, "min": 92.72427426220128, "max": 93.06145845088606 }, "generation_tps_intertoken": { "values": [ 92.95449125726437, 92.9313227397716, 92.62975410495133 ], "median": 92.9313227397716, "min": 92.62975410495133, "max": 92.95449125726437 }, "input_seconds_native": { "values": [ 9.163030624389648, 8.46372389793396, 8.459462881088257 ], "median": 8.46372389793396, "min": 8.459462881088257, "max": 9.163030624389648 }, "input_tps_native": { "values": [ 3576.1094056348506, 3871.581870481248, 3873.5319795840987 ], "median": 3871.581870481248, "min": 3576.1094056348506, "max": 3873.5319795840987 }, "first_generated_seconds": { "values": [ 9.192164542968385, 8.492621793993749, 8.48864426300861 ], "median": 8.492621793993749, "min": 8.48864426300861, "max": 9.192164542968385 }, "first_visible_seconds": { "values": [ 9.192164542968385, 8.492649144027382, 8.488666923018172 ], "median": 8.492649144027382, "min": 8.488666923018172, "max": 9.192164542968385 }, "first_answer_visible_seconds": { "values": [ 9.907278175000101, 8.492649144027382, 8.488666923018172 ], "median": 8.492649144027382, "min": 8.488666923018172, "max": 9.907278175000101 }, "end_to_end_seconds": { "values": [ 18.51600027899258, 19.47699537104927, 19.044399930979125 ], "median": 19.044399930979125, "min": 18.51600027899258, "max": 19.47699537104927 }, "draft_acceptance_ratio": { "values": [ 0.4767225325884544, 0.4755134281200632, 0.4720853858784893 ], "median": 0.4755134281200632, "min": 0.4720853858784893, "max": 0.4767225325884544 } }, "mtp_speedup": 1.5588659007562684 } ], "memory_peak_GiB": { "mtp_off": [ 62.453125, 62.4921875 ], "mtp_on": [ 66.2265625, 62.509765625 ] }, "configurations": [ { "id": "llama_cpp_g1", "label": "llama.cpp / B200×1", "quantization": "UD-IQ3_XXS GGUF", "transport": "HTTP", "cpu_embedding": "CPU-mapped embedding buffer retained in baseline" }, { "id": "exl3_g2_off", "label": "ExLlamaV3 / B200×2 / MTPなし", "quantization": "EXL3 sc_3.10bpw", "transport": "in-process API", "embedding": "isolated GPU embedding adaptation" }, { "id": "exl3_g2_mtp3", "label": "ExLlamaV3 / B200×2 / MTPあり", "quantization": "EXL3 sc_3.10bpw", "transport": "in-process API", "draft_tokens": 3, "embedding": "same isolated GPU embedding adaptation" } ], "editorial_limits": [ "Engine comparison changes GPU count, quantization, embedding placement and measurement interface; not engine-only causation.", "TTFT is request/enqueue-to-first-token, not model file loading, not isolated prefill. Native engine prefill rates are separately retained.", "MTPon/off uses the same model, quantization, engine variant, GPUs, inputs and sampling; draft/cache allocation is the intended changed condition.", "All displayed speeds are medians of per-trial generation averages, include reasoning/control tokens, and are not instantaneous minima or production guarantees.", "All6MTP trials retained;2hit1024tokens and end incomplete. MTPoff andMTPon both have quality findings; no attribution of quality differences to MTP without a controlled quality study.", "H100, H200 NVL and RTX PRO6000 are not measured by this package; no numeric forecast.", "B200 results and underlying raw runs retained; old stopped campaigns remain stopped." ], "llamacpp_mtp_B200": { "host": "HGX B200", "GPUs": 2, "concurrent_users": 1, "model": "GLM-5.3-Flash", "quantization": "GGUF UD-IQ3_XXS", "context_capacity": 40960, "generation_limit": 1024, "target_split": "layer 1:1", "MTP_on": { "spec_type": "draft-mtp", "max_draft_tokens": 3, "sampling": "probabilistic" }, "MTP_off": "none", "groups": [ { "input_tokens": 8193, "repetitions_each": 3, "off": { "actual_input_tokens": 8193, "repetitions": 3, "generation_tokens_per_second": { "all": [ 53.01411468020781, 62.42567300017588, 62.50841860176169 ], "median": 62.42567300017588, "min": 53.01411468020781, "max": 62.50841860176169 }, "first_streamed_generated_text_seconds": { "all": [ 15.459843400050886, 15.278459863038734, 15.262090574018657 ], "median": 15.278459863038734, "min": 15.262090574018657, "max": 15.459843400050886 }, "first_answer_text_seconds": { "all": [ 18.094800171093084, 15.278459863038734, 15.262090574018657 ], "median": 15.278459863038734, "min": 15.262090574018657, "max": 18.094800171093084 }, "input_processing_seconds": { "all": [ 15.349368, 15.236589, 15.232861999999999 ], "median": 15.236589, "min": 15.232861999999999, "max": 15.349368 }, "end_to_end_seconds": { "all": [ 31.475104280980304, 30.525371531024575, 30.840471785049886 ], "median": 30.840471785049886, "min": 30.525371531024575, "max": 31.475104280980304 }, "capped_trials": [], "draft_acceptance_fraction": null }, "on": { "actual_input_tokens": 8193, "repetitions": 3, "generation_tokens_per_second": { "all": [ 76.22982713921762, 79.82800810527158, 80.5982898508317 ], "median": 79.82800810527158, "min": 76.22982713921762, "max": 80.5982898508317 }, "first_streamed_generated_text_seconds": { "all": [ 16.9353461220162, 16.871887013083324, 16.84939561400097 ], "median": 16.871887013083324, "min": 16.84939561400097, "max": 16.9353461220162 }, "first_answer_text_seconds": { "all": [ 17.78652513108682, 16.871887013083324, 16.84939561400097 ], "median": 16.871887013083324, "min": 16.84939561400097, "max": 17.78652513108682 }, "input_processing_seconds": { "all": [ 16.829689, 16.803206, 16.802743999999997 ], "median": 16.803206, "min": 16.802743999999997, "max": 16.829689 }, "end_to_end_seconds": { "all": [ 29.37180881202221, 27.645839194068685, 29.505253204028122 ], "median": 29.37180881202221, "min": 27.645839194068685, "max": 29.505253204028122 }, "capped_trials": [ "ctx8192-r3" ], "draft_acceptance_fraction": { "all": [ 0.49476439790575916, 0.5367573011077543, 0.5424164524421594 ], "median": 0.5367573011077543, "min": 0.49476439790575916, "max": 0.5424164524421594 } }, "generation_median_ratio": 1.2787688825564871, "generation_median_change_percent": 27.876888255648712, "first_streamed_text_median_change_seconds": 1.5934271500445902, "on_all_trial_means_above_50": true }, { "input_tokens": 32768, "repetitions_each": 3, "off": { "actual_input_tokens": 32768, "repetitions": 3, "generation_tokens_per_second": { "all": [ 60.63912368648685, 60.6719484029221, 60.70372479179465 ], "median": 60.6719484029221, "min": 60.63912368648685, "max": 60.70372479179465 }, "first_streamed_generated_text_seconds": { "all": [ 58.53948803897947, 58.624004732002504, 58.55625737295486 ], "median": 58.55625737295486, "min": 58.53948803897947, "max": 58.624004732002504 }, "first_answer_text_seconds": { "all": [ 58.72643397108186, 60.29133868601639, 59.43423853290733 ], "median": 59.43423853290733, "min": 58.72643397108186, "max": 60.29133868601639 }, "input_processing_seconds": { "all": [ 58.501602, 58.566722, 58.518316 ], "median": 58.518316, "min": 58.501602, "max": 58.566722 }, "end_to_end_seconds": { "all": [ 72.73888725508004, 74.59567479300313, 72.7900477029616 ], "median": 72.7900477029616, "min": 72.73888725508004, "max": 74.59567479300313 }, "capped_trials": [], "draft_acceptance_fraction": null }, "on": { "actual_input_tokens": 32768, "repetitions": 3, "generation_tokens_per_second": { "all": [ 73.52232967010374, 72.59330832738225, 72.77739825119912 ], "median": 72.77739825119912, "min": 72.59330832738225, "max": 73.52232967010374 }, "first_streamed_generated_text_seconds": { "all": [ 64.41275359608699, 64.5085239650216, 64.46981710090768 ], "median": 64.46981710090768, "min": 64.41275359608699, "max": 64.5085239650216 }, "first_answer_text_seconds": { "all": [ 64.56360230804421, 65.41622533102054, 65.02215314190835 ], "median": 65.02215314190835, "min": 64.56360230804421, "max": 65.41622533102054 }, "input_processing_seconds": { "all": [ 64.382014, 64.44702, 64.44107699999999 ], "median": 64.44107699999999, "min": 64.382014, "max": 64.44702 }, "end_to_end_seconds": { "all": [ 77.8379915910773, 78.4913438380463, 76.97445666091517 ], "median": 77.8379915910773, "min": 76.97445666091517, "max": 78.4913438380463 }, "capped_trials": [], "draft_acceptance_fraction": { "all": [ 0.5154639175257731, 0.5020576131687243, 0.505524861878453 ], "median": 0.505524861878453, "min": 0.5020576131687243, "max": 0.5154639175257731 } }, "generation_median_ratio": 1.1995230113245217, "generation_median_change_percent": 19.952301132452167, "first_streamed_text_median_change_seconds": 5.913559727952816, "on_all_trial_means_above_50": true } ], "GPU_peak_MiB": { "off": { "0": 58218, "1": 56692 }, "on": { "0": 58458, "1": 59854 } }, "limits": [ "GPU memory is a B200 observation, not proof of H100 execution or speed.", "Reported generation includes reasoning tokens; first streamed text and first answer text are separate.", "Trial-average >50 is not an instantaneous speed or service-level guarantee.", "MTP probabilistic sampling can produce different outputs even with the same seed.", "No removal of slow, capped, or qualitatively incorrect outputs.", "Fixed 3-draft comparison, not an exhaustive optimization search.", "Earlier stopped 1M/256K/layer/row campaigns are unchanged.", "ExLlamaV3 uses a different quantization and measurement interface; compare its MTP effect within its own paired conditions." ] } }