{ "metadata": { "version": "0.4.29", "engine": "vllm", "model": "GLM-5.3-Flash-EXL3-4bpw", "server": "127.0.0.1:5001", "timestamp": "2026-08-28T06:45:45.924295", "decode_mode": "duration", "primary_decode_layer": "sustained_decode", "duration_per_test": 10.0, "request_count": 0, "warmup_request_count": 0, "run_burst": false, "prefill_mode": "skipped", "standalone_prefill": false, "prefill_only": false, "skip_prefill": true, "burst_e2e_status": "not_run_use_--run-burst", "burst_request_count": 0, "burst_warmup_request_count": 0, "burst_requests_per_concurrency": 5, "decode_warmup_seconds": 3.0, "decode_warmup_context": 65536, "decode_warmup_concurrency": 1, "cell_warmup_timeout_seconds": 0.0, "cell_warmup_timeout_policy": "<=32k:60s,64k:120s,>=128k:180s when override is 0", "show_capacity_limited_values": false, "max_tokens": 4096, "temperature": null, "ignore_eos": true, "max_total_tokens": 129473, "dcp_size": 0, "metrics_available": true, "metrics_warning": "", "concurrency_levels": [ 1 ], "context_lengths": [ 0, 32768, 65536 ], "startup_diagnostics_available": true, "nvidia_p2p_override_effective": true, "p2pmark_status": "not_run", "amd_fabric_status": "not_run" }, "startup_diagnostics": { "version": "0.4.29", "server_url": "http://127.0.0.1:5001", "hostname": "pop-os", "uname": "Linux pop-os 6.18.7-76061807-generic #202601231045~1769703228~24.04~cb87b5b SMP PREEMPT_DYNAMIC Thu J x86_64 x86_64 x86_64 GNU/Linux", "env": {}, "args": { "concurrency": "1", "contexts": "0,32k,64k", "max_tokens": 4096, "duration": 10.0, "request_count": 0, "run_burst": false, "standalone_prefill": false, "prefill_only": false, "skip_prefill": true, "prefill_contexts": "8k,64k,128k", "prefill_metric": "client", "dcp_size": 0, "kv_budget": 129473 }, "nvidia_p2p_override": { "effective": true, "configured": true, "params_path": "/proc/driver/nvidia/params", "params_available": true, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": true, "runtime": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1", "DmaRemapPeerMmio": "1" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [], "mismatched": {}, "registry_dwords": "ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "nvidia_smi_query": { "cmd": [ "nvidia-smi", "--query-gpu=index,name,driver_version,pci.bus_id,pcie.link.gen.current,pcie.link.width.current,power.limit", "--format=csv,noheader,nounits" ], "returncode": 0, "stdout": "0, NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.57.04, 00000000:01:00.0, 1, 16, 300.00\n1, NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 610.57.04, 00000000:21:00.0, 2, 16, 600.00\n2, NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.57.04, 00000000:81:00.0, 1, 16, 300.00\n3, NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 610.57.04, 00000000:C1:00.0, 2, 16, 600.00", "stderr": "" }, "nvidia_smi_topo": { "cmd": [ "nvidia-smi", "topo", "-m" ], "returncode": 0, "stdout": "\u001b[4mGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\u001b[0m\nGPU0\t X \tNODE\tNODE\tNODE\t0-47\t0\t\tN/A\nGPU1\tNODE\t X \tNODE\tNODE\t0-47\t0\t\tN/A\nGPU2\tNODE\tNODE\t X \tNODE\t0-47\t0\t\tN/A\nGPU3\tNODE\tNODE\tNODE\t X \t0-47\t0\t\tN/A\n\nLegend:\n\n X = Self\n SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)\n NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node\n PHB = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)\n PXB = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)\n PIX = Connection traversing at most a single PCIe bridge\n NV# = Connection traversing a bonded set of # NVLinks", "stderr": "" } }, "nvidia_p2p_override": { "effective": true, "configured": true, "params_path": "/proc/driver/nvidia/params", "params_available": true, "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", "modprobe_available": true, "runtime": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1", "DmaRemapPeerMmio": "1" }, "expected": { "ForceP2P": "0x11", "RMForceP2PType": "1", "RMPcieP2PType": "2", "GrdmaPciTopoCheckOverride": "1", "EnableResizableBar": "1" }, "missing": [], "mismatched": {}, "registry_dwords": "ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1", "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" }, "p2pmark": { "status": "not_run" }, "amd_fabric": { "status": "not_run" }, "hardware_run_summary": { "samples": 47, "duration_seconds": 109.922, "gpu_count": 4, "cpu_util_avg_pct": 7.77, "cpu_temp_max_c": 67.25, "gpu_util_avg_pct": 45.13, "gpu_util_max_pct": 100.0, "mem_util_avg_pct": 19.02, "mem_util_max_pct": 63.0, "temp_avg_c": 57.82, "temp_max_c": 93.0, "power_total_avg_w": 910.58, "power_total_max_w": 1028.72, "power_limit_total_w": 1800.0, "vram_used_avg_mb": 189586.0, "vram_used_max_mb": 189586.0, "vram_total_mb": 391548.0, "vram_used_avg_pct": 48.42, "vram_used_max_pct": 48.42, "pcie_rx_avg_mb_s": 12035.43, "pcie_rx_max_mb_s": 28897.0, "pcie_tx_avg_mb_s": 11422.64, "pcie_tx_max_mb_s": 27898.0 }, "event_log": [ "06:43:54 benchmark start engine=vllm", "06:43:54 startup server=http://127.0.0.1:5001 model=GLM-5.3-Flash-EXL3-4bpw", "06:43:54 startup decode concurrency=1 contexts=0,32k,64k", "06:43:54 startup NVIDIA P2P override: enabled: runtime NVIDIA P2P override matches expected RegistryDwords", "06:43:54 startup KV cache budget manual: 129,473 tokens", "06:43:54 startup engine vLLM 0.1.dev20111+g7f1e92bec.d20260827 models=['GLM-5.3-Flash-EXL3-4bpw']", "06:43:54 startup model context length: 98,304 tokens", "06:43:54 startup prefill tests: skipped", "06:43:54 startup calibrating padding text run=mcbixrdixlnd up_to=64k", "06:43:54 startup token targeting: estimate from 8k", "06:43:54 startup calibrated: 6.18 chars/token (cached, source=8k)", "06:43:54 startup context 32k: 202,404 chars (~32,767 tokens)", "06:43:54 startup context 64k: 404,809 chars (~65,535 tokens)", "06:43:54 startup startup preparation done", "06:43:54 hardware monitor interval=2s", "06:43:54 decode warmup start", "06:43:55 decode warmup start C=1 ctx=64k 3s", "06:43:55 cell start C=1 ctx=64k", "06:44:19 ready C=1 ctx=64k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "06:44:22 cell done C=1 ctx=64k 137.4 tok/s | norm 51.8 step/s len=2.65", "06:44:22 decode warmup done C=1 ctx=64k", "06:44:24 cell start C=1 ctx=0", "06:44:29 ready C=1 ctx=0 running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "06:44:39 cell done C=1 ctx=0 145.5 tok/s | norm 52.5 step/s len=2.77", "06:44:41 cell start C=1 ctx=32k", "06:44:56 ready C=1 ctx=32k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "06:45:06 cell done C=1 ctx=32k 147.2 tok/s | norm 51.1 step/s len=2.88", "06:45:08 cell start C=1 ctx=64k", "06:45:33 ready C=1 ctx=64k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "06:45:43 cell done C=1 ctx=64k 151.5 tok/s | norm 50.4 step/s len=3.00" ], "prefill": {}, "results": [ { "concurrency": 1, "context_tokens": 0, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.993703, "measurement_wall_seconds": 10.000804, "client_output_tokens": 1454, "server_output_tokens": 1454, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 145.49161652209398, "per_request_avg_tps": 145.49161652209398, "ttft_avg": 0.12390130397398025, "ttft_p50": 0.12390130397398025, "ttft_p90": 0.12390130397398025, "ttft_p99": 0.12390130397398025, "time_to_second_token_avg": 0.01941704103955999, "time_to_second_token_p50": 0.01941704103955999, "time_to_second_token_p90": 0.01941704103955999, "time_to_second_token_p99": 0.01941704103955999, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.006799128897581661, "inter_token_latency_p50": 0.006799128897581661, "inter_token_latency_p90": 0.006799128897581661, "inter_token_latency_p99": 0.006799128897581661, "output_tps_per_user_avg": 147.0776646631429, "output_tps_per_user_p50": 147.0776646631429, "output_tps_per_user_p90": 147.0776646631429, "output_tps_per_user_p99": 147.0776646631429, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.019130468264624174, "chunk_inter_token_latency_p50": 0.019130468264624174, "chunk_inter_token_latency_p90": 0.019130468264624174, "chunk_inter_token_latency_p99": 0.019130468264624174, "input_seq_len_avg": 78.0, "output_seq_len_avg": 2266.0, "output_seq_len_p50": 2266.0, "output_seq_len_p90": 2266.0, "output_seq_len_p99": 2266.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 0.12390130397398025, "time_to_second_token": 0.01941704103955999, "latency": 0.0, "inter_token_latency_avg": 0.006799128897581661, "chunk_inter_token_latency_avg": 0.019130468264624174, "input_tokens": 78, "output_tokens": 2266, "output_tps_per_user": 147.0776646631429, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 1454, "wall_time": 15.543849968002178, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 145.31791904792772, "server_utilization": 0.6603773584905661, "server_spec_accept_rate": 0.2527891156462585, "server_spec_accept_length": 2.7695238095238093, "server_spec_drafts": 525, "server_spec_draft_tokens": 3675, "server_spec_accepted_tokens": 929, "server_spec_pos_accept": [ 0.7162, 0.459, 0.2762, 0.1505, 0.0895, 0.0514, 0.0267 ], "server_engine_steps": 525.0, "server_steps_per_s": 52.53308024353462, "server_accept_len_effective": 2.7695238095238097, "accept_norm_tps": 0.0, "accept_norm_ref_len": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 5.53, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 7.179, "gpu_count": 4, "cpu_util_avg_pct": 7.1, "cpu_temp_max_c": 67.12, "gpu_util_avg_pct": 49.5, "gpu_util_max_pct": 99.0, "mem_util_avg_pct": 29.56, "mem_util_max_pct": 63.0, "temp_avg_c": 56.69, "temp_max_c": 84.0, "power_total_avg_w": 936.38, "power_total_max_w": 937.87, "power_limit_total_w": 1800.0, "vram_used_avg_mb": 189586.0, "vram_used_max_mb": 189586.0, "vram_total_mb": 391548.0, "vram_used_avg_pct": 48.42, "vram_used_max_pct": 48.42, "pcie_rx_avg_mb_s": 3023.0, "pcie_rx_max_mb_s": 3065.0, "pcie_tx_avg_mb_s": 2752.25, "pcie_tx_max_mb_s": 2819.0 } }, { "concurrency": 1, "context_tokens": 32768, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.984746, "measurement_wall_seconds": 10.000889, "client_output_tokens": 1470, "server_output_tokens": 1470, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 147.22458371168588, "per_request_avg_tps": 147.22458371168588, "ttft_avg": 5.372631194011774, "ttft_p50": 5.372631194011774, "ttft_p90": 5.372631194011774, "ttft_p99": 5.372631194011774, "time_to_second_token_avg": 0.010886964970268309, "time_to_second_token_p50": 0.010886964970268309, "time_to_second_token_p90": 0.010886964970268309, "time_to_second_token_p99": 0.010886964970268309, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.0064921210413876685, "inter_token_latency_p50": 0.0064921210413876685, "inter_token_latency_p90": 0.0064921210413876685, "inter_token_latency_p99": 0.0064921210413876685, "output_tps_per_user_avg": 154.03286439438497, "output_tps_per_user_p50": 154.03286439438497, "output_tps_per_user_p90": 154.03286439438497, "output_tps_per_user_p99": 154.03286439438497, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.01996819047578328, "chunk_inter_token_latency_p50": 0.01996819047578328, "chunk_inter_token_latency_p90": 0.01996819047578328, "chunk_inter_token_latency_p99": 0.01996819047578328, "input_seq_len_avg": 32321.0, "output_seq_len_avg": 2031.0, "output_seq_len_p50": 2031.0, "output_seq_len_p90": 2031.0, "output_seq_len_p99": 2031.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 5.372631194011774, "time_to_second_token": 0.010886964970268309, "latency": 0.0, "inter_token_latency_avg": 0.0064921210413876685, "chunk_inter_token_latency_avg": 0.01996819047578328, "input_tokens": 32321, "output_tokens": 2031, "output_tps_per_user": 154.03286439438497, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 1470, "wall_time": 24.152813333028462, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 146.92232203610482, "server_utilization": 0.7169811320754718, "server_spec_accept_rate": 0.2689075630252101, "server_spec_accept_length": 2.8823529411764706, "server_spec_drafts": 510, "server_spec_draft_tokens": 3570, "server_spec_accepted_tokens": 960, "server_spec_pos_accept": [ 0.7588, 0.502, 0.2745, 0.1647, 0.0922, 0.051, 0.0392 ], "server_engine_steps": 510.0, "server_steps_per_s": 51.077916797931834, "server_accept_len_effective": 2.8823529411764706, "accept_norm_tps": 0.0, "accept_norm_ref_len": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 14.147, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 7.183, "gpu_count": 4, "cpu_util_avg_pct": 7.15, "cpu_temp_max_c": 67.25, "gpu_util_avg_pct": 49.5, "gpu_util_max_pct": 99.0, "mem_util_avg_pct": 28.31, "mem_util_max_pct": 59.0, "temp_avg_c": 58.38, "temp_max_c": 87.0, "power_total_avg_w": 931.73, "power_total_max_w": 939.71, "power_limit_total_w": 1800.0, "vram_used_avg_mb": 189586.0, "vram_used_max_mb": 189586.0, "vram_total_mb": 391548.0, "vram_used_avg_pct": 48.42, "vram_used_max_pct": 48.42, "pcie_rx_avg_mb_s": 2903.5, "pcie_rx_max_mb_s": 2917.0, "pcie_tx_avg_mb_s": 2706.5, "pcie_tx_max_mb_s": 2713.0 } }, { "concurrency": 1, "context_tokens": 65536, "benchmark_mode": "duration", "request_count_target": 0, "warmup_request_count": 0, "measurement_seconds": 9.992618, "measurement_wall_seconds": 10.000719, "client_output_tokens": 1514, "server_output_tokens": 1514, "aggregate_source": "openai_continuous_usage", "aggregate_tps": 151.51185116953508, "per_request_avg_tps": 151.51185116953508, "ttft_avg": 10.824413124995772, "ttft_p50": 10.824413124995772, "ttft_p90": 10.824413124995772, "ttft_p99": 10.824413124995772, "time_to_second_token_avg": 0.00351814302848652, "time_to_second_token_p50": 0.00351814302848652, "time_to_second_token_p90": 0.00351814302848652, "time_to_second_token_p99": 0.00351814302848652, "request_latency_avg": 0.0, "request_latency_p50": 0.0, "request_latency_p90": 0.0, "request_latency_p99": 0.0, "inter_token_latency_avg": 0.006552730234684932, "inter_token_latency_p50": 0.006552730234684932, "inter_token_latency_p90": 0.006552730234684932, "inter_token_latency_p99": 0.006552730234684932, "output_tps_per_user_avg": 152.60814411476866, "output_tps_per_user_p50": 152.60814411476866, "output_tps_per_user_p90": 152.60814411476866, "output_tps_per_user_p99": 152.60814411476866, "e2e_output_tps_per_user_avg": 0.0, "e2e_output_tps_per_user_p50": 0.0, "e2e_output_tps_per_user_p90": 0.0, "e2e_output_tps_per_user_p99": 0.0, "chunk_inter_token_latency_avg": 0.01984675848059249, "chunk_inter_token_latency_p50": 0.01984675848059249, "chunk_inter_token_latency_p90": 0.01984675848059249, "chunk_inter_token_latency_p99": 0.01984675848059249, "input_seq_len_avg": 64513.0, "output_seq_len_avg": 2106.0, "output_seq_len_p50": 2106.0, "output_seq_len_p90": 2106.0, "output_seq_len_p99": 2106.0, "request_count": 1, "completed_request_count": 0, "request_samples": [ { "ttft": 10.824413124995772, "time_to_second_token": 0.00351814302848652, "latency": 0.0, "inter_token_latency_avg": 0.006552730234684932, "chunk_inter_token_latency_avg": 0.01984675848059249, "input_tokens": 64513, "output_tokens": 2106, "output_tps_per_user": 152.60814411476866, "e2e_output_tps_per_user": 0.0, "completed": false } ], "total_tokens": 1514, "wall_time": 35.809400569007266, "num_completed": 1, "num_errors": 0, "server_gen_throughput": 151.32474166571015, "server_utilization": 0.7547169811320755, "server_spec_accept_rate": 0.286281179138322, "server_spec_accept_length": 3.003968253968254, "server_spec_drafts": 504, "server_spec_draft_tokens": 3528, "server_spec_accepted_tokens": 1010, "server_spec_pos_accept": [ 0.7381, 0.5099, 0.3115, 0.1964, 0.119, 0.0774, 0.0516 ], "server_engine_steps": 504.0, "server_steps_per_s": 50.43723447123229, "server_accept_len_effective": 3.003968253968254, "accept_norm_tps": 0.0, "accept_norm_ref_len": 0.0, "avg_running_reqs": 1, "max_running_reqs": 1, "effective_concurrency": 1, "avg_queue_reqs": 0, "max_queue_reqs": 0, "queue_fraction": 0.0, "underfilled": false, "warmup_timed_out": false, "warmup_duration": 25.797, "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", "timeout_reason": "", "capacity_limited": false, "hardware_summary": { "samples": 4, "duration_seconds": 7.176, "gpu_count": 4, "cpu_util_avg_pct": 7.08, "cpu_temp_max_c": 66.75, "gpu_util_avg_pct": 49.5, "gpu_util_max_pct": 99.0, "mem_util_avg_pct": 28.44, "mem_util_max_pct": 60.0, "temp_avg_c": 59.69, "temp_max_c": 88.0, "power_total_avg_w": 946.31, "power_total_max_w": 950.64, "power_limit_total_w": 1800.0, "vram_used_avg_mb": 189586.0, "vram_used_max_mb": 189586.0, "vram_total_mb": 391548.0, "vram_used_avg_pct": 48.42, "vram_used_max_pct": 48.42, "pcie_rx_avg_mb_s": 2866.75, "pcie_rx_max_mb_s": 2950.0, "pcie_tx_avg_mb_s": 2665.25, "pcie_tx_max_mb_s": 2731.0 } } ], "summary_table": { "0": { "1": 145.49161652209398 }, "32768": { "1": 147.22458371168588 }, "65536": { "1": 151.51185116953508 } }, "burst_results": [], "burst_summary_table": {}, "methodology": { "prefill": { "name": "Prefill", "present": false, "mode": "skipped", "formula": "prompt_tokens / TTFT", "notes": "Default mode records the required decode scout request for each non-zero decode context, so normal runs do not pay for a separate prefill phase. Standalone mode repeats cold-prefill samples. Prometheus prefill counters, when available and uncontaminated, are stored as validation." }, "sustained_decode": { "name": "Sustained Decode", "present": true, "formula": "OpenAI stream usage completion_tokens per measured window; client chunk fallback only when continuous usage is unavailable", "notes": "Duration-based steady-state cell after warmup. This is the main tuning/regression signal for kernels, NCCL, DCP, MTP, and scheduling. Prometheus metrics are stored as validation and scheduler state, not the default headline." }, "burst_e2e_decode": { "name": "Burst / E2E Decode", "present": false, "status": "not run; use --run-burst", "formula": "sum(completion_tokens) / profiling_wall_time", "notes": "Finite client-facing request burst using OpenAI stream usage. It includes request admission, scheduling, prefill/cache behavior, and completion." }, "acceptance_normalization": { "name": "Acceptance-normalized decode (MTP / speculative)", "present": true, "formula": "engine_steps = spec_drafts + max(0, output_tokens - (accepted_tokens + spec_drafts)); accept_len_effective = output_tokens / engine_steps; steps_per_s = aggregate_tps / accept_len_effective", "notes": "With speculative decoding tok/s = steps_per_s * accept_len, so raw tok/s mixes engine speed with data-dependent acceptance. steps_per_s (target-model forward passes per second) is the acceptance-independent speed used to compare runs; server_spec_pos_accept holds per-draft-position acceptance probabilities. Counters are vLLM window deltas; SGLang falls back to its lifetime accept-length gauge." } } }