| { |
| "metadata": { |
| "version": "0.4.29", |
| "engine": "vllm", |
| "model": "GLM-5.3-Flash-EXL3-4bpw", |
| "server": "127.0.0.1:5001", |
| "timestamp": "2026-08-28T01:14:26.824486", |
| "decode_mode": "duration", |
| "primary_decode_layer": "sustained_decode", |
| "duration_per_test": 10.0, |
| "request_count": 0, |
| "warmup_request_count": 0, |
| "run_burst": false, |
| "prefill_mode": "integrated_decode_scout", |
| "standalone_prefill": false, |
| "prefill_only": false, |
| "skip_prefill": false, |
| "burst_e2e_status": "not_run_use_--run-burst", |
| "burst_request_count": 0, |
| "burst_warmup_request_count": 0, |
| "burst_requests_per_concurrency": 5, |
| "decode_warmup_seconds": 3.0, |
| "decode_warmup_context": 65536, |
| "decode_warmup_concurrency": 1, |
| "cell_warmup_timeout_seconds": 0.0, |
| "cell_warmup_timeout_policy": "<=32k:60s,64k:120s,>=128k:180s when override is 0", |
| "show_capacity_limited_values": false, |
| "max_tokens": 1024, |
| "temperature": null, |
| "ignore_eos": true, |
| "max_total_tokens": 884736, |
| "dcp_size": 0, |
| "metrics_available": true, |
| "metrics_warning": "", |
| "concurrency_levels": [ |
| 1, |
| 2, |
| 4 |
| ], |
| "context_lengths": [ |
| 0, |
| 65536 |
| ], |
| "startup_diagnostics_available": true, |
| "nvidia_p2p_override_effective": true, |
| "p2pmark_status": "not_run", |
| "amd_fabric_status": "not_run" |
| }, |
| "startup_diagnostics": { |
| "version": "0.4.29", |
| "server_url": "http://127.0.0.1:5001", |
| "hostname": "pop-os", |
| "uname": "Linux pop-os 6.18.7-76061807-generic #202601231045~1769703228~24.04~cb87b5b SMP PREEMPT_DYNAMIC Thu J x86_64 x86_64 x86_64 GNU/Linux", |
| "env": {}, |
| "args": { |
| "concurrency": "1,2,4", |
| "contexts": "0,64k", |
| "max_tokens": 1024, |
| "duration": 10.0, |
| "request_count": 0, |
| "run_burst": false, |
| "standalone_prefill": false, |
| "prefill_only": false, |
| "skip_prefill": false, |
| "prefill_contexts": "8k,64k,128k", |
| "prefill_metric": "client", |
| "dcp_size": 0, |
| "kv_budget": 0 |
| }, |
| "nvidia_p2p_override": { |
| "effective": true, |
| "configured": true, |
| "params_path": "/proc/driver/nvidia/params", |
| "params_available": true, |
| "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", |
| "modprobe_available": true, |
| "runtime": { |
| "ForceP2P": "0x11", |
| "RMForceP2PType": "1", |
| "RMPcieP2PType": "2", |
| "GrdmaPciTopoCheckOverride": "1", |
| "EnableResizableBar": "1", |
| "DmaRemapPeerMmio": "1" |
| }, |
| "expected": { |
| "ForceP2P": "0x11", |
| "RMForceP2PType": "1", |
| "RMPcieP2PType": "2", |
| "GrdmaPciTopoCheckOverride": "1", |
| "EnableResizableBar": "1" |
| }, |
| "missing": [], |
| "mismatched": {}, |
| "registry_dwords": "ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1", |
| "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", |
| "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" |
| }, |
| "p2pmark": { |
| "status": "not_run" |
| }, |
| "amd_fabric": { |
| "status": "not_run" |
| }, |
| "nvidia_smi_query": { |
| "cmd": [ |
| "nvidia-smi", |
| "--query-gpu=index,name,driver_version,pci.bus_id,pcie.link.gen.current,pcie.link.width.current,power.limit", |
| "--format=csv,noheader,nounits" |
| ], |
| "returncode": 0, |
| "stdout": "0, NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.57.04, 00000000:01:00.0, 1, 16, 300.00\n1, NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 610.57.04, 00000000:21:00.0, 1, 16, 300.00\n2, NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.57.04, 00000000:81:00.0, 1, 16, 300.00\n3, NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 610.57.04, 00000000:C1:00.0, 1, 16, 300.00", |
| "stderr": "" |
| }, |
| "nvidia_smi_topo": { |
| "cmd": [ |
| "nvidia-smi", |
| "topo", |
| "-m" |
| ], |
| "returncode": 0, |
| "stdout": "\u001b[4mGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\u001b[0m\nGPU0\t X \tNODE\tNODE\tNODE\t0-47\t0\t\tN/A\nGPU1\tNODE\t X \tNODE\tNODE\t0-47\t0\t\tN/A\nGPU2\tNODE\tNODE\t X \tNODE\t0-47\t0\t\tN/A\nGPU3\tNODE\tNODE\tNODE\t X \t0-47\t0\t\tN/A\n\nLegend:\n\n X = Self\n SYS = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)\n NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node\n PHB = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)\n PXB = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)\n PIX = Connection traversing at most a single PCIe bridge\n NV# = Connection traversing a bonded set of # NVLinks", |
| "stderr": "" |
| } |
| }, |
| "nvidia_p2p_override": { |
| "effective": true, |
| "configured": true, |
| "params_path": "/proc/driver/nvidia/params", |
| "params_available": true, |
| "modprobe_path": "/etc/modprobe.d/nvidia-p2p-override.conf", |
| "modprobe_available": true, |
| "runtime": { |
| "ForceP2P": "0x11", |
| "RMForceP2PType": "1", |
| "RMPcieP2PType": "2", |
| "GrdmaPciTopoCheckOverride": "1", |
| "EnableResizableBar": "1", |
| "DmaRemapPeerMmio": "1" |
| }, |
| "expected": { |
| "ForceP2P": "0x11", |
| "RMForceP2PType": "1", |
| "RMPcieP2PType": "2", |
| "GrdmaPciTopoCheckOverride": "1", |
| "EnableResizableBar": "1" |
| }, |
| "missing": [], |
| "mismatched": {}, |
| "registry_dwords": "ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1", |
| "suggested_modprobe_line": "options nvidia NVreg_RegistryDwords=\"ForceP2P=0x11;RMForceP2PType=1;RMPcieP2PType=2;GrdmaPciTopoCheckOverride=1;EnableResizableBar=1\"", |
| "suggested_reload": "stop GPU workloads, then reload NVIDIA modules or reboot; the modprobe file alone is not enough until the nvidia module is reloaded" |
| }, |
| "p2pmark": { |
| "status": "not_run" |
| }, |
| "amd_fabric": { |
| "status": "not_run" |
| }, |
| "hardware_run_summary": { |
| "samples": 243, |
| "duration_seconds": 577.387, |
| "gpu_count": 4, |
| "cpu_util_avg_pct": 6.97, |
| "cpu_temp_max_c": 67.5, |
| "gpu_util_avg_pct": 48.04, |
| "gpu_util_max_pct": 100.0, |
| "mem_util_avg_pct": 16.63, |
| "mem_util_max_pct": 62.0, |
| "temp_avg_c": 50.75, |
| "temp_max_c": 75.0, |
| "power_total_avg_w": 604.36, |
| "power_total_max_w": 624.86, |
| "power_limit_total_w": 1200.0, |
| "vram_used_avg_mb": 189600.28, |
| "vram_used_max_mb": 189606.0, |
| "vram_total_mb": 391548.0, |
| "vram_used_avg_pct": 48.42, |
| "vram_used_max_pct": 48.42, |
| "pcie_rx_avg_mb_s": 9462.3, |
| "pcie_rx_max_mb_s": 24512.0, |
| "pcie_tx_avg_mb_s": 9641.98, |
| "pcie_tx_max_mb_s": 25466.0 |
| }, |
| "event_log": [ |
| "01:04:47 benchmark start engine=vllm", |
| "01:04:47 startup server=http://127.0.0.1:5001 model=GLM-5.3-Flash-EXL3-4bpw", |
| "01:04:47 startup decode concurrency=1,2,4 contexts=0,64k", |
| "01:04:47 startup NVIDIA P2P override: enabled: runtime NVIDIA P2P override matches expected RegistryDwords", |
| "01:04:47 startup engine vLLM 0.1.dev20111+g7f1e92bec.d20260827 models=['GLM-5.3-Flash-EXL3-4bpw']", |
| "01:04:47 startup KV cache budget from vLLM metrics: 884,736 tokens (54 blocks x 8192; local 442,368 \u00d7 CP 2; CP source: local process)", |
| "01:04:47 startup model context length: 98,304 tokens", |
| "01:04:47 startup prefill tests: integrated from decode scout requests ['64k']; scout-only extras ['8k']", |
| "01:04:47 startup calibrating padding text run=erjqltqqoilw up_to=64k", |
| "01:04:47 startup token targeting: estimate from 8k", |
| "01:04:47 startup calibrated: 6.18 chars/token (cached, source=8k)", |
| "01:04:47 startup context 8k: 50,601 chars (~8,191 tokens)", |
| "01:04:47 startup context 64k: 404,809 chars (~65,535 tokens)", |
| "01:04:47 startup startup preparation done", |
| "01:04:47 hardware monitor interval=2s", |
| "01:04:47 decode warmup start", |
| "01:04:48 prefill warmup start ctx=8k", |
| "01:04:51 prefill warmup done ctx=8k", |
| "01:04:51 prefill scout-only start ctx=8k", |
| "01:04:54 prefill scout-only done ctx=8k 3,897 tok/s", |
| "01:04:54 decode warmup start C=1 ctx=64k 3s", |
| "01:04:54 cell start C=1 ctx=64k", |
| "01:05:28 ready C=1 ctx=64k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", |
| "01:05:31 cell done C=1 ctx=64k 107.8 tok/s | norm 43.2 step/s len=2.50", |
| "01:05:31 decode warmup done C=1 ctx=64k", |
| "01:05:33 cell start C=1 ctx=0", |
| "01:05:39 ready C=1 ctx=0 running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", |
| "01:05:49 cell done C=1 ctx=0 129.5 tok/s | norm 44.1 step/s len=2.94", |
| "01:05:51 cell start C=1 ctx=64k", |
| "01:05:51 integrated prefill start ctx=64k", |
| "01:06:08 integrated prefill done ctx=64k 4,297 tok/s", |
| "01:06:27 ready C=1 ctx=64k running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", |
| "01:06:46 cell done C=1 ctx=64k 122.2 tok/s | norm 41.9 step/s len=2.92", |
| "01:06:48 cell start C=2 ctx=0", |
| "01:07:49 warmup timeout C=2 ctx=0 running_reqs=1/2", |
| "01:07:59 cell done C=2 ctx=0 115.9 tok/s | norm 42.2 step/s len=2.75", |
| "01:08:01 cell start C=4 ctx=0", |
| "01:09:02 warmup timeout C=4 ctx=0 running_reqs=1/4", |
| "01:09:13 cell done C=4 ctx=0 119.0 tok/s | norm 42.7 step/s len=2.78", |
| "01:09:15 cell start C=2 ctx=64k", |
| "01:11:16 warmup timeout C=2 ctx=64k running_reqs=1/2", |
| "01:11:42 cell done C=2 ctx=64k 12.1 tok/s | norm 3.9 step/s len=3.10", |
| "01:11:44 cell start C=4 ctx=64k", |
| "01:13:44 warmup timeout C=4 ctx=64k running_reqs=1/4", |
| "01:14:24 cell done C=4 ctx=64k 0.1 tok/s" |
| ], |
| "prefill": { |
| "8192": { |
| "ttft_seconds": 2.104, |
| "prefill_seconds": 2.104, |
| "tok_per_sec": 3897.0, |
| "client_ttft_seconds": 2.104, |
| "client_tok_per_sec": 3897.0, |
| "prompt_tokens": 8199, |
| "samples": 1, |
| "method": "scout_only", |
| "server_validation": { |
| "method": "", |
| "tok_per_sec": 0.0, |
| "prefill_seconds": 0.0, |
| "prompt_tokens": 0, |
| "request_prompt_tokens": 0, |
| "cached_tokens": 0, |
| "token_source": "", |
| "samples": 0, |
| "invalid_reason": "" |
| }, |
| "hardware_summary": {} |
| }, |
| "65536": { |
| "ttft_seconds": 15.015, |
| "prefill_seconds": 15.015, |
| "tok_per_sec": 4297.0, |
| "client_ttft_seconds": 15.015, |
| "client_tok_per_sec": 4297.0, |
| "prompt_tokens": 64513, |
| "samples": 1, |
| "method": "integrated_scout", |
| "server_validation": { |
| "method": "prometheus:kv_computed", |
| "tok_per_sec": 4320.0, |
| "prefill_seconds": 14.934, |
| "prompt_tokens": 64513, |
| "request_prompt_tokens": 0, |
| "cached_tokens": 0, |
| "token_source": "", |
| "samples": 1, |
| "invalid_reason": "" |
| }, |
| "hardware_summary": {} |
| } |
| }, |
| "results": [ |
| { |
| "concurrency": 1, |
| "context_tokens": 0, |
| "benchmark_mode": "duration", |
| "request_count_target": 0, |
| "warmup_request_count": 0, |
| "measurement_seconds": 9.988261, |
| "measurement_wall_seconds": 10.000404, |
| "client_output_tokens": 1293, |
| "server_output_tokens": 1293, |
| "aggregate_source": "openai_continuous_usage", |
| "aggregate_tps": 129.45196578500713, |
| "per_request_avg_tps": 129.45196578500713, |
| "ttft_avg": 0.13393967098090798, |
| "ttft_p50": 0.13393967098090798, |
| "ttft_p90": 0.1397153286030516, |
| "ttft_p99": 0.14101485156803392, |
| "time_to_second_token_avg": 0.020580344484187663, |
| "time_to_second_token_p50": 0.020580344484187663, |
| "time_to_second_token_p90": 0.021969041670672596, |
| "time_to_second_token_p99": 0.022281498537631707, |
| "request_latency_avg": 8.68479720497271, |
| "request_latency_p50": 8.68479720497271, |
| "request_latency_p90": 8.68479720497271, |
| "request_latency_p99": 8.68479720497271, |
| "inter_token_latency_avg": 0.0078036521248696505, |
| "inter_token_latency_p50": 0.0078036521248696505, |
| "inter_token_latency_p90": 0.00825326384121595, |
| "inter_token_latency_p99": 0.008354426477393866, |
| "output_tps_per_user_avg": 128.81325648749555, |
| "output_tps_per_user_p50": 128.81325648749555, |
| "output_tps_per_user_p90": 136.2349032255562, |
| "output_tps_per_user_p99": 137.90477374161983, |
| "e2e_output_tps_per_user_avg": 117.9071860668988, |
| "e2e_output_tps_per_user_p50": 117.9071860668988, |
| "e2e_output_tps_per_user_p90": 117.9071860668988, |
| "e2e_output_tps_per_user_p99": 117.9071860668988, |
| "chunk_inter_token_latency_avg": 0.022398087895907616, |
| "chunk_inter_token_latency_p50": 0.022398087895907616, |
| "chunk_inter_token_latency_p90": 0.02262544274799395, |
| "chunk_inter_token_latency_p99": 0.022676597589713375, |
| "input_seq_len_avg": 78.0, |
| "output_seq_len_avg": 1024.0, |
| "output_seq_len_p50": 1024.0, |
| "output_seq_len_p90": 1024.0, |
| "output_seq_len_p99": 1024.0, |
| "request_count": 2, |
| "completed_request_count": 1, |
| "request_samples": [ |
| { |
| "ttft": 0.12672009895322844, |
| "time_to_second_token": 0.018844473001081496, |
| "latency": 8.68479720497271, |
| "inter_token_latency_avg": 0.008365666770302524, |
| "chunk_inter_token_latency_avg": 0.022113894330799695, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 119.53619806491974, |
| "e2e_output_tps_per_user": 117.9071860668988, |
| "completed": true |
| }, |
| { |
| "ttft": 0.1411592430085875, |
| "time_to_second_token": 0.02231621596729383, |
| "latency": 0.0, |
| "inter_token_latency_avg": 0.007241637479436777, |
| "chunk_inter_token_latency_avg": 0.022682281461015533, |
| "input_tokens": 78, |
| "output_tokens": 925, |
| "output_tps_per_user": 138.09031491007136, |
| "e2e_output_tps_per_user": 0.0, |
| "completed": false |
| } |
| ], |
| "total_tokens": 1293, |
| "wall_time": 15.541175424004905, |
| "num_completed": 1, |
| "num_errors": 0, |
| "server_gen_throughput": 129.23415974403488, |
| "server_utilization": 0.6603773584905661, |
| "server_spec_accept_rate": 0.2775789131142206, |
| "server_spec_accept_length": 2.9430523917995446, |
| "server_spec_drafts": 439, |
| "server_spec_draft_tokens": 3073, |
| "server_spec_accepted_tokens": 853, |
| "server_spec_pos_accept": [ |
| 0.7699, |
| 0.5011, |
| 0.2825, |
| 0.18, |
| 0.1071, |
| 0.0638, |
| 0.0387 |
| ], |
| "server_engine_steps": 440.0, |
| "server_steps_per_s": 44.05171302815401, |
| "server_accept_len_effective": 2.9386363636363635, |
| "accept_norm_tps": 0.0, |
| "accept_norm_ref_len": 0.0, |
| "avg_running_reqs": 1, |
| "max_running_reqs": 1, |
| "effective_concurrency": 1, |
| "avg_queue_reqs": 0, |
| "max_queue_reqs": 0, |
| "queue_fraction": 0.0, |
| "underfilled": false, |
| "warmup_timed_out": false, |
| "warmup_duration": 5.529, |
| "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", |
| "timeout_reason": "", |
| "capacity_limited": false, |
| "hardware_summary": { |
| "samples": 4, |
| "duration_seconds": 7.182, |
| "gpu_count": 4, |
| "cpu_util_avg_pct": 7.22, |
| "cpu_temp_max_c": 66.62, |
| "gpu_util_avg_pct": 49.5, |
| "gpu_util_max_pct": 99.0, |
| "mem_util_avg_pct": 26.75, |
| "mem_util_max_pct": 62.0, |
| "temp_avg_c": 44.25, |
| "temp_max_c": 57.0, |
| "power_total_avg_w": 619.0, |
| "power_total_max_w": 619.23, |
| "power_limit_total_w": 1200.0, |
| "vram_used_avg_mb": 189604.0, |
| "vram_used_max_mb": 189604.0, |
| "vram_total_mb": 391548.0, |
| "vram_used_avg_pct": 48.42, |
| "vram_used_max_pct": 48.42, |
| "pcie_rx_avg_mb_s": 2535.0, |
| "pcie_rx_max_mb_s": 2601.0, |
| "pcie_tx_avg_mb_s": 2322.5, |
| "pcie_tx_max_mb_s": 2407.0 |
| } |
| }, |
| { |
| "concurrency": 1, |
| "context_tokens": 65536, |
| "benchmark_mode": "duration", |
| "request_count_target": 0, |
| "warmup_request_count": 0, |
| "measurement_seconds": 4.417692, |
| "measurement_wall_seconds": 10.0022, |
| "client_output_tokens": 540, |
| "server_output_tokens": 540, |
| "aggregate_source": "openai_continuous_usage", |
| "aggregate_tps": 122.23577506588123, |
| "per_request_avg_tps": 122.23577506588123, |
| "ttft_avg": 15.145516593009233, |
| "ttft_p50": 15.145516593009233, |
| "ttft_p90": 15.145516593009233, |
| "ttft_p99": 15.145516593009233, |
| "time_to_second_token_avg": 0.004005067981779575, |
| "time_to_second_token_p50": 0.004005067981779575, |
| "time_to_second_token_p90": 0.004005067981779575, |
| "time_to_second_token_p99": 0.004005067981779575, |
| "request_latency_avg": 23.09197867201874, |
| "request_latency_p50": 23.09197867201874, |
| "request_latency_p90": 23.09197867201874, |
| "request_latency_p99": 23.09197867201874, |
| "inter_token_latency_avg": 0.007767802618777622, |
| "inter_token_latency_p50": 0.007767802618777622, |
| "inter_token_latency_p90": 0.007767802618777622, |
| "inter_token_latency_p99": 0.007767802618777622, |
| "output_tps_per_user_avg": 128.73653581035055, |
| "output_tps_per_user_p50": 128.73653581035055, |
| "output_tps_per_user_p90": 128.73653581035055, |
| "output_tps_per_user_p99": 128.73653581035055, |
| "e2e_output_tps_per_user_avg": 44.34440264059365, |
| "e2e_output_tps_per_user_p50": 44.34440264059365, |
| "e2e_output_tps_per_user_p90": 44.34440264059365, |
| "e2e_output_tps_per_user_p99": 44.34440264059365, |
| "chunk_inter_token_latency_avg": 0.024080188118210628, |
| "chunk_inter_token_latency_p50": 0.024080188118210628, |
| "chunk_inter_token_latency_p90": 0.024080188118210628, |
| "chunk_inter_token_latency_p99": 0.024080188118210628, |
| "input_seq_len_avg": 64513.0, |
| "output_seq_len_avg": 1024.0, |
| "output_seq_len_p50": 1024.0, |
| "output_seq_len_p90": 1024.0, |
| "output_seq_len_p99": 1024.0, |
| "request_count": 1, |
| "completed_request_count": 1, |
| "request_samples": [ |
| { |
| "ttft": 15.145516593009233, |
| "time_to_second_token": 0.004005067981779575, |
| "latency": 23.09197867201874, |
| "inter_token_latency_avg": 0.007767802618777622, |
| "chunk_inter_token_latency_avg": 0.024080188118210628, |
| "input_tokens": 64513, |
| "output_tokens": 1024, |
| "output_tps_per_user": 128.73653581035055, |
| "e2e_output_tps_per_user": 44.34440264059365, |
| "completed": true |
| } |
| ], |
| "total_tokens": 540, |
| "wall_time": 55.68818276398815, |
| "num_completed": 1, |
| "num_errors": 0, |
| "server_gen_throughput": 53.96348511679134, |
| "server_utilization": 0.7169811320754718, |
| "server_spec_accept_rate": 0.277992277992278, |
| "server_spec_accept_length": 2.945945945945946, |
| "server_spec_drafts": 185, |
| "server_spec_draft_tokens": 1295, |
| "server_spec_accepted_tokens": 360, |
| "server_spec_pos_accept": [ |
| 0.7351, |
| 0.4919, |
| 0.3351, |
| 0.2, |
| 0.0865, |
| 0.0595, |
| 0.0378 |
| ], |
| "server_engine_steps": 185.0, |
| "server_steps_per_s": 41.87707108738524, |
| "server_accept_len_effective": 2.918918918918919, |
| "accept_norm_tps": 0.0, |
| "accept_norm_ref_len": 0.0, |
| "avg_running_reqs": 1, |
| "max_running_reqs": 1, |
| "effective_concurrency": 1, |
| "avg_queue_reqs": 0, |
| "max_queue_reqs": 0, |
| "queue_fraction": 0.0, |
| "underfilled": false, |
| "warmup_timed_out": false, |
| "warmup_duration": 35.944, |
| "ready_reason": "running_reqs=1/1, queue_reqs=0, active_streams=1/1, stable=3.0s", |
| "timeout_reason": "", |
| "capacity_limited": false, |
| "hardware_summary": { |
| "samples": 8, |
| "duration_seconds": 16.689, |
| "gpu_count": 4, |
| "cpu_util_avg_pct": 7.11, |
| "cpu_temp_max_c": 66.88, |
| "gpu_util_avg_pct": 49.81, |
| "gpu_util_max_pct": 100.0, |
| "mem_util_avg_pct": 14.59, |
| "mem_util_max_pct": 59.0, |
| "temp_avg_c": 47.62, |
| "temp_max_c": 65.0, |
| "power_total_avg_w": 617.6, |
| "power_total_max_w": 622.46, |
| "power_limit_total_w": 1200.0, |
| "vram_used_avg_mb": 189604.0, |
| "vram_used_max_mb": 189604.0, |
| "vram_total_mb": 391548.0, |
| "vram_used_avg_pct": 48.42, |
| "vram_used_max_pct": 48.42, |
| "pcie_rx_avg_mb_s": 10950.75, |
| "pcie_rx_max_mb_s": 18611.0, |
| "pcie_tx_avg_mb_s": 15013.5, |
| "pcie_tx_max_mb_s": 23519.0 |
| } |
| }, |
| { |
| "concurrency": 2, |
| "context_tokens": 0, |
| "benchmark_mode": "duration", |
| "request_count_target": 0, |
| "warmup_request_count": 0, |
| "measurement_seconds": 9.995853, |
| "measurement_wall_seconds": 10.000935, |
| "client_output_tokens": 1159, |
| "server_output_tokens": 1159, |
| "aggregate_source": "openai_continuous_usage", |
| "aggregate_tps": 115.94808445857444, |
| "per_request_avg_tps": 57.97404222928722, |
| "ttft_avg": 7.834794788103965, |
| "ttft_p50": 8.624857554968912, |
| "ttft_p90": 9.220339812606108, |
| "ttft_p99": 9.429953900745604, |
| "time_to_second_token_avg": 0.022789318120986637, |
| "time_to_second_token_p50": 0.023204773024190217, |
| "time_to_second_token_p90": 0.024304261978249996, |
| "time_to_second_token_p99": 0.024516121998894958, |
| "request_latency_avg": 16.199721001743455, |
| "request_latency_p50": 17.21378530547372, |
| "request_latency_p90": 17.579868975607678, |
| "request_latency_p99": 17.613597658531507, |
| "inter_token_latency_avg": 0.008167093486157814, |
| "inter_token_latency_p50": 0.00816347410365027, |
| "inter_token_latency_p90": 0.008771674752686963, |
| "inter_token_latency_p99": 0.008959866448893655, |
| "output_tps_per_user_avg": 123.1466847171223, |
| "output_tps_per_user_p50": 122.49686681223787, |
| "output_tps_per_user_p90": 130.61500965849436, |
| "output_tps_per_user_p99": 145.5801546760414, |
| "e2e_output_tps_per_user_avg": 66.49566993826983, |
| "e2e_output_tps_per_user_p50": 59.48782372259443, |
| "e2e_output_tps_per_user_p90": 77.83971971955552, |
| "e2e_output_tps_per_user_p99": 112.64835775550863, |
| "chunk_inter_token_latency_avg": 0.023320615398816368, |
| "chunk_inter_token_latency_p50": 0.023428110182258776, |
| "chunk_inter_token_latency_p90": 0.023711158726643578, |
| "chunk_inter_token_latency_p99": 0.024126927350817735, |
| "input_seq_len_avg": 78.0, |
| "output_seq_len_avg": 1024.0, |
| "output_seq_len_p50": 1024.0, |
| "output_seq_len_p90": 1024.0, |
| "output_seq_len_p99": 1024.0, |
| "request_count": 9, |
| "completed_request_count": 8, |
| "request_samples": [ |
| { |
| "ttft": 0.1269236400257796, |
| "time_to_second_token": 0.01867156900698319, |
| "latency": 8.788493758998811, |
| "inter_token_latency_avg": 0.008466832960872953, |
| "chunk_inter_token_latency_avg": 0.022853747015759977, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 118.10791645721771, |
| "e2e_output_tps_per_user": 116.51598420394788, |
| "completed": true |
| }, |
| { |
| "ttft": 8.624857554968912, |
| "time_to_second_token": 0.02298608800629154, |
| "latency": 16.714498371002264, |
| "inter_token_latency_avg": 0.007907762283512563, |
| "chunk_inter_token_latency_avg": 0.023448234249372035, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 126.45802493139794, |
| "e2e_output_tps_per_user": 61.26417779767309, |
| "completed": true |
| }, |
| { |
| "ttft": 9.453244354983326, |
| "time_to_second_token": 0.023422114027198404, |
| "latency": 17.61734528996749, |
| "inter_token_latency_avg": 0.007980548323542681, |
| "chunk_inter_token_latency_avg": 0.023595667442150758, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 125.30467324531975, |
| "e2e_output_tps_per_user": 58.12453483460616, |
| "completed": true |
| }, |
| { |
| "ttft": 8.935068062972277, |
| "time_to_second_token": 0.023124370025470853, |
| "latency": 17.158334736945108, |
| "inter_token_latency_avg": 0.008038383845525738, |
| "chunk_inter_token_latency_avg": 0.023428110182258776, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 124.4031162503657, |
| "e2e_output_tps_per_user": 59.679451164636404, |
| "completed": true |
| }, |
| { |
| "ttft": 9.162113677011803, |
| "time_to_second_token": 0.024539662001188844, |
| "latency": 0.0, |
| "inter_token_latency_avg": 0.006791496704820366, |
| "chunk_inter_token_latency_avg": 0.024173123864614864, |
| "input_tokens": 78, |
| "output_tokens": 211, |
| "output_tps_per_user": 147.24294856687996, |
| "e2e_output_tps_per_user": 0.0, |
| "completed": false |
| }, |
| { |
| "ttft": 8.918001865968108, |
| "time_to_second_token": 0.021153510024305433, |
| "latency": 17.269235874002334, |
| "inter_token_latency_avg": 0.00816347410365027, |
| "chunk_inter_token_latency_avg": 0.02275540601644203, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 122.49686681223787, |
| "e2e_output_tps_per_user": 59.296196280552444, |
| "completed": true |
| }, |
| { |
| "ttft": 8.376473198004533, |
| "time_to_second_token": 0.023204773024190217, |
| "latency": 17.563807698024902, |
| "inter_token_latency_avg": 0.008980776637361066, |
| "chunk_inter_token_latency_avg": 0.022968336250050923, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 111.34894457121723, |
| "e2e_output_tps_per_user": 58.30170869584, |
| "completed": true |
| }, |
| { |
| "ttft": 8.430108908971306, |
| "time_to_second_token": 0.02375636500073597, |
| "latency": 17.079744989983737, |
| "inter_token_latency_avg": 0.008455167234616258, |
| "chunk_inter_token_latency_avg": 0.023189372871347, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 118.2708717937482, |
| "e2e_output_tps_per_user": 59.95405672628693, |
| "completed": true |
| }, |
| { |
| "ttft": 8.486361830029637, |
| "time_to_second_token": 0.024245411972515285, |
| "latency": 17.406307295022998, |
| "inter_token_latency_avg": 0.008719399281518438, |
| "chunk_inter_token_latency_avg": 0.023473540697350952, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 114.68679982571635, |
| "e2e_output_tps_per_user": 58.829249802615706, |
| "completed": true |
| } |
| ], |
| "total_tokens": 1159, |
| "wall_time": 71.05695495504187, |
| "num_completed": 2, |
| "num_errors": 0, |
| "server_gen_throughput": 115.83716967539169, |
| "server_utilization": 0.6603773584905661, |
| "server_spec_accept_rate": 0.24949221394719026, |
| "server_spec_accept_length": 2.746445497630332, |
| "server_spec_drafts": 422, |
| "server_spec_draft_tokens": 2954, |
| "server_spec_accepted_tokens": 737, |
| "server_spec_pos_accept": [ |
| 0.7322, |
| 0.4692, |
| 0.263, |
| 0.154, |
| 0.0782, |
| 0.0332, |
| 0.0166 |
| ], |
| "server_engine_steps": 422.0, |
| "server_steps_per_s": 42.217507887418826, |
| "server_accept_len_effective": 2.7464454976303316, |
| "accept_norm_tps": 0.0, |
| "accept_norm_ref_len": 0.0, |
| "avg_running_reqs": 1, |
| "max_running_reqs": 1, |
| "effective_concurrency": 1, |
| "avg_queue_reqs": 1, |
| "max_queue_reqs": 1, |
| "queue_fraction": 1.0, |
| "underfilled": true, |
| "warmup_timed_out": true, |
| "warmup_duration": 60.872, |
| "ready_reason": "warmup_timeout", |
| "timeout_reason": "running_reqs=1/2", |
| "capacity_limited": true, |
| "hardware_summary": { |
| "samples": 5, |
| "duration_seconds": 9.555, |
| "gpu_count": 4, |
| "cpu_util_avg_pct": 7.12, |
| "cpu_temp_max_c": 66.38, |
| "gpu_util_avg_pct": 49.4, |
| "gpu_util_max_pct": 99.0, |
| "mem_util_avg_pct": 25.45, |
| "mem_util_max_pct": 61.0, |
| "temp_avg_c": 49.65, |
| "temp_max_c": 67.0, |
| "power_total_avg_w": 619.02, |
| "power_total_max_w": 619.47, |
| "power_limit_total_w": 1200.0, |
| "vram_used_avg_mb": 189606.0, |
| "vram_used_max_mb": 189606.0, |
| "vram_total_mb": 391548.0, |
| "vram_used_avg_pct": 48.42, |
| "vram_used_max_pct": 48.42, |
| "pcie_rx_avg_mb_s": 2422.0, |
| "pcie_rx_max_mb_s": 2640.0, |
| "pcie_tx_avg_mb_s": 2273.8, |
| "pcie_tx_max_mb_s": 2384.0 |
| } |
| }, |
| { |
| "concurrency": 4, |
| "context_tokens": 0, |
| "benchmark_mode": "duration", |
| "request_count_target": 0, |
| "warmup_request_count": 0, |
| "measurement_seconds": 9.993663, |
| "measurement_wall_seconds": 10.000769, |
| "client_output_tokens": 1189, |
| "server_output_tokens": 1189, |
| "aggregate_source": "openai_continuous_usage", |
| "aggregate_tps": 118.97538984991404, |
| "per_request_avg_tps": 29.74384746247851, |
| "ttft_avg": 19.78742812310035, |
| "ttft_p50": 24.75903701898642, |
| "ttft_p90": 26.1186341956025, |
| "ttft_p99": 26.15733930614777, |
| "time_to_second_token_avg": 0.02317402323630328, |
| "time_to_second_token_p50": 0.023638906015548855, |
| "time_to_second_token_p90": 0.02446553821209818, |
| "time_to_second_token_p99": 0.024968253751285373, |
| "request_latency_avg": 27.448205057356972, |
| "request_latency_p50": 33.088808445987524, |
| "request_latency_p90": 33.99806104596937, |
| "request_latency_p99": 34.4808259572566, |
| "inter_token_latency_avg": 0.008139797620579165, |
| "inter_token_latency_p50": 0.008184581259989297, |
| "inter_token_latency_p90": 0.008731430765029264, |
| "inter_token_latency_p99": 0.008955405133080832, |
| "output_tps_per_user_avg": 123.39942798393562, |
| "output_tps_per_user_p50": 122.18096054449921, |
| "output_tps_per_user_p90": 134.3267922406224, |
| "output_tps_per_user_p99": 138.45432623611825, |
| "e2e_output_tps_per_user_avg": 46.01047960697323, |
| "e2e_output_tps_per_user_p50": 30.947136506158543, |
| "e2e_output_tps_per_user_p90": 77.39302160517252, |
| "e2e_output_tps_per_user_p99": 111.22101751525767, |
| "chunk_inter_token_latency_avg": 0.02353125415264528, |
| "chunk_inter_token_latency_p50": 0.02348133677266808, |
| "chunk_inter_token_latency_p90": 0.02384579067365582, |
| "chunk_inter_token_latency_p99": 0.02408855324857224, |
| "input_seq_len_avg": 78.0, |
| "output_seq_len_avg": 1024.0, |
| "output_seq_len_p50": 1024.0, |
| "output_seq_len_p90": 1024.0, |
| "output_seq_len_p99": 1024.0, |
| "request_count": 9, |
| "completed_request_count": 8, |
| "request_samples": [ |
| { |
| "ttft": 0.1278693950152956, |
| "time_to_second_token": 0.01804825698491186, |
| "latency": 8.90592117496999, |
| "inter_token_latency_avg": 0.008580695777081813, |
| "chunk_inter_token_latency_avg": 0.0241155268680074, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 116.54066592955093, |
| "e2e_output_tps_per_user": 114.97968372748937, |
| "completed": true |
| }, |
| { |
| "ttft": 25.77864516695263, |
| "time_to_second_token": 0.02290963102132082, |
| "latency": 33.76817299297545, |
| "inter_token_latency_avg": 0.0078099001231894645, |
| "chunk_inter_token_latency_avg": 0.023778356625067925, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 128.0426105617869, |
| "e2e_output_tps_per_user": 30.32441228647506, |
| "completed": true |
| }, |
| { |
| "ttft": 25.00813842198113, |
| "time_to_second_token": 0.023656431993003935, |
| "latency": 0.0, |
| "inter_token_latency_avg": 0.008246520938160926, |
| "chunk_inter_token_latency_avg": 0.023019065640334093, |
| "input_tokens": 78, |
| "output_tokens": 389, |
| "output_tps_per_user": 121.26325846969985, |
| "e2e_output_tps_per_user": 0.0, |
| "completed": false |
| }, |
| { |
| "ttft": 9.027650000003632, |
| "time_to_second_token": 0.023537531029433012, |
| "latency": 16.70896882499801, |
| "inter_token_latency_avg": 0.00750862055229167, |
| "chunk_inter_token_latency_avg": 0.023347473632201757, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 133.18025501965136, |
| "e2e_output_tps_per_user": 61.2844521241796, |
| "completed": true |
| }, |
| { |
| "ttft": 26.10788277600659, |
| "time_to_second_token": 0.023854755039792508, |
| "latency": 33.4722074510064, |
| "inter_token_latency_avg": 0.007198753347995902, |
| "chunk_inter_token_latency_avg": 0.023603604727563485, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 138.9129411245067, |
| "e2e_output_tps_per_user": 30.592544620752573, |
| "completed": true |
| }, |
| { |
| "ttft": 16.83144874899881, |
| "time_to_second_token": 0.02357069100253284, |
| "latency": 26.018286619975697, |
| "inter_token_latency_avg": 0.008980291173975452, |
| "chunk_inter_token_latency_avg": 0.023316847388266213, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 111.35496395684392, |
| "e2e_output_tps_per_user": 39.35693441142345, |
| "completed": true |
| }, |
| { |
| "ttft": 24.284541705972515, |
| "time_to_second_token": 0.025024111033417284, |
| "latency": 33.153149329009466, |
| "inter_token_latency_avg": 0.008669215662792717, |
| "chunk_inter_token_latency_avg": 0.02340002011355396, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 115.35068902390854, |
| "e2e_output_tps_per_user": 30.886960084482403, |
| "completed": true |
| }, |
| { |
| "ttft": 26.161639873986132, |
| "time_to_second_token": 0.023638906015548855, |
| "latency": 34.53446650295518, |
| "inter_token_latency_avg": 0.008184581259989297, |
| "chunk_inter_token_latency_avg": 0.02371905560614462, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 122.18096054449921, |
| "e2e_output_tps_per_user": 29.651536673148673, |
| "completed": true |
| }, |
| { |
| "ttft": 24.75903701898642, |
| "time_to_second_token": 0.024325895006768405, |
| "latency": 33.02446756296558, |
| "inter_token_latency_avg": 0.008079599749735253, |
| "chunk_inter_token_latency_avg": 0.02348133677266808, |
| "input_tokens": 78, |
| "output_tokens": 1024, |
| "output_tps_per_user": 123.76850722497329, |
| "e2e_output_tps_per_user": 31.007312927834686, |
| "completed": true |
| } |
| ], |
| "total_tokens": 1189, |
| "wall_time": 71.4120287669939, |
| "num_completed": 4, |
| "num_errors": 0, |
| "server_gen_throughput": 118.84137161719322, |
| "server_utilization": 0.6603773584905661, |
| "server_spec_accept_rate": 0.25553319919517103, |
| "server_spec_accept_length": 2.788732394366197, |
| "server_spec_drafts": 426, |
| "server_spec_draft_tokens": 2982, |
| "server_spec_accepted_tokens": 762, |
| "server_spec_pos_accept": [ |
| 0.73, |
| 0.4695, |
| 0.277, |
| 0.1714, |
| 0.0751, |
| 0.0399, |
| 0.0258 |
| ], |
| "server_engine_steps": 427.0, |
| "server_steps_per_s": 42.72707440362766, |
| "server_accept_len_effective": 2.7845433255269323, |
| "accept_norm_tps": 0.0, |
| "accept_norm_ref_len": 0.0, |
| "avg_running_reqs": 1, |
| "max_running_reqs": 1, |
| "effective_concurrency": 1, |
| "avg_queue_reqs": 3, |
| "max_queue_reqs": 3, |
| "queue_fraction": 1.0, |
| "underfilled": true, |
| "warmup_timed_out": true, |
| "warmup_duration": 60.889, |
| "ready_reason": "warmup_timeout", |
| "timeout_reason": "running_reqs=1/4", |
| "capacity_limited": true, |
| "hardware_summary": { |
| "samples": 4, |
| "duration_seconds": 7.185, |
| "gpu_count": 4, |
| "cpu_util_avg_pct": 7.05, |
| "cpu_temp_max_c": 67.12, |
| "gpu_util_avg_pct": 49.5, |
| "gpu_util_max_pct": 99.0, |
| "mem_util_avg_pct": 26.0, |
| "mem_util_max_pct": 59.0, |
| "temp_avg_c": 51.38, |
| "temp_max_c": 69.0, |
| "power_total_avg_w": 619.54, |
| "power_total_max_w": 619.93, |
| "power_limit_total_w": 1200.0, |
| "vram_used_avg_mb": 189606.0, |
| "vram_used_max_mb": 189606.0, |
| "vram_total_mb": 391548.0, |
| "vram_used_avg_pct": 48.42, |
| "vram_used_max_pct": 48.42, |
| "pcie_rx_avg_mb_s": 2439.75, |
| "pcie_rx_max_mb_s": 2649.0, |
| "pcie_tx_avg_mb_s": 2278.25, |
| "pcie_tx_max_mb_s": 2448.0 |
| } |
| }, |
| { |
| "concurrency": 2, |
| "context_tokens": 65536, |
| "benchmark_mode": "duration", |
| "request_count_target": 0, |
| "warmup_request_count": 0, |
| "measurement_seconds": 9.984335, |
| "measurement_wall_seconds": 10.000465, |
| "client_output_tokens": 121, |
| "server_output_tokens": 121, |
| "aggregate_source": "openai_continuous_usage", |
| "aggregate_tps": 12.118983865880503, |
| "per_request_avg_tps": 6.059491932940252, |
| "ttft_avg": 35.443730441189835, |
| "ttft_p50": 40.205350022006314, |
| "ttft_p90": 40.74598405077122, |
| "ttft_p99": 41.04840777766425, |
| "time_to_second_token_avg": 0.008084286202210933, |
| "time_to_second_token_p50": 0.008268840028904378, |
| "time_to_second_token_p90": 0.009591756807640194, |
| "time_to_second_token_p99": 0.009597312689293177, |
| "request_latency_avg": 42.43303290549375, |
| "request_latency_p50": 48.06976508401567, |
| "request_latency_p90": 48.95471742947702, |
| "request_latency_p99": 49.295935525210226, |
| "inter_token_latency_avg": 0.00817623119573374, |
| "inter_token_latency_p50": 0.008107823757570827, |
| "inter_token_latency_p90": 0.008595517486809978, |
| "inter_token_latency_p99": 0.008858294394729843, |
| "output_tps_per_user_avg": 122.5777308761553, |
| "output_tps_per_user_p50": 123.33765877264314, |
| "output_tps_per_user_p90": 127.79598606592414, |
| "output_tps_per_user_p99": 129.85589972455693, |
| "e2e_output_tps_per_user_avg": 26.393212866280486, |
| "e2e_output_tps_per_user_p50": 21.302371631183085, |
| "e2e_output_tps_per_user_p90": 35.93885086168265, |
| "e2e_output_tps_per_user_p99": 41.58429651962935, |
| "chunk_inter_token_latency_avg": 0.0248280156995747, |
| "chunk_inter_token_latency_p50": 0.024617049442487427, |
| "chunk_inter_token_latency_p90": 0.025319819759822115, |
| "chunk_inter_token_latency_p99": 0.02538532188198598, |
| "input_seq_len_avg": 64513.0, |
| "output_seq_len_avg": 1024.0, |
| "output_seq_len_p50": 1024.0, |
| "output_seq_len_p90": 1024.0, |
| "output_seq_len_p99": 1024.0, |
| "request_count": 5, |
| "completed_request_count": 4, |
| "request_samples": [ |
| { |
| "ttft": 15.913573045982048, |
| "time_to_second_token": 0.008091808995231986, |
| "latency": 24.258752806985285, |
| "inter_token_latency_avg": 0.008157555973610203, |
| "chunk_inter_token_latency_avg": 0.024617049442487427, |
| "input_tokens": 64513, |
| "output_tokens": 1024, |
| "output_tps_per_user": 122.58573563393406, |
| "e2e_output_tps_per_user": 42.211568259401204, |
| "completed": true |
| }, |
| { |
| "ttft": 39.77577421802562, |
| "time_to_second_token": 0.004880354972556233, |
| "latency": 48.07007792202057, |
| "inter_token_latency_avg": 0.008107823757570827, |
| "chunk_inter_token_latency_avg": 0.025210649556215672, |
| "input_tokens": 64513, |
| "output_tokens": 1024, |
| "output_tps_per_user": 123.33765877264314, |
| "e2e_output_tps_per_user": 21.302232995360146, |
| "completed": true |
| }, |
| { |
| "ttft": 41.0820104139857, |
| "time_to_second_token": 0.008268840028904378, |
| "latency": 0.0, |
| "inter_token_latency_avg": 0.008040989966927252, |
| "chunk_inter_token_latency_avg": 0.025392599895559743, |
| "input_tokens": 64513, |
| "output_tokens": 121, |
| "output_tps_per_user": 124.36279663486951, |
| "e2e_output_tps_per_user": 0.0, |
| "completed": false |
| }, |
| { |
| "ttft": 40.205350022006314, |
| "time_to_second_token": 0.009582497004885226, |
| "latency": 48.06945224601077, |
| "inter_token_latency_avg": 0.007687294451617258, |
| "chunk_inter_token_latency_avg": 0.02434706570899212, |
| "input_tokens": 64513, |
| "output_tokens": 1024, |
| "output_tps_per_user": 130.08477901996056, |
| "e2e_output_tps_per_user": 21.30251026700602, |
| "completed": true |
| }, |
| { |
| "ttft": 40.241944505949505, |
| "time_to_second_token": 0.00959793000947684, |
| "latency": 49.33384864695836, |
| "inter_token_latency_avg": 0.008887491828943161, |
| "chunk_inter_token_latency_avg": 0.024572713894618525, |
| "input_tokens": 64513, |
| "output_tokens": 1024, |
| "output_tps_per_user": 112.51768431936922, |
| "e2e_output_tps_per_user": 20.75653994335457, |
| "completed": true |
| } |
| ], |
| "total_tokens": 121, |
| "wall_time": 146.70948266400956, |
| "num_completed": 2, |
| "num_errors": 0, |
| "server_gen_throughput": 12.093841253636528, |
| "server_utilization": 0.7169811320754718, |
| "server_spec_accept_rate": 0.3082706766917293, |
| "server_spec_accept_length": 3.1578947368421053, |
| "server_spec_drafts": 38, |
| "server_spec_draft_tokens": 266, |
| "server_spec_accepted_tokens": 82, |
| "server_spec_pos_accept": [ |
| 0.7368, |
| 0.5526, |
| 0.3684, |
| 0.1842, |
| 0.1316, |
| 0.1053, |
| 0.0789 |
| ], |
| "server_engine_steps": 39.0, |
| "server_steps_per_s": 3.906118766688757, |
| "server_accept_len_effective": 3.1025641025641026, |
| "accept_norm_tps": 0.0, |
| "accept_norm_ref_len": 0.0, |
| "avg_running_reqs": 1, |
| "max_running_reqs": 1, |
| "effective_concurrency": 1, |
| "avg_queue_reqs": 1, |
| "max_queue_reqs": 1, |
| "queue_fraction": 1.0, |
| "underfilled": true, |
| "warmup_timed_out": true, |
| "warmup_duration": 120.609, |
| "ready_reason": "warmup_timeout", |
| "timeout_reason": "running_reqs=1/2", |
| "capacity_limited": true, |
| "hardware_summary": { |
| "samples": 11, |
| "duration_seconds": 23.837, |
| "gpu_count": 4, |
| "cpu_util_avg_pct": 7.09, |
| "cpu_temp_max_c": 67.25, |
| "gpu_util_avg_pct": 49.75, |
| "gpu_util_max_pct": 100.0, |
| "mem_util_avg_pct": 10.07, |
| "mem_util_max_pct": 29.0, |
| "temp_avg_c": 54.2, |
| "temp_max_c": 74.0, |
| "power_total_avg_w": 617.96, |
| "power_total_max_w": 618.92, |
| "power_limit_total_w": 1200.0, |
| "vram_used_avg_mb": 189606.0, |
| "vram_used_max_mb": 189606.0, |
| "vram_total_mb": 391548.0, |
| "vram_used_avg_pct": 48.42, |
| "vram_used_max_pct": 48.42, |
| "pcie_rx_avg_mb_s": 14405.09, |
| "pcie_rx_max_mb_s": 22097.0, |
| "pcie_tx_avg_mb_s": 16793.09, |
| "pcie_tx_max_mb_s": 22792.0 |
| } |
| }, |
| { |
| "concurrency": 4, |
| "context_tokens": 65536, |
| "benchmark_mode": "duration", |
| "request_count_target": 0, |
| "warmup_request_count": 0, |
| "measurement_seconds": 10.002486, |
| "measurement_wall_seconds": 10.002486, |
| "client_output_tokens": 0, |
| "server_output_tokens": 0, |
| "aggregate_source": "prometheus_fallback", |
| "aggregate_tps": 0.06453558471444981, |
| "per_request_avg_tps": 0.016133896178612453, |
| "ttft_avg": 53.79123655226431, |
| "ttft_p50": 53.92967229100759, |
| "ttft_p90": 83.8944338750327, |
| "ttft_p99": 90.48217335203546, |
| "time_to_second_token_avg": 0.006969800742808729, |
| "time_to_second_token_p50": 0.008464488462777808, |
| "time_to_second_token_p90": 0.00998570021474734, |
| "time_to_second_token_p99": 0.010018377252854406, |
| "request_latency_avg": 62.69665365225228, |
| "request_latency_p50": 62.90479161249823, |
| "request_latency_p90": 92.55194512308226, |
| "request_latency_p99": 99.28581670898885, |
| "inter_token_latency_avg": 0.008705197556195476, |
| "inter_token_latency_p50": 0.008637062442312125, |
| "inter_token_latency_p90": 0.009213662504711036, |
| "inter_token_latency_p99": 0.009430095063974691, |
| "output_tps_per_user_avg": 115.22638877102585, |
| "output_tps_per_user_p50": 115.78047725913596, |
| "output_tps_per_user_p90": 121.29599886317772, |
| "output_tps_per_user_p99": 123.34338172040091, |
| "e2e_output_tps_per_user_avg": 21.279332857760934, |
| "e2e_output_tps_per_user_p50": 16.913611610276735, |
| "e2e_output_tps_per_user_p90": 34.79479489896622, |
| "e2e_output_tps_per_user_p99": 40.42771152572509, |
| "chunk_inter_token_latency_avg": 0.02482740656725064, |
| "chunk_inter_token_latency_p50": 0.02478472501408651, |
| "chunk_inter_token_latency_p90": 0.025181052141669647, |
| "chunk_inter_token_latency_p99": 0.02530338106269595, |
| "input_seq_len_avg": 64513.0, |
| "output_seq_len_avg": 1024.0, |
| "output_seq_len_p50": 1024.0, |
| "output_seq_len_p90": 1024.0, |
| "output_seq_len_p99": 1024.0, |
| "request_count": 4, |
| "completed_request_count": 4, |
| "request_samples": [ |
| { |
| "ttft": 16.091457222006284, |
| "time_to_second_token": 0.010022008034866303, |
| "latency": 24.94300672103418, |
| "inter_token_latency_avg": 0.008652541054768228, |
| "chunk_inter_token_latency_avg": 0.024863903087157014, |
| "input_tokens": 64513, |
| "output_tokens": 1024, |
| "output_tps_per_user": 115.57298528493219, |
| "e2e_output_tps_per_user": 41.053591150920525, |
| "completed": true |
| }, |
| { |
| "ttft": 41.04423527698964, |
| "time_to_second_token": 0.0009282180108129978, |
| "latency": 50.71582369500538, |
| "inter_token_latency_avg": 0.009454143126115097, |
| "chunk_inter_token_latency_avg": 0.024423203075797335, |
| "input_tokens": 64513, |
| "output_tokens": 1024, |
| "output_tps_per_user": 105.77373186129462, |
| "e2e_output_tps_per_user": 20.190936977739472, |
| "completed": true |
| }, |
| { |
| "ttft": 66.81510930502554, |
| "time_to_second_token": 0.00990098196780309, |
| "latency": 75.09375952999108, |
| "inter_token_latency_avg": 0.008092522214042552, |
| "chunk_inter_token_latency_avg": 0.025316973165032206, |
| "input_tokens": 64513, |
| "output_tokens": 1024, |
| "output_tps_per_user": 123.57086870453684, |
| "e2e_output_tps_per_user": 13.636286242814, |
| "completed": true |
| }, |
| { |
| "ttft": 91.21414440503577, |
| "time_to_second_token": 0.007027994957752526, |
| "latency": 100.03402466297848, |
| "inter_token_latency_avg": 0.008621583829856025, |
| "chunk_inter_token_latency_avg": 0.024705546941016007, |
| "input_tokens": 64513, |
| "output_tokens": 1024, |
| "output_tps_per_user": 115.98796923333974, |
| "e2e_output_tps_per_user": 10.236517059569747, |
| "completed": true |
| } |
| ], |
| "total_tokens": 4096, |
| "wall_time": 160.65691649296787, |
| "num_completed": 4, |
| "num_errors": 0, |
| "server_gen_throughput": 0.06453558471444981, |
| "server_utilization": 0.6981132075471699, |
| "server_spec_accept_rate": 0.12315270935960591, |
| "server_spec_accept_length": 1.8620689655172413, |
| "server_spec_drafts": 0, |
| "server_spec_draft_tokens": 0, |
| "server_spec_accepted_tokens": 0, |
| "server_spec_pos_accept": [], |
| "server_engine_steps": 0.0, |
| "server_steps_per_s": 0.0, |
| "server_accept_len_effective": 0.0, |
| "accept_norm_tps": 0.0, |
| "accept_norm_ref_len": 0.0, |
| "avg_running_reqs": 1, |
| "max_running_reqs": 1, |
| "effective_concurrency": 1, |
| "avg_queue_reqs": 3, |
| "max_queue_reqs": 3, |
| "queue_fraction": 1.0, |
| "underfilled": true, |
| "warmup_timed_out": true, |
| "warmup_duration": 120.589, |
| "ready_reason": "warmup_timeout", |
| "timeout_reason": "running_reqs=1/4", |
| "capacity_limited": true, |
| "hardware_summary": { |
| "samples": 17, |
| "duration_seconds": 38.153, |
| "gpu_count": 4, |
| "cpu_util_avg_pct": 7.11, |
| "cpu_temp_max_c": 67.12, |
| "gpu_util_avg_pct": 49.72, |
| "gpu_util_max_pct": 100.0, |
| "mem_util_avg_pct": 10.35, |
| "mem_util_max_pct": 30.0, |
| "temp_avg_c": 54.96, |
| "temp_max_c": 75.0, |
| "power_total_avg_w": 618.56, |
| "power_total_max_w": 620.17, |
| "power_limit_total_w": 1200.0, |
| "vram_used_avg_mb": 189606.0, |
| "vram_used_max_mb": 189606.0, |
| "vram_total_mb": 391548.0, |
| "vram_used_avg_pct": 48.42, |
| "vram_used_max_pct": 48.42, |
| "pcie_rx_avg_mb_s": 14517.06, |
| "pcie_rx_max_mb_s": 23537.0, |
| "pcie_tx_avg_mb_s": 17394.76, |
| "pcie_tx_max_mb_s": 25466.0 |
| } |
| } |
| ], |
| "summary_table": { |
| "0": { |
| "1": 129.45196578500713, |
| "2": 115.94808445857444, |
| "4": 118.97538984991404 |
| }, |
| "65536": { |
| "1": 122.23577506588123, |
| "2": 12.118983865880503, |
| "4": 0.06453558471444981 |
| } |
| }, |
| "burst_results": [], |
| "burst_summary_table": {}, |
| "methodology": { |
| "prefill": { |
| "name": "Prefill", |
| "present": true, |
| "mode": "integrated_decode_scout", |
| "formula": "prompt_tokens / TTFT", |
| "notes": "Default mode records the required decode scout request for each non-zero decode context, so normal runs do not pay for a separate prefill phase. Standalone mode repeats cold-prefill samples. Prometheus prefill counters, when available and uncontaminated, are stored as validation." |
| }, |
| "sustained_decode": { |
| "name": "Sustained Decode", |
| "present": true, |
| "formula": "OpenAI stream usage completion_tokens per measured window; client chunk fallback only when continuous usage is unavailable", |
| "notes": "Duration-based steady-state cell after warmup. This is the main tuning/regression signal for kernels, NCCL, DCP, MTP, and scheduling. Prometheus metrics are stored as validation and scheduler state, not the default headline." |
| }, |
| "burst_e2e_decode": { |
| "name": "Burst / E2E Decode", |
| "present": false, |
| "status": "not run; use --run-burst", |
| "formula": "sum(completion_tokens) / profiling_wall_time", |
| "notes": "Finite client-facing request burst using OpenAI stream usage. It includes request admission, scheduling, prefill/cache behavior, and completion." |
| }, |
| "acceptance_normalization": { |
| "name": "Acceptance-normalized decode (MTP / speculative)", |
| "present": true, |
| "formula": "engine_steps = spec_drafts + max(0, output_tokens - (accepted_tokens + spec_drafts)); accept_len_effective = output_tokens / engine_steps; steps_per_s = aggregate_tps / accept_len_effective", |
| "notes": "With speculative decoding tok/s = steps_per_s * accept_len, so raw tok/s mixes engine speed with data-dependent acceptance. steps_per_s (target-model forward passes per second) is the acceptance-independent speed used to compare runs; server_spec_pos_accept holds per-draft-position acceptance probabilities. Counters are vLLM window deltas; SGLang falls back to its lifetime accept-length gauge." |
| } |
| } |
| } |
