From 2040465b9df2a2368fc7627642fc8ee6f8c99ff8 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 18 Aug 2026 11:10:07 -0700 Subject: [PATCH 1/6] feat(config): add GLM-5.2 GB300 Dynamo-TRT AgentX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加 GLM-5.2 GB300 Dynamo-TRT AgentX 配置和多节点 srt-slurm 配方。 --- benchmarks/benchmark_lib.sh | 2 +- benchmarks/multi_node/agentic_srt.sh | 9 +- ...mo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml | 187 +++++++++++++++++ ...amo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml | 193 +++++++++++++++++ ...mo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml | 187 +++++++++++++++++ ...mo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml | 187 +++++++++++++++++ ...disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml | 188 +++++++++++++++++ ...-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml | 187 +++++++++++++++++ ...disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml | 194 ++++++++++++++++++ configs/nvidia-master.yaml | 122 +++++++++++ perf-changelog.yaml | 7 + runners/launch_gb300-nv.sh | 29 +++ 12 files changed, 1489 insertions(+), 3 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 68cd412217..b73fd9364d 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1996,7 +1996,7 @@ build_replay_cmd() { # rolling TTFT/ITL/throughput block and emit it every 30 seconds. export AIPERF_UI_REALTIME_METRICS_ENABLED=true REPLAY_CMD="$AIPERF_CLI profile --scenario inferencex-agentx-mvp" - REPLAY_CMD+=" --url http://localhost:$PORT" + REPLAY_CMD+=" --url ${AIPERF_SERVER_URL:-http://localhost:$PORT}" REPLAY_CMD+=" --endpoint /v1/chat/completions" REPLAY_CMD+=" --endpoint-type chat" REPLAY_CMD+=" --streaming" diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 79a36da524..77c196c133 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -9,7 +9,12 @@ set -x INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-/infmax-workspace}" source "$INFMAX_CONTAINER_WORKSPACE/benchmarks/benchmark_lib.sh" -check_env_vars MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION +check_env_vars \ + MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION \ + SRT_FRONTEND_HOST SRT_FRONTEND_PORT + +export AIPERF_SERVER_URL="http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" +echo "Using srt-slurm frontend endpoint: $AIPERF_SERVER_URL" BASE_RESULT_DIR="${RESULT_DIR:-/logs/agentic}" BASE_RESULT_FILENAME="$RESULT_FILENAME" @@ -32,7 +37,7 @@ install_agentic_deps wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}" - local frontend_metrics_url="http://localhost:${PORT}/metrics" + local frontend_metrics_url="${AIPERF_SERVER_URL%/}/metrics" "$AIPERF_PYTHON" - \ "$timeout_seconds" \ diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml new file mode 100644 index 0000000000..ffcda09100 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml @@ -0,0 +1,187 @@ +name: dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0.dev20260807 + tensorrt_llm: 1.3.0rc22.post1 + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 5] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml new file mode 100644 index 0000000000..294d70d1fb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml @@ -0,0 +1,193 @@ +name: dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0.dev20260807 + tensorrt_llm: 1.3.0rc22.post1 + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 6 + decode_workers: 3 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.91" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 7 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 1 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 7 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '1' + DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml new file mode 100644 index 0000000000..f54effc250 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml @@ -0,0 +1,187 @@ +name: dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0.dev20260807 + tensorrt_llm: 1.3.0rc22.post1 + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 4 + gpus_per_decode: 4 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 2 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml new file mode 100644 index 0000000000..0c61f63e17 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml @@ -0,0 +1,187 @@ +name: dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0.dev20260807 + tensorrt_llm: 1.3.0rc22.post1 + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + prefill_workers: 3 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 4 + gpus_per_decode: 4 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 5] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml new file mode 100644 index 0000000000..397a42744f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml @@ -0,0 +1,188 @@ +name: dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0.dev20260807 + tensorrt_llm: 1.3.0rc22.post1 + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 5 + prefill_workers: 5 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 16 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml new file mode 100644 index 0000000000..6dbb97914b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml @@ -0,0 +1,187 @@ +name: dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0.dev20260807 + tensorrt_llm: 1.3.0rc22.post1 + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 6 + prefill_workers: 6 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml new file mode 100644 index 0000000000..b773b8ff0f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml @@ -0,0 +1,194 @@ +name: dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + frameworks: + dynamo: 1.4.0.dev20260807 + tensorrt_llm: 1.3.0rc22.post1 + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 8 + prefill_workers: 8 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 16 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: first_decode + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '259' + DURATION: '3600' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c8d55ad5d5..33f84327f4 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8453,6 +8453,128 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 16 dp-attn: true +# GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving. +# Seven topology variants use NIXL KV transfer and MTP3/MTP5/MTP7 decoding. +glm5.2-fp4-gb300-dynamo-trt-agentic-mtp: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.4.0.dev20260807" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [1] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: mtp + conc-list: [30] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: mtp + conc-list: [20] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: mtp + conc-list: [60] + kv-offloading: none + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: mtp + conc-list: [152] + kv-offloading: none + prefill: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [259] + kv-offloading: none + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [227] + kv-offloading: none + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + qwen3.5-fp8-gb200-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 73648d9b99..d6a8421f1c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6076,3 +6076,10 @@ - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX mirroring the GB300 PR #2571 MTP tuning, with every GB300 4-GPU worker sized to 8 GPUs on GB200." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2636 +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 NVFP4 GB300 AgentX recipes for Dynamo and TensorRT-LLM disaggregated serving with NIXL KV transfer and MTP speculative decoding." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 88b964a8ad..2c5420810a 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -65,6 +65,10 @@ elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then # in our GLM-5.1 sglang recipes. export MODEL_PATH=/scratch/models/GLM-5.1-NVFP4 export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" +elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-trt" ]]; then + export SERVED_MODEL_NAME="GLM-5.2-NVFP4" + export MODEL_PATH=/scratch/models/GLM-5.2-NVFP4 + export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5.2-NVFP4" elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then export MODEL_PATH=/scratch/models/GLM-5.2-NVFP4 export SRT_SLURM_MODEL_PREFIX="glm-5.2-fp4" @@ -202,6 +206,18 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qw find "$TRTLLM_RECIPES_DIR" -name "*.yaml" \ -exec sed -i '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' {} + fi +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.38 + TRTLLM_RECIPES_DIR="benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2" + mkdir -p "$TRTLLM_RECIPES_DIR" + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2" \ + "$TRTLLM_RECIPES_DIR" + if [[ "${EVAL_ONLY:-false}" == "true" ]]; then + find "$TRTLLM_RECIPES_DIR" -name "*.yaml" \ + -exec sed -i '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' {} + + fi elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.38: the two features the # cquil11 fork was pinned for are merged upstream (present in v1.0.36) — @@ -641,6 +657,19 @@ if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then else echo "WARNING: RUN_EVAL=true but no eval results found at $EVAL_DIR" fi + + # srt-slurm stages eval artifacts but does not write the metadata file + # consumed by score validation. Reuse the canonical metadata writer so + # topology and recipe identity stay aligned with the workflow inputs. + eval_conc_value="${EVAL_CONC:-${CONC:-1}}" + ( + export IS_MULTINODE=true + # shellcheck source=benchmarks/benchmark_lib.sh + source "$GITHUB_WORKSPACE/benchmarks/benchmark_lib.sh" + _write_lm_eval_meta_json \ + "$GITHUB_WORKSPACE/meta_env.json" "" "$eval_conc_value" + ) + echo "Wrote meta_env.json (conc=${eval_conc_value}, prefix=${MODEL_PREFIX:-unknown})" fi # Snapshot logs to GITHUB_WORKSPACE BEFORE cleanup, so the EXIT trap's From b22e6ef516b5285b4e255055649134ca2ae14817 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 18 Aug 2026 11:12:10 -0700 Subject: [PATCH 2/6] chore(changelog): link PR #2657 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在变更日志中补充 PR #2657 链接。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 00a159d55b..08ad94407f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6093,4 +6093,4 @@ - agentic-coding description: - "Add GLM-5.2 NVFP4 GB300 AgentX recipes for Dynamo and TensorRT-LLM disaggregated serving with NIXL KV transfer and MTP speculative decoding." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2657 From a9359a6c897f6c85335f91725bb334aa1121344f Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 18 Aug 2026 15:20:51 -0700 Subject: [PATCH 3/6] feat(config): update GLM-5.2 GB300 Pareto01 recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新 GLM-5.2 GB300 Pareto01 配方，并移除重复的可选身份信息。 --- ...mo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml} | 29 +++++++------------ ...mo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml | 9 ------ ...mo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml | 9 ------ ...mo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml | 9 ------ ...disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml | 9 ------ ...-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml | 9 ------ ...disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml | 9 ------ configs/nvidia-master.yaml | 8 ++--- 8 files changed, 14 insertions(+), 77 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/{dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml => dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml} (90%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml similarity index 90% rename from benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml index 294d70d1fb..cfdf1eb69c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml @@ -1,4 +1,4 @@ -name: dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7 +name: dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8 model: path: nvidia/GLM-5.2-NVFP4 @@ -10,15 +10,6 @@ dynamo: wheel: 1.4.0.dev20260807 request_plane: tcp -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 - frameworks: - dynamo: 1.4.0.dev20260807 - tensorrt_llm: 1.3.0rc22.post1 - health_check: max_attempts: 270 interval_seconds: 10 @@ -29,8 +20,8 @@ resources: prefill_nodes: 1 prefill_workers: 1 gpus_per_prefill: 4 - decode_nodes: 6 - decode_workers: 3 + decode_nodes: 2 + decode_workers: 1 gpus_per_decode: 8 backend: @@ -54,7 +45,7 @@ backend: UCX_RNDV_SCHEME: put_zcopy TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" - TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.91" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "3.06" DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" DYN_ENGINE_CONV_AFFINITY: "1" DYN_TOKENIZER: fastokens @@ -101,7 +92,7 @@ backend: enable_heuristic_topk: true speculative_config: decoding_type: MTP - max_draft_len: 7 + max_draft_len: 8 tensor_parallel_size: 4 decode: cache_transceiver_config: @@ -112,7 +103,7 @@ backend: kv_transfer_timeout_ms: 600000 cuda_graph_config: batch_sizes: [1] - enable_padding: true + enable_padding: false enable_attention_dp: false trust_remote_code: true kv_cache_config: @@ -127,8 +118,8 @@ backend: moe_config: backend: TRTLLM use_low_precision_moe_combine: true - moe_expert_parallel_size: 8 - num_postprocess_workers: 4 + moe_expert_parallel_size: 1 + num_postprocess_workers: 8 pipeline_parallel_size: 1 print_iter_log: true return_perf_metrics: true @@ -139,8 +130,8 @@ backend: use_cute_dsl_topk: true speculative_config: decoding_type: MTP - max_draft_len: 7 - stream_interval: 20 + max_draft_len: 8 + stream_interval: 60 tensor_parallel_size: 8 publish_events_and_metrics: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml index ffcda09100..958baec770 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml @@ -10,15 +10,6 @@ dynamo: wheel: 1.4.0.dev20260807 request_plane: tcp -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 - frameworks: - dynamo: 1.4.0.dev20260807 - tensorrt_llm: 1.3.0rc22.post1 - health_check: max_attempts: 270 interval_seconds: 10 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml index f54effc250..ceb0062eb6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml @@ -10,15 +10,6 @@ dynamo: wheel: 1.4.0.dev20260807 request_plane: tcp -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 - frameworks: - dynamo: 1.4.0.dev20260807 - tensorrt_llm: 1.3.0rc22.post1 - health_check: max_attempts: 270 interval_seconds: 10 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml index 0c61f63e17..f974d14032 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml @@ -10,15 +10,6 @@ dynamo: wheel: 1.4.0.dev20260807 request_plane: tcp -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 - frameworks: - dynamo: 1.4.0.dev20260807 - tensorrt_llm: 1.3.0rc22.post1 - health_check: max_attempts: 270 interval_seconds: 10 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml index 397a42744f..43605b1dc9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml @@ -10,15 +10,6 @@ dynamo: wheel: 1.4.0.dev20260807 request_plane: tcp -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 - frameworks: - dynamo: 1.4.0.dev20260807 - tensorrt_llm: 1.3.0rc22.post1 - health_check: max_attempts: 270 interval_seconds: 10 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml index 6dbb97914b..7cd10df6b7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml @@ -10,15 +10,6 @@ dynamo: wheel: 1.4.0.dev20260807 request_plane: tcp -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 - frameworks: - dynamo: 1.4.0.dev20260807 - tensorrt_llm: 1.3.0rc22.post1 - health_check: max_attempts: 270 interval_seconds: 10 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml index b773b8ff0f..d4d52a7e82 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml @@ -10,15 +10,6 @@ dynamo: wheel: 1.4.0.dev20260807 request_plane: tcp -identity: - model: - repo: nvidia/GLM-5.2-NVFP4 - container: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 - frameworks: - dynamo: 1.4.0.dev20260807 - tensorrt_llm: 1.3.0rc22.post1 - health_check: max_attempts: 270 interval_seconds: 10 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0d3d067fa4..8246a98fcb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8470,7 +8470,7 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: dp-attn: true # GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving. -# Seven topology variants use NIXL KV transfer and MTP3/MTP5/MTP7 decoding. +# Seven topology variants use NIXL KV transfer and MTP3/MTP5/MTP8 decoding. glm5.2-fp4-gb300-dynamo-trt-agentic-mtp: image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 model: nvidia/GLM-5.2-NVFP4 @@ -8494,11 +8494,11 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p3d-tep8-c1-b1-mtp7.yaml" + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml" decode: - num-worker: 3 + num-worker: 1 tp: 8 - ep: 8 + ep: 1 dp-attn: false - spec-decoding: mtp conc-list: [30] From 15c9fff7b6c0de41df313cb6c58cb822afc68390 Mon Sep 17 00:00:00 2001 From: Ankur-singh Date: Wed, 19 Aug 2026 13:16:25 -0700 Subject: [PATCH 4/6] fix(config): refresh GLM-5.2 GB300 TRT Pareto points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新 GLM-5.2 GB300 TensorRT-LLM AgentX Pareto 配置，并关闭所有点的性能指标响应。 --- ...l => dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml} | 88 +++------------ ...mo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml | 4 +- ...mo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml | 4 +- ...mo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml | 4 +- ...disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml | 4 +- ...-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml | 4 +- ...disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml | 4 +- configs/nvidia-master.yaml | 36 ++++-- perf-changelog.yaml | 106 +++++++++++++++++- 9 files changed, 160 insertions(+), 94 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/{dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml => dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml} (61%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml similarity index 61% rename from benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml rename to benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml index cfdf1eb69c..eb3e3d2889 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml @@ -1,4 +1,4 @@ -name: dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8 +name: dynamo-agg-gb300-tp8-c1-b2-mtp8 model: path: nvidia/GLM-5.2-NVFP4 @@ -17,16 +17,13 @@ health_check: resources: gpu_type: gb300 gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 backend: type: trtllm - prefill_environment: &server_environment + aggregated_environment: HF_HUB_OFFLINE: "1" TRANSFORMERS_OFFLINE: "1" TQDM_DISABLE: "1" @@ -40,98 +37,51 @@ backend: PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TRTLLM_SERVE_ENABLE_MSGSPEC: "1" TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" - TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" UCX_CUDA_IPC_ENABLE_MNNVL: "y" UCX_RNDV_SCHEME: put_zcopy - TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" - TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "3.06" + TRTLLM_DISAGG_IDLE_CHECK: pr17324 DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" DYN_ENGINE_CONV_AFFINITY: "1" DYN_TOKENIZER: fastokens DYN_PUBLISH_KV_EVENTS: "0" - decode_environment: *server_environment trtllm_config: - prefill: - attention_dp_config: - enable_kv_cache_aware_routing: false - kv_cache_routing_conversation_affinity: true - kv_cache_routing_max_sessions: 65536 - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: null + aggregated: + cuda_graph_config: + batch_sizes: [1, 2] + enable_padding: false disable_overlap_scheduler: false - enable_attention_dp: true + enable_attention_dp: false enable_chunked_prefill: true + enable_lm_head_tp_in_adp: false trust_remote_code: true kv_cache_config: dtype: fp8 enable_block_reuse: true event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.75 - host_cache_size: 137438953472 + free_gpu_memory_fraction: 0.85 tokens_per_block: 64 - max_batch_size: 256 + max_batch_size: 2 max_num_tokens: 8192 max_seq_len: 1048576 - moe_config: - backend: CUTEDSL - moe_expert_parallel_size: 4 - num_postprocess_workers: 8 - pipeline_parallel_size: 1 - print_iter_log: true - return_perf_metrics: true - scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - sparse_attention_config: - algorithm: dsa - enable_heuristic_topk: true - speculative_config: - decoding_type: MTP - max_draft_len: 8 - tensor_parallel_size: 4 - decode: - cache_transceiver_config: - backend: NIXL - transceiver_runtime: PYTHON - kv_cache_bounce_size_mb: 5120 - max_tokens_in_buffer: 1048576 - kv_transfer_timeout_ms: 600000 - cuda_graph_config: - batch_sizes: [1] - enable_padding: false - enable_attention_dp: false - trust_remote_code: true - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - event_buffer_max_size: 0 - free_gpu_memory_fraction: 0.8 - tokens_per_block: 64 - max_batch_size: 1 - max_num_tokens: 128 - max_seq_len: 1048576 moe_config: backend: TRTLLM use_low_precision_moe_combine: true moe_expert_parallel_size: 1 num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false sparse_attention_config: algorithm: dsa enable_heuristic_topk: true use_cute_dsl_paged_mqa_logits: true - use_cute_dsl_topk: true speculative_config: decoding_type: MTP max_draft_len: 8 - stream_interval: 60 + num_nextn_predict_layers: 8 + stream_interval: 20 tensor_parallel_size: 8 publish_events_and_metrics: false @@ -154,11 +104,9 @@ frontend: active-decode-blocks-threshold: None active-prefill-tokens-threshold: None active-prefill-tokens-threshold-frac: None - orchestrator_placement: first_decode benchmark: type: custom - client_placement: last_decode command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml index 958baec770..e2f1815ef5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml @@ -84,7 +84,7 @@ backend: num_postprocess_workers: 8 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false scheduler_config: capacity_scheduler_policy: MAX_UTILIZATION sparse_attention_config: @@ -122,7 +122,7 @@ backend: num_postprocess_workers: 4 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false sparse_attention_config: algorithm: dsa enable_heuristic_topk: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml index ceb0062eb6..c172fffe20 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml @@ -84,7 +84,7 @@ backend: num_postprocess_workers: 8 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false scheduler_config: capacity_scheduler_policy: MAX_UTILIZATION sparse_attention_config: @@ -122,7 +122,7 @@ backend: num_postprocess_workers: 4 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false sparse_attention_config: algorithm: dsa enable_heuristic_topk: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml index f974d14032..fe36720786 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml @@ -84,7 +84,7 @@ backend: num_postprocess_workers: 8 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false scheduler_config: capacity_scheduler_policy: MAX_UTILIZATION sparse_attention_config: @@ -122,7 +122,7 @@ backend: num_postprocess_workers: 4 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false sparse_attention_config: algorithm: dsa enable_heuristic_topk: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml index 43605b1dc9..1bdf69fc7d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml @@ -84,7 +84,7 @@ backend: num_postprocess_workers: 8 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false scheduler_config: capacity_scheduler_policy: MAX_UTILIZATION sparse_attention_config: @@ -123,7 +123,7 @@ backend: num_postprocess_workers: 4 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false sparse_attention_config: algorithm: dsa enable_heuristic_topk: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml index 7cd10df6b7..9703e10128 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml @@ -84,7 +84,7 @@ backend: num_postprocess_workers: 8 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false scheduler_config: capacity_scheduler_policy: MAX_UTILIZATION sparse_attention_config: @@ -123,7 +123,7 @@ backend: num_postprocess_workers: 4 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false sparse_attention_config: algorithm: dsa enable_heuristic_topk: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml index d4d52a7e82..a5653b1793 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml @@ -84,7 +84,7 @@ backend: num_postprocess_workers: 8 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false scheduler_config: capacity_scheduler_policy: MAX_UTILIZATION sparse_attention_config: @@ -123,7 +123,7 @@ backend: num_postprocess_workers: 4 pipeline_parallel_size: 1 print_iter_log: true - return_perf_metrics: true + return_perf_metrics: false sparse_attention_config: algorithm: dsa enable_heuristic_topk: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8246a98fcb..8694ff7c96 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8469,9 +8469,9 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 16 dp-attn: true -# GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving. -# Seven topology variants use NIXL KV transfer and MTP3/MTP5/MTP8 decoding. -glm5.2-fp4-gb300-dynamo-trt-agentic-mtp: +# GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo aggregated serving. +# One TP8 worker serves prefill and decode on the same eight GPUs at concurrency 1. +glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp: image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 model: nvidia/GLM-5.2-NVFP4 model-prefix: glm5.2 @@ -8479,9 +8479,8 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-mtp: precision: fp4 framework: dynamo-trt router: { name: dynamo-router, version: "1.4.0.dev20260807" } - kv-p2p-transfer: nixl multinode: true - disagg: true + disagg: false scenarios: agentic-coding: - search-space: @@ -8490,16 +8489,33 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-mtp: kv-offloading: none prefill: num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c1-b1-mtp8.yaml" + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 8 ep: 1 dp-attn: false + +# GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving. +# Six topology variants use NIXL KV transfer and MTP3/MTP5 decoding. +glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.4.0.dev20260807" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: - spec-decoding: mtp conc-list: [30] kv-offloading: none diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 699680aec4..708689e1d4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5926,6 +5926,30 @@ - "Rides on the NVFP4-V2 checkpoint switch from #2205" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550 +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang + - dsv4-fp4-gb300-dynamo-sglang + - qwen3.5-fp4-gb300-dynamo-sglang + scenario-type: + - fixed-seq-len + description: + - "Extend official DCGM power collection to the recipe-gated FP4 GB200 and GB300 Dynamo-SGLang lanes." + - "Synchronize all DSV4 GB300 recipes on lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3." + - "Pin the srt-slurm power producer and record its exact checkout SHA for strict provenance validation." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2507 + +- config-keys: + - qwen3.5-fp8-b300-sglang-agentic-power-ab + - qwen3.5-fp4-b300-sglang-agentic-power-ab + scenario-type: + - agentic-coding + description: + - "Add a controlled Qwen3.5 B300 AgentX power matrix across FP8/FP4, HBM-only/HiCache, and concurrency 16/32 using identical SGLang v0.5.16 TP2/EP2 MTP settings." + - "新增受控的 Qwen3.5 B300 AgentX 功耗矩阵：在相同的 SGLang v0.5.16、TP2/EP2 和 MTP 配置下，对比 FP8/FP4、仅 HBM/HiCache，以及并发 16/32。" + - "Allow workflow_dispatch AgentX jobs to fail closed when strict power validation is requested." + - "允许通过 workflow_dispatch 启动的 AgentX 任务在启用严格功耗校验时遇到无效遥测后直接失败。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2601 + - config-keys: - dsv4-fp4-mi355x-sglang-agentic-mtp description: @@ -6105,9 +6129,87 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2652 - config-keys: - - glm5.2-fp4-gb300-dynamo-trt-agentic-mtp + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Add Kimi-K3 GB300 AgentX configurations." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2637 + +- config-keys: + - qwen3.5-fp8-mi355x-sglang-disagg + scenario-type: + - fixed-seq-len + description: + - "Bump qwen3.5-fp8-mi355x-sglang-disagg image from v0.5.14 to v0.5.16 (3-5% throughput improvement) and switch from TP8P+TP8D (16 GPU) to TP4P+TP8D (12 GPU) — TP4 prefill delivers identical decode interactivity with 24-31% better throughput/GPU." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2606 + +- config-keys: + - minimaxm3-fp4-b200-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 NVFP4 B200 single-node TensorRT-LLM AgentX with EAGLE3-GQA speculative decoding." + - "Use the nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 image." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2659 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Repin the image to the 2026-08-17 ROCm nightly (nightly-311b3513af33bc29b4acb2fde2e9313e5e9966a0, digest sha256:4b403315f9a0d9b0ed4221dfc7ea04538701b4de0ce283c0e2cee622b217ff89). Docker Hub last pushed it at 2026-08-17T05:32:09Z and the tag commit is vllm-project/vllm@311b3513." + - "Update the pure TP8 AgentX search-space to concurrency 1, 4, 8, 16, 32, 40, 48, 56, and 64; keep only concurrency 64 for the DP-attention arm." + - "The pinned nightly includes vLLM #51473 (native 384-shard MXFP4 TP8 allocation), #52212 (gfx950 Triton sparse-MLA decode), and #52401, the merged equivalent of Fangzhou's #52488 regression fix for ROCm DeepSeek V4 MRV1 plus the wide eager attention region." + - "Pin max-num-batched-tokens to 8192 for both arms instead of the nightly default of 16384; the TP8 initialization check increased GPU KV cache capacity from 4,730,981 to 8,524,228 tokens and reduced peak activation memory from 11.44 GiB to 8.9 GiB." + - "Remove the MoE padding from 384 to 512 and replace it with the native 384-expert kernels, save 30% redundant dummy computation." + - "Optimze the MLA kernel under long context scenario, where the num_kv_split is further tuned for best hardware utilization." + - "Cache fp32 upcast of static e8m0 weight scale in AITER scaled_mm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2590 + +- config-keys: + - minimaxm3-fp4-b300-trtllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 NVFP4 B300 single-node TensorRT-LLM AgentX with EAGLE3-GQA speculative decoding (3 draft tokens)." + - "Use the nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1 image." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2658 + +- config-keys: + - dsv4-fp4-b200-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Update the B200 SGLang AgentX HiCache MTP DEP8 configuration." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2656 + +- config-keys: + - glm5.2-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the MI355X GLM-5.2 FP4 ATOM AgentX MTP submission: TP4 at concurrency 2, 4, 8, and 10 uses LMCache DRAM offload, while GPU-resident TP8 at concurrency 1, 2, and 4 uses no KV offload." + - "Use AITER INT4 quick-reduce, ptpc_fp8 online quantization excluding embeddings, lm_head, gates, and experts, an FP8 KV cache, and three-token MTP with golden synthetic acceptance length 2.99 for benchmark runs." + - "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add new B200 multi-node configs with PP2, DCP8" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2672 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp scenario-type: - agentic-coding description: - - "Add GLM-5.2 NVFP4 GB300 AgentX recipes for Dynamo and TensorRT-LLM disaggregated serving with NIXL KV transfer and MTP speculative decoding." + - "Add one aggregated TP8 and six disaggregated GLM-5.2 NVFP4 GB300 AgentX recipes for Dynamo and TensorRT-LLM with MTP speculative decoding." + - "Use the concurrency-1 aggregated Pareto point and disable TensorRT-LLM performance-metrics responses for all seven points." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2657 From 03a01cdf2ce9adcccacd278ed96e5f383e7e3812 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Wed, 19 Aug 2026 23:37:28 -0700 Subject: [PATCH 5/6] fix(config): scope GB300 batch_3 partition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅为 GLM-5.2 GB300 Dynamo-TRT AgentX 配置启用 batch_3 分区。 --- configs/nvidia-master.yaml | 7 +++++++ perf-changelog.yaml | 15 +-------------- runners/launch_gb300-nv.sh | 2 +- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0efdd8f270..0b994f3c6a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8868,6 +8868,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp: dp-attn: false additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml" + - "SLURM_PARTITION=batch_3" decode: num-worker: 0 tp: 8 @@ -8900,6 +8901,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml" + - "SLURM_PARTITION=batch_3" decode: num-worker: 4 tp: 4 @@ -8915,6 +8917,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml" + - "SLURM_PARTITION=batch_3" decode: num-worker: 1 tp: 8 @@ -8930,6 +8933,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml" + - "SLURM_PARTITION=batch_3" decode: num-worker: 4 tp: 4 @@ -8945,6 +8949,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml" + - "SLURM_PARTITION=batch_3" decode: num-worker: 1 tp: 16 @@ -8960,6 +8965,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml" + - "SLURM_PARTITION=batch_3" decode: num-worker: 1 tp: 16 @@ -8975,6 +8981,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: dp-attn: true additional-settings: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml" + - "SLURM_PARTITION=batch_3" decode: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 292f6d3b3a..92fe447a81 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6271,23 +6271,10 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2657 - config-keys: - - qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg - - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg - - qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg - - dsv4-fp4-gb300-dynamo-vllm-agentic - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg - - dsv4-fp4-gb300-dynamo-sglang-agentic-agg - - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg - - glm5.2-fp4-gb300-dynamo-sglang-agentic-agg - - glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg - glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp - - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg - - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg - - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg scenario-type: - agentic-coding description: - - "Use the batch_3 Slurm partition for the GB300 launcher." + - "Use the batch_3 Slurm partition for the GLM-5.2 GB300 Dynamo-TRT AgentX configuration." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2657 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index b2f3edbb66..ed193dcc2d 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -7,7 +7,7 @@ set -exo pipefail # shellcheck source=runners/slurm_utils.sh source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" -export SLURM_PARTITION="batch_3" +export SLURM_PARTITION="${SLURM_PARTITION:-batch_1}" export SLURM_ACCOUNT="benchmark" export ENROOT_ROOTFS_WRITABLE=1 From 7049e7133561068a6a237b9c6826cef3f1b93f8b Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 20 Aug 2026 01:00:15 -0700 Subject: [PATCH 6/6] fix(gb300): align sbatch partition with launcher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 使 srtctl 提交的作业使用启动脚本解析后的 Slurm 分区。 --- runners/launch_gb300-nv.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index ed193dcc2d..d5a58c5fcc 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -8,6 +8,7 @@ set -exo pipefail source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" export SLURM_PARTITION="${SLURM_PARTITION:-batch_1}" +export SBATCH_PARTITION="$SLURM_PARTITION" export SLURM_ACCOUNT="benchmark" export ENROOT_ROOTFS_WRITABLE=1