diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index a4fe70803f..9cec822cca 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1996,7 +1996,7 @@ build_replay_cmd() { # rolling TTFT/ITL/throughput block and emit it every 30 seconds. export AIPERF_UI_REALTIME_METRICS_ENABLED=true REPLAY_CMD="$AIPERF_CLI profile --scenario inferencex-agentx-mvp" - REPLAY_CMD+=" --url http://localhost:$PORT" + REPLAY_CMD+=" --url ${AIPERF_SERVER_URL:-http://localhost:$PORT}" REPLAY_CMD+=" --endpoint /v1/chat/completions" REPLAY_CMD+=" --endpoint-type chat" REPLAY_CMD+=" --streaming" diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 79a36da524..77c196c133 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -9,7 +9,12 @@ set -x INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-/infmax-workspace}" source "$INFMAX_CONTAINER_WORKSPACE/benchmarks/benchmark_lib.sh" -check_env_vars MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION +check_env_vars \ + MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION \ + SRT_FRONTEND_HOST SRT_FRONTEND_PORT + +export AIPERF_SERVER_URL="http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" +echo "Using srt-slurm frontend endpoint: $AIPERF_SERVER_URL" BASE_RESULT_DIR="${RESULT_DIR:-/logs/agentic}" BASE_RESULT_FILENAME="$RESULT_FILENAME" @@ -32,7 +37,7 @@ install_agentic_deps wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}" - local frontend_metrics_url="http://localhost:${PORT}/metrics" + local frontend_metrics_url="${AIPERF_SERVER_URL%/}/metrics" "$AIPERF_PYTHON" - \ "$timeout_seconds" \ diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml new file mode 100644 index 0000000000..eb3e3d2889 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml @@ -0,0 +1,132 @@ +name: dynamo-agg-gb300-tp8-c1-b2-mtp8 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +backend: + type: trtllm + aggregated_environment: + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "3.06" + TRTLLM_DISAGG_IDLE_CHECK: pr17324 + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + trtllm_config: + aggregated: + cuda_graph_config: + batch_sizes: [1, 2] + enable_padding: false + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + tokens_per_block: 64 + max_batch_size: 2 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 8 + num_nextn_predict_layers: 8 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '1' + DURATION: '3600' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml new file mode 100644 index 0000000000..033048cf9b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml @@ -0,0 +1,178 @@ +name: dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 5] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: head + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml new file mode 100644 index 0000000000..8f6fdcc2cf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml @@ -0,0 +1,178 @@ +name: dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 4 + gpus_per_decode: 4 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 2 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: head + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml new file mode 100644 index 0000000000..c618bf7f89 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml @@ -0,0 +1,178 @@ +name: dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 3 + prefill_workers: 3 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 4 + gpus_per_decode: 4 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 5] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: head + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml new file mode 100644 index 0000000000..8407308128 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml @@ -0,0 +1,179 @@ +name: dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 5 + prefill_workers: 5 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 16 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: head + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml new file mode 100644 index 0000000000..4941ce4270 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml @@ -0,0 +1,178 @@ +name: dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 6 + prefill_workers: 6 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: head + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml new file mode 100644 index 0000000000..f291bda5c4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml @@ -0,0 +1,185 @@ +name: dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 8 + prefill_workers: 8 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "1.99" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 16 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: head + +benchmark: + type: custom + client_placement: last_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: none + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + FRAMEWORK: dynamo-trt + PRECISION: fp4 + CONC: '259' + DURATION: '3600' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 50ed9b4713..0b994f3c6a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8843,6 +8843,151 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 16 dp-attn: true +# GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo aggregated serving. +# One TP8 worker serves prefill and decode on the same eight GPUs at concurrency 1. +glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.4.0.dev20260807" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [1] + kv-offloading: none + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml" + - "SLURM_PARTITION=batch_3" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + +# GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving. +# Six topology variants use NIXL KV transfer and MTP3/MTP5 decoding. +glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.4.0.dev20260807" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [30] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p4d-tep4-c30-b2-mtp5.yaml" + - "SLURM_PARTITION=batch_3" + decode: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: mtp + conc-list: [20] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tep8-c20-b5-mtp5.yaml" + - "SLURM_PARTITION=batch_3" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: false + - spec-decoding: mtp + conc-list: [60] + kv-offloading: none + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-3p4d-tep4-c60-b5-mtp5.yaml" + - "SLURM_PARTITION=batch_3" + decode: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: mtp + conc-list: [152] + kv-offloading: none + prefill: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-5p1d-dep16-c152-b16-mtp3.yaml" + - "SLURM_PARTITION=batch_3" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [259] + kv-offloading: none + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-8p1d-dep16-c259-b16-mtp3.yaml" + - "SLURM_PARTITION=batch_3" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: mtp + conc-list: [227] + kv-offloading: none + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-6p1d-dep8-c227-b16-mtp3.yaml" + - "SLURM_PARTITION=batch_3" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + qwen3.5-fp8-gb200-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 93b6a7c1b3..92fe447a81 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6251,3 +6251,30 @@ description: - "Refresh with lower stream interval to collect correct client metrics" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2686 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Add one aggregated TP8 and six disaggregated GLM-5.2 NVFP4 GB300 AgentX recipes for Dynamo and TensorRT-LLM with MTP speculative decoding." + - "Use the concurrency-1 aggregated Pareto point and disable TensorRT-LLM performance-metrics responses for all seven points." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2657 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Place the Dynamo frontend on the Slurm head node for eval routing across the six disaggregated recipes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2657 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Use the batch_3 Slurm partition for the GLM-5.2 GB300 Dynamo-TRT AgentX configuration." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2657 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index ff6597d0bb..d5a58c5fcc 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -7,7 +7,8 @@ set -exo pipefail # shellcheck source=runners/slurm_utils.sh source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" -export SLURM_PARTITION="batch_1" +export SLURM_PARTITION="${SLURM_PARTITION:-batch_1}" +export SBATCH_PARTITION="$SLURM_PARTITION" export SLURM_ACCOUNT="benchmark" export ENROOT_ROOTFS_WRITABLE=1 @@ -65,6 +66,10 @@ elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then # in our GLM-5.1 sglang recipes. export MODEL_PATH=/scratch/models/GLM-5.1-NVFP4 export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" +elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo-trt" ]]; then + export SERVED_MODEL_NAME="GLM-5.2-NVFP4" + export MODEL_PATH=/scratch/models/GLM-5.2-NVFP4 + export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5.2-NVFP4" elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp4" ]]; then export MODEL_PATH=/scratch/models/GLM-5.2-NVFP4 export SRT_SLURM_MODEL_PREFIX="glm-5.2-fp4" @@ -206,6 +211,18 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qw find "$TRTLLM_RECIPES_DIR" -name "*.yaml" \ -exec sed -i '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' {} + fi +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "glm5.2" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.38 + TRTLLM_RECIPES_DIR="benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2" + mkdir -p "$TRTLLM_RECIPES_DIR" + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2" \ + "$TRTLLM_RECIPES_DIR" + if [[ "${EVAL_ONLY:-false}" == "true" ]]; then + find "$TRTLLM_RECIPES_DIR" -name "*.yaml" \ + -exec sed -i '/TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS/d' {} + + fi elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.38: the two features the # cquil11 fork was pinned for are merged upstream (present in v1.0.36) — @@ -662,6 +679,19 @@ if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then else echo "WARNING: RUN_EVAL=true but no eval results found at $EVAL_DIR" fi + + # srt-slurm stages eval artifacts but does not write the metadata file + # consumed by score validation. Reuse the canonical metadata writer so + # topology and recipe identity stay aligned with the workflow inputs. + eval_conc_value="${EVAL_CONC:-${CONC:-1}}" + ( + export IS_MULTINODE=true + # shellcheck source=benchmarks/benchmark_lib.sh + source "$GITHUB_WORKSPACE/benchmarks/benchmark_lib.sh" + _write_lm_eval_meta_json \ + "$GITHUB_WORKSPACE/meta_env.json" "" "$eval_conc_value" + ) + echo "Wrote meta_env.json (conc=${eval_conc_value}, prefix=${MODEL_PREFIX:-unknown})" fi # Snapshot logs to GITHUB_WORKSPACE BEFORE cleanup, so the EXIT trap's