From ecede7b9cee4194612665a183ba477d994e45af0 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 02:01:47 +0000 Subject: [PATCH 01/25] perf(gb300): refresh AgentX vLLM MTP P/D recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 刷新 GB300 AgentX vLLM MTP P/D 配方。 --- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 147 ------------- .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 148 ------------- ...gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml | 10 +- ...gg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml | 203 ++++++++++++++++++ ...300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml | 10 +- ...300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml | 10 +- configs/nvidia-master.yaml | 82 ++----- perf-changelog.yaml | 9 + 8 files changed, 245 insertions(+), 374 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml deleted file mode 100644 index 9559ffb75b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ /dev/null @@ -1,147 +0,0 @@ -name: "svf-vllm-agg-gb300-tp4-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode at concurrency 8. Size max-num-seqs at -# 4x concurrency and expand the MTP CUDA-graph envelope to match. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - wheel: "1.2.1" - install: true - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}" - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml deleted file mode 100644 index 4eebce98fa..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ /dev/null @@ -1,148 +0,0 @@ -name: "svf-vllm-agg-gb300-tp8-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU -# nodes and serves both prefill and decode at concurrency 4. Keep at least -# 16 sequence slots and otherwise size the scheduler at 4x concurrency. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - wheel: "1.2.1" - install: true - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - # The frontend shares Grace CPU capacity with the long TP8 cold start. - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}" - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml index e5cc1c76ea..e9edbc5aa1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml @@ -5,19 +5,19 @@ name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" frameworks: - dynamo: "1.3.0.dev20260720" + dynamo: "1.4.0" dynamo: - wheel: "1.3.0.dev20260720" + wheel: "1.4.0" install: true setup_script: vllm-container-deps.sh @@ -77,6 +77,7 @@ backend: prefill: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 @@ -104,6 +105,7 @@ backend: decode: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml new file mode 100644 index 0000000000..8b9c58298d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml @@ -0,0 +1,203 @@ +# Source: https://github.com/Inferact/srt-slurm-sa/blob/4a870cd5bc333bf8a312fc65ca1ea82cdab9b2df/recipes/vllm/deepseek-v4-pro/GB300/agentic/agentx-v1.0.1/1p3d-pdep4-dtp8-c3-kv-nixl-mtp-flashinfer-ar-lpt512-psi1.yaml +# Adapted to the InferenceX AgentX harness and throughput-only synthetic MTP +# acceptance injection so eval-only runs continue to verify real MTP output. +name: "svf-vllm-disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic" + +# GB300 high-interactivity AgentX MTP3 topology: one DEP4 prefill worker +# feeds three TP8 decode workers at concurrency 3 through NIXL. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + frameworks: + dynamo: "1.4.0" + +dynamo: + install: true + wheel: "1.4.0" + request_plane: tcp + +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p3d-{job_id}" + +slurm: + time_limit: "08:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 1 + decode_nodes: 6 + prefill_workers: 1 + decode_workers: 3 + gpus_per_prefill: 4 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "3" + DURATION: "3600" + KV_OFFLOADING: "none" + ETCD_LEASE_TTL: "120" + DYN_ROUTER_QUEUE_THRESHOLD: "None" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: "14400" + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + args: + router-mode: "kv" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + orchestrator_placement: first_decode + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + # vLLM KV routing needs prefill KV events so Dynamo can select the + # cache-owning PDEP4 rank. + kv_events_config: + prefill: true + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 4 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 512 + prefill-schedule-interval: 1 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + # MTP capture sizes are tokens: 64 seqs * (1 target + 3 drafts). + max-cudagraph-capture-size: 256 + decode: + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + # MTP capture sizes are tokens: 256 seqs * (1 target + 3 drafts). + max-cudagraph-capture-size: 1024 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + prefill_environment: + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + +setup_script: vllm-container-deps.sh + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + client_placement: last_decode + type: custom + use_chat_template: true + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + AGENTIC_OUTPUT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + MODEL: "deepseek-ai/DeepSeek-V4-Pro" + MODEL_PREFIX: "dsv4" + SERVED_MODEL_NAME: "DeepSeek-V4-Pro" + FRAMEWORK: "dynamo-vllm" + PRECISION: "fp4" + CONC: "3" + DURATION: "3600" + RESULT_FILENAME: "dsv4_agentx101_p1dep4_d3tp8_c3_kv_nixl_mtp_fi_ar_lpt512_psi1" + RUNNER_TYPE: "gb300" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + SPEC_DECODING: "mtp" + DISAGG: "true" + OFFLOADING: "none" + KV_OFFLOADING: "none" + TP: "8" + PREFILL_TP: "1" + PREFILL_NUM_WORKERS: "1" + PREFILL_EP: "4" + DECODE_TP: "8" + DECODE_NUM_WORKERS: "3" + DECODE_EP: "1" + EP_SIZE: "1" + AIPERF_MAX_OSL: "none" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" + NUM_DATASET_ENTRIES: "393" + HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" + PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml index d5a0b44161..ef389239a5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml @@ -6,19 +6,19 @@ name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" frameworks: - dynamo: "1.3.0.dev20260720" + dynamo: "1.4.0" dynamo: - wheel: "1.3.0.dev20260720" + wheel: "1.4.0" install: true environment: @@ -78,6 +78,7 @@ backend: prefill: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 @@ -104,6 +105,7 @@ backend: decode: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml index f770241932..1b23cbdf94 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml @@ -6,19 +6,19 @@ name: "svf-vllm-disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" frameworks: - dynamo: "1.3.0.dev20260720" + dynamo: "1.4.0" dynamo: - wheel: "1.3.0.dev20260720" + wheel: "1.4.0" install: true environment: @@ -78,6 +78,7 @@ backend: prefill: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 @@ -104,6 +105,7 @@ backend: decode: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 50ed9b4713..4155df883e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7735,94 +7735,42 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true -dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f +dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.1" } + kv-p2p-transfer: nixl multinode: true - disagg: false + disagg: true scenarios: agentic-coding: - search-space: # Keep the checked-in recipes on real MTP verification. Throughput jobs # inject the committed golden AL at launch; EVAL_ONLY leaves them real. + # Source: Inferact/srt-slurm-sa@4a870cd5bc333bf8a312fc65ca1ea82cdab9b2df. - spec-decoding: mtp - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" - # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts eight GPUs only once. - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: mtp - conc-list: [4] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "SYNTHETIC_ACCEPTANCE=true" - - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" - # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts eight GPUs only once. - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: mtp - conc-list: [8] + conc-list: [3] + router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 1 tp: 4 - ep: 1 - dp-attn: false + ep: 4 + dp-attn: true additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" - # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts four GPUs only once. + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml" decode: - num-worker: 0 - tp: 4 + num-worker: 3 + tp: 8 ep: 1 dp-attn: false - -dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-vllm - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - # Keep the checked-in recipes on real MTP verification. Throughput jobs - # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - spec-decoding: mtp conc-list: [1152] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } + router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 2 tp: 8 @@ -7839,7 +7787,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: dp-attn: true - spec-decoding: mtp conc-list: [1024] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } + router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 2 tp: 8 @@ -7856,7 +7804,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: dp-attn: true - spec-decoding: mtp conc-list: [256] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } + router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 1 tp: 4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 93b6a7c1b3..ad849c70fe 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6251,3 +6251,12 @@ description: - "Refresh with lower stream interval to collect correct client metrics" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2686 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Update the GB300 DeepSeek-V4-Pro Dynamo-vLLM AgentX P/D image from 426e59f to d62ad46, move Dynamo to 1.4.0, and enable safetensors prefetch loading." + - "Replace the aggregate TP data points with the upstream 1P DEP4 + 3D TP8 concurrency-3 NIXL point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 9bfb59ebd8783974e39a7389c8844fce6753b37a Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 02:09:54 +0000 Subject: [PATCH 02/25] perf(gb300): add AgentX c512 P/D point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 GB300 AgentX 并发度 512 的 P/D 测试点。 --- ...b300-1p1d-dep8-dep16-c512-mtp-agentic.yaml | 215 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 8 + 3 files changed, 241 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml new file mode 100644 index 0000000000..68221b2c2d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml @@ -0,0 +1,215 @@ +# Source: https://github.com/SemiAnalysisAI/InferenceX/blob/0c33d4615792705ed12bfc204e3a54cfa436cf02/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml +# Runtime and performance arguments follow the current DEP8/DEP16 GB300 P/D +# recipes; throughput-only synthetic MTP acceptance is injected at launch. +name: "svf-vllm-disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP16 decode worker at concurrency 512. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + frameworks: + dynamo: "1.4.0" + +dynamo: + wheel: "1.4.0" + install: true + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c512-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep16-decode-c512-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 4155df883e..c3be749aec 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7802,6 +7802,24 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: tp: 16 ep: 16 dp-attn: true + # Source: SemiAnalysisAI/InferenceX@0c33d4615792705ed12bfc204e3a54cfa436cf02. + - spec-decoding: mtp + conc-list: [512] + router: { name: dynamo-router, version: "1.4.0" } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true - spec-decoding: mtp conc-list: [256] router: { name: dynamo-router, version: "1.4.0" } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ad849c70fe..f765b76e6a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6260,3 +6260,11 @@ - "Update the GB300 DeepSeek-V4-Pro Dynamo-vLLM AgentX P/D image from 426e59f to d62ad46, move Dynamo to 1.4.0, and enable safetensors prefetch loading." - "Replace the aggregate TP data points with the upstream 1P DEP4 + 3D TP8 concurrency-3 NIXL point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Port the GB300 1P DEP8 + 1D DEP16 concurrency-512 MTP P/D point and align its runtime tuning with the retained DEP8/DEP16 recipes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 882575412c579a45346488432013cb549b1334c1 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 06:11:22 +0000 Subject: [PATCH 03/25] perf(gb300): restore AgentX aggregate TP sweeps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 恢复 GB300 AgentX 聚合 TP4/TP8 扫描，并在高并发点启用 vLLM Simple CPU KV 卸载。 --- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 150 +++++++++++++++++ ...0-tp4-vllm-simple-offload-mtp-agentic.yaml | 136 ++++++++++++++++ .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 151 ++++++++++++++++++ ...0-tp8-vllm-simple-offload-mtp-agentic.yaml | 137 ++++++++++++++++ configs/nvidia-master.yaml | 95 +++++++++++ perf-changelog.yaml | 9 ++ 6 files changed, 678 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml new file mode 100644 index 0000000000..bbc79d0dda --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -0,0 +1,150 @@ +name: "svf-vllm-agg-gb300-tp4-mtp-agentic" + +# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node +# and serves both prefill and decode for the GPU-resident c1-c4 sweep. Size +# max-num-seqs at 4x the maximum concurrency and the MTP graph at four tokens +# per sequence (one target plus three draft tokens). + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + frameworks: + dynamo: "1.4.0" + +dynamo: + wheel: "1.4.0" + install: true + +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + ETCD_LEASE_TTL: "7200" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 64 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml new file mode 100644 index 0000000000..d52103916e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml @@ -0,0 +1,136 @@ +name: "svf-vllm-agg-gb300-tp4-vllm-simple-offload-mtp-agentic" + +# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node +# and serves both prefill and decode for the c6-c16 sweep. Each rank reserves +# 128 GiB of host DRAM for SimpleCPUOffloadConnector (512 GiB per node). + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + frameworks: + dynamo: "1.4.0" + +dynamo: + wheel: "1.4.0" + install: true + +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + ETCD_LEASE_TTL: "7200" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: null + aggregated_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "64" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-simple-offload-mtp-{job_id}" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml new file mode 100644 index 0000000000..a235ac61fd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -0,0 +1,151 @@ +name: "svf-vllm-agg-gb300-tp8-mtp-agentic" + +# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU nodes +# and serves both prefill and decode for the GPU-resident c1-c8 sweep. Size +# max-num-seqs at 4x the maximum concurrency and the MTP graph at four tokens +# per sequence (one target plus three draft tokens). + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + frameworks: + dynamo: "1.4.0" + +dynamo: + wheel: "1.4.0" + install: true + +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + # The frontend shares Grace CPU capacity with the long TP8 cold start. + ETCD_LEASE_TTL: "7200" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp8-mtp-{job_id}" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep aggregate workers in the multinode result schema so ingestion uses + # the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml new file mode 100644 index 0000000000..fee6d9402d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml @@ -0,0 +1,137 @@ +name: "svf-vllm-agg-gb300-tp8-vllm-simple-offload-mtp-agentic" + +# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU nodes +# and serves both prefill and decode for the c12-c24 sweep. Each rank reserves +# 128 GiB of host DRAM for SimpleCPUOffloadConnector (512 GiB per node). + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + frameworks: + dynamo: "1.4.0" + +dynamo: + wheel: "1.4.0" + install: true + +environment: + DYNAMO_WHEEL_DIRS: "/srtctl-wheels" + # The frontend shares Grace CPU capacity with the long TP8 cold start. + ETCD_LEASE_TTL: "7200" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: null + aggregated_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "96" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + NCCL_P2P_LEVEL: "NVL" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp8-simple-offload-mtp-{job_id}" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 96 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 384 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + gpu-memory-utilization: 0.94 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep aggregate workers in the multinode result schema so ingestion uses + # the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c3be749aec..c0bbc2f4bf 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7735,6 +7735,101 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true +dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.4.0" } + multinode: true + disagg: false + scenarios: + agentic-coding: + # 0.6095238 resolves to 512 GiB of host DRAM per four-GPU GB300 node, + # matching the SimpleCPUOffloadConnector pool in the offload recipes. + - dram-utilization: 0.6095238 + search-space: + # Keep the checked-in recipes on real MTP verification. Throughput jobs + # inject the committed golden AL at launch; EVAL_ONLY leaves them real. + # TP4 remains GPU-resident through c4, then uses vLLM Simple CPU offload. + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 2, 4] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts four GPUs only once. + decode: + num-worker: 0 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [6, 8, 12, 16] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml" + decode: + num-worker: 0 + tp: 4 + ep: 1 + dp-attn: false + # TP8 remains GPU-resident through c8, then uses vLLM Simple CPU offload. + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 2, 4, 8] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts eight GPUs only once. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple } + conc-list: [12, 16, 20, 24] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f765b76e6a..3d7112f809 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6268,3 +6268,12 @@ description: - "Port the GB300 1P DEP8 + 1D DEP16 concurrency-512 MTP P/D point and align its runtime tuning with the retained DEP8/DEP16 recipes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Restore the GB300 DeepSeek-V4-Pro aggregate TP4 and TP8 AgentX MTP recipes on image d62ad46 with Dynamo 1.4.0 and safetensors prefetch loading." + - "Sweep TP4 at concurrency 1, 2, 4, 6, 8, 12, and 16 with vLLM Simple CPU KV offload from concurrency 6; sweep TP8 at concurrency 1, 2, 4, 8, 12, 16, 20, and 24 with offload from concurrency 12." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 62f2955ba6cc186543a0ec907fc3a08d5f640001 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 06:17:28 +0000 Subject: [PATCH 04/25] perf(gb300): align AgentX runtime tuning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将全部 GB300 AgentX 配方回退到 426e59f 镜像，禁用前缀缓存保留间隔，并对齐聚合 TP 的低延迟归约配置。 --- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 12 ++++++------ ...gg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml | 12 ++++++------ .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 12 ++++++------ ...gg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml | 12 ++++++------ ...disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml | 8 ++++---- ...isagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml | 8 ++++---- .../disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml | 7 ++++--- ...sagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml | 8 ++++---- ...sagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml | 8 ++++---- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 10 ++++++++++ 11 files changed, 56 insertions(+), 45 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index bbc79d0dda..afe67180b0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -7,14 +7,14 @@ name: "svf-vllm-agg-gb300-tp4-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.4.0" @@ -80,7 +80,7 @@ backend: VLLM_LOG_STATS_INTERVAL: "1" VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -89,9 +89,9 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" UCX_MEMTYPE_CACHE: "n" @@ -122,7 +122,7 @@ backend: compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 64 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 + gpu-memory-utilization: 0.92 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml index d52103916e..b22d8af9b2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml @@ -6,14 +6,14 @@ name: "svf-vllm-agg-gb300-tp4-vllm-simple-offload-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.4.0" @@ -70,7 +70,7 @@ backend: VLLM_LOG_STATS_INTERVAL: "1" VLLM_V2_WARMUP_MAX_NUM_SEQS: "64" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -79,8 +79,8 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" UCX_MEMTYPE_CACHE: "n" @@ -108,7 +108,7 @@ backend: max-cudagraph-capture-size: 256 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' - gpu-memory-utilization: 0.94 + gpu-memory-utilization: 0.92 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index a235ac61fd..228905ce31 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -7,14 +7,14 @@ name: "svf-vllm-agg-gb300-tp8-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.4.0" @@ -81,7 +81,7 @@ backend: VLLM_LOG_STATS_INTERVAL: "1" VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -90,9 +90,9 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" UCX_MEMTYPE_CACHE: "n" @@ -123,7 +123,7 @@ backend: compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 128 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.94 + gpu-memory-utilization: 0.92 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml index fee6d9402d..a6e62fa5c1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml @@ -6,14 +6,14 @@ name: "svf-vllm-agg-gb300-tp8-vllm-simple-offload-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.4.0" @@ -71,7 +71,7 @@ backend: VLLM_LOG_STATS_INTERVAL: "1" VLLM_V2_WARMUP_MAX_NUM_SEQS: "96" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -80,8 +80,8 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" UCX_MEMTYPE_CACHE: "n" @@ -109,7 +109,7 @@ backend: max-cudagraph-capture-size: 384 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' - gpu-memory-utilization: 0.94 + gpu-memory-utilization: 0.92 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml index e9edbc5aa1..1184b49e53 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml @@ -5,14 +5,14 @@ name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.4.0" @@ -150,7 +150,7 @@ backend: NCCL_NVLS_ENABLE: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" @@ -179,7 +179,7 @@ backend: NCCL_NVLS_ENABLE: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_DSV4_MEGA_FP8_COMBINE: "1" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml index 68221b2c2d..aa43ddaf9b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml @@ -9,14 +9,14 @@ name: "svf-vllm-disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.4.0" @@ -154,7 +154,7 @@ backend: NCCL_NVLS_ENABLE: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" @@ -183,7 +183,7 @@ backend: NCCL_NVLS_ENABLE: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_DSV4_MEGA_FP8_COMBINE: "1" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep16-decode-c512-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml index 8b9c58298d..6917346319 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml @@ -7,14 +7,14 @@ name: "svf-vllm-disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic" # feeds three TP8 decode workers at concurrency 3 through NIXL. model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.4.0" @@ -35,6 +35,7 @@ environment: VLLM_ALLREDUCE_USE_FLASHINFER: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p3d-{job_id}" slurm: @@ -178,7 +179,7 @@ benchmark: DURATION: "3600" RESULT_FILENAME: "dsv4_agentx101_p1dep4_d3tp8_c3_kv_nixl_mtp_fi_ar_lpt512_psi1" RUNNER_TYPE: "gb300" - IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" SPEC_DECODING: "mtp" DISAGG: "true" OFFLOADING: "none" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml index ef389239a5..bc377b7c48 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml @@ -6,14 +6,14 @@ name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.4.0" @@ -151,7 +151,7 @@ backend: NCCL_NVLS_ENABLE: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" @@ -180,7 +180,7 @@ backend: NCCL_NVLS_ENABLE: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_DSV4_MEGA_FP8_COMBINE: "1" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml index 1b23cbdf94..7a7b731dad 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml @@ -6,14 +6,14 @@ name: "svf-vllm-disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic" model: path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46" + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: dynamo: "1.4.0" @@ -151,7 +151,7 @@ backend: NCCL_NVLS_ENABLE: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" @@ -180,7 +180,7 @@ backend: NCCL_NVLS_ENABLE: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_DSV4_MEGA_FP8_COMBINE: "1" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep16-decode-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c0bbc2f4bf..451f18f789 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7736,7 +7736,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: dp-attn: true dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46 + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv @@ -7831,7 +7831,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: dp-attn: false dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-d62ad46 + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3d7112f809..5c591e9b05 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6277,3 +6277,13 @@ - "Restore the GB300 DeepSeek-V4-Pro aggregate TP4 and TP8 AgentX MTP recipes on image d62ad46 with Dynamo 1.4.0 and safetensors prefetch loading." - "Sweep TP4 at concurrency 1, 2, 4, 6, 8, 12, and 16 with vLLM Simple CPU KV offload from concurrency 6; sweep TP8 at concurrency 1, 2, 4, 8, 12, 16, 20, and 24 with offload from concurrency 12." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Revert all GB300 DeepSeek-V4-Pro AgentX recipes from image d62ad46 to the previous 426e59f image." + - "Disable prefix-cache retention intervals across aggregate and P/D workers; align aggregate TP all-reduce and GPU-memory tuning with the low-latency 1P3D TP8 decode recipe." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From bb4c9aea4a42e9712579658ce109751ed9979c6b Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 06:30:44 +0000 Subject: [PATCH 05/25] fix(recipes): restore aggregate GPU memory utilization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore the GB300 DeepSeek-V4-Pro aggregate TP4 and TP8 recipes to 0.94 GPU memory utilization. The 0.92 setting belongs to the separate low-latency 1P3D decode topology. 将 GB300 DeepSeek-V4-Pro 聚合式 TP4 和 TP8 配方的 GPU 显存利用率恢复为 0.94。0.92 设置仅适用于独立的低延迟 1P3D 解码拓扑。 --- .../deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml | 2 +- .../agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml | 2 +- .../deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml | 2 +- .../agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml | 2 +- perf-changelog.yaml | 8 ++++++++ 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index afe67180b0..e1d6cfdf59 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -122,7 +122,7 @@ backend: compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 64 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.92 + gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml index b22d8af9b2..592e881a8a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml @@ -108,7 +108,7 @@ backend: max-cudagraph-capture-size: 256 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' - gpu-memory-utilization: 0.92 + gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 228905ce31..84f9c5c99c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -123,7 +123,7 @@ backend: compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 128 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - gpu-memory-utilization: 0.92 + gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml index a6e62fa5c1..f40689f938 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml @@ -109,7 +109,7 @@ backend: max-cudagraph-capture-size: 384 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' - gpu-memory-utilization: 0.92 + gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5c591e9b05..5e0b5bc76f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6287,3 +6287,11 @@ - "Revert all GB300 DeepSeek-V4-Pro AgentX recipes from image d62ad46 to the previous 426e59f image." - "Disable prefix-cache retention intervals across aggregate and P/D workers; align aggregate TP all-reduce and GPU-memory tuning with the low-latency 1P3D TP8 decode recipe." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Restore aggregate TP4 and TP8 GPU memory utilization from 0.92 to 0.94; the lower value was specific to the separate low-latency 1P3D decode topology." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From bb0d6cfa79aa7989d21a0f07cb52ead7e4f3fcb3 Mon Sep 17 00:00:00 2001 From: Roy Wang Date: Wed, 19 Aug 2026 14:37:20 +0800 Subject: [PATCH 06/25] Update agg-gb300-tp4-mtp-agentic.yaml --- .../deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index e1d6cfdf59..d396883570 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -81,7 +81,6 @@ backend: VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" @@ -94,9 +93,6 @@ backend: VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" NCCL_P2P_LEVEL: "NVL" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" @@ -111,7 +107,6 @@ backend: kv-cache-dtype: "fp8" tensor-parallel-size: 4 pipeline-parallel-size: 1 - disable-custom-all-reduce: true attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 16 @@ -119,11 +114,9 @@ backend: trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 64 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' gpu-memory-utilization: 0.94 - stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" From 40a87df53480fad6fe117c53409523a3d1e3c818 Mon Sep 17 00:00:00 2001 From: Roy Wang Date: Wed, 19 Aug 2026 14:38:51 +0800 Subject: [PATCH 07/25] Remove no-enable-flashinfer-autotune setting --- .../vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index d396883570..efdd3c0e00 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -112,7 +112,6 @@ backend: max-num-seqs: 16 max-num-batched-tokens: 8192 trust-remote-code: true - no-enable-flashinfer-autotune: true block-size: 256 max-cudagraph-capture-size: 64 speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' From 3b111809f9ee62b0c44a1bea13b34865f047ea4b Mon Sep 17 00:00:00 2001 From: Roy Wang Date: Wed, 19 Aug 2026 14:42:08 +0800 Subject: [PATCH 08/25] Update agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml --- .../agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml index 592e881a8a..b5c0a279ed 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml @@ -71,7 +71,6 @@ backend: VLLM_V2_WARMUP_MAX_NUM_SEQS: "64" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" @@ -83,9 +82,6 @@ backend: VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" NCCL_P2P_LEVEL: "NVL" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-simple-offload-mtp-{job_id}" @@ -102,14 +98,11 @@ backend: max-num-seqs: 64 max-num-batched-tokens: 8192 trust-remote-code: true - no-enable-flashinfer-autotune: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 256 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' gpu-memory-utilization: 0.94 - stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" From 242ae2f86a5bf25d91ba944e1fde6265a1690830 Mon Sep 17 00:00:00 2001 From: Roy Wang Date: Wed, 19 Aug 2026 14:42:44 +0800 Subject: [PATCH 09/25] Update agg-gb300-tp8-mtp-agentic.yaml --- .../deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 84f9c5c99c..3b411b7192 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -82,7 +82,6 @@ backend: VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" @@ -95,9 +94,6 @@ backend: VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" NCCL_P2P_LEVEL: "NVL" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" @@ -120,11 +116,9 @@ backend: trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 128 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' gpu-memory-utilization: 0.94 - stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" From 9998456c5e4708130c5be9b9dccb7d3eb589886c Mon Sep 17 00:00:00 2001 From: Roy Wang Date: Wed, 19 Aug 2026 14:43:19 +0800 Subject: [PATCH 10/25] Update agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml --- .../agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml index f40689f938..f6fd9e2914 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml @@ -72,7 +72,6 @@ backend: VLLM_V2_WARMUP_MAX_NUM_SEQS: "96" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" @@ -84,9 +83,6 @@ backend: VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" NCCL_P2P_LEVEL: "NVL" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp8-simple-offload-mtp-{job_id}" @@ -103,14 +99,11 @@ backend: max-num-seqs: 96 max-num-batched-tokens: 8192 trust-remote-code: true - no-enable-flashinfer-autotune: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 384 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' gpu-memory-utilization: 0.94 - stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" From 7f0e9ddcdf899018da17d3d886ce2202831ea064 Mon Sep 17 00:00:00 2001 From: Roy Wang Date: Wed, 19 Aug 2026 14:43:44 +0800 Subject: [PATCH 11/25] Update agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml --- .../agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml index b5c0a279ed..1aeb1c076e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml @@ -92,7 +92,6 @@ backend: kv-cache-dtype: "fp8" tensor-parallel-size: 4 pipeline-parallel-size: 1 - disable-custom-all-reduce: true attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 64 From 7023b0f59c2fc8f3afdccbeeeed661df055c918f Mon Sep 17 00:00:00 2001 From: Roy Wang Date: Wed, 19 Aug 2026 14:44:05 +0800 Subject: [PATCH 12/25] Update agg-gb300-tp8-mtp-agentic.yaml --- .../vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 3b411b7192..9749d01a22 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -108,13 +108,11 @@ backend: kv-cache-dtype: "fp8" tensor-parallel-size: 8 pipeline-parallel-size: 1 - disable-custom-all-reduce: true attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 32 max-num-batched-tokens: 8192 trust-remote-code: true - no-enable-flashinfer-autotune: true block-size: 256 max-cudagraph-capture-size: 128 speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' From aa1b77306fd42d3769be4aeac650a1a153561bee Mon Sep 17 00:00:00 2001 From: Roy Wang Date: Wed, 19 Aug 2026 14:44:25 +0800 Subject: [PATCH 13/25] Update agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml --- .../agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml index f6fd9e2914..f92ad30fb7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml @@ -93,7 +93,6 @@ backend: kv-cache-dtype: "fp8" tensor-parallel-size: 8 pipeline-parallel-size: 1 - disable-custom-all-reduce: true attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 96 From 435a2ac453a5d2b80a5077db701e348fdbcfe973 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 07:06:01 +0000 Subject: [PATCH 14/25] fix(recipes): keep aggregate MTP verification real MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove checked-in synthetic acceptance fields from the four GB300 aggregate recipes. Throughput jobs inject acceptance length 2.49 from nvidia-master.yaml, while eval-only runs retain real target-model verification. 从四个 GB300 聚合式配方中移除检入的合成接受率字段。吞吐任务由 nvidia-master.yaml 注入 2.49 的接受长度，而仅评测任务保留真实目标模型验证。 --- .../deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml | 2 +- .../agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml | 2 +- .../deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml | 2 +- .../agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml | 2 +- perf-changelog.yaml | 8 ++++++++ 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index efdd3c0e00..46a2e3ad94 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -114,7 +114,7 @@ backend: trust-remote-code: true block-size: 256 max-cudagraph-capture-size: 64 - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' gpu-memory-utilization: 0.94 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml index 1aeb1c076e..ca7bdb54fd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml @@ -99,7 +99,7 @@ backend: trust-remote-code: true block-size: 256 max-cudagraph-capture-size: 256 - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' gpu-memory-utilization: 0.94 no-disable-hybrid-kv-cache-manager: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 9749d01a22..50a2a8c568 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -115,7 +115,7 @@ backend: trust-remote-code: true block-size: 256 max-cudagraph-capture-size: 128 - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' gpu-memory-utilization: 0.94 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml index f92ad30fb7..e962b56f4b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml @@ -100,7 +100,7 @@ backend: trust-remote-code: true block-size: 256 max-cudagraph-capture-size: 384 - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' gpu-memory-utilization: 0.94 no-disable-hybrid-kv-cache-manager: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5e0b5bc76f..9089c1e0ce 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6295,3 +6295,11 @@ description: - "Restore aggregate TP4 and TP8 GPU memory utilization from 0.92 to 0.94; the lower value was specific to the separate low-latency 1P3D decode topology." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Keep the checked-in aggregate TP4 and TP8 recipes on real MTP verification; throughput runs inject synthetic acceptance length 2.49 from nvidia-master.yaml, while EVAL_ONLY leaves the recipes unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 1fd4b55d4dec66fa001f81385bdd08ab503f83ae Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 07:37:07 +0000 Subject: [PATCH 15/25] perf(agentx): tune aggregate KV connectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为聚合 TP4/TP8 配方启用 Mooncake Store，并调整 Simple CPU Offload 调度容量。 --- .../deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml | 1 + .../agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml | 9 +++++---- .../deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml | 1 + .../agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml | 9 +++++---- perf-changelog.yaml | 9 +++++++++ 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index 46a2e3ad94..7a1b8e4b78 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -115,6 +115,7 @@ backend: block-size: 256 max-cudagraph-capture-size: 64 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}' gpu-memory-utilization: 0.94 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml index ca7bdb54fd..331601ecba 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml @@ -2,7 +2,8 @@ name: "svf-vllm-agg-gb300-tp4-vllm-simple-offload-mtp-agentic" # GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node # and serves both prefill and decode for the c6-c16 sweep. Each rank reserves -# 128 GiB of host DRAM for SimpleCPUOffloadConnector (512 GiB per node). +# 128 GiB of host DRAM for SimpleCPUOffloadConnector (512 GiB per node). Keep +# 32 scheduler slots and capture the corresponding 128-token MTP graph envelope. model: path: "deepseek-v4-pro" @@ -68,7 +69,7 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "64" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" NCCL_CUMEM_ENABLE: "1" @@ -94,11 +95,11 @@ backend: pipeline-parallel-size: 1 attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 - max-num-seqs: 64 + max-num-seqs: 32 max-num-batched-tokens: 8192 trust-remote-code: true block-size: 256 - max-cudagraph-capture-size: 256 + max-cudagraph-capture-size: 128 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' gpu-memory-utilization: 0.94 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 50a2a8c568..7519565ad6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -116,6 +116,7 @@ backend: block-size: 256 max-cudagraph-capture-size: 128 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}' gpu-memory-utilization: 0.94 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml index e962b56f4b..6b0daec8e0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml @@ -2,7 +2,8 @@ name: "svf-vllm-agg-gb300-tp8-vllm-simple-offload-mtp-agentic" # GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU nodes # and serves both prefill and decode for the c12-c24 sweep. Each rank reserves -# 128 GiB of host DRAM for SimpleCPUOffloadConnector (512 GiB per node). +# 128 GiB of host DRAM for SimpleCPUOffloadConnector (512 GiB per node). Keep +# 64 scheduler slots and capture the corresponding 256-token MTP graph envelope. model: path: "deepseek-v4-pro" @@ -69,7 +70,7 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "96" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "64" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" NCCL_CUMEM_ENABLE: "1" @@ -95,11 +96,11 @@ backend: pipeline-parallel-size: 1 attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 - max-num-seqs: 96 + max-num-seqs: 64 max-num-batched-tokens: 8192 trust-remote-code: true block-size: 256 - max-cudagraph-capture-size: 384 + max-cudagraph-capture-size: 256 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' gpu-memory-utilization: 0.94 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9089c1e0ce..cbc0d5c413 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6303,3 +6303,12 @@ description: - "Keep the checked-in aggregate TP4 and TP8 recipes on real MTP verification; throughput runs inject synthetic acceptance length 2.49 from nvidia-master.yaml, while EVAL_ONLY leaves the recipes unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Activate MooncakeStoreConnector for the GPU-resident aggregate TP4 and TP8 recipes without proactive KV offload." + - "Set the SimpleCPUOffload TP4 and TP8 max-num-seqs capacities to 32 and 64 with matching 128- and 256-token MTP graph envelopes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 7e3919c16ce45f87f3538a7c6772000fd4fcb6b1 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 07:38:52 +0000 Subject: [PATCH 16/25] perf(agentx): raise aggregate DRAM budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将聚合 Simple CPU Offload 的主机 DRAM 预算提高到 80%。 --- configs/nvidia-master.yaml | 6 +++--- perf-changelog.yaml | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 451f18f789..15a34c41a8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7747,9 +7747,9 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: disagg: false scenarios: agentic-coding: - # 0.6095238 resolves to 512 GiB of host DRAM per four-GPU GB300 node, - # matching the SimpleCPUOffloadConnector pool in the offload recipes. - - dram-utilization: 0.6095238 + # Reserve 80% (672 GiB) of host DRAM per four-GPU GB300 node, leaving + # headroom above the 512 GiB SimpleCPUOffloadConnector pool. + - dram-utilization: 0.80 search-space: # Keep the checked-in recipes on real MTP verification. Throughput jobs # inject the committed golden AL at launch; EVAL_ONLY leaves them real. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cbc0d5c413..cbadde2a37 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6312,3 +6312,11 @@ - "Activate MooncakeStoreConnector for the GPU-resident aggregate TP4 and TP8 recipes without proactive KV offload." - "Set the SimpleCPUOffload TP4 and TP8 max-num-seqs capacities to 32 and 64 with matching 128- and 256-token MTP graph envelopes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Raise the aggregate SimpleCPUOffload DRAM budget from 0.6095238 to 0.80, exposing 672 GiB per four-GPU GB300 node." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 6f8f76cc7b99bbd2690eaccb9fc975d33c390596 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 07:40:22 +0000 Subject: [PATCH 17/25] perf(agentx): delay aggregate CPU offload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 TP4 并发 6、8 和 TP8 并发 12 调整为无 CPU 卸载运行。 --- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 5 ++--- ...gg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml | 2 +- .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 5 ++--- ...gg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml | 2 +- configs/nvidia-master.yaml | 12 ++++++------ perf-changelog.yaml | 8 ++++++++ 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index 7a1b8e4b78..b76a1eb37f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -1,9 +1,8 @@ name: "svf-vllm-agg-gb300-tp4-mtp-agentic" # GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode for the GPU-resident c1-c4 sweep. Size -# max-num-seqs at 4x the maximum concurrency and the MTP graph at four tokens -# per sequence (one target plus three draft tokens). +# and serves both prefill and decode for the GPU-resident c1-c8 sweep. Keep +# 16 scheduler slots and capture the corresponding 64-token MTP graph envelope. model: path: "deepseek-v4-pro" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml index 331601ecba..f77cc0a48f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml @@ -1,7 +1,7 @@ name: "svf-vllm-agg-gb300-tp4-vllm-simple-offload-mtp-agentic" # GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode for the c6-c16 sweep. Each rank reserves +# and serves both prefill and decode for the c12-c16 sweep. Each rank reserves # 128 GiB of host DRAM for SimpleCPUOffloadConnector (512 GiB per node). Keep # 32 scheduler slots and capture the corresponding 128-token MTP graph envelope. diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 7519565ad6..8e3fa777cf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -1,9 +1,8 @@ name: "svf-vllm-agg-gb300-tp8-mtp-agentic" # GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU nodes -# and serves both prefill and decode for the GPU-resident c1-c8 sweep. Size -# max-num-seqs at 4x the maximum concurrency and the MTP graph at four tokens -# per sequence (one target plus three draft tokens). +# and serves both prefill and decode for the GPU-resident c1-c12 sweep. Keep +# 32 scheduler slots and capture the corresponding 128-token MTP graph envelope. model: path: "deepseek-v4-pro" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml index 6b0daec8e0..16a686084b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml @@ -1,7 +1,7 @@ name: "svf-vllm-agg-gb300-tp8-vllm-simple-offload-mtp-agentic" # GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU nodes -# and serves both prefill and decode for the c12-c24 sweep. Each rank reserves +# and serves both prefill and decode for the c16-c24 sweep. Each rank reserves # 128 GiB of host DRAM for SimpleCPUOffloadConnector (512 GiB per node). Keep # 64 scheduler slots and capture the corresponding 256-token MTP graph envelope. diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 15a34c41a8..bba4e326c8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7753,10 +7753,10 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: search-space: # Keep the checked-in recipes on real MTP verification. Throughput jobs # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - # TP4 remains GPU-resident through c4, then uses vLLM Simple CPU offload. + # TP4 remains GPU-resident through c8, then uses vLLM Simple CPU offload. - spec-decoding: mtp kv-offloading: none - conc-list: [1, 2, 4] + conc-list: [1, 2, 4, 6, 8] prefill: num-worker: 1 tp: 4 @@ -7776,7 +7776,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple } - conc-list: [6, 8, 12, 16] + conc-list: [12, 16] prefill: num-worker: 1 tp: 4 @@ -7791,10 +7791,10 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: tp: 4 ep: 1 dp-attn: false - # TP8 remains GPU-resident through c8, then uses vLLM Simple CPU offload. + # TP8 remains GPU-resident through c12, then uses vLLM Simple CPU offload. - spec-decoding: mtp kv-offloading: none - conc-list: [1, 2, 4, 8] + conc-list: [1, 2, 4, 8, 12] prefill: num-worker: 1 tp: 8 @@ -7814,7 +7814,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple } - conc-list: [12, 16, 20, 24] + conc-list: [16, 20, 24] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cbadde2a37..6ebf5254ea 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6320,3 +6320,11 @@ description: - "Raise the aggregate SimpleCPUOffload DRAM budget from 0.6095238 to 0.80, exposing 672 GiB per four-GPU GB300 node." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Run aggregate TP4 concurrency 6 and 8 and TP8 concurrency 12 on the GPU-resident Mooncake Store recipes; retain SimpleCPUOffload for TP4 concurrency 12 and 16 and TP8 concurrency 16, 20, and 24." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 5b274e7369ed69041550e5ed043026d84a548cf1 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 07:45:27 +0000 Subject: [PATCH 18/25] fix(agentx): use Mooncake aggregate offload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将聚合高并发卸载路径从 vLLM Simple CPU Offload 更正为 Mooncake Store。 --- ...> agg-gb300-tp4-mooncake-mtp-agentic.yaml} | 26 ++++++++++++++----- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 1 - ...> agg-gb300-tp8-mooncake-mtp-agentic.yaml} | 26 ++++++++++++++----- .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 1 - configs/nvidia-master.yaml | 16 ++++++------ perf-changelog.yaml | 9 +++++++ 6 files changed, 57 insertions(+), 22 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/{agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml => agg-gb300-tp4-mooncake-mtp-agentic.yaml} (76%) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/{agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml => agg-gb300-tp8-mooncake-mtp-agentic.yaml} (77%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml similarity index 76% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml index f77cc0a48f..5556848c88 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml @@ -1,9 +1,9 @@ -name: "svf-vllm-agg-gb300-tp4-vllm-simple-offload-mtp-agentic" +name: "svf-vllm-agg-gb300-tp4-mooncake-mtp-agentic" # GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode for the c12-c16 sweep. Each rank reserves -# 128 GiB of host DRAM for SimpleCPUOffloadConnector (512 GiB per node). Keep -# 32 scheduler slots and capture the corresponding 128-token MTP graph envelope. +# and serves both prefill and decode for the c12-c16 sweep with Mooncake Store +# as the external KV tier. Keep 32 scheduler slots and capture the corresponding +# 128-token MTP graph envelope. model: path: "deepseek-v4-pro" @@ -62,6 +62,15 @@ frontend: backend: type: vllm connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false aggregated_environment: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" @@ -79,13 +88,18 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" NCCL_P2P_LEVEL: "NVL" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-simple-offload-mtp-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mooncake-mtp-{job_id}" vllm_config: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro" @@ -101,7 +115,7 @@ backend: block-size: 256 max-cudagraph-capture-size: 128 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}' gpu-memory-utilization: 0.94 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index b76a1eb37f..d452907b58 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -114,7 +114,6 @@ backend: block-size: 256 max-cudagraph-capture-size: 64 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}' gpu-memory-utilization: 0.94 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml similarity index 77% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml index 16a686084b..f4767c2fa6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml @@ -1,9 +1,9 @@ -name: "svf-vllm-agg-gb300-tp8-vllm-simple-offload-mtp-agentic" +name: "svf-vllm-agg-gb300-tp8-mooncake-mtp-agentic" # GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU nodes -# and serves both prefill and decode for the c16-c24 sweep. Each rank reserves -# 128 GiB of host DRAM for SimpleCPUOffloadConnector (512 GiB per node). Keep -# 64 scheduler slots and capture the corresponding 256-token MTP graph envelope. +# and serves both prefill and decode for the c16-c24 sweep with Mooncake Store +# as the external KV tier. Keep 64 scheduler slots and capture the corresponding +# 256-token MTP graph envelope. model: path: "deepseek-v4-pro" @@ -63,6 +63,15 @@ frontend: backend: type: vllm connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false aggregated_environment: HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" @@ -80,13 +89,18 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" NCCL_P2P_LEVEL: "NVL" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp8-simple-offload-mtp-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp8-mooncake-mtp-{job_id}" vllm_config: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro" @@ -102,7 +116,7 @@ backend: block-size: 256 max-cudagraph-capture-size: 256 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}' gpu-memory-utilization: 0.94 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 8e3fa777cf..bc6e6a5882 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -115,7 +115,6 @@ backend: block-size: 256 max-cudagraph-capture-size: 128 speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}' gpu-memory-utilization: 0.94 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bba4e326c8..547a4c8f1c 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7747,13 +7747,13 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: disagg: false scenarios: agentic-coding: - # Reserve 80% (672 GiB) of host DRAM per four-GPU GB300 node, leaving - # headroom above the 512 GiB SimpleCPUOffloadConnector pool. + # Reserve 80% (672 GiB) of host DRAM per four-GPU GB300 node for the + # Mooncake Store external KV tier. - dram-utilization: 0.80 search-space: # Keep the checked-in recipes on real MTP verification. Throughput jobs # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - # TP4 remains GPU-resident through c8, then uses vLLM Simple CPU offload. + # TP4 remains GPU-resident through c8, then uses Mooncake Store. - spec-decoding: mtp kv-offloading: none conc-list: [1, 2, 4, 6, 8] @@ -7775,7 +7775,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: dp-attn: false - spec-decoding: mtp kv-offloading: dram - kv-offload-backend: { name: vllm-simple } + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } conc-list: [12, 16] prefill: num-worker: 1 @@ -7785,13 +7785,13 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-vllm-simple-offload-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml" decode: num-worker: 0 tp: 4 ep: 1 dp-attn: false - # TP8 remains GPU-resident through c12, then uses vLLM Simple CPU offload. + # TP8 remains GPU-resident through c12, then uses Mooncake Store. - spec-decoding: mtp kv-offloading: none conc-list: [1, 2, 4, 8, 12] @@ -7813,7 +7813,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: dp-attn: false - spec-decoding: mtp kv-offloading: dram - kv-offload-backend: { name: vllm-simple } + kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } conc-list: [16, 20, 24] prefill: num-worker: 1 @@ -7823,7 +7823,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-vllm-simple-offload-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml" decode: num-worker: 0 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6ebf5254ea..64529bdae4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6328,3 +6328,12 @@ description: - "Run aggregate TP4 concurrency 6 and 8 and TP8 concurrency 12 on the GPU-resident Mooncake Store recipes; retain SimpleCPUOffload for TP4 concurrency 12 and 16 and TP8 concurrency 16, 20, and 24." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Correct the aggregate offload backend to Mooncake Store 0.3.11.post1 for TP4 concurrency 12 and 16 and TP8 concurrency 16, 20, and 24; keep the lower-concurrency recipes GPU-resident without a KV connector." + - "Rename the two offload recipes from vllm-simple to mooncake and mirror the disaggregated prefill-side MooncakeStoreConnector settings." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 62ab34bb046566b3e83c6b3ef00ceaf401edcfdc Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 08:08:50 +0000 Subject: [PATCH 19/25] fix(agentx): pin Mooncake hash seed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在两个聚合式 Mooncake Store 配置中固定 PYTHONHASHSEED，以确保跨进程和节点的前缀块键保持一致。 --- .../agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml | 2 ++ .../agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml | 2 ++ perf-changelog.yaml | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml index 5556848c88..f93db4ad5b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml @@ -25,6 +25,8 @@ dynamo: environment: DYNAMO_WHEEL_DIRS: "/srtctl-wheels" ETCD_LEASE_TTL: "7200" + # Keep Mooncake prefix-block keys deterministic across vLLM processes. + PYTHONHASHSEED: "0" setup_script: vllm-container-deps.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml index f4767c2fa6..d92bbb5e35 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml @@ -26,6 +26,8 @@ environment: DYNAMO_WHEEL_DIRS: "/srtctl-wheels" # The frontend shares Grace CPU capacity with the long TP8 cold start. ETCD_LEASE_TTL: "7200" + # Keep Mooncake prefix-block keys deterministic across vLLM processes. + PYTHONHASHSEED: "0" setup_script: vllm-container-deps.sh diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 64529bdae4..08a99408f5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6337,3 +6337,11 @@ - "Correct the aggregate offload backend to Mooncake Store 0.3.11.post1 for TP4 concurrency 12 and 16 and TP8 concurrency 16, 20, and 24; keep the lower-concurrency recipes GPU-resident without a KV connector." - "Rename the two offload recipes from vllm-simple to mooncake and mirror the disaggregated prefill-side MooncakeStoreConnector settings." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Pin PYTHONHASHSEED=0 in both aggregate Mooncake Store recipes so prefix-block keys remain deterministic across vLLM processes and nodes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 2171291d4af6b3c3d9f3dc5a60eb7bee2c01df0d Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 09:43:14 +0000 Subject: [PATCH 20/25] perf(gb300): cap TP sweeps and add c388 P/D point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将聚合式 TP4 和 TP8 扫描上限统一为并发 16，并添加经过当前分支运行时配置适配的 1P1D DEP8/DEP32 并发 388 P/D 点。 --- ...b300-1p1d-dep8-dep32-c388-mtp-agentic.yaml | 215 ++++++++++++++++++ configs/nvidia-master.yaml | 20 +- perf-changelog.yaml | 10 + 3 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml new file mode 100644 index 0000000000..066b884314 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml @@ -0,0 +1,215 @@ +# Source: https://github.com/SemiAnalysisAI/InferenceX/blob/c0fce8a2682364c68858e5ed8995bf94bbb70e30/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml +# Preserve the source point's topology-specific batching while aligning its +# shared runtime settings with the current GB300 P/D recipes in this branch. +name: "svf-vllm-disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP32 decode worker at concurrency 388. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.4.0" + +dynamo: + wheel: "1.4.0" + install: true + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 8 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 32 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 16384 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 32 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 4 + max-num-batched-tokens: 16 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 16 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c388-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c388-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 547a4c8f1c..c65f64788d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7814,7 +7814,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } - conc-list: [16, 20, 24] + conc-list: [16] prefill: num-worker: 1 tp: 8 @@ -7915,6 +7915,24 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: tp: 16 ep: 16 dp-attn: true + # Source: SemiAnalysisAI/InferenceX@c0fce8a2682364c68858e5ed8995bf94bbb70e30. + - spec-decoding: mtp + conc-list: [388] + router: { name: dynamo-router, version: "1.4.0" } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true - spec-decoding: mtp conc-list: [256] router: { name: dynamo-router, version: "1.4.0" } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 08a99408f5..98674c9b12 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6345,3 +6345,13 @@ description: - "Pin PYTHONHASHSEED=0 in both aggregate Mooncake Store recipes so prefix-block keys remain deterministic across vLLM processes and nodes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Cap both aggregate TP4 and TP8 sweeps at concurrency 16 by removing TP8 concurrency 20 and 24; TP4 already ends at concurrency 16." + - "Port the GB300 1P DEP8 + 1D DEP32 concurrency-388 P/D point from c0fce8a, preserving its topology-specific batching while aligning Dynamo 1.4.0, safetensors prefetch, prefix retention, JIT cache paths, and synthetic acceptance length with this branch." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From c041fdf72489914f140f21da4566b26e00a2845c Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 09:50:09 +0000 Subject: [PATCH 21/25] perf(gb300): reduce c388 decode Mooncake threads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 c388 DEP32 解码 worker 的 Mooncake 接收线程数从 20 降至 4，并保持 DEP8 预填充 worker 为 20。 --- ...b300-1p1d-dep8-dep32-c388-mtp-agentic.yaml | 2 +- perf-changelog.yaml | 97 +------------------ 2 files changed, 2 insertions(+), 97 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml index 066b884314..2635753629 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml @@ -173,7 +173,7 @@ backend: VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" VLLM_SERVER_DEV_MODE: "1" VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" UCX_MEMTYPE_CACHE: "n" UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 98674c9b12..f207c0a15d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6252,106 +6252,11 @@ - "Refresh with lower stream interval to collect correct client metrics" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2686 -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg - scenario-type: - - agentic-coding - description: - - "Update the GB300 DeepSeek-V4-Pro Dynamo-vLLM AgentX P/D image from 426e59f to d62ad46, move Dynamo to 1.4.0, and enable safetensors prefetch loading." - - "Replace the aggregate TP data points with the upstream 1P DEP4 + 3D TP8 concurrency-3 NIXL point." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg - scenario-type: - - agentic-coding - description: - - "Port the GB300 1P DEP8 + 1D DEP16 concurrency-512 MTP P/D point and align its runtime tuning with the retained DEP8/DEP16 recipes." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - scenario-type: - - agentic-coding - description: - - "Restore the GB300 DeepSeek-V4-Pro aggregate TP4 and TP8 AgentX MTP recipes on image d62ad46 with Dynamo 1.4.0 and safetensors prefetch loading." - - "Sweep TP4 at concurrency 1, 2, 4, 6, 8, 12, and 16 with vLLM Simple CPU KV offload from concurrency 6; sweep TP8 at concurrency 1, 2, 4, 8, 12, 16, 20, and 24 with offload from concurrency 12." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg - scenario-type: - - agentic-coding - description: - - "Revert all GB300 DeepSeek-V4-Pro AgentX recipes from image d62ad46 to the previous 426e59f image." - - "Disable prefix-cache retention intervals across aggregate and P/D workers; align aggregate TP all-reduce and GPU-memory tuning with the low-latency 1P3D TP8 decode recipe." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - scenario-type: - - agentic-coding - description: - - "Restore aggregate TP4 and TP8 GPU memory utilization from 0.92 to 0.94; the lower value was specific to the separate low-latency 1P3D decode topology." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - scenario-type: - - agentic-coding - description: - - "Keep the checked-in aggregate TP4 and TP8 recipes on real MTP verification; throughput runs inject synthetic acceptance length 2.49 from nvidia-master.yaml, while EVAL_ONLY leaves the recipes unchanged." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - scenario-type: - - agentic-coding - description: - - "Activate MooncakeStoreConnector for the GPU-resident aggregate TP4 and TP8 recipes without proactive KV offload." - - "Set the SimpleCPUOffload TP4 and TP8 max-num-seqs capacities to 32 and 64 with matching 128- and 256-token MTP graph envelopes." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - scenario-type: - - agentic-coding - description: - - "Raise the aggregate SimpleCPUOffload DRAM budget from 0.6095238 to 0.80, exposing 672 GiB per four-GPU GB300 node." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - scenario-type: - - agentic-coding - description: - - "Run aggregate TP4 concurrency 6 and 8 and TP8 concurrency 12 on the GPU-resident Mooncake Store recipes; retain SimpleCPUOffload for TP4 concurrency 12 and 16 and TP8 concurrency 16, 20, and 24." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - scenario-type: - - agentic-coding - description: - - "Correct the aggregate offload backend to Mooncake Store 0.3.11.post1 for TP4 concurrency 12 and 16 and TP8 concurrency 16, 20, and 24; keep the lower-concurrency recipes GPU-resident without a KV connector." - - "Rename the two offload recipes from vllm-simple to mooncake and mirror the disaggregated prefill-side MooncakeStoreConnector settings." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - -- config-keys: - - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - scenario-type: - - agentic-coding - description: - - "Pin PYTHONHASHSEED=0 in both aggregate Mooncake Store recipes so prefix-block keys remain deterministic across vLLM processes and nodes." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 - - config-keys: - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg scenario-type: - agentic-coding description: - - "Cap both aggregate TP4 and TP8 sweeps at concurrency 16 by removing TP8 concurrency 20 and 24; TP4 already ends at concurrency 16." - - "Port the GB300 1P DEP8 + 1D DEP32 concurrency-388 P/D point from c0fce8a, preserving its topology-specific batching while aligning Dynamo 1.4.0, safetensors prefetch, prefix retention, JIT cache paths, and synthetic acceptance length with this branch." + - "Refresh GB300 AgentX MTP aggregate/P-D recipes: cap TP4/TP8 at c16 and add c388 DEP8/DEP32 with 4 decode Mooncake receive threads." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 0c01661034e2dbf9e0242bc90f9debe4966d53e5 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 10:35:05 +0000 Subject: [PATCH 22/25] perf(gb300): change AgentX DEP32 point to c384 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 GB300 AgentX 的 1P1D DEP8/DEP32 数据点从并发 388 调整为并发 384，并同步配方名称、路径和 JIT 缓存标识。 --- ...=> disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml} | 8 ++++---- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/{disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml => disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml} (98%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml similarity index 98% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml index 2635753629..4a404a062b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml @@ -1,10 +1,10 @@ # Source: https://github.com/SemiAnalysisAI/InferenceX/blob/c0fce8a2682364c68858e5ed8995bf94bbb70e30/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml # Preserve the source point's topology-specific batching while aligning its # shared runtime settings with the current GB300 P/D recipes in this branch. -name: "svf-vllm-disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic" +name: "svf-vllm-disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic" # GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP32 decode worker at concurrency 388. Decode consumes P/D KV through NIXL +# DEP32 decode worker at concurrency 384. Decode consumes P/D KV through NIXL # and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. model: @@ -158,7 +158,7 @@ backend: VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c388-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c384-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" @@ -185,7 +185,7 @@ backend: NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c388-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c384-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c65f64788d..e9597d75ce 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7917,7 +7917,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: dp-attn: true # Source: SemiAnalysisAI/InferenceX@c0fce8a2682364c68858e5ed8995bf94bbb70e30. - spec-decoding: mtp - conc-list: [388] + conc-list: [384] router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 1 @@ -7927,7 +7927,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml" decode: num-worker: 1 tp: 32 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f207c0a15d..d3b96d44d3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6258,5 +6258,5 @@ scenario-type: - agentic-coding description: - - "Refresh GB300 AgentX MTP aggregate/P-D recipes: cap TP4/TP8 at c16 and add c388 DEP8/DEP32 with 4 decode Mooncake receive threads." + - "Refresh GB300 AgentX MTP aggregate/P-D recipes: cap TP4/TP8 at c16 and add c384 DEP8/DEP32 with 4 decode Mooncake receive threads." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 75e1819c8174a26e604063a6a0a6114c0630c9fb Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 19 Aug 2026 10:45:09 +0000 Subject: [PATCH 23/25] perf(gb300): raise AgentX DEP32 point to c448 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 GB300 AgentX 的 1P1D DEP8/DEP32 数据点调整到并发 448，并将解码侧 max-num-seqs 提高到 8、max-num-batched-tokens 提高到 32。 --- ...sagg-gb300-1p1d-dep8-dep32-c448-mtp-agentic.yaml} | 12 ++++++------ configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/{disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml => disagg-gb300-1p1d-dep8-dep32-c448-mtp-agentic.yaml} (97%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c448-mtp-agentic.yaml similarity index 97% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c448-mtp-agentic.yaml index 4a404a062b..31a84a9cfd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c448-mtp-agentic.yaml @@ -1,10 +1,10 @@ # Source: https://github.com/SemiAnalysisAI/InferenceX/blob/c0fce8a2682364c68858e5ed8995bf94bbb70e30/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c388-mtp-agentic.yaml # Preserve the source point's topology-specific batching while aligning its # shared runtime settings with the current GB300 P/D recipes in this branch. -name: "svf-vllm-disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic" +name: "svf-vllm-disagg-gb300-1p1d-dep8-dep32-c448-mtp-agentic" # GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one -# DEP32 decode worker at concurrency 384. Decode consumes P/D KV through NIXL +# DEP32 decode worker at concurrency 448. Decode consumes P/D KV through NIXL # and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. model: @@ -118,8 +118,8 @@ backend: enable-expert-parallel: true enable-ep-weight-filter: true max-model-len: 1048576 - max-num-seqs: 4 - max-num-batched-tokens: 16 + max-num-seqs: 8 + max-num-batched-tokens: 32 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 @@ -158,7 +158,7 @@ backend: VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c384-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c448-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" @@ -185,7 +185,7 @@ backend: NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c384-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c448-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e9597d75ce..3b95d1ca77 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7917,7 +7917,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: dp-attn: true # Source: SemiAnalysisAI/InferenceX@c0fce8a2682364c68858e5ed8995bf94bbb70e30. - spec-decoding: mtp - conc-list: [384] + conc-list: [448] router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 1 @@ -7927,7 +7927,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c384-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c448-mtp-agentic.yaml" decode: num-worker: 1 tp: 32 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d3b96d44d3..768372dd9e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6258,5 +6258,5 @@ scenario-type: - agentic-coding description: - - "Refresh GB300 AgentX MTP aggregate/P-D recipes: cap TP4/TP8 at c16 and add c384 DEP8/DEP32 with 4 decode Mooncake receive threads." + - "Refresh GB300 AgentX MTP aggregate/P-D recipes: cap TP4/TP8 at c16 and add c448 DEP8/DEP32 with decode max-num-seqs 8, max-num-batched-tokens 32, and 4 Mooncake receive threads." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 09f19201731f8ee4432019c85e08d81bd9e3ede5 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Thu, 20 Aug 2026 05:00:38 +0000 Subject: [PATCH 24/25] perf(gb300): add AgentX DEP32 c256 point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加 GB300 AgentX DEP8 预填充和 DEP32 解码的 c256 配置点，并复用 c448 运行时调优。 --- ...b300-1p1d-dep8-dep32-c256-mtp-agentic.yaml | 215 ++++++++++++++++++ configs/nvidia-master.yaml | 18 ++ perf-changelog.yaml | 2 +- 3 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c256-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c256-mtp-agentic.yaml new file mode 100644 index 0000000000..8ff57beb5c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c256-mtp-agentic.yaml @@ -0,0 +1,215 @@ +# Source: https://github.com/SemiAnalysisAI/InferenceX/blob/2454f220d618232013baa4b68d19b0f9f39c7bc5/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c448-mtp-agentic.yaml +# Preserve the c448 point's DEP8/DEP32 topology and runtime tuning while adding +# a separate c256 operating point with isolated JIT cache paths. +name: "svf-vllm-disagg-gb300-1p1d-dep8-dep32-c256-mtp-agentic" + +# GB300 AgentX MTP3 topology: one DEP8 prefill worker feeds one +# DEP32 decode worker at concurrency 256. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.4.0" + +dynamo: + wheel: "1.4.0" + install: true + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 8 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 32 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 16384 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + safetensors-load-strategy: "prefetch" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 32 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 8 + max-num-batched-tokens: 32 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 16 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep8-prefill-c256-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep32-decode-c256-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3b95d1ca77..9bba002e60 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7933,6 +7933,24 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: tp: 32 ep: 32 dp-attn: true + # Same DEP8/DEP32 runtime as c448 above, with a lower concurrency point. + - spec-decoding: mtp + conc-list: [256] + router: { name: dynamo-router, version: "1.4.0" } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep32-c256-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true - spec-decoding: mtp conc-list: [256] router: { name: dynamo-router, version: "1.4.0" } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 768372dd9e..b612cb063a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6258,5 +6258,5 @@ scenario-type: - agentic-coding description: - - "Refresh GB300 AgentX MTP aggregate/P-D recipes: cap TP4/TP8 at c16 and add c448 DEP8/DEP32 with decode max-num-seqs 8, max-num-batched-tokens 32, and 4 Mooncake receive threads." + - "Refresh GB300 AgentX MTP aggregate/P-D recipes: cap TP4/TP8 at c16 and add c448 and c256 1P DEP8 / 1D DEP32 operating points with the same runtime tuning, decode max-num-seqs 8, max-num-batched-tokens 32, 4 Mooncake receive threads, and isolated JIT cache paths." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 From 2a8bf7beb0ba9faa72032e21147bddf24f6e1fd4 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Thu, 20 Aug 2026 05:32:42 +0000 Subject: [PATCH 25/25] perf(gb300): make AgentX additions append-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留现有聚合与解聚配置不变，仅新增 TP4 并发点、1P4D c4 以及 DEP16/DEP32 配置，并将性能变更日志标记为 append-only。 --- .../agg-gb300-tp4-mooncake-mtp-agentic.yaml | 145 ----------------- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 28 ++-- .../agg-gb300-tp8-mooncake-mtp-agentic.yaml | 146 ------------------ .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 32 ++-- ...gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml | 10 +- ...b300-1p1d-dep8-dep16-c512-mtp-agentic.yaml | 2 +- ...g-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml} | 22 +-- ...300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml | 10 +- ...300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml | 10 +- configs/nvidia-master.yaml | 71 ++++----- perf-changelog.yaml | 4 +- 11 files changed, 94 insertions(+), 386 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/{disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml => disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml} (92%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml deleted file mode 100644 index f93db4ad5b..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml +++ /dev/null @@ -1,145 +0,0 @@ -name: "svf-vllm-agg-gb300-tp4-mooncake-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode for the c12-c16 sweep with Mooncake Store -# as the external KV tier. Keep 32 scheduler slots and capture the corresponding -# 128-token MTP graph envelope. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - wheel: "1.4.0" - install: true - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - ETCD_LEASE_TTL: "7200" - # Keep Mooncake prefix-block keys deterministic across vLLM processes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mooncake-mtp-{job_id}" - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - block-size: 256 - max-cudagraph-capture-size: 128 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}' - gpu-memory-utilization: 0.94 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - # Avoid concurrent readers observing a mismatched mmap data/index pair. - AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index d452907b58..9559ffb75b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -1,8 +1,8 @@ name: "svf-vllm-agg-gb300-tp4-mtp-agentic" # GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode for the GPU-resident c1-c8 sweep. Keep -# 16 scheduler slots and capture the corresponding 64-token MTP graph envelope. +# and serves both prefill and decode at concurrency 8. Size max-num-seqs at +# 4x concurrency and expand the MTP CUDA-graph envelope to match. model: path: "deepseek-v4-pro" @@ -15,10 +15,10 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: - dynamo: "1.4.0" + dynamo: "1.2.1" dynamo: - wheel: "1.4.0" + wheel: "1.2.1" install: true environment: @@ -77,9 +77,10 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" @@ -87,11 +88,14 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" NCCL_P2P_LEVEL: "NVL" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" @@ -102,19 +106,21 @@ backend: vllm_config: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 4 pipeline-parallel-size: 1 + disable-custom-all-reduce: true attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 - max-num-seqs: 16 + max-num-seqs: 32 max-num-batched-tokens: 8192 trust-remote-code: true + no-enable-flashinfer-autotune: true block-size: 256 - max-cudagraph-capture-size: 64 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' speculative-config: '{"method":"mtp","num_speculative_tokens":3}' gpu-memory-utilization: 0.94 + stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml deleted file mode 100644 index d92bbb5e35..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml +++ /dev/null @@ -1,146 +0,0 @@ -name: "svf-vllm-agg-gb300-tp8-mooncake-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU nodes -# and serves both prefill and decode for the c16-c24 sweep with Mooncake Store -# as the external KV tier. Keep 64 scheduler slots and capture the corresponding -# 256-token MTP graph envelope. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.4.0" - -dynamo: - wheel: "1.4.0" - install: true - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - # The frontend shares Grace CPU capacity with the long TP8 cold start. - ETCD_LEASE_TTL: "7200" - # Keep Mooncake prefix-block keys deterministic across vLLM processes. - PYTHONHASHSEED: "0" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "64" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp8-mooncake-mtp-{job_id}" - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - trust-remote-code: true - block-size: 256 - max-cudagraph-capture-size: 256 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' - kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}' - gpu-memory-utilization: 0.94 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index bc6e6a5882..4eebce98fa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -1,8 +1,8 @@ name: "svf-vllm-agg-gb300-tp8-mtp-agentic" -# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU nodes -# and serves both prefill and decode for the GPU-resident c1-c12 sweep. Keep -# 32 scheduler slots and capture the corresponding 128-token MTP graph envelope. +# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 4. Keep at least +# 16 sequence slots and otherwise size the scheduler at 4x concurrency. model: path: "deepseek-v4-pro" @@ -15,10 +15,10 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: - dynamo: "1.4.0" + dynamo: "1.2.1" dynamo: - wheel: "1.4.0" + wheel: "1.2.1" install: true environment: @@ -78,9 +78,10 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" @@ -88,34 +89,39 @@ backend: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_USE_RUST_FRONTEND: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" NCCL_P2P_LEVEL: "NVL" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp8-mtp-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}" vllm_config: aggregated: served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 8 pipeline-parallel-size: 1 + disable-custom-all-reduce: true attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 - max-num-seqs: 32 + max-num-seqs: 16 max-num-batched-tokens: 8192 trust-remote-code: true + no-enable-flashinfer-autotune: true block-size: 256 - max-cudagraph-capture-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' speculative-config: '{"method":"mtp","num_speculative_tokens":3}' gpu-memory-utilization: 0.94 + stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml index 1184b49e53..e5cc1c76ea 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml @@ -14,10 +14,10 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: - dynamo: "1.4.0" + dynamo: "1.3.0.dev20260720" dynamo: - wheel: "1.4.0" + wheel: "1.3.0.dev20260720" install: true setup_script: vllm-container-deps.sh @@ -77,7 +77,6 @@ backend: prefill: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 @@ -105,7 +104,6 @@ backend: decode: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 @@ -150,7 +148,7 @@ backend: NCCL_NVLS_ENABLE: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" @@ -179,7 +177,7 @@ backend: NCCL_NVLS_ENABLE: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_DSV4_MEGA_FP8_COMBINE: "1" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml index aa43ddaf9b..f632eec5ee 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep16-c512-mtp-agentic.yaml @@ -92,7 +92,7 @@ backend: enable-ep-weight-filter: true max-model-len: 1048576 max-num-seqs: 64 - max-num-batched-tokens: 8192 + max-num-batched-tokens: 16384 long-prefill-token-threshold: 1024 trust-remote-code: true no-enable-flashinfer-autotune: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml similarity index 92% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml index 6917346319..c289cb5655 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml @@ -1,10 +1,10 @@ # Source: https://github.com/Inferact/srt-slurm-sa/blob/4a870cd5bc333bf8a312fc65ca1ea82cdab9b2df/recipes/vllm/deepseek-v4-pro/GB300/agentic/agentx-v1.0.1/1p3d-pdep4-dtp8-c3-kv-nixl-mtp-flashinfer-ar-lpt512-psi1.yaml -# Adapted to the InferenceX AgentX harness and throughput-only synthetic MTP -# acceptance injection so eval-only runs continue to verify real MTP output. -name: "svf-vllm-disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic" +# Adapted from the source topology to a 1P4D concurrency-4 point for the +# InferenceX AgentX harness; eval-only runs continue to verify real MTP output. +name: "svf-vllm-disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic" # GB300 high-interactivity AgentX MTP3 topology: one DEP4 prefill worker -# feeds three TP8 decode workers at concurrency 3 through NIXL. +# feeds four TP8 decode workers at concurrency 4 through NIXL. model: path: "deepseek-v4-pro" container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" @@ -36,7 +36,7 @@ environment: VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p3d-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p4d-{job_id}" slurm: time_limit: "08:00:00" @@ -51,9 +51,9 @@ resources: het_jobs: false spread_workers: false prefill_nodes: 1 - decode_nodes: 6 + decode_nodes: 8 prefill_workers: 1 - decode_workers: 3 + decode_workers: 4 gpus_per_prefill: 4 gpus_per_decode: 8 @@ -69,7 +69,7 @@ frontend: MODEL_PREFIX: "dsv4" FRAMEWORK: "dynamo-vllm" PRECISION: "fp4" - CONC: "3" + CONC: "4" DURATION: "3600" KV_OFFLOADING: "none" ETCD_LEASE_TTL: "120" @@ -175,9 +175,9 @@ benchmark: SERVED_MODEL_NAME: "DeepSeek-V4-Pro" FRAMEWORK: "dynamo-vllm" PRECISION: "fp4" - CONC: "3" + CONC: "4" DURATION: "3600" - RESULT_FILENAME: "dsv4_agentx101_p1dep4_d3tp8_c3_kv_nixl_mtp_fi_ar_lpt512_psi1" + RESULT_FILENAME: "dsv4_agentx101_p1dep4_d4tp8_c4_kv_nixl_mtp_fi_ar_lpt512_psi1" RUNNER_TYPE: "gb300" IMAGE: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" SPEC_DECODING: "mtp" @@ -189,7 +189,7 @@ benchmark: PREFILL_NUM_WORKERS: "1" PREFILL_EP: "4" DECODE_TP: "8" - DECODE_NUM_WORKERS: "3" + DECODE_NUM_WORKERS: "4" DECODE_EP: "1" EP_SIZE: "1" AIPERF_MAX_OSL: "none" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml index bc377b7c48..d5a0b44161 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml @@ -15,10 +15,10 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: - dynamo: "1.4.0" + dynamo: "1.3.0.dev20260720" dynamo: - wheel: "1.4.0" + wheel: "1.3.0.dev20260720" install: true environment: @@ -78,7 +78,6 @@ backend: prefill: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 @@ -105,7 +104,6 @@ backend: decode: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 @@ -151,7 +149,7 @@ backend: NCCL_NVLS_ENABLE: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" @@ -180,7 +178,7 @@ backend: NCCL_NVLS_ENABLE: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_DSV4_MEGA_FP8_COMBINE: "1" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml index 7a7b731dad..f770241932 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml @@ -15,10 +15,10 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" frameworks: - dynamo: "1.4.0" + dynamo: "1.3.0.dev20260720" dynamo: - wheel: "1.4.0" + wheel: "1.3.0.dev20260720" install: true environment: @@ -78,7 +78,6 @@ backend: prefill: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 @@ -105,7 +104,6 @@ backend: decode: kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - safetensors-load-strategy: "prefetch" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 @@ -151,7 +149,7 @@ backend: NCCL_NVLS_ENABLE: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" @@ -180,7 +178,7 @@ backend: NCCL_NVLS_ENABLE: "1" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_DSV4_MEGA_FP8_COMBINE: "1" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep16-decode-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9bba002e60..b455b0562a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7742,91 +7742,85 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: runner: cluster:gb300-nv precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.4.0" } + router: { name: dynamo-router, version: "1.2.1" } multinode: true disagg: false scenarios: agentic-coding: - # Reserve 80% (672 GiB) of host DRAM per four-GPU GB300 node for the - # Mooncake Store external KV tier. - - dram-utilization: 0.80 - search-space: + - search-space: # Keep the checked-in recipes on real MTP verification. Throughput jobs # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - # TP4 remains GPU-resident through c8, then uses Mooncake Store. - spec-decoding: mtp - kv-offloading: none - conc-list: [1, 2, 4, 6, 8] + conc-list: [1] prefill: num-worker: 1 - tp: 4 + tp: 8 ep: 1 dp-attn: false additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts four GPUs only once. + # count at zero so result aggregation counts eight GPUs only once. decode: num-worker: 0 - tp: 4 + tp: 8 ep: 1 dp-attn: false - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } - conc-list: [12, 16] + conc-list: [4] prefill: num-worker: 1 - tp: 4 + tp: 8 ep: 1 dp-attn: false additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mooncake-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts eight GPUs only once. decode: num-worker: 0 - tp: 4 + tp: 8 ep: 1 dp-attn: false - # TP8 remains GPU-resident through c12, then uses Mooncake Store. - spec-decoding: mtp - kv-offloading: none - conc-list: [1, 2, 4, 8, 12] + conc-list: [8] prefill: num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts eight GPUs only once. + # count at zero so result aggregation counts four GPUs only once. decode: num-worker: 0 - tp: 8 + tp: 4 ep: 1 dp-attn: false - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } - conc-list: [16] + kv-offloading: none + conc-list: [1, 2, 4, 6] prefill: num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mooncake-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts four GPUs only once. decode: num-worker: 0 - tp: 8 + tp: 4 ep: 1 dp-attn: false @@ -7845,9 +7839,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - search-space: # Keep the checked-in recipes on real MTP verification. Throughput jobs # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - # Source: Inferact/srt-slurm-sa@4a870cd5bc333bf8a312fc65ca1ea82cdab9b2df. - spec-decoding: mtp - conc-list: [3] + conc-list: [4] router: { name: dynamo-router, version: "1.4.0" } prefill: num-worker: 1 @@ -7857,15 +7850,15 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: additional-settings: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p3d-dep4-tp8-c3-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p4d-dep4-tp8-c4-mtp-agentic.yaml" decode: - num-worker: 3 + num-worker: 4 tp: 8 ep: 1 dp-attn: false - spec-decoding: mtp conc-list: [1152] - router: { name: dynamo-router, version: "1.4.0" } + router: { name: dynamo-router, version: "1.3.0.dev20260720" } prefill: num-worker: 2 tp: 8 @@ -7882,7 +7875,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: dp-attn: true - spec-decoding: mtp conc-list: [1024] - router: { name: dynamo-router, version: "1.4.0" } + router: { name: dynamo-router, version: "1.3.0.dev20260720" } prefill: num-worker: 2 tp: 8 @@ -7897,7 +7890,6 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: tp: 16 ep: 16 dp-attn: true - # Source: SemiAnalysisAI/InferenceX@0c33d4615792705ed12bfc204e3a54cfa436cf02. - spec-decoding: mtp conc-list: [512] router: { name: dynamo-router, version: "1.4.0" } @@ -7915,7 +7907,6 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: tp: 16 ep: 16 dp-attn: true - # Source: SemiAnalysisAI/InferenceX@c0fce8a2682364c68858e5ed8995bf94bbb70e30. - spec-decoding: mtp conc-list: [448] router: { name: dynamo-router, version: "1.4.0" } @@ -7953,7 +7944,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: dp-attn: true - spec-decoding: mtp conc-list: [256] - router: { name: dynamo-router, version: "1.4.0" } + router: { name: dynamo-router, version: "1.3.0.dev20260720" } prefill: num-worker: 1 tp: 4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b612cb063a..aa1bb026e0 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6258,5 +6258,7 @@ scenario-type: - agentic-coding description: - - "Refresh GB300 AgentX MTP aggregate/P-D recipes: cap TP4/TP8 at c16 and add c448 and c256 1P DEP8 / 1D DEP32 operating points with the same runtime tuning, decode max-num-seqs 8, max-num-batched-tokens 32, 4 Mooncake receive threads, and isolated JIT cache paths." + - "Keep the aggregate runtime recipe and existing TP4 c8 point unchanged, and add GPU-resident TP4 points at c1, c2, c4, and c6." + - "Add 1P DEP4 + 4D TP8 at c4, 1P DEP8 + 1D DEP16 at c512, and 1P DEP8 + 1D DEP32 at c448 and c256 using isolated recipes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2665 + append-only: true