From 8511297c09f093df09c281fa77fb219b6c070154 Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Wed, 19 Aug 2026 00:54:30 -0400 Subject: [PATCH] feat(gb300): add Kimi-K3 aggregate Pareto points Extend the GB300 AgentX aggregate sweep with concurrency 2, 8, 16, 24, 32, 40 and 56, alongside the existing 1, 4, 48, 70 and 74 points. Each concurrency gets its own recipe: max-num-seqs is sized at 2x concurrency and the cudagraph ladder is built from the draft length, which steps down with concurrency -- DSpark K=7 at golden AL 3.84 up to 8, K=3 at 3.00 for 16, no drafting above. gpu-memory-utilization is 0.92 and derates to 0.90 from concurrency 56 up. The Mooncake DRAM tier stays on the GB300 constraint, a 150 GB per-rank segment at dram-utilization 0.1664. The changelog entry lists every Kimi-K3 GB300 key, aggregate and disaggregated, so the sweep re-measures all 14 points in one run rather than mixing fresh results with older ones. Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-Authored-By: Claude Opus 5 (1M context) --- ...300-dcp8-c16-dspark3-mooncake-agentic.yaml | 145 ++++++++++++++++++ ...b300-dcp8-c2-dspark7-mooncake-agentic.yaml | 145 ++++++++++++++++++ ...b300-dcp8-c24-nospec-mooncake-agentic.yaml | 142 +++++++++++++++++ ...b300-dcp8-c32-nospec-mooncake-agentic.yaml | 142 +++++++++++++++++ ...b300-dcp8-c40-nospec-mooncake-agentic.yaml | 142 +++++++++++++++++ ...b300-dcp8-c56-nospec-mooncake-agentic.yaml | 142 +++++++++++++++++ ...b300-dcp8-c8-dspark7-mooncake-agentic.yaml | 145 ++++++++++++++++++ configs/nvidia-master.yaml | 119 ++++++++++++++ perf-changelog.yaml | 11 ++ 9 files changed, 1133 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c16-dspark3-mooncake-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c2-dspark7-mooncake-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c24-nospec-mooncake-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c32-nospec-mooncake-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c40-nospec-mooncake-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c56-nospec-mooncake-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c8-dspark7-mooncake-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c16-dspark3-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c16-dspark3-mooncake-agentic.yaml new file mode 100644 index 0000000000..b3058586a2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c16-dspark3-mooncake-agentic.yaml @@ -0,0 +1,145 @@ +# GB300 aggregate point at concurrency 16: one TP8 x DCP8 worker across +# two 4-GPU nodes, drafting with DSpark K=3 at golden AL 3.00, max-num-seqs 32. +name: "kimi-k3-vllm-agg-gb300-dcp8-c16-dspark3-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 32 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Throughput jobs inject the committed K=3 golden AL (3.00); EVAL_ONLY + # preserves this real target-verification configuration. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + # Capture ladder: multiples of (1 + K=3) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c2-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c2-dspark7-mooncake-agentic.yaml new file mode 100644 index 0000000000..4a834116b9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c2-dspark7-mooncake-agentic.yaml @@ -0,0 +1,145 @@ +# GB300 aggregate point at concurrency 2: one TP8 x DCP8 worker across +# two 4-GPU nodes, drafting with DSpark K=7 at golden AL 3.84, max-num-seqs 4. +name: "kimi-k3-vllm-agg-gb300-dcp8-c2-dspark7-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 4 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Throughput jobs inject the committed K=7 golden AL (3.84); EVAL_ONLY + # preserves this real target-verification configuration. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + # Capture ladder: multiples of (1 + K=7) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,64,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c24-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c24-nospec-mooncake-agentic.yaml new file mode 100644 index 0000000000..52c9425168 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c24-nospec-mooncake-agentic.yaml @@ -0,0 +1,142 @@ +# GB300 aggregate point at concurrency 24: one TP8 x DCP8 worker across +# two 4-GPU nodes, no drafting, max-num-seqs 48. +name: "kimi-k3-vllm-agg-gb300-dcp8-c24-nospec-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 48 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Capture ladder: multiples of (1 + K=0) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,64,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c32-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c32-nospec-mooncake-agentic.yaml new file mode 100644 index 0000000000..3607543226 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c32-nospec-mooncake-agentic.yaml @@ -0,0 +1,142 @@ +# GB300 aggregate point at concurrency 32: one TP8 x DCP8 worker across +# two 4-GPU nodes, no drafting, max-num-seqs 64. +name: "kimi-k3-vllm-agg-gb300-dcp8-c32-nospec-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 64 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Capture ladder: multiples of (1 + K=0) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c40-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c40-nospec-mooncake-agentic.yaml new file mode 100644 index 0000000000..d77b3becd8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c40-nospec-mooncake-agentic.yaml @@ -0,0 +1,142 @@ +# GB300 aggregate point at concurrency 40: one TP8 x DCP8 worker across +# two 4-GPU nodes, no drafting, max-num-seqs 80. +name: "kimi-k3-vllm-agg-gb300-dcp8-c40-nospec-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 80 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Capture ladder: multiples of (1 + K=0) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c56-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c56-nospec-mooncake-agentic.yaml new file mode 100644 index 0000000000..a104c8ad48 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c56-nospec-mooncake-agentic.yaml @@ -0,0 +1,142 @@ +# GB300 aggregate point at concurrency 56: one TP8 x DCP8 worker across +# two 4-GPU nodes, no drafting, max-num-seqs 112. +name: "kimi-k3-vllm-agg-gb300-dcp8-c56-nospec-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 112 + gpu-memory-utilization: 0.90 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Capture ladder: multiples of (1 + K=0) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c8-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c8-dspark7-mooncake-agentic.yaml new file mode 100644 index 0000000000..0d80ed0006 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c8-dspark7-mooncake-agentic.yaml @@ -0,0 +1,145 @@ +# GB300 aggregate point at concurrency 8: one TP8 x DCP8 worker across +# two 4-GPU nodes, drafting with DSpark K=7 at golden AL 3.84, max-num-seqs 16. +name: "kimi-k3-vllm-agg-gb300-dcp8-c8-dspark7-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 16 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Throughput jobs inject the committed K=7 golden AL (3.84); EVAL_ONLY + # preserves this real target-verification configuration. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + # Capture ladder: multiples of (1 + K=7) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 238aaf320f..423626ac42 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8873,6 +8873,125 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml" decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } +# Additional aggregate Pareto points, at concurrency 2, 8, 16, 24, 32, 40 and +# 56. Each concurrency gets its own recipe because max-num-seqs is sized at 2x +# concurrency and the cudagraph ladder is built from the draft length, which +# steps down with concurrency: DSpark K=7 up to 8, K=3 at 16, no drafting above. +# Concurrencies stay disjoint from the keys above so generated exp-names remain +# unique. +kimik3-fp4-gb300-dynamo-vllm-agentic-pareto-mooncake-dcp8-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + # GB300 host-DRAM budget, unchanged from the keys above: a 150 GB Mooncake + # segment per rank. + - dram-utilization: 0.1664 + search-space: + # DSpark K=7, golden AL 3.84, for concurrency <= 8. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [2] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c2-dspark7-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [8] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c8-dspark7-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # DSpark K=3, golden AL 3.00, at concurrency 16. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [16] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.00" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c16-dspark3-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # No drafting above concurrency 16. + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [24] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c24-nospec-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [32] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c32-nospec-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [40] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c40-nospec-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # gpu-memory-utilization derates to 0.90 from concurrency 56 up. + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [56] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c56-nospec-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # Kimi-K3 MXFP4 B200 aggregated vLLM via Dynamo (TP8 x PP2, 2 nodes / 16 # GPUs), agentic bring-up. The native MXFP4 checkpoint (2.8T total params, diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 52c74bf168..f0255986ff 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6195,3 +6195,14 @@ - "Use AITER INT4 quick-reduce, ptpc_fp8 online quantization excluding embeddings, lm_head, gates, and experts, an FP8 KV cache, and three-token MTP with golden synthetic acceptance length 2.99 for benchmark runs." - "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-pareto-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + scenario-type: + - agentic-coding + description: + - "Add more configs to the Kimi K3 GB300 AgentX sweep." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2669