diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c16-dspark3-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c16-dspark3-mooncake-agentic.yaml new file mode 100644 index 0000000000..b3058586a2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c16-dspark3-mooncake-agentic.yaml @@ -0,0 +1,145 @@ +# GB300 aggregate point at concurrency 16: one TP8 x DCP8 worker across +# two 4-GPU nodes, drafting with DSpark K=3 at golden AL 3.00, max-num-seqs 32. +name: "kimi-k3-vllm-agg-gb300-dcp8-c16-dspark3-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 32 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Throughput jobs inject the committed K=3 golden AL (3.00); EVAL_ONLY + # preserves this real target-verification configuration. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + # Capture ladder: multiples of (1 + K=3) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c2-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c2-dspark7-mooncake-agentic.yaml new file mode 100644 index 0000000000..4a834116b9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c2-dspark7-mooncake-agentic.yaml @@ -0,0 +1,145 @@ +# GB300 aggregate point at concurrency 2: one TP8 x DCP8 worker across +# two 4-GPU nodes, drafting with DSpark K=7 at golden AL 3.84, max-num-seqs 4. +name: "kimi-k3-vllm-agg-gb300-dcp8-c2-dspark7-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 4 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Throughput jobs inject the committed K=7 golden AL (3.84); EVAL_ONLY + # preserves this real target-verification configuration. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + # Capture ladder: multiples of (1 + K=7) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,64,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c24-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c24-nospec-mooncake-agentic.yaml new file mode 100644 index 0000000000..52c9425168 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c24-nospec-mooncake-agentic.yaml @@ -0,0 +1,142 @@ +# GB300 aggregate point at concurrency 24: one TP8 x DCP8 worker across +# two 4-GPU nodes, no drafting, max-num-seqs 48. +name: "kimi-k3-vllm-agg-gb300-dcp8-c24-nospec-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 48 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Capture ladder: multiples of (1 + K=0) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,64,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c32-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c32-nospec-mooncake-agentic.yaml new file mode 100644 index 0000000000..3607543226 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c32-nospec-mooncake-agentic.yaml @@ -0,0 +1,142 @@ +# GB300 aggregate point at concurrency 32: one TP8 x DCP8 worker across +# two 4-GPU nodes, no drafting, max-num-seqs 64. +name: "kimi-k3-vllm-agg-gb300-dcp8-c32-nospec-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 64 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Capture ladder: multiples of (1 + K=0) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c40-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c40-nospec-mooncake-agentic.yaml new file mode 100644 index 0000000000..d77b3becd8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c40-nospec-mooncake-agentic.yaml @@ -0,0 +1,142 @@ +# GB300 aggregate point at concurrency 40: one TP8 x DCP8 worker across +# two 4-GPU nodes, no drafting, max-num-seqs 80. +name: "kimi-k3-vllm-agg-gb300-dcp8-c40-nospec-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 80 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Capture ladder: multiples of (1 + K=0) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c56-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c56-nospec-mooncake-agentic.yaml new file mode 100644 index 0000000000..a104c8ad48 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c56-nospec-mooncake-agentic.yaml @@ -0,0 +1,142 @@ +# GB300 aggregate point at concurrency 56: one TP8 x DCP8 worker across +# two 4-GPU nodes, no drafting, max-num-seqs 112. +name: "kimi-k3-vllm-agg-gb300-dcp8-c56-nospec-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 112 + gpu-memory-utilization: 0.90 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Capture ladder: multiples of (1 + K=0) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c8-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c8-dspark7-mooncake-agentic.yaml new file mode 100644 index 0000000000..0d80ed0006 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c8-dspark7-mooncake-agentic.yaml @@ -0,0 +1,145 @@ +# GB300 aggregate point at concurrency 8: one TP8 x DCP8 worker across +# two 4-GPU nodes, drafting with DSpark K=7 at golden AL 3.84, max-num-seqs 16. +name: "kimi-k3-vllm-agg-gb300-dcp8-c8-dspark7-mooncake-agentic" + +model: + path: "moonshotai/Kimi-K3" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef" + frameworks: + dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +slurm: + time_limit: "04:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TOKENIZER_CACHE_BYTES: "8589934592" + +backend: + type: vllm + connector: null + mooncake_kv_store: + master_extra_args: + - "--eviction_high_watermark_ratio=0.95" + - "--eviction_ratio=0.10" + store_config: + metadata_server: "P2PHANDSHAKE" + # The GB300 nodes are limited to a 150 GB per-rank segment. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + # Use the direct DCP a2a/gather kernels rather than the collective + # fallback. + VLLM_USE_DIRECT_DCP_A2A: "1" + VLLM_USE_DIRECT_DCP_Q_GATHER: "1" + VLLM_USE_DIRECT_DCP_KV_GATHER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0" + PYTHONNOUSERSITE: "1" + TORCH_CUDA_ARCH_LIST: "10.0" + PYTHONHASHSEED: "42" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900" + MC_GID_INDEX: "3" + MC_STORE_MEMCPY: "1" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_SLICE_SIZE: "1048576" + MC_WORKERS_PER_CTX: "4" + WITH_NVIDIA_PEERMEM: "0" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + dcp-comm-backend: "a2a" + # max-num-seqs is sized at 2x concurrency. + max-num-seqs: 16 + gpu-memory-utilization: 0.92 + max-num-batched-tokens: 16384 + trust-remote-code: true + language-model-only: true + load-format: "fastsafetensors" + moe-backend: "auto" + no-enable-flashinfer-autotune: true + enable-cumem-allocator: true + enable-prefix-caching: true + prefix-match-unit: 128 + kv-cache-dtype: "fp8" + stream-interval: 10 + attention-backend: "TOKENSPEED_MLA" + attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + # Throughput jobs inject the committed K=7 golden AL (3.84); EVAL_ONLY + # preserves this real target-verification configuration. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}' + # Capture ladder: multiples of (1 + K=7) up to + # min(max-num-seqs, 128) entries, then the fixed powers of two above it. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,256,512,1024,2048,4096,8192]}' + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + comment: >- + '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}' + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + AIPERF_HTTP_TCP_USER_TIMEOUT: "900000" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 238aaf320f..423626ac42 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8873,6 +8873,125 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml" decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } +# Additional aggregate Pareto points, at concurrency 2, 8, 16, 24, 32, 40 and +# 56. Each concurrency gets its own recipe because max-num-seqs is sized at 2x +# concurrency and the cudagraph ladder is built from the draft length, which +# steps down with concurrency: DSpark K=7 up to 8, K=3 at 16, no drafting above. +# Concurrencies stay disjoint from the keys above so generated exp-names remain +# unique. +kimik3-fp4-gb300-dynamo-vllm-agentic-pareto-mooncake-dcp8-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + # GB300 host-DRAM budget, unchanged from the keys above: a 150 GB Mooncake + # segment per rank. + - dram-utilization: 0.1664 + search-space: + # DSpark K=7, golden AL 3.84, for concurrency <= 8. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [2] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c2-dspark7-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [8] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c8-dspark7-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # DSpark K=3, golden AL 3.00, at concurrency 16. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [16] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.00" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c16-dspark3-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # No drafting above concurrency 16. + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [24] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c24-nospec-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [32] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c32-nospec-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [40] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c40-nospec-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # gpu-memory-utilization derates to 0.90 from concurrency 56 up. + - kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } + conc-list: [56] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c56-nospec-mooncake-agentic.yaml" + decode: { num-worker: 0, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + # Kimi-K3 MXFP4 B200 aggregated vLLM via Dynamo (TP8 x PP2, 2 nodes / 16 # GPUs), agentic bring-up. The native MXFP4 checkpoint (2.8T total params, diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 52c74bf168..f0255986ff 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6195,3 +6195,14 @@ - "Use AITER INT4 quick-reduce, ptpc_fp8 online quantization excluding embeddings, lm_head, gates, and experts, an FP8 KV cache, and three-token MTP with golden synthetic acceptance length 2.99 for benchmark runs." - "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-pareto-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + scenario-type: + - agentic-coding + description: + - "Add more configs to the Kimi K3 GB300 AgentX sweep." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2669