Perf tuning for GB300 agentx Kimi K3

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c16-dspark3-mooncake-agentic.yaml

@@ -0,0 +1,145 @@
+# GB300 aggregate point at concurrency 16: one TP8 x DCP8 worker across
+# two 4-GPU nodes, drafting with DSpark K=3 at golden AL 3.00, max-num-seqs 32.
+name: "kimi-k3-vllm-agg-gb300-dcp8-c16-dspark3-mooncake-agentic"
+
+model:
+  path: "moonshotai/Kimi-K3"
+  container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+  container:
+    image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef"
+  frameworks:
+    dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1"
+
+dynamo:
+  hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1"
+  install: true
+
+slurm:
+  time_limit: "04:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "gb300"
+  gpus_per_node: 4
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 8
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  args:
+    router-mode: "random"
+    router-session-affinity-ttl-secs: 900
+  env:
+    DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600"
+    DYN_TOKENIZER_CACHE_BYTES: "8589934592"
+
+backend:
+  type: vllm
+  connector: null
+  mooncake_kv_store:
+    master_extra_args:
+      - "--eviction_high_watermark_ratio=0.95"
+      - "--eviction_ratio=0.10"
+    store_config:
+      metadata_server: "P2PHANDSHAKE"
+      # The GB300 nodes are limited to a 150 GB per-rank segment.
+      global_segment_size: "150GB"
+      local_buffer_size: "4GB"
+      protocol: "rdma"
+      device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+      mode: "embedded"
+      enable_offload: false
+  aggregated_environment:
+    VLLM_ALLREDUCE_USE_FLASHINFER: "1"
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    # Use the direct DCP a2a/gather kernels rather than the collective
+    # fallback.
+    VLLM_USE_DIRECT_DCP_A2A: "1"
+    VLLM_USE_DIRECT_DCP_Q_GATHER: "1"
+    VLLM_USE_DIRECT_DCP_KV_GATHER: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0"
+    VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0"
+    PYTHONNOUSERSITE: "1"
+    TORCH_CUDA_ARCH_LIST: "10.0"
+    PYTHONHASHSEED: "42"
+    VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900"
+    MC_GID_INDEX: "3"
+    MC_STORE_MEMCPY: "1"
+    MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
+    MC_SLICE_SIZE: "1048576"
+    MC_WORKERS_PER_CTX: "4"
+    WITH_NVIDIA_PEERMEM: "0"
+    VLLM_MOONCAKE_LOAD_RECV_THREADS: "4"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 8
+      decode-context-parallel-size: 8
+      dcp-comm-backend: "a2a"
+      # max-num-seqs is sized at 2x concurrency.
+      max-num-seqs: 32
+      gpu-memory-utilization: 0.92
+      max-num-batched-tokens: 16384
+      trust-remote-code: true
+      language-model-only: true
+      load-format: "fastsafetensors"
+      moe-backend: "auto"
+      no-enable-flashinfer-autotune: true
+      enable-cumem-allocator: true
+      enable-prefix-caching: true
+      prefix-match-unit: 128
+      kv-cache-dtype: "fp8"
+      stream-interval: 10
+      attention-backend: "TOKENSPEED_MLA"
+      attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}'
+      # Throughput jobs inject the committed K=3 golden AL (3.00); EVAL_ONLY
+      # preserves this real target-verification configuration.
+      speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
+      # Capture ladder: multiples of (1 + K=3) up to
+      # min(max-num-seqs, 128) entries, then the fixed powers of two above it.
+      compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128,256,512,1024,2048,4096,8192]}'
+      kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}'
+
+sbatch_directives:
+  mem: "0"
+  cpus-per-task: "72"
+  comment: >-
+    '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}'
+
+srun_options:
+  mem: "0"
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
+    AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
+    AIPERF_HTTP_TCP_USER_TIMEOUT: "900000"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-c2-dspark7-mooncake-agentic.yaml

@@ -0,0 +1,145 @@
+# GB300 aggregate point at concurrency 2: one TP8 x DCP8 worker across
+# two 4-GPU nodes, drafting with DSpark K=7 at golden AL 3.84, max-num-seqs 4.
+name: "kimi-k3-vllm-agg-gb300-dcp8-c2-dspark7-mooncake-agentic"
+
+model:
+  path: "moonshotai/Kimi-K3"
+  container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef"
+  precision: "fp4"
+
+identity:
+  model:
+    repo: "moonshotai/Kimi-K3"
+  container:
+    image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-75c2eef"
+  frameworks:
+    dynamo: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1"
+
+dynamo:
+  hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1"
+  install: true
+
+slurm:
+  time_limit: "04:00:00"
+
+health_check:
+  max_attempts: 720
+  interval_seconds: 10
+
+resources:
+  gpu_type: "gb300"
+  gpus_per_node: 4
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 8
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  args:
+    router-mode: "random"
+    router-session-affinity-ttl-secs: 900
+  env:
+    DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600"
+    DYN_TOKENIZER_CACHE_BYTES: "8589934592"
+
+backend:
+  type: vllm
+  connector: null
+  mooncake_kv_store:
+    master_extra_args:
+      - "--eviction_high_watermark_ratio=0.95"
+      - "--eviction_ratio=0.10"
+    store_config:
+      metadata_server: "P2PHANDSHAKE"
+      # The GB300 nodes are limited to a 150 GB per-rank segment.
+      global_segment_size: "150GB"
+      local_buffer_size: "4GB"
+      protocol: "rdma"
+      device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+      mode: "embedded"
+      enable_offload: false
+  aggregated_environment:
+    VLLM_ALLREDUCE_USE_FLASHINFER: "1"
+    VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1"
+    VLLM_USE_V2_MODEL_RUNNER: "1"
+    # Use the direct DCP a2a/gather kernels rather than the collective
+    # fallback.
+    VLLM_USE_DIRECT_DCP_A2A: "1"
+    VLLM_USE_DIRECT_DCP_Q_GATHER: "1"
+    VLLM_USE_DIRECT_DCP_KV_GATHER: "1"
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0"
+    VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: "0"
+    PYTHONNOUSERSITE: "1"
+    TORCH_CUDA_ARCH_LIST: "10.0"
+    PYTHONHASHSEED: "42"
+    VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "900"
+    MC_GID_INDEX: "3"
+    MC_STORE_MEMCPY: "1"
+    MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
+    MC_SLICE_SIZE: "1048576"
+    MC_WORKERS_PER_CTX: "4"
+    WITH_NVIDIA_PEERMEM: "0"
+    VLLM_MOONCAKE_LOAD_RECV_THREADS: "4"
+  vllm_config:
+    aggregated:
+      served-model-name: "moonshotai/Kimi-K3"
+      tensor-parallel-size: 8
+      decode-context-parallel-size: 8
+      dcp-comm-backend: "a2a"
+      # max-num-seqs is sized at 2x concurrency.
+      max-num-seqs: 4
+      gpu-memory-utilization: 0.92
+      max-num-batched-tokens: 16384
+      trust-remote-code: true
+      language-model-only: true
+      load-format: "fastsafetensors"
+      moe-backend: "auto"
+      no-enable-flashinfer-autotune: true
+      enable-cumem-allocator: true
+      enable-prefix-caching: true
+      prefix-match-unit: 128
+      kv-cache-dtype: "fp8"
+      stream-interval: 10
+      attention-backend: "TOKENSPEED_MLA"
+      attention-config: '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}'
+      # Throughput jobs inject the committed K=7 golden AL (3.84); EVAL_ONLY
+      # preserves this real target-verification configuration.
+      speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":7,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'
+      # Capture ladder: multiples of (1 + K=7) up to
+      # min(max-num-seqs, 128) entries, then the fixed powers of two above it.
+      compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","cudagraph_capture_sizes":[8,16,24,32,64,128,256,512,1024,2048,4096,8192]}'
+      kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}'
+
+sbatch_directives:
+  mem: "0"
+  cpus-per-task: "72"
+  comment: >-
+    '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}'
+
+srun_options:
+  mem: "0"
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
+    AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
+    AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
+    AIPERF_HTTP_TCP_USER_TIMEOUT: "900000"
+    RESULT_DIR: "/logs/agentic"
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
+    HF_HUB_CACHE: "/hf_hub_cache"
+    WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"