From 9e987b673af2c98dd132099f5624e9ad6ad05bea Mon Sep 17 00:00:00 2001 From: Sirra Date: Wed, 19 Aug 2026 11:35:01 +0530 Subject: [PATCH 1/2] [AMD] [AGENTX] Minimax-M3 Perf Tuning Signed-off-by: Sirra --- .../agentic/minimaxm3_fp4_mi355x_mtp.sh | 140 +++++------------- configs/amd-master.yaml | 2 +- 2 files changed, 37 insertions(+), 105 deletions(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh index b735220fc7..5a2d5346a5 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh @@ -1,16 +1,21 @@ #!/usr/bin/env bash -set -eo pipefail +set -euo pipefail set -x # Agentic trace replay benchmark for MiniMax-M3 FP4 on MI355X using vLLM -# and EAGLE3 speculative decoding. +# EAGLE3 speculative decoding. # # Required env vars: -# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, +# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, # TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION source "$(dirname "$0")/../../benchmark_lib.sh" +# Force the eval framework to lm-eval for this recipe. run_eval derives its +# default as swebench for agentic scenarios (scenario_default=swebench when +# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence +# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so +# setting it here makes the effective framework always lm-eval, never swebench. export EVAL_FRAMEWORK="lm-eval" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION @@ -23,16 +28,16 @@ NUM_SPEC_TOKENS=3 # minimax-m3.thinking_on[3] SYNTHETIC_ACCEPT_LEN=2.78 -if [[ -v SLURM_JOB_ID ]]; then +if [[ -n "${SLURM_JOB_ID+x}" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi # ROCR/HIP visibility for vLLM 0.14+ -if [[ -v ROCR_VISIBLE_DEVICES ]]; then +if [[ -n "${ROCR_VISIBLE_DEVICES+x}" ]]; then export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" fi -if [[ -n "$MODEL_PATH" ]]; then +if [[ -n "${MODEL_PATH:-}" ]]; then if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then hf download "$MODEL" --local-dir "$MODEL_PATH" fi @@ -49,117 +54,51 @@ amd-smi || true resolve_trace_source install_agentic_deps -# Require the vLLM Prometheus stream in every official result. AIPerf -# deduplicates this endpoint against its automatic localhost discovery. -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" - # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" SERVER_PID="" -LMCACHE_PIDS=() cleanup_agentic_services() { local exit_code=$? trap - EXIT INT TERM set +e stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - local i - for i in "${!LMCACHE_PIDS[@]}"; do - stop_background_process_tree "${LMCACHE_PIDS[$i]}" "LMCache server $i" - done exit "$exit_code" } trap cleanup_agentic_services EXIT trap 'exit 130' INT trap 'exit 143' TERM +# AgentX replays growing multi-turn prefixes, so keep prefix caching enabled +# for both GPU-resident and native-offload configurations. OFFLOAD_ARGS=() -case "$KV_OFFLOAD_BACKEND" in +case "${KV_OFFLOAD_BACKEND:-}" in "") require_agentic_kv_offload_none ;; - lmcache) - require_agentic_kv_offload_backend lmcache - - # Keep the image's tested torch/ROCm stack and install only LMCache's - # missing pure-Python runtime dependencies. - LMCACHE_VERSION="0.5.3" - LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-rocm" - agentic_pip_install --quiet --no-cache-dir --no-deps \ - "sortedcontainers==2.4.0" \ - "opentelemetry-exporter-prometheus==0.61b0" \ - "cupy-rocm-7-0==14.1.1" \ - "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX" - python3 -c \ - "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ - >/dev/null - - # Split the node-level DRAM limit evenly across one MP server per TP rank. - LMCACHE_N_SERVERS="$TP" - LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" - SHM_FREE_GB=$(df -BG --output=avail /dev/shm 2>/dev/null | tail -1 | tr -dc '0-9') - if [ -n "$SHM_FREE_GB" ] && [ "$SHM_FREE_GB" -gt 0 ]; then - SHM_CAP_GB=$((SHM_FREE_GB * 90 / 100)) - if [ "$LMCACHE_L1_SIZE_GB" -gt "$SHM_CAP_GB" ]; then - echo "Error: LMCache L1 ${LMCACHE_L1_SIZE_GB} GB exceeds 90% of free /dev/shm (${SHM_CAP_GB} GB)." >&2 - exit 1 - fi - fi - LMCACHE_L1_SHARD_GB=$((LMCACHE_L1_SIZE_GB / LMCACHE_N_SERVERS)) - if [ "$LMCACHE_L1_SHARD_GB" -lt 1 ]; then - echo "Error: LMCache DRAM budget is less than 1 GB per TP rank." >&2 - exit 1 - fi - - LMCACHE_SERVER_URLS=() - LMCACHE_HTTP_PORTS=() - LMCACHE_LOGS=() - : > "$RESULT_DIR/lmcache_command.txt" - for shard in $(seq 0 $((LMCACHE_N_SERVERS - 1))); do - shard_port=$((5555 + shard)) - shard_http_port=$((8080 + shard)) - shard_log="${LMCACHE_LOG%.log}_${shard}.log" - LMCACHE_CMD=( - lmcache server - --host 127.0.0.1 - --port "$shard_port" - --http-host 127.0.0.1 - --http-port "$shard_http_port" - --l1-size-gb "$LMCACHE_L1_SHARD_GB" - --l1-init-size-gb 10 - --l1-read-ttl-seconds 7200 - --chunk-size 256 - --max-workers 2 - --eviction-policy LRU - --supported-transfer-mode lmcache_driven - ) - append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" - "${LMCACHE_CMD[@]}" > "$shard_log" 2>&1 & - LMCACHE_PIDS+=($!) - LMCACHE_HTTP_PORTS+=("$shard_http_port") - LMCACHE_LOGS+=("$shard_log") - LMCACHE_SERVER_URLS+=("tcp://127.0.0.1:${shard_port}") - done - for shard in "${!LMCACHE_PIDS[@]}"; do - wait_for_ready \ - --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORTS[$shard]}/healthcheck" \ - --log "${LMCACHE_LOGS[$shard]}" \ - --pid "${LMCACHE_PIDS[$shard]}" \ - --sleep-interval 1 \ - --timeout 600 - done - LMCACHE_SERVER_URLS_CSV=$(IFS=,; echo "${LMCACHE_SERVER_URLS[*]}") + vllm-native) + require_agentic_kv_offload_backend vllm-native + unset VLLM_USE_SIMPLE_KV_OFFLOAD + # Use vLLM's regular native KV-offload path (OffloadingConnector), + # NOT the SimpleCPUOffloadConnector. The "vllm-native" backend resolves to + # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 + # would switch it to SimpleCPUOffloadConnector. We intentionally leave + # that env var UNSET here so the regular OffloadingConnector path is + # used. The shortcut --kv_offloading_backend native + --kv_offloading_size + # form constructs the KVTransferConfig at engine startup + # (vllm/config/vllm.py:662). + + # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) + # This gives extra cache hit than disabling hybrid kv cache manager OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.server_urls\":\"$LMCACHE_SERVER_URLS_CSV\",\"lmcache.mp.mq_timeout\":6000.0}}" + --kv_offloading_backend native + --kv_offloading_size "$TOTAL_CPU_DRAM_GB" ) ;; *) - echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND (expected empty or lmcache)" >&2 + echo "Unsupported KV_OFFLOAD_BACKEND: ${KV_OFFLOAD_BACKEND:-}" >&2 exit 1 ;; esac @@ -187,14 +126,7 @@ export VLLM_USE_BREAKABLE_CUDAGRAPH=0 export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_USE_AITER_MOE=1 export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -# The AITER page-16 sparse-attention path requires exactly one KV head per -# tensor-parallel rank. MiniMax-M3 has four KV heads, so TP4 uses that fast -# path while TP2 uses vLLM's supported Triton sparse-attention fallback. -if [ "$TP" -eq 4 ]; then - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 -else - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0 -fi +export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 @@ -218,13 +150,14 @@ VLLM_CMD=( --tool-call-parser minimax_m3 --enable-auto-tool-choice --default-chat-template-kwargs '{"thinking_mode":"enabled"}' - --max-num-seqs "$((2 * CONC))" + --max-num-seqs "$CONC" --stream-interval 20 --hf-overrides '{"text_config": {"use_index_cache": true, "index_topk_freq": 4}}' --speculative-config "$SPEC_CONFIG" "${OFFLOAD_ARGS[@]}" ) -write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! echo "Server PID: $SERVER_PID" @@ -236,6 +169,5 @@ if [ "${EVAL_ONLY}" = "true" ]; then run_eval --port "$PORT" else build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --apply-chat-template" run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi +fi \ No newline at end of file diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index cf5cdfeb6d..c62047144f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1628,7 +1628,7 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: multinode: false scenarios: agentic-coding: - - dram-utilization: 0.20 + - dram-utilization: 0.50 search-space: - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 5, 8, 10, 12, 15, 20, 24, 28, 32], spec-decoding: mtp } - { tp: 2, kv-offloading: none, conc-list: [1, 2, 5], spec-decoding: mtp } From 62bf882f2df0d732752bc9d83caa3ee2324bda79 Mon Sep 17 00:00:00 2001 From: Sirra Date: Wed, 19 Aug 2026 11:42:19 +0530 Subject: [PATCH 2/2] [AMD] [AGENTX] Minimax-M3 Perf Tuning Signed-off-by: Sirra --- perf-changelog.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7500c62923..4218d37290 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6234,3 +6234,11 @@ description: - "Add GB200 Kimi K3 AgentX config." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2639 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add the MI355X Minimax-M3 FP4 Agentic MTP" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2671