diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh new file mode 100755 index 000000000..0fbe6619a --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on MI355X using +# ATOM MTP. Throughput runs use the committed golden synthetic acceptance; +# eval-only runs use the model's real MTP acceptance. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +if [ "$TP" -ne 8 ] || [ "$EP_SIZE" -ne 1 ] || [ "$DP_ATTENTION" != "false" ]; then + echo "This recipe requires TP=8, EP_SIZE=1, and DP_ATTENTION=false" >&2 + exit 1 +fi +require_agentic_kv_offload_none + +if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +rocm-smi || true +amd-smi || true + +resolve_trace_source +install_agentic_deps + +# ATOM runtime settings validated with the DeepSeek-V4-Pro AgentX baseline. +export AITER_BF16_FP8_MOE_BOUND=0 +export AITER_LOG_LEVEL=WARNING +export ATOM_MOE_GU_ITLV=1 +export ATOM_DISABLE_MMAP=true +export ATOM_DEBUG_PREFIX_HITS=1 +export ATOM_PROFILER_MORE=0 +export ATOM_PROFILER_TIMEOUT=1200 + +# AgentX/AIPerf network, failure, warmup, and trace-gap settings from the +# validated one-hour baseline. +export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 +export AIPERF_FAILED_REQUEST_THRESHOLD=0.10 +export AIPERF_LIVE_FAILED_REQUEST_THRESHOLD=0.10 +export AIPERF_TRACE_IDLE_GAP_CAP_SECONDS=300 +export AIPERF_WARMUP_REQUESTS_PER_LANE=10 +export AIPERF_BENCHMARK_GRACE_PERIOD=30 + +# Require ATOM Prometheus metrics in every official result. +export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="atom:" + +wait_for_amd_gpu_clean + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +cleanup_atom_server() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "ATOM server" 60 + exit "$exit_code" +} +trap cleanup_atom_server EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# AgentX concurrency counts session trees. Keep 2x scheduler headroom for the +# request bursts produced by subagent fan-out. +MAX_NUM_SEQS=$((2 * CONC)) + +# golden_al_distribution/dsv4_mtp.yaml: thinking_on, 3 draft tokens -> AL 2.49 +# --spec-decode-acceptance-length 2.49. +# https://github.com/ROCm/ATOM/pull/1948 +NUM_SPEC_TOKENS=3 +SPEC_DECODE_AL=2.49 +SPEC_ARGS=( + --method mtp + --num-speculative-tokens "$NUM_SPEC_TOKENS" +) +if [ "${EVAL_ONLY:-false}" != "true" ]; then + SPEC_ARGS+=(--spec-decode-acceptance-length "$SPEC_DECODE_AL") +fi + +echo "Starting ATOM server with MAX_NUM_SEQS=$MAX_NUM_SEQS NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS SPEC_DECODE_AL=$SPEC_DECODE_AL EVAL_ONLY=${EVAL_ONLY:-false}" +ATOM_CMD=( + python3 -u -m atom.entrypoints.openai_server + --model "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --server-port "$PORT" + --tensor-parallel-size "$TP" + --kv-cache-dtype fp8 + --index-cache-dtype fp4 + --enable-prefix-caching + --gpu-memory-utilization 0.9 + --max-num-batched-tokens 16384 + --attn-prefill-chunk-size 16384 + --state-checkpoint-interval-tokens 8192 + --level 3 + --cudagraph-mode FULL + "${SPEC_ARGS[@]}" + --max-num-seqs "$MAX_NUM_SEQS" +) +write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" +"${ATOM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "${EVAL_ONLY:-false}" = "true" ]; then + run_eval --port "$PORT" +else + # AgentX DSv4 traces already carry fully formed chat payloads; do not apply + # AIPerf's generic chat template on top of them. + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 59f655945..10604c32b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1284,6 +1284,22 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: # while MTP creates two. Restore these points after the upstream hybrid # KV recovery fix lands: https://github.com/vllm-project/vllm/pull/45497 +# DeepSeek-V4-Pro FP4 AgentX on one MI355X node using ATOM MTP. Throughput +# uses the thinking_on golden AL 2.49 for three draft tokens; eval uses real +# MTP acceptance. max-num-seqs is set to 2x concurrency by the recipe. +dsv4-fp4-mi355x-atom-agentic-mtp: + image: rocm/atom-dev:nightly_202608201032 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom + multinode: false + scenarios: + agentic-coding: + - search-space: + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 48] } + dsr1-fp4-mi355x-sglang-disagg-mtp: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 model: amd/DeepSeek-R1-0528-MXFP4-v2 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 93b6a7c1b..b2a5962b5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6196,6 +6196,18 @@ - "Set max-num-seqs to twice the concurrency, select CUDA graph capture sizes by concurrency, and cap max-num-batched-tokens at 16384." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2576 +- config-keys: + - dsv4-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4-Pro FP4 ATOM MTP AgentX on one 8x MI355X node at concurrency 1, 2, 4, 8, 16, 32, and 48, with no KV offload and max-num-seqs set to twice concurrency." + - "Use rocm/atom-dev:nightly_202608201032, FP8 KV/index caches, prefix caching, 32K state checkpoints, 16K batching/prefill chunks, and FULL cudagraph mode." + - "Use three-token MTP with the committed DeepSeek-V4 thinking-mode golden AL 2.49 (synthetic acceptance length 3) for throughput, while eval-only runs measure real MTP acceptance." + - "Increase the AgentX warmup grace period from the 1800-second default to 3600 seconds at concurrency 32 and 48, matching the DeepSeek-V4-Pro SGLang saturation recipe so in-flight warmup requests can drain and reused long prefixes are fully primed before profiling." + - "Keep concurrency 1 through 16 at the default 1800 seconds; the profiling duration remains 3600 seconds for every concurrency arm." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2698 + - config-keys: - kimik3-fp4-b200-dynamo-vllm-agentic-dspark scenario-type: