From b5057a9696cd580480b0e4152625a43cf6a324c2 Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Thu, 20 Aug 2026 05:58:05 +0000 Subject: [PATCH 1/2] [AMD][MI35X] Add HiCache and TP2/EP1 arms to the Qwen3.5 MXFP4 MI355X AgentX sweep Add a HiCache host-DRAM KV tier to the AgentX script, gated on KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=hicache. The ratio 1.5 / write_through / direct / page_first_direct combination is the one already validated on cluster:mi355x-amds by the dsv4 and glm5.2 AgentX recipes, and every knob stays overridable. The arm keeps --page-size 16 so only the cache tier differs between the kvnone and kvdram points. Replace the TP2/EP2 arm with TP2/EP1 to match qwen3.5-fp4-b200-sglang-agentic-mtp's 2-GPU point, and add kvdram arms at TP4 concurrency 40-64 and TP2 concurrency 20-32. Each kvdram arm repeats its kvnone neighbour's last concurrency so the host tier's gain is measured at a fixed operating point. --- .../agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh | 20 ++++++++++++++++++- configs/amd-master.yaml | 4 +++- perf-changelog.yaml | 9 +++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index c1d3d4da9..2cdcaa8cf 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -11,7 +11,8 @@ source "$(dirname "$0")/../../benchmark_lib.sh" export EVAL_FRAMEWORK="lm-eval" check_env_vars \ - MODEL TP CONC EP_SIZE RESULT_DIR DURATION + MODEL TP CONC EP_SIZE KV_OFFLOADING \ + TOTAL_CPU_DRAM_GB RESULT_DIR DURATION SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} @@ -53,6 +54,22 @@ trap cleanup_agentic_services EXIT trap 'exit 130' INT trap 'exit 143' TERM +CACHE_ARGS=() +if require_agentic_kv_offload_backend hicache; then + HICACHE_RATIO="${HICACHE_RATIO:-1.5}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + echo "HiCache CPU tier: ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT, dram_budget=${TOTAL_CPU_DRAM_GB} GB, tp=$TP" + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-ratio "$HICACHE_RATIO" + --hicache-write-policy "$HICACHE_WRITE_POLICY" + --hicache-io-backend "$HICACHE_IO_BACKEND" + --hicache-mem-layout "$HICACHE_MEM_LAYOUT" + ) +fi + PARALLEL_ARGS=( --tp "$TP" --dp 1 @@ -111,6 +128,7 @@ SGLANG_CMD=( --speculative-num-draft-tokens 4 --enable-metrics --enable-cache-report + "${CACHE_ARGS[@]}" ) printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 59f655945..f6c5df3a4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -372,8 +372,10 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [40, 48, 56, 64] } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32] } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 93b6a7c1b..66bacb2d1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6251,3 +6251,12 @@ description: - "Refresh with lower stream interval to collect correct client metrics" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2686 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Replace the TP2/EP2 arm with TP2/EP1 so the 2-GPU point matches qwen3.5-fp4-b200-sglang-agentic-mtp." + - "Add HiCache host-DRAM KV tier arms at TP4 concurrency 40, 48, 56, and 64 and TP2 concurrency 20, 24, 28, and 32, using hicache ratio 1.5 with write_through, direct io, and page_first_direct layout." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2677 From 9ac23c757ed5abce91789dd1d4108049cd485a73 Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Thu, 20 Aug 2026 05:59:04 +0000 Subject: [PATCH 2/2] [AMD][MI35X] Point the Qwen3.5 AgentX HiCache changelog entry to PR #2693 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 66bacb2d1..747928a7a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6259,4 +6259,4 @@ description: - "Replace the TP2/EP2 arm with TP2/EP1 so the 2-GPU point matches qwen3.5-fp4-b200-sglang-agentic-mtp." - "Add HiCache host-DRAM KV tier arms at TP4 concurrency 40, 48, 56, and 64 and TP2 concurrency 20, 24, 28, and 32, using hicache ratio 1.5 with write_through, direct io, and page_first_direct layout." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2677 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2693