diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 97169ae65b..747b76e28e 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -8,8 +8,9 @@ on: - '.github/workflows/e2e-tests.yml' - '.github/workflows/test-process-result.yml' - 'benchmarks/benchmark_lib.sh' - - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml' - - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-*/**/*.yaml' + - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-*/**/*.yaml' - 'runners/launch_gb200-nv.sh' - 'runners/launch_gb300-nv.sh' - 'utils/aggregate_power.py' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml index 84349c2775..6f8a7c6646 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml @@ -140,3 +140,16 @@ benchmark: concurrencies: "64" req_rate: "inf" use_chat_template: false + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml index 478c91b046..f248456c34 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml @@ -154,3 +154,16 @@ benchmark: concurrencies: "1536" req_rate: "inf" use_chat_template: false + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml index 11434ca534..2614ebe28c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml @@ -154,3 +154,16 @@ benchmark: concurrencies: "4096" req_rate: "inf" use_chat_template: false + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml index 9962318e8a..fcdcecd48a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml @@ -154,3 +154,16 @@ benchmark: concurrencies: "8192" req_rate: "inf" use_chat_template: false + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml index e88d4b7d53..1e1aefb3f6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml @@ -154,3 +154,16 @@ benchmark: concurrencies: "8192" req_rate: "inf" use_chat_template: false + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml index 0b2423a8ec..8c09c416e6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml @@ -122,3 +122,16 @@ benchmark: use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml index 79c9a46bda..20723ca5ee 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml @@ -129,3 +129,16 @@ benchmark: use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml index 1bf4f0e859..e243a595d7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml @@ -141,3 +141,16 @@ benchmark: use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml index 82519e378c..9f58ec8558 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml @@ -141,3 +141,16 @@ benchmark: use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml index e69c5e6043..664311f9d9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml @@ -141,3 +141,16 @@ benchmark: use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml index 73bcecaec5..9d9bf62146 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml @@ -141,3 +141,16 @@ benchmark: use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml index 66829c4044..0b3df1f7f3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml @@ -141,3 +141,16 @@ benchmark: use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml index 34b71a9180..42efef6b44 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml @@ -141,3 +141,16 @@ benchmark: use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml index 632097ab4c..1d21b7c248 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml @@ -153,3 +153,16 @@ benchmark: req_rate: "inf" use_chat_template: false custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml index f2cfb33808..93d0fbe9ca 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml @@ -153,3 +153,16 @@ benchmark: req_rate: "inf" use_chat_template: false custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml index f9cc35cd3e..f0e217acaa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml @@ -153,3 +153,16 @@ benchmark: req_rate: "inf" use_chat_template: false custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml index 6ce313dc9b..b548d99a72 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml @@ -153,3 +153,16 @@ benchmark: req_rate: "inf" use_chat_template: false custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml index 9783174a87..6ec739ec22 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml @@ -154,3 +154,16 @@ benchmark: req_rate: "inf" use_chat_template: false custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml index 39e11b7199..de669e6930 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml @@ -139,3 +139,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml index a4cf477ed3..fc2538b89b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml @@ -139,3 +139,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml index be5c4cf389..cd10fab296 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml @@ -119,3 +119,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml index 5657ad84d6..84b07b553f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml @@ -128,3 +128,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml index f4ae3076ce..8fcc5e88fc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml @@ -139,3 +139,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml index 4f9f902769..c28a29d9e5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml @@ -139,3 +139,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml index 17018a57ea..57d16b47ce 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml @@ -139,3 +139,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml index 15578537a4..1986557c27 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml @@ -139,3 +139,16 @@ benchmark: req_rate: "inf" use_chat_template: true custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml index c110e15993..296ea92ffd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml @@ -152,4 +152,17 @@ benchmark: osl: 1024 req_rate: "inf" random_range_ratio: 0.8 - concurrencies: "512x1024x2048" \ No newline at end of file + concurrencies: "512x1024x2048" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml index 143f339d0e..18bce2d9db 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml @@ -120,4 +120,17 @@ benchmark: num_prompts_mult: 10 num_warmup_mult: 1 random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64" \ No newline at end of file + concurrencies: "1x2x4x8x16x32x64" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml index 7b4ae6a03e..11b68b2ee1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml @@ -152,4 +152,17 @@ benchmark: osl: 1024 req_rate: "inf" random_range_ratio: 0.8 - concurrencies: "4096" \ No newline at end of file + concurrencies: "4096" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml index 0eb6c58817..3a38a20df2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml @@ -158,3 +158,16 @@ benchmark: num_warmup_mult: 2 random_range_ratio: 0.8 concurrencies: "1024" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml index df82d47501..c5e3064b10 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml @@ -141,3 +141,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "1x2x8" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml index 544b98a962..2c0b987497 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml @@ -155,3 +155,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "32x48x80" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml index 486efbdfe9..1463d31f6f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml @@ -171,3 +171,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "480" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml index f4a1432310..6d6ebc1f51 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml @@ -171,3 +171,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "768" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml index bde189a965..76fa20380f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml @@ -171,3 +171,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "1280" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml index 0fc8db0c2b..b41436b7aa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml @@ -171,3 +171,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "1344" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml index 2d8bacfc92..d52f707f47 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml @@ -171,3 +171,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "1920x2304" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml index fa43632fbc..7885feda54 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml @@ -175,3 +175,16 @@ benchmark: concurrencies: "2048" req_rate: "inf" random_range_ratio: 0.8 + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml index 691d0d72cf..d52f03b6d0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml @@ -173,3 +173,16 @@ benchmark: concurrencies: "5120" req_rate: "inf" random_range_ratio: 0.8 + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml index 44aa3934d8..a57b199849 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml @@ -173,3 +173,16 @@ benchmark: concurrencies: "5120" req_rate: "inf" random_range_ratio: 0.8 + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml index 39e5315059..4f63950743 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-dep4-dep16.yaml @@ -151,4 +151,17 @@ benchmark: osl: 1024 req_rate: "inf" random_range_ratio: 0.8 - concurrencies: "512x1024x2048" \ No newline at end of file + concurrencies: "512x1024x2048" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml index 90eaa12aae..7d2845969b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/1p1d-tp4-tp4.yaml @@ -120,4 +120,17 @@ benchmark: num_prompts_mult: 10 num_warmup_mult: 1 random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64" \ No newline at end of file + concurrencies: "1x2x4x8x16x32x64" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml index 334636ebb9..bff4f73b0b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/1k1k/2p1d-dep4-dep16.yaml @@ -151,4 +151,17 @@ benchmark: osl: 1024 req_rate: "inf" random_range_ratio: 0.8 - concurrencies: "4096" \ No newline at end of file + concurrencies: "4096" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml index 7ad092cfd4..8346f0effa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/4p1d-dep4-dep16.yaml @@ -157,3 +157,16 @@ benchmark: num_warmup_mult: 2 random_range_ratio: 0.8 concurrencies: "1024" + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml index f7fe5eac58..281780271a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml @@ -141,3 +141,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "1x2x8" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml index 39f46b93bf..5cb0d79679 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml @@ -155,3 +155,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "32x48x80" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml index fc156cc7c2..a4ff3dbe2f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_0.yaml @@ -171,3 +171,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "480" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml index e4a689c8aa..48f64a8fc2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_1.yaml @@ -171,3 +171,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "768" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml index 77b083db57..714f796468 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_2.yaml @@ -171,3 +171,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "1280" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml index a58d6d6c0a..a9b014a9b0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_3.yaml @@ -171,3 +171,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "1344" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml index 80d34ee1b9..05691230c0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_maxtpt_4.yaml @@ -171,3 +171,16 @@ benchmark: random_range_ratio: 0.8 concurrencies: "1920x2304" use_chat_template: true + +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: false + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 10 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 93b6a7c1b3..87af36fd8d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6251,3 +6251,18 @@ description: - "Refresh with lower stream interval to collect correct client metrics" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2686 + +- config-keys: + - dsv4-fp4-gb200-dynamo-sglang + - dsv4-fp4-gb200-dynamo-sglang-mtp + - dsv4-fp4-gb300-dynamo-sglang + - dsv4-fp4-gb300-dynamo-sglang-mtp + - qwen3.5-fp4-gb300-dynamo-sglang + - qwen3.5-fp8-gb200-dynamo-sglang + - qwen3.5-fp8-gb200-dynamo-sglang-mtp + - qwen3.5-fp8-gb300-dynamo-sglang + - qwen3.5-fp8-gb300-dynamo-sglang-mtp + description: + - "Enable optional dcgm-power telemetry on 51 eligible GB200 and GB300 recipes." + - "Keep 8 hardware-proven recipes required, and leave the 2 dedicated-infra recipes disabled." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2687 diff --git a/utils/test_gb200_power_official_contract.py b/utils/test_gb200_power_official_contract.py index 6913c5f815..706ddbec9f 100644 --- a/utils/test_gb200_power_official_contract.py +++ b/utils/test_gb200_power_official_contract.py @@ -18,6 +18,12 @@ GB200_LAUNCHER = REPO_ROOT / "runners/launch_gb200-nv.sh" GB300_LAUNCHER = REPO_ROOT / "runners/launch_gb300-nv.sh" TMPL_PATH = REPO_ROOT / ".github/workflows/benchmark-multinode-tmpl.yml" +PROCESS_RESULT_WORKFLOW = REPO_ROOT / ".github/workflows/test-process-result.yml" +PROCESS_RESULT_RECIPE_GLOBS = ( + "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml", + "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-*/**/*.yaml", + "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-*/**/*.yaml", +) STAMP_NAME = "power-producer-sha.txt" EXPORTER_IMAGE = "nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" @@ -149,9 +155,10 @@ def test_pin_literal_lives_only_in_the_two_launchers(): ] -# Every recipe that opts into dcgm-power, with the exporter port its cluster -# requires (im-gb300 nodes already bind 9401; gb200 keeps the default). -POWER_RECIPES = { +# Recipes with hardware proof may fail closed immediately. Newly eligible +# recipes start optional until their own cluster smoke establishes the same +# artifact contract. +REQUIRED_POWER_RECIPES = { "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml": 9401, "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml": 9401, "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml": 9401, @@ -163,22 +170,109 @@ def test_pin_literal_lives_only_in_the_two_launchers(): } +def _config_file_values(value): + if isinstance(value, dict): + for child in value.values(): + yield from _config_file_values(child) + elif isinstance(value, list): + for child in value: + yield from _config_file_values(child) + elif isinstance(value, str) and value.startswith("CONFIG_FILE="): + yield value.removeprefix("CONFIG_FILE=").split(":", 1)[0] + + +def _master_gb_recipe_runners(): + master = yaml.safe_load((REPO_ROOT / "configs/nvidia-master.yaml").read_text()) + runners = {} + for config in master.values(): + if not isinstance(config, dict): + continue + if config.get("runner") not in ("gb200", "gb300"): + continue + if config.get("framework") != "dynamo-sglang" or not config.get("disagg"): + continue + for config_file in _config_file_values(config.get("scenarios", {})): + runners[config_file.removeprefix("recipes/")] = config["runner"] + return runners + + +def _gb_power_candidates(): + recipe_root = REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes/sglang" + master_runners = _master_gb_recipe_runners() + candidates = {} + for family in ("deepseek-v4", "qwen3.5"): + for path in sorted((recipe_root / family).rglob("*.yaml")): + relative = path.relative_to(REPO_ROOT) + relative_recipe = relative.relative_to( + "benchmarks/multi_node/srt-slurm-recipes" + ).as_posix() + path_text = relative.as_posix() + recipe = yaml.safe_load(path.read_text()) + benchmark = recipe.get("benchmark", {}) + resources = recipe.get("resources", {}) + + runner = master_runners.get(relative_recipe) + if runner is None and "gb200" in path_text: + runner = "gb200" + elif runner is None and "gb300" in path_text: + runner = "gb300" + + if ( + runner not in ("gb200", "gb300") + or "/agentic/" in path_text + or benchmark.get("type") != "sa-bench" + or resources.get("prefill_nodes", 0) <= 0 + or resources.get("decode_nodes", 0) <= 0 + ): + continue + candidates[relative.as_posix()] = (runner, recipe) + return candidates + + +GB_POWER_CANDIDATES = _gb_power_candidates() +ELIGIBLE_POWER_RECIPES = { + path: 19401 if runner == "gb300" else 9401 + for path, (runner, recipe) in GB_POWER_CANDIDATES.items() + if recipe.get("benchmark", {}).get("client_placement", "head") == "head" + and not recipe.get("infra", {}).get("etcd_nats_dedicated_node", False) +} +INELIGIBLE_POWER_RECIPES = set(GB_POWER_CANDIDATES) - set(ELIGIBLE_POWER_RECIPES) + + def test_exactly_the_declared_recipes_opt_into_dcgm_power(): hits = git_grep_lines( "-lF", "provider: dcgm-power", "--", "benchmarks/multi_node/srt-slurm-recipes" ) - assert sorted(hits) == sorted(POWER_RECIPES) + relevant_hits = set(hits) & set(GB_POWER_CANDIDATES) + + assert len(GB_POWER_CANDIDATES) == 61 + assert len(ELIGIBLE_POWER_RECIPES) == 59 + assert len(REQUIRED_POWER_RECIPES) == 8 + assert len(ELIGIBLE_POWER_RECIPES) - len(REQUIRED_POWER_RECIPES) == 51 + assert relevant_hits == set(ELIGIBLE_POWER_RECIPES) + assert INELIGIBLE_POWER_RECIPES == { + "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml", + "benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/8p1d-dep4-dep16.yaml", + } def test_every_power_recipe_declares_the_same_telemetry_contract(): - for path, port in POWER_RECIPES.items(): + for path, port in ELIGIBLE_POWER_RECIPES.items(): telemetry = yaml.safe_load((REPO_ROOT / path).read_text())["telemetry"] - assert telemetry["enabled"] is True, path - assert telemetry["provider"] == "dcgm-power", path - assert telemetry["default_frequency"] == 1.0, path - assert telemetry["required"] is True, path - assert telemetry["dcgm_exporter"]["container_image"] == "dcgm-exporter", path - assert telemetry["dcgm_exporter"]["port"] == port, path + assert telemetry == { + "enabled": True, + "provider": "dcgm-power", + "default_frequency": 1.0, + "storage_subdir": "power", + "required": path in REQUIRED_POWER_RECIPES, + "startup_timeout_seconds": 120, + "request_timeout_seconds": 2, + "collector_join_timeout_seconds": 10, + "dcgm_exporter": { + "container_image": "dcgm-exporter", + "port": port, + }, + }, path def test_lane_and_pin_have_no_env_override_backdoor(): @@ -223,3 +317,21 @@ def test_stamp_filename_agrees_between_launchers_and_template(): # The manual input stays the override: the stamp only fills an empty env. assert f'if [ -z "$POWER_PRODUCER_SHA" ] && [ -f {STAMP_NAME} ]; then' in tmpl assert "POWER_PRODUCER_SHA: ${{ inputs.power-producer-sha }}" in tmpl + + +def test_process_result_workflow_watches_all_gb_power_recipe_families(): + workflow = PROCESS_RESULT_WORKFLOW.read_text() + covered = set() + + for recipe_glob in PROCESS_RESULT_RECIPE_GLOBS: + assert f"- '{recipe_glob}'" in workflow + result = subprocess.run( + ["git", "ls-files", f":(glob){recipe_glob}"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + covered.update(result.stdout.splitlines()) + + assert set(GB_POWER_CANDIDATES) <= covered