[Power] Add optional B200 and B300 multinode telemetry / 添加可选 B200 和 B300 多节点功耗采集

.github/workflows/test-process-result.yml

@@ -8,10 +8,20 @@ on:
       - '.github/workflows/e2e-tests.yml'
       - '.github/workflows/test-process-result.yml'
       - 'benchmarks/benchmark_lib.sh'
+      - 'configs/nvidia-master.yaml'
+      - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml'
+      - 'benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/**/*.yaml'
+      - 'benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/**/*.yaml'
+      - 'benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml'
       - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml'
       - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml'
       - 'runners/launch_gb200-nv.sh'
       - 'runners/launch_gb300-nv.sh'
+      - 'runners/launch_b200-dgxc.sh'
+      - 'runners/launch_b200-nscale-slurm.sh'
+      - 'runners/launch_b300-nv.sh'
+      - 'runners/launch_h200-dgxc-slurm.sh'
+      - 'runners/inject_srt_power_concurrencies.py'
       - 'utils/aggregate_power.py'
       - 'utils/aggregate_power_multinode.py'
       - 'utils/agentic/aggregation/power_adapter.py'
@@ -24,8 +34,11 @@ on:
       - 'utils/process_result.py'
       - 'utils/test_aggregate_power.py'
       - 'utils/test_aggregate_power_multinode.py'
+      - 'utils/test_b200_b300_power_official_contract.py'
       - 'utils/test_gb200_power_official_contract.py'
       - 'utils/test_gb300_power_official_contract.py'
+      - 'utils/test_h200_power_official_contract.py'
+      - 'utils/test_inject_srt_power_concurrencies.py'
       - 'utils/test_process_result.py'
 
 permissions:
@@ -54,4 +67,4 @@ jobs:
       - name: Run pytest
         run: |
           cd utils
-          python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/test_power_adapter.py agentic/aggregation/test_power_lifecycle.py agentic/aggregation/test_process_agentic_result.py test_gb200_power_official_contract.py test_gb300_power_official_contract.py test_process_result.py -v
+          python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/test_power_adapter.py agentic/aggregation/test_power_lifecycle.py agentic/aggregation/test_process_agentic_result.py test_b200_b300_power_official_contract.py test_gb200_power_official_contract.py test_gb300_power_official_contract.py test_h200_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v

benchmarks/benchmark_lib.sh

@@ -2190,11 +2190,16 @@ run_agentic_replay_and_write_outputs() (
     local validation_rc
     local power_rc=0
     local agentx_power_enabled=0
+    local agentx_multinode_power_enabled=0
     local agentx_monitor_stopped=1
 
     case "${ENABLE_AGENTX_POWER:-1}" in
         1|true|TRUE|yes|YES)
-            if [ "${IS_MULTINODE:-false}" != "true" ]; then
+            if [ "${IS_MULTINODE:-false}" = "true" ]; then
+                if [ -n "${SRT_MEASUREMENT_WINDOW_DIR:-}" ]; then
+                    agentx_multinode_power_enabled=1
+                fi
+            else
                 agentx_power_enabled=1
             fi
             ;;
@@ -2207,11 +2212,42 @@ run_agentic_replay_and_write_outputs() (
         fi
     }
 
-    if [ "$agentx_power_enabled" = "1" ]; then
+    _write_agentx_multinode_window() {
+        local state="$1"
+        local -a power_args
+        power_args=(
+            --result-dir "$result_dir"
+            --concurrency "${CONC:?CONC must be set for multinode AgentX power}"
+            --write-multinode-window "$state"
+        )
+        case "${REQUIRE_POWER:-0}" in
+            1|true|TRUE|yes|YES) power_args+=(--require-power) ;;
+        esac
+        (
+            cd "$INFMAX_CONTAINER_WORKSPACE"
+            "$AIPERF_PYTHON" -m utils.agentic.aggregation.power_adapter "${power_args[@]}"
+        )
+    }
+
+    if [ "$agentx_power_enabled" = "1" ] || [ "$agentx_multinode_power_enabled" = "1" ]; then
         # AIPerf currently exports naive local datetimes while SMI emits the
         # same host wall clock. Capture the launch-time offset so the adapter
         # can attach it explicitly before normalizing the profiling window.
         date +%z > "$result_dir/agentic_power_timezone_offset.txt"
+    fi
+
+    if [ "$agentx_multinode_power_enabled" = "1" ]; then
+        set +e
+        _write_agentx_multinode_window running
+        power_rc=$?
+        set -e
+        if [ "$power_rc" -ne 0 ]; then
+            echo "ERROR: failed to publish the AgentX formal running power window" >&2
+            return "$power_rc"
+        fi
+    fi
+
+    if [ "$agentx_power_enabled" = "1" ]; then
         start_gpu_monitor --output "$result_dir/gpu_metrics.csv"
         agentx_monitor_stopped=0
         # This function runs in a subshell, so these handlers cannot replace
@@ -2238,6 +2274,13 @@ run_agentic_replay_and_write_outputs() (
 
     write_agentic_result_json "$result_dir"
 
+    if [ "$agentx_multinode_power_enabled" = "1" ] && [ "$replay_rc" -eq 0 ]; then
+        set +e
+        _write_agentx_multinode_window completed
+        power_rc=$?
+        set -e
+    fi
+
     if [ "$agentx_power_enabled" = "1" ]; then
         local expected_num_gpus
         local -a power_args

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml

@@ -118,3 +118,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml

@@ -125,3 +125,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml

@@ -126,3 +126,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p2d-dep8-dep8-mtp.yaml

@@ -137,3 +137,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-5p3d-dep8-dep8-mtp.yaml

@@ -126,3 +126,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml

@@ -132,3 +132,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml

@@ -143,3 +143,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: false
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4-mtp.yaml

@@ -123,3 +123,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-tp4-tp4.yaml

@@ -118,3 +118,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: false
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p2d-dep4-dep8-mtp.yaml

@@ -133,3 +133,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p6d-dep4-tp4-mtp.yaml

@@ -127,3 +127,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8-mtp.yaml

@@ -133,3 +133,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-2p1d-dep4-dep8.yaml

@@ -143,3 +143,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: false
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8-mtp.yaml

@@ -133,3 +133,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-4p1d-dep4-dep8.yaml

@@ -143,3 +143,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: false
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-6p1d-dep4-dep8-mtp.yaml

@@ -133,3 +133,16 @@ benchmark:
   req_rate: "inf"
   use_chat_template: true
   custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"
+
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: false
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 10
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401