[Power] Wire H200 multinode AgentX power / 接入 H200 多节点 AgentX 功耗链路

.github/workflows/test-process-result.yml

@@ -8,10 +8,13 @@ on:
       - '.github/workflows/e2e-tests.yml'
       - '.github/workflows/test-process-result.yml'
       - 'benchmarks/benchmark_lib.sh'
+      - 'benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-2p2d-pcp8-tp8-dp8-mtp.yaml'
       - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml'
       - 'benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/1p1d-tp4-tp4.yaml'
       - 'runners/launch_gb200-nv.sh'
       - 'runners/launch_gb300-nv.sh'
+      - 'runners/launch_h200-dgxc-slurm.sh'
+      - 'runners/inject_srt_power_concurrencies.py'
       - 'utils/aggregate_power.py'
       - 'utils/aggregate_power_multinode.py'
       - 'utils/agentic/aggregation/power_adapter.py'
@@ -26,6 +29,8 @@ on:
       - 'utils/test_aggregate_power_multinode.py'
       - 'utils/test_gb200_power_official_contract.py'
       - 'utils/test_gb300_power_official_contract.py'
+      - 'utils/test_h200_power_official_contract.py'
+      - 'utils/test_inject_srt_power_concurrencies.py'
       - 'utils/test_process_result.py'
 
 permissions:
@@ -54,4 +59,4 @@ jobs:
       - name: Run pytest
         run: |
           cd utils
-          python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/test_power_adapter.py agentic/aggregation/test_power_lifecycle.py agentic/aggregation/test_process_agentic_result.py test_gb200_power_official_contract.py test_gb300_power_official_contract.py test_process_result.py -v
+          python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/test_power_adapter.py agentic/aggregation/test_power_lifecycle.py agentic/aggregation/test_process_agentic_result.py test_gb200_power_official_contract.py test_gb300_power_official_contract.py test_h200_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v

benchmarks/benchmark_lib.sh

@@ -2190,11 +2190,16 @@ run_agentic_replay_and_write_outputs() (
     local validation_rc
     local power_rc=0
     local agentx_power_enabled=0
+    local agentx_multinode_power_enabled=0
     local agentx_monitor_stopped=1
 
     case "${ENABLE_AGENTX_POWER:-1}" in
         1|true|TRUE|yes|YES)
-            if [ "${IS_MULTINODE:-false}" != "true" ]; then
+            if [ "${IS_MULTINODE:-false}" = "true" ]; then
+                if [ -n "${SRT_MEASUREMENT_WINDOW_DIR:-}" ]; then
+                    agentx_multinode_power_enabled=1
+                fi
+            else
                 agentx_power_enabled=1
             fi
             ;;
@@ -2207,11 +2212,42 @@ run_agentic_replay_and_write_outputs() (
         fi
     }
 
-    if [ "$agentx_power_enabled" = "1" ]; then
+    _write_agentx_multinode_window() {
+        local state="$1"
+        local -a power_args
+        power_args=(
+            --result-dir "$result_dir"
+            --concurrency "${CONC:?CONC must be set for multinode AgentX power}"
+            --write-multinode-window "$state"
+        )
+        case "${REQUIRE_POWER:-0}" in
+            1|true|TRUE|yes|YES) power_args+=(--require-power) ;;
+        esac
+        (
+            cd "$INFMAX_CONTAINER_WORKSPACE"
+            "$AIPERF_PYTHON" -m utils.agentic.aggregation.power_adapter "${power_args[@]}"
+        )
+    }
+
+    if [ "$agentx_power_enabled" = "1" ] || [ "$agentx_multinode_power_enabled" = "1" ]; then
         # AIPerf currently exports naive local datetimes while SMI emits the
         # same host wall clock. Capture the launch-time offset so the adapter
         # can attach it explicitly before normalizing the profiling window.
         date +%z > "$result_dir/agentic_power_timezone_offset.txt"
+    fi
+
+    if [ "$agentx_multinode_power_enabled" = "1" ]; then
+        set +e
+        _write_agentx_multinode_window running
+        power_rc=$?
+        set -e
+        if [ "$power_rc" -ne 0 ]; then
+            echo "ERROR: failed to publish the AgentX formal running power window" >&2
+            return "$power_rc"
+        fi
+    fi
+
+    if [ "$agentx_power_enabled" = "1" ]; then
         start_gpu_monitor --output "$result_dir/gpu_metrics.csv"
         agentx_monitor_stopped=0
         # This function runs in a subshell, so these handlers cannot replace
@@ -2238,6 +2274,13 @@ run_agentic_replay_and_write_outputs() (
 
     write_agentic_result_json "$result_dir"
 
+    if [ "$agentx_multinode_power_enabled" = "1" ] && [ "$replay_rc" -eq 0 ]; then
+        set +e
+        _write_agentx_multinode_window completed
+        power_rc=$?
+        set -e
+    fi
+
     if [ "$agentx_power_enabled" = "1" ]; then
         local expected_num_gpus
         local -a power_args

runners/inject_srt_power_concurrencies.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Inject exact matrix concurrencies into a runtime srt-slurm recipe copy."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _validate_concurrencies(concurrencies: list[Any]) -> list[int]:
+    if (
+        not concurrencies
+        or any(isinstance(value, bool) or not isinstance(value, int) for value in concurrencies)
+        or any(value <= 0 for value in concurrencies)
+        or len(set(concurrencies)) != len(concurrencies)
+    ):
+        raise ValueError("concurrencies must be positive unique integers")
+    return concurrencies
+
+
+def inject_concurrencies(recipe_path: Path, concurrencies: list[Any]) -> None:
+    """Atomically set benchmark.concurrencies on a disposable recipe copy."""
+    values = _validate_concurrencies(concurrencies)
+    try:
+        recipe = yaml.safe_load(recipe_path.read_text(encoding="utf-8"))
+    except (OSError, yaml.YAMLError) as exc:
+        raise ValueError(f"failed to load recipe: {exc}") from exc
+    if not isinstance(recipe, dict) or not isinstance(recipe.get("benchmark"), dict):
+        raise ValueError("recipe must contain a benchmark mapping")
+
+    recipe["benchmark"]["concurrencies"] = values
+    fd, temporary_name = tempfile.mkstemp(
+        dir=recipe_path.parent,
+        prefix=f".{recipe_path.name}.",
+        text=True,
+    )
+    temporary_path = Path(temporary_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            yaml.safe_dump(recipe, handle, sort_keys=False)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary_path, recipe_path)
+    except BaseException:
+        temporary_path.unlink(missing_ok=True)
+        raise
+
+
+def _positive_integer(raw: str) -> int:
+    try:
+        value = int(raw)
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError("must be a positive integer") from exc
+    if value <= 0 or str(value) != raw:
+        raise argparse.ArgumentTypeError("must be a canonical positive integer")
+    return value
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("recipe", type=Path)
+    parser.add_argument("concurrencies", nargs="+", type=_positive_integer)
+    args = parser.parse_args()
+    try:
+        inject_concurrencies(args.recipe, args.concurrencies)
+    except ValueError as exc:
+        parser.error(str(exc))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

runners/launch_h200-dgxc-slurm.sh

@@ -7,6 +7,12 @@ SLURM_ACCOUNT="sa-shared"
 HF_HUB_CACHE_MOUNT="${HF_HUB_CACHE_MOUNT:-/models/gharunners/hf-hub-cache}"
 AIPERF_MMAP_CACHE_HOST_PATH="${AIPERF_MMAP_CACHE_HOST_PATH:-/home/sa-shared/gharunners/ai-perf-cache}"
 
+# Immutable producer prerequisite for the GLM-5.2 AgentX lane. This fork is
+# intentionally long-lived; update the SHA only after reviewing a new fork
+# commit and re-running the H200 hardware gate.
+POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git"
+POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270"
+
 set -x
 
 source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"
@@ -20,6 +26,33 @@ if [[ "$IS_MULTINODE" == "true" ]]; then
     CONFIG_PATH="${CONFIG_FILE%%:*}"
     LOCAL_CONFIG_FILE="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}"
 
+    # The producer pin decision is recipe-driven. Upstream-only recipes have
+    # no workspace mirror and remain non-power.
+    USES_DCGM_POWER=0
+    _RECIPE_REL="${CONFIG_FILE%%:*}"
+    _RECIPE_SRC="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${_RECIPE_REL#recipes/}"
+    if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk '
+        /^telemetry:/ { t = 1; next }
+        t && /^[^ ]/  { t = 0 }
+        t && /^  provider: dcgm-power$/ { p = 1 }
+        t && /^  enabled: true$/        { e = 1 }
+        END { exit !(p && e) }
+    ' "$_RECIPE_SRC"; then
+        USES_DCGM_POWER=1
+    fi
+
+    # PR-A supports exactly the GLM-5.2 FP8 AgentX topology. Future recipes
+    # must earn a separate cluster smoke instead of inheriting this lane.
+    if [[ "$USES_DCGM_POWER" == "1" && (
+        "$IS_AGENTIC" != "1" ||
+        "$FRAMEWORK" != "dynamo-sglang" ||
+        "$MODEL_PREFIX" != "glm5.2" ||
+        "$PRECISION" != "fp8"
+    ) ]]; then
+        echo "Error: H200 dcgm-power is validated only for AgentX dynamo-sglang glm5.2/fp8" >&2
+        exit 1
+    fi
+
     # MODEL_PATH: Override with pre-downloaded paths on H200 runner
     # The yaml files specify HuggingFace model IDs for portability, but we use
     # local paths to avoid repeated downloading on the shared H200 cluster.
@@ -75,10 +108,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then
     fi
 
     if [[ $IS_AGENTIC == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" ]]; then
-        # v1.0.44 includes the AgentX custom benchmark integration and passes
-        # every logical SGLang worker's Prometheus URL to AIPerf.
-        git clone --branch v1.0.44 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
+        # The pinned fork carries the v1.0.44 AgentX lifecycle plus the formal
+        # custom-benchmark dcgm-power contract needed by PR-A.
+        git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR"
         cd "$SRT_REPO_DIR"
+        git checkout "$POWER_SRT_SLURM_PIN" || exit 1
+        test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; }
+        if [[ "$USES_DCGM_POWER" == "1" ]]; then
+            git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt"
+        fi
     elif [[ $IS_AGENTIC == "1" && $FRAMEWORK == "vllm" && $MODEL_PREFIX == "kimik3" ]]; then
         git clone https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR"
         cd "$SRT_REPO_DIR"
@@ -162,6 +200,32 @@ if [[ "$IS_MULTINODE" == "true" ]]; then
             "
     fi
 
+    if [[ "$USES_DCGM_POWER" == "1" ]]; then
+        DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless"
+        DCGM_EXPORTER_SQSH="/data/gharunners/containers/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
+        if ! unsquashfs -l "$DCGM_EXPORTER_SQSH" >/dev/null 2>&1; then
+            DCGM_EXPORTER_LOCK="${DCGM_EXPORTER_SQSH}.lock"
+            mkdir -p "$(dirname "$DCGM_EXPORTER_SQSH")"
+            srun --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \
+                --nodes=1 --ntasks=1 --time=30 --job-name="$RUNNER_NAME" \
+                bash -c "
+                    set -euo pipefail
+                    exec 9>\"$DCGM_EXPORTER_LOCK\"
+                    flock -w 1800 9
+                    if unsquashfs -l \"$DCGM_EXPORTER_SQSH\" >/dev/null 2>&1; then
+                        exit 0
+                    fi
+                    rm -f \"$DCGM_EXPORTER_SQSH\"
+                    export ENROOT_CACHE_PATH=\${HOME}/.cache/enroot
+                    mkdir -p \"\$ENROOT_CACHE_PATH\"
+                    enroot import -o \"$DCGM_EXPORTER_SQSH\" docker://$DCGM_EXPORTER_IMAGE
+                "
+        fi
+        test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash is not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; }
+        unsquashfs -l "$DCGM_EXPORTER_SQSH" >/dev/null || { echo "Error: DCGM exporter squash is invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; }
+        sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256"
+    fi
+
     export ISL="$ISL"
     export OSL="$OSL"
     export EVAL_ONLY="${EVAL_ONLY:-false}"
@@ -213,6 +277,11 @@ use_exclusive_sbatch_directive: false
 ${DEFAULT_MOUNTS_BLOCK}
 EOF
 
+    if [[ "$USES_DCGM_POWER" == "1" ]]; then
+        sed -i "/^  nginx-sqsh:/a\\  dcgm-exporter: ${DCGM_EXPORTER_SQSH}" srtslurm.yaml
+        grep -q "^  dcgm-exporter: " srtslurm.yaml || { echo "Error: dcgm-exporter injection failed: nginx-sqsh anchor not found in srtslurm.yaml" >&2; exit 1; }
+    fi
+
     echo "Generated srtslurm.yaml:"
     cat srtslurm.yaml
 
@@ -224,6 +293,12 @@ EOF
         cp "$LOCAL_CONFIG_FILE" "$CONFIG_PATH"
     fi
 
+    if [[ "$USES_DCGM_POWER" == "1" ]]; then
+        read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST"
+        python "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \
+            "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}"
+    fi
+
     # Export eval-related env vars for srt-slurm post-benchmark eval
     export INFMAX_WORKSPACE="$GITHUB_WORKSPACE"
 
@@ -272,6 +347,30 @@ EOF
 
     echo "Found logs directory: $LOGS_DIR"
 
+    if [[ "$USES_DCGM_POWER" == "1" ]]; then
+        POWER_LOGS_ROOT=$(cd "$LOGS_DIR" && pwd -P)
+        read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST"
+        for concurrency in "${POWER_CONCURRENCIES[@]}"; do
+            power_args=(
+                --result-dir "$POWER_LOGS_ROOT/agentic/conc_${concurrency}"
+                --agg-result "$GITHUB_WORKSPACE/${RESULT_FILENAME}_conc${concurrency}.json"
+                --power-dir "$POWER_LOGS_ROOT/power"
+                --logs-root "$POWER_LOGS_ROOT"
+                --expected-producer-sha "$POWER_SRT_SLURM_PIN"
+            )
+            case "${REQUIRE_POWER:-0}" in
+                1|true|TRUE|yes|YES) power_args+=(--require-power) ;;
+            esac
+            (
+                cd "$GITHUB_WORKSPACE"
+                python -m utils.agentic.aggregation.power_adapter "${power_args[@]}"
+            ) || exit 1
+        done
+        mkdir -p "$LOGS_DIR/power"
+        cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256"
+        cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt"
+    fi
+
     cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS"
     bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"