diff --git a/.agents/scripts/gdpval-sif.sh b/.agents/scripts/gdpval-sif.sh new file mode 100755 index 00000000000..00034ff5888 --- /dev/null +++ b/.agents/scripts/gdpval-sif.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# gdpval-sif.sh — ensure the GDPVal Stirrup Apptainer SIF exists on THIS cluster. +# +# Build-if-absent, reuse-if-present. Self-contained: the SIF is built and reused +# on the TARGET cluster's own filesystem — this NEVER copies a SIF from another +# cluster. Idempotent, so it's safe to run before every `nel run`; a subsequent +# run reuses the built SIF instantly. +# +# Usage: +# .agents/scripts/gdpval-sif.sh [] [--commit ] [--force|--check] +# Persistent path on the target cluster's shared FS. +# DEFAULTS to $GDPVAL_SIF_DIR (from .env) when omitted. A +# directory -> /$GDPVAL_SIF_NAME (default python-3.13.gdpval.sif, +# matching the example config); a *.sif path +# is used verbatim. Bind-mount this SAME dir into the eval +# container at /gdpval/sif (see recipes/examples/gym_gdpval/). +# --commit NeMo Gym commit whose gdpval.def to build. Keep in sync +# with the config's install_on_the_fly.commit. +# --force Rebuild even if the SIF already exists. +# --check Verify-only preflight: exit 0 if the expected SIF exists, +# nonzero (listing what IS there) if not. Never builds. +# Use before `nel run` — NEL's mount validation is `test -d` +# and cannot see a missing/misnamed SIF file. +# +# Requires `apptainer` (or `singularity`) on PATH with unprivileged/fakeroot +# build support, plus network egress to GitHub/base image. Run on a node that has +# it — a login node, or (preferred for the ~30-min build) the CPU partition: +# srun -p cpu -t 01:00:00 --pty \ +# .agents/scripts/gdpval-sif.sh /lustre/<...>/gdpval/sif +# +# Env overrides: GDPVAL_GYM_COMMIT, GDPVAL_SIF_NAME, APPTAINER_BIN. +set -euo pipefail + +# Keep GDPVAL_GYM_COMMIT in sync with install_on_the_fly.commit in the config. +GDPVAL_GYM_COMMIT="${GDPVAL_GYM_COMMIT:-dd41196f620f2af99947d776cbe5da9439d2a08d}" # pragma: allowlist secret +GDPVAL_SIF_NAME="${GDPVAL_SIF_NAME:-python-3.13.gdpval.sif}" +APPTAINER_BIN="${APPTAINER_BIN:-}" + +_log() { printf '\033[2m %s\033[0m\n' "$*" >&2; } +_die() { printf '\033[31mgdpval-sif: %s\033[0m\n' "$*" >&2; exit 1; } +_usage() { sed -n '/^# gdpval-sif\.sh/,/^set -euo/p' "$0" | sed 's/^# \{0,1\}//; /^set -euo/d'; } + +# --- parse args --- +target=""; force=0; check=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --commit) GDPVAL_GYM_COMMIT="${2:?--commit needs a value}"; shift 2 ;; + --force) force=1; shift ;; + --check) check=1; shift ;; + -h|--help) _usage; exit 0 ;; + -*) _die "unknown flag: $1 (see --help)" ;; + *) [[ -z "$target" ]] || _die "unexpected extra arg: $1"; target="$1"; shift ;; + esac +done +# Default to $GDPVAL_SIF_DIR (.env) when no path is given, so agents run it hands-free. +target="${target:-${GDPVAL_SIF_DIR:-}}" +[[ -n "$target" ]] || { _usage; _die "no path given and GDPVAL_SIF_DIR is unset — pass a dir or set GDPVAL_SIF_DIR (see recipes/env.example)"; } + +# --- resolve dir vs *.sif --- +if [[ "$target" == *.sif ]]; then + sif="$target"; sif_dir="$(dirname "$target")" +else + sif_dir="$target"; sif="$sif_dir/$GDPVAL_SIF_NAME" +fi +# --- verify-only mode (preflight) --- +# NEL's submit-time mount validation runs `test -d`, so it only proves the SIF *dir* +# exists — a dir holding the WRONG sif name (e.g. python-3.12 after a gym bump to a +# 3.13 def) passes validation, and the Stirrup agent then SILENTLY falls back to +# non-sandboxed exec. Run this before submitting to fail loudly instead. +if [[ "$check" -eq 1 ]]; then + if [[ -f "$sif" ]]; then + _log "SIF present: $sif ($(du -h "$sif" 2>/dev/null | cut -f1))" + echo "$sif"; exit 0 + fi + printf '\033[31mgdpval-sif: MISSING expected SIF: %s\033[0m\n' "$sif" >&2 + [[ -d "$sif_dir" ]] && { echo " dir exists but does not contain it; found:" >&2 + ls -1 "$sif_dir"/*.sif 2>/dev/null | sed 's/^/ /' >&2 || echo " (no .sif files)" >&2; } + echo " Build it with: $0 ${sif_dir} (or --commit for a different def)" >&2 + exit 1 +fi + +mkdir -p "$sif_dir" || _die "cannot create SIF dir: $sif_dir" + +# --- reuse if present --- +if [[ -f "$sif" && "$force" -eq 0 ]]; then + _log "reusing existing SIF (no rebuild): $sif" + echo "$sif"; exit 0 +fi + +# --- locate apptainer/singularity --- +if [[ -z "$APPTAINER_BIN" ]]; then + APPTAINER_BIN="$(command -v apptainer || command -v singularity || true)" +fi +[[ -n "$APPTAINER_BIN" ]] || _die "apptainer/singularity not found on PATH. Run on a node that has it \ +(e.g. 'module load apptainer', or inside the eval image). This script does NOT copy a SIF from another cluster." + +def_url="https://raw.githubusercontent.com/NVIDIA-NeMo/Gym/${GDPVAL_GYM_COMMIT}/responses_api_agents/stirrup_agent/containers/gdpval.def" +tmp="${sif_dir}/.build.$$.${GDPVAL_SIF_NAME}" +def_local="${sif_dir}/.gdpval.$$.def" +lock="${sif_dir}/.gdpval-sif.lock" + +# --- build under a flock (double-checked) so concurrent runs don't double-build --- +exec 9>"$lock" || _die "cannot open lock file: $lock" +_log "acquiring build lock ($lock) ..." +flock 9 +# Re-check inside the lock: another builder may have finished while we waited. +if [[ -f "$sif" && "$force" -eq 0 ]]; then + _log "another builder produced it: $sif" + echo "$sif"; exit 0 +fi + +_log "building GDPVal SIF (this can take ~20-40 min)" +_log " gym commit: ${GDPVAL_GYM_COMMIT}" +_log " def: ${def_url}" +_log " dest: ${sif}" +rm -f "$tmp" "$def_local" +# apptainer build cannot take a remote def URL as its source — fetch the def to a +# local file first, then build from it. +if command -v curl >/dev/null 2>&1; then curl -fsSL "$def_url" -o "$def_local" +else wget -qO "$def_local" "$def_url"; fi +[ -s "$def_local" ] || { rm -f "$def_local"; _die "failed to download def from $def_url"; } +# Prefer --fakeroot (needs an /etc/subuid entry for the build user); fall back to an +# unprivileged build where fakeroot is unavailable. +if "$APPTAINER_BIN" build --fakeroot "$tmp" "$def_local"; then + : +elif "$APPTAINER_BIN" build "$tmp" "$def_local"; then + _log "built without --fakeroot (unprivileged mode)" +else + rm -f "$tmp" "$def_local" + _die "apptainer build failed (see output above)." +fi +rm -f "$def_local" + +# Atomic publish: a partial build never looks complete. +mv -f "$tmp" "$sif" || { rm -f "$tmp"; _die "failed to move built SIF into place: $sif"; } +_log "done: $sif" +echo "$sif" diff --git a/.agents/skills/evaluation/SKILL.md b/.agents/skills/evaluation/SKILL.md index 6776315949d..a35771b0839 100644 --- a/.agents/skills/evaluation/SKILL.md +++ b/.agents/skills/evaluation/SKILL.md @@ -49,6 +49,29 @@ Steps 1–9 below are the 0.2.6 path — use them for everything else. --- +### GDPVal (NeMo Gym "Stirrup" agent) path — branch here too + +GDPVal **does** run on the 0.2.6 `nel` launcher (as a `nemo_gym` task, not +nel-next), so Steps 1–9 apply — but it is mechanically special and **standalone** +(one gym eval per config; never mix it with `aa/` tasks). If the user asks for +GDPVal: + +1. Read **`references/gym-gdpval.md`** (Apptainer SIF sandbox, `_gym_prepare.yaml` + machinery, deploy sizing, rubric-vs-comparison scoring, MLflow deliverables trap, + failure modes) + **`recipes/tasks/aa_gym/gdpval.md`**. +2. Start from the self-contained **`recipes/examples/gym_gdpval/`** dir — copy the + **whole dir** (the `_gym_prepare.yaml` include must travel next to the config). +3. Prerequisite: set `GDPVAL_SIF_DIR` in `.env`, then ensure the SIF exists with + `.agents/scripts/gdpval-sif.sh` (uses `$GDPVAL_SIF_DIR`; build-if-absent, + reuse-if-present, no cross-cluster copy). The config bind-mounts `$GDPVAL_SIF_DIR` + at exactly `/gdpval/sif/python-3.12.gdpval.sif`, or the agent silently runs + unsandboxed. `.env` needs `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, + `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR`, and `NEMO_EVALUATOR_TRUST_PRE_CMD=1` (the + config has a `pre_cmd`). Thinking mode is mandatory (non-thinking loses ~86%). +4. Dry-run → canary (`limit_samples=2`, verify the SIF sandbox + judge) → full. + +--- + ### Step 1 — Prerequisites Run `nel --version`; if missing, instruct `pip install nemo-evaluator-launcher`. If user has an existing config, skip to Step 8 (optionally review for `???` and quantization flags first). @@ -62,8 +85,9 @@ Run `nel --version`; if missing, instruct `pip install nemo-evaluator-launcher`. - AA Index v2 suite (default for quantized-checkpoint validation, see `references/quantization-benchmarks.md`): `recipes/tasks/aa/{gpqa_diamond,hle,lcr,scicode,ifbench,mmmu_pro,tau2_bench_telecom,omniscience}.md` - Optional: `recipes/tasks/mmlu_pro.md`, `recipes/tasks/aime_2025.md`, `recipes/tasks/livecodebench.md` - **nel-next only** (different evaluator — see the nel-next section below, NOT the 0.2.6 steps): shared reference `references/nel-next.md` + per-benchmark recipes `recipes/tasks/aa_next/{terminal_bench_2_1,swebench_verified}.md` (agentic). The `aa_next/` dir holds tasks that require nemo-evaluator-next (0.3.x); `aa/` is the 0.2.6 suite. +- **GDPVal (NeMo Gym / agentic)** — **part of the AA suite** but a 0.2.6 `nemo_gym` task on a different harness, so it's **standalone** (see the GDPVal branch above): recipe `recipes/tasks/aa_gym/gdpval.md` + shared reference `references/gym-gdpval.md` + self-contained example `recipes/examples/gym_gdpval/`. Generated as its **own config** from the example, **never merged into the `aa/` multi-task `tasks` list**. The `aa_gym/` dir holds the NeMo Gym Stirrup-agent tasks. -**AA rule:** If the user mentions "AA" / "Artificial Analysis", generate **only** tasks under `recipes/tasks/aa/`. Do not add MMLU-Pro, AIME 2025, or LiveCodeBench unless explicitly asked. +**AA rule:** If the user mentions "AA" / "Artificial Analysis", generate the `recipes/tasks/aa/` tasks (one multi-task config) **plus a companion standalone GDPVal config** (`recipes/tasks/aa_gym/gdpval.md`, via the GDPVal branch) — GDPVal is part of the AA suite but a different harness, so it's its own config, never added to the `aa/` `tasks` list. Do not add MMLU-Pro, AIME 2025, or LiveCodeBench unless explicitly asked. GDPVal is the heaviest AA task (standalone, multi-hour, needs the SIF sandbox + judge) — surface it and let the user opt out per run. **Shortcut path** (when task list is known up front, e.g. "run AA"): diff --git a/.agents/skills/evaluation/recipes/env.example b/.agents/skills/evaluation/recipes/env.example index 1e9f2e4750a..bf1d3515d1d 100644 --- a/.agents/skills/evaluation/recipes/env.example +++ b/.agents/skills/evaluation/recipes/env.example @@ -37,8 +37,19 @@ NEMO_EVALUATOR_TRUST_PRE_CMD=1 # /v1 base; tau2-bench needs the full /v1/chat/completions. # HLE + AA-LCR + AA-Omniscience judges (ns_hle_aa, ns_aa_lcr, ns_omniscience) — shared inference host +# GDPVal (nemo_gym) also reuses INFERENCE_JUDGE_URL for its pairwise judge. # INFERENCE_JUDGE_URL=https:///v1 +# GDPVal (nemo_gym Stirrup agent) — agent web search. Secret; exported and read +# by the harness. See recipes/tasks/aa_gym/gdpval.md + references/gym-gdpval.md. +# TAVILY_API_KEY= + +# GDPVal (nemo_gym) — persistent Apptainer SIF cache dir on the TARGET cluster's +# shared FS (a path, not a secret). .agents/scripts/gdpval-sif.sh builds the SIF +# here if absent and reuses it otherwise; the config bind-mounts this dir at +# /gdpval/sif. Convention: a per-user .cache dir. +# GDPVAL_SIF_DIR=//.cache/gdpval/sif + # Tau2 (tau2_bench_telecom) — judger + user-simulator model_ids are hardcoded in # the recipe; only the shared endpoint URL comes from here # TAU2_ENDPOINT_URL=https:///v1/chat/completions # user + judger diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml new file mode 100644 index 00000000000..bd9fde37fbf --- /dev/null +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Shared prepare/teardown snippets — include via: defaults: [_gym_prepare, _self_] +# +# Compensates for the eval image's deployment-oriented packaging when run +# standalone. Interpolate into each benchmark's command: +# ${gym_prepare.prepare} at the start (activate venv, checkout pin, repair venvs) +# ${gym_prepare.run} as the last line (data prep + rollout collection run in +# its own session, then process-group reaped so the run +# finalizes — see the run: comment for why) +# Remove once the eval image ships complete, ray-consistent venvs and Gym's +# shutdown reaps server process groups. + +gym_prepare: + # Common command preamble shared by every benchmark: activate the baked Gym + # venv, optionally checkout the install_on_the_fly pin (only when /opt/Gym is a + # git repo; baked images aren't), then repair the image's incomplete per-server + # venvs. The repair: (1) rewrites each component requirements.txt — drops the + # editable "-e nemo-gym[dev]" line (it forces a ray>=2.55.1 re-resolve vs the + # image's pinned ray 2.49.2, breaking venv-less servers like arena_judge) and + # ensures ray==2.49.2 + tqdm; (2) installs the fixed requirements into each + # baked (skeleton) sub-venv; (3) fronts the main venv on PYTHONPATH so nemo_gym + # + framework deps resolve for server processes. + # NOTE: avoid bash ${VAR} here — OmegaConf parses ${...}; $(...) / $r / $v are fine. + # NOTE: every step is guarded (|| true / -q grep) so it is safe under set -e. + prepare: |- + set -ex + cd /opt/Gym + export UV_CACHE_DIR=/opt/cache/uv + source .venv/bin/activate + # install_on_the_fly: checkout the pin when /opt/Gym is a git repo; some + # images bake Gym at a fixed version (not a git repo) — use it as-is. + if [ -d .git ]; then + git remote add oss_pin "{{config.params.extra.nemo_gym.install_on_the_fly.url}}" 2>/dev/null || true + git fetch oss_pin + git checkout "{{config.params.extra.nemo_gym.install_on_the_fly.commit}}" + echo "=== NeMo Gym commit ===" && git rev-parse HEAD + else + echo "=== /opt/Gym is not a git repo; using baked-in Gym version ===" + fi + for r in /opt/Gym/responses_api_models/*/requirements.txt \ + /opt/Gym/responses_api_agents/*/requirements.txt \ + /opt/Gym/resources_servers/*/requirements.txt; do + [ -f "$r" ] || continue + grep -vE '^[[:space:]]*-e ' "$r" > "$r.fixed" || true + grep -qiE '^ray([<>=[]|$)' "$r.fixed" 2>/dev/null || echo 'ray==2.49.2' >> "$r.fixed" + grep -qiE '^tqdm' "$r.fixed" 2>/dev/null || echo 'tqdm' >> "$r.fixed" + mv "$r.fixed" "$r" 2>/dev/null || true + done + for v in /opt/Gym/responses_api_models/*/.venv \ + /opt/Gym/responses_api_agents/*/.venv \ + /opt/Gym/resources_servers/*/.venv; do + [ -d "$v" ] || continue + d="$(dirname "$v")" + [ -f "$d/requirements.txt" ] && uv pip install --python "$v/bin/python" -q -r "$d/requirements.txt" || true + done + export PYTHONPATH="/opt/Gym:$(/opt/Gym/.venv/bin/python -c 'import site; print(site.getsitepackages()[0])')" + + # Data prep + rollout collection. Interpolate as the LAST line of each + # benchmark's command via ${gym_prepare.run} (after ${gym_prepare.prepare} and + # any benchmark-specific prep). + # + # Rollout collection is run in its OWN session (setsid) so its entire process + # tree — servers + their multiprocessing pools + Ray workers — can be reaped by + # process group. Why: ng_e2e's own cleanup (cli.py shutdown) only SIGINTs/SIGKILLs + # the tracked server PIDs after a 1s grace; it does NOT reap each server's child + # tree, so pools/Ray workers orphan and re-parent to the launcher (which is + # blocked reading our stdout) — they hold stdout open and the run never finalizes. + # (Invisible in the deployment flow because the whole node is discarded; only bites + # the deployment-free inline path here.) The inner shell records the session PGID + # (its own $$, before exec) so we can target it; real exit code is captured from + # wait and propagated via exit. + # + # Ray ALSO daemonizes gcs_server/raylet into their own session, so they escape the + # process-group reap; if left running they keep the launcher's stdout open and the + # run hangs in post-eval. So after the group reap we additionally stop Ray and kill + # its daemons by name (safe: matches only Ray daemons, not the launcher's python). + # Remove once Gym's shutdown reaps server process groups + Ray. + run: |- + ng_prepare_benchmark {{config.params.extra.nemo_gym.data_prep_params}} {{config.params.extra.nemo_gym.common_params}} + # The rollout command is written to a script and run from it, rather than passed + # to `bash -c '...'`. Gym params legitimately CONTAIN single quotes — comparison + # mode's ++multistage.stages='[{num_tasks: 45}, ...]' and ++...judge_panel='[{...}]' + # — and those would close a single-quoted `bash -c` wrapper early, so Hydra then + # receives the value split on spaces and dies with + # "no viable alternative at input '[{num_tasks:'". + # The heredoc delimiter is QUOTED ('GYM_RUN_EOF') so nothing expands while writing: + # $$ and $TAVILY_API_KEY / $INFERENCE_API_KEY land in the script literally and are + # expanded at run time by the shell that executes it, which is what we want. + cat > /tmp/gym_run.sh <<'GYM_RUN_EOF' + echo $$ > /tmp/gym_eval_pgid + exec ng_e2e_collect_rollouts {{config.params.extra.nemo_gym.collect_rollout_params}} {{config.params.extra.nemo_gym.common_params}} + GYM_RUN_EOF + setsid --wait bash /tmp/gym_run.sh & + __ev=$! + __rc=0; wait "$__ev" || __rc=$? + echo "Evaluator Gym finished!" + __pg=$(cat /tmp/gym_eval_pgid 2>/dev/null || echo) + [ -n "$__pg" ] && kill -9 -"$__pg" 2>/dev/null || true + timeout 60 ray stop --force >/dev/null 2>&1 || true + pkill -9 -f raylet >/dev/null 2>&1 || true + pkill -9 -f gcs_server >/dev/null 2>&1 || true + pkill -9 -f plasma_store >/dev/null 2>&1 || true + exit $__rc diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml new file mode 100644 index 00000000000..f9df5023764 --- /dev/null +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -0,0 +1,322 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================= +# Example: GDPVal (NeMo Gym "Stirrup" agent) — single-task gym eval template. +# +# GDPVal is an AGENTIC benchmark: the Stirrup agent produces office/PDF +# deliverables inside a per-task Apptainer code-exec sandbox, then a pairwise/ +# rubric judge (Gemini 3.1 Pro) scores them. It runs on the 0.2.6 `nel` launcher +# as a `nemo_gym` task (NOT nel-next), but it is STANDALONE — one gym eval per +# config, no other tasks. Read recipes/tasks/aa_gym/gdpval.md and +# references/gym-gdpval.md before editing this file. +# +# This template SELF-DEPLOYS a (quantized) checkpoint via vLLM on ONE SLURM node +# and runs GDPVal against it. For the full 220-task run of a large MoE you will +# likely need multi-node — see references/gym-gdpval.md (deployment sizing). +# +# PREREQUISITES (see references/gym-gdpval.md): +# 1. Set GDPVAL_SIF_DIR in .env (persistent SIF cache dir on this cluster), then +# ensure the SIF exists (build-if-absent, reuse-if-present — never copied from +# another cluster): +# srun -p cpu -t 01:00:00 --pty .agents/scripts/gdpval-sif.sh # uses $GDPVAL_SIF_DIR +# The mount below binds $GDPVAL_SIF_DIR at /gdpval/sif so the SIF lands at EXACTLY +# /gdpval/sif/python-3.13.gdpval.sif (matches GDPVAL_CONTAINER_PATH below). +# Without it, the agent SILENTLY falls back to non-sandboxed local exec. +# 2. .env has HF_TOKEN, INFERENCE_API_KEY (judge auth), TAVILY_API_KEY (agent web +# search), INFERENCE_JUDGE_URL (judge host), and GDPVAL_SIF_DIR (item 1). See +# recipes/env.example. +# 3. This file's `defaults` include `_gym_prepare` — the _gym_prepare.yaml in +# THIS directory MUST travel next to this config (Hydra resolves it +# relative to the config dir). Copy the whole gym_gdpval/ dir to your +# workspace, don't copy the yaml alone. +# +# Canary (validates SIF sandbox + judge + gym plumbing on a couple of tasks): +# nel run --config example_gym_gdpval.yaml --env-file .env \ +# -o ++evaluation.nemo_evaluator_config.config.params.limit_samples=2 +# ============================================================================= +defaults: + # slurm/default works anywhere; if your install ships a predefined + # internal/slurm/ config, prefer it (pre-fills hostname/partition/ + # gres — see SKILL.md Step 4). + - execution: slurm/default + - deployment: vllm + - _gym_prepare # provides ${gym_prepare.prepare} / ${gym_prepare.run} + - _self_ + +# GDPVal scoring mode. This template is RUBRIC-ONLY, deliberately: rubric needs no +# reference deliverables and runs on the public gym image, so it works standalone. +# rubric — standalone LLM-judge scoring, 0-1 reward per deliverable. (this template) +# comparison — pairwise vs anchored reference deliverables; the only mode that yields +# an AA-comparable ELO / win-rate. It needs a reference set, a newer gym +# image, and its own overrides — do NOT just flip reward_mode here. +# To run comparison, NVIDIA-internal users should follow `modelopttools:eval-config` +# Step 3c, which converts this config (container override + reference_models map + +# mounts + multistage). See references/gym-gdpval.md "Scoring modes". +gdpval: + reward_mode: rubric + +# GDPVal pairwise judge. base_url is config (from .env), not a secret, so no +# export needed; only api_key (INFERENCE_API_KEY) is exported and read by the +# harness. Keep the judge fixed across comparable runs. +gdpval_judge: + base_url: # from .env (/v1 base); shared inference host + model: gcp/google/gemini-3.1-pro-preview # Gemini 3.1 Pro; use an equivalent on your endpoint if needed + api_key: INFERENCE_API_KEY # doc only — the gym injects the VALUE as $INFERENCE_API_KEY + # in common_params (NOT ${gdpval_judge.api_key}, which passes + # the literal NAME "INFERENCE_API_KEY" → judge 500). + +cluster: + sbatch_comment: '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"480","reason":"benchmarking","description":"Eval benchmark low GPU utilization"}}' + +execution: + hostname: ??? + username: ${oc.env:USER} + account: ??? + output_dir: ??? # absolute host path; keeps deliverables + response cache across resumes + walltime: "04:00:00" + # gres: a predefined internal/slurm/ config sets this. On slurm/default + # it's gpu:8 — set to the node's GPU count (match --tensor/--data-parallel-size) + # or sbatch fails "Requested node configuration is not available". + mounts: + # mount_home ALWAYS false — see example_eval.yaml / SKILL Step 4 for why. + mount_home: false + deployment: + # Real HF cache -> /hf-cache (paired with HF_HOME below). + : /hf-cache + evaluation: + : /hf-cache + : /cache/uv + # GDPVal Stirrup SIF dir — substitute the literal $GDPVAL_SIF_DIR value (.env), + # the persistent shared-FS dir gdpval-sif.sh builds python-3.13.gdpval.sif into + # (build-if-absent/reuse). Use the literal path here, NOT ${oc.env:...}: mount + # KEYS are not interpolated (same rule as the judge URLs). Mounting the DIR lands + # the SIF at /gdpval/sif/python-3.13.gdpval.sif == GDPVAL_CONTAINER_PATH. + : /gdpval/sif + # Writable shared-FS staging for ref files. Node-local /tmp breaks multi-node Ray. + : /gdpval_ref_files + # Comparison mode additionally mounts one dir per reference model at + # /gdpval/refs/ (both `deployment` and `evaluation`) — added by + # modelopttools:eval-config Step 3c, not here. + auto_export: # REQUIRED trigger for MLflow upload (see example_eval.yaml). + destinations: + - mlflow + # Auto-export is a separate CPU-only sbatch; GPU-only partitions reject it. + cpu_partition: ??? + +deployment: + env_vars: + HF_TOKEN: host:HF_TOKEN + HF_HOME: lit:/hf-cache + # vLLM backend toggles go HERE (not in command). Uncomment per model card — + # e.g. NVFP4 MoE on Blackwell needs FlashInfer FP4 kernels: + # VLLM_USE_FLASHINFER_MOE_FP4: lit:1 + # VLLM_FLASHINFER_MOE_BACKEND: lit:throughput + checkpoint_path: ??? # prefer a path already on the cluster over hf_model_handle + hf_model_handle: + served_model_name: ??? + image: vllm/vllm-openai:v0.19.1 # bump to the EXACT model's recipes.vllm.ai minimum (SKILL Step 3) + # GDPVal REQUIRES thinking mode from the policy (non-thinking loses ~86% of + # pairwise judgements). Serve with the model's --reasoning-parser so vLLM emits + # a separate reasoning channel; thinking is forced on via the adapter_config + # chat_template_kwargs below. Add --enable-expert-parallel for MoE, and + # --trust-remote-code for custom-code models. + # After filling `parallelism`, append --max-num-seqs N (N = ceil(parallelism / data_parallel_size)). + command: >- + vllm serve /checkpoint + --served-model-name ${deployment.served_model_name} + --host 0.0.0.0 + --port ${deployment.port} + --tensor-parallel-size 1 + --data-parallel-size 1 + --max-model-len 131072 + --reasoning-parser + --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 128}' + --max-num-batched-tokens 8192 + --enable-chunked-prefill + +evaluation: + env_vars: + HF_TOKEN: host:HF_TOKEN + HF_HOME: lit:/hf-cache + DUMMY_API_KEY: lit:dummy # the deployed vLLM endpoint's (policy) key + INFERENCE_API_KEY: host:INFERENCE_API_KEY # GDPVal judge auth (shared inference host) + TAVILY_API_KEY: host:TAVILY_API_KEY # Stirrup agent web search + UV_CACHE_DIR: lit:/cache/uv + # Apptainer SIF for the Stirrup per-task code-exec sandbox (must match the mount above). + GDPVAL_CONTAINER_PATH: lit:/gdpval/sif/python-3.13.gdpval.sif + # Shared-FS staging for ref files; node-local /tmp breaks multi-node Ray. + GDPVAL_REF_FILES_DIR: lit:/gdpval_ref_files + # Deliverables land under /results (already mounted) so they persist. The + # "*cache*" basename matches the mlflow exporter's exclusion, so the (large) + # deliverables are NOT auto-uploaded — they stay on disk for inspection. + # Drop "_cache" if you WANT them uploaded as artifacts. + PERSIST_DELIVERABLES_DIR: lit:/results/gdpval/deliverables_cache + # Stirrup agent turn cap (optional; default 100). + # GDPVAL_MAX_TURNS: lit:100 + NEL_INVOCATION_ID: runtime:NEL_INVOCATION_ID + # Installs apptainer + squashfuse into the eval container (needs + # NEMO_EVALUATOR_TRUST_PRE_CMD=1 in the launching shell). See references/gym-gdpval.md. + # Installs the apptainer RUNTIME into the eval container (it is not baked in) so the + # Stirrup agent can exec the prebuilt SIF. ARCH-AWARE: Blackwell/Grace clusters are + # aarch64, so never hardcode an amd64 .deb — the Ubuntu PPA builds both arches. + pre_cmd: | + set -ex + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq software-properties-common squashfuse fuse3 ca-certificates || true + add-apt-repository -y ppa:apptainer/ppa + apt-get update -qq + apt-get install -y -qq apptainer + apptainer --version + mkdir -p /usr/local/var/apptainer/mnt/session + nemo_evaluator_config: + config: + params: + temperature: 1.0 + top_p: 0.95 + parallelism: 16384 # gym-internal concurrency, NOT a model-server cap + request_timeout: 36000 # gym rollouts are long-running + max_retries: 10 + target: + api_endpoint: + api_key_name: DUMMY_API_KEY + adapter_config: + use_system_prompt: false + process_reasoning_traces: true + params_to_remove: + - max_tokens + - max_completion_tokens + # Thinking mode MUST be on — non-thinking loses ~86% of pairwise + # judgements. Keep the toggle key your model family uses (enable_thinking + # -> Qwen3.x/GLM; thinking -> Kimi/DeepSeek); see SKILL "Reasoning adapter config". + params_to_add: + chat_template_kwargs: + enable_thinking: true + skip_special_tokens: false + use_caching: false + use_progress_tracking: true + tracking_requests_stats: true + log_failed_requests: true + use_request_logging: true + max_logged_requests: 10 + use_response_logging: true + max_logged_responses: 10 + # STANDALONE: exactly one gym task. Do NOT add other tasks to this list. + tasks: + - name: nemo_gym + # Public image — fine for RUBRIC mode (this template's default). + # COMPARISON mode needs a NEWER Gym than this image ships and must OVERRIDE this + # line; see references/gym-gdpval.md "Scoring modes". NOTE: this image bakes Gym + # as a non-git directory, so `install_on_the_fly.commit` below is silently + # ignored here (the prepare step logs "/opt/Gym is not a git repo; using baked-in + # Gym version") — the pin only applies on images where /opt/Gym IS a git repo. + container: nvcr.io/nvidia/eval-factory/nemo-gym:26.05 # pin a verified tag + nemo_evaluator_config: + config: + params: + extra: + nemo_gym: + install_on_the_fly: + url: https://github.com/NVIDIA-NeMo/Gym + # Gym version. BUMPING THIS REQUIRES REBUILDING THE SIF from the matching + # commit — gdpval.def (the sandbox) is versioned with the gym repo, and an + # old SIF + new gym silently degrades deliverables. Rebuild with + # `gdpval-sif.sh --commit ` to a new name + repoint GDPVAL_CONTAINER_PATH. + # See references/gym-gdpval.md "Rebuild the SIF when the Gym version changes". + # Current golden pin (updated GDPVal task-sampling algo, which drives + # multistage stage-1 selection). Its gdpval.def is byte-identical to + # 049b1fd0…, so a python-3.13 SIF built from either commit is valid — + # no rebuild when moving between them. + # NOTE: this pin is INERT on the public nemo-gym image (see the + # `container:` comment above) — it only takes effect on an image where + # /opt/Gym is a git repo. Confirm via "=== NeMo Gym commit ===" + SHA + # in the client log before crediting it with any behaviour change. + commit: dd41196f620f2af99947d776cbe5da9439d2a08d # pragma: allowlist secret + command: | + ${gym_prepare.prepare} + + # Writable staging dir for ref files (bind-mounted, see execution). + mkdir -p /gdpval_ref_files + + # num_repeats: this template defaults to 1 (halves cost). The + # reviewed GOLDEN uses num_repeats=2 (220 tasks x 2 = 440 rollouts); + # for golden-comparable / REPORTED scores, DELETE the sed line below + # to keep the checked-in default of 2. num_repeats can NOT be set via + # a ++ override (OmegaConf ListConfig merge error) — patch the file. + sed -i 's/num_repeats: 2$/num_repeats: 1/' benchmarks/gdpval/config.yaml + + ${gym_prepare.run} + data_prep_params: >- + "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" + +hf_token=$HF_TOKEN + ++use_cached_prepared_benchmarks=true + collect_rollout_params: >- + "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" + ++port_range_low=63000 + ++port_range_high=64000 + ++global_aiohttp_connector_limit_per_host={{config.params.parallelism}} + ++uv_cache_dir=$UV_CACHE_DIR + ++skip_venv_if_present=true + ++output_jsonl_fpath={{config.output_dir}}/evaluator_rollouts.jsonl + ++nemo_gym_log_dir={{config.output_dir}}/nemo_gym_logs + ++overwrite_metrics_conflicts=true + ++split=benchmark + ++resume_from_cache=true + ++reuse_existing_data_preparation=true + ++upload_rollouts_to_wandb=false + ++responses_create_params.temperature={{config.params.temperature}} + ++responses_create_params.top_p={{config.params.top_p}} + # NOTE: no reference_* overrides below — rubric mode needs none, and the + # single-reference keys (reference_deliverables_dir / reference_elo) + # CONFLICT with comparison mode's reference_models map (added by + # modelopttools:eval-config Step 3c). + # Do NOT put '#' comments INSIDE this folded (>-) scalar: YAML keeps + # them as literal text, the block folds to one line, and the first '#' + # comments out every override after it in the shell command. + common_params: >- + ++use_absolute_ip=true + ++policy_base_url={{target.api_endpoint.url}} + ++policy_api_key=$DUMMY_API_KEY + ++policy_model_name={{target.api_endpoint.model_id}} + ++gdpval_judge_model.responses_api_models.openai_model.openai_base_url=${gdpval_judge.base_url} + ++gdpval_judge_model.responses_api_models.openai_model.openai_model=${gdpval_judge.model} + ++gdpval_judge_model.responses_api_models.openai_model.openai_api_key=$INFERENCE_API_KEY + ++gdpval_judge_model.responses_api_models.openai_model.max_concurrent_requests=10 + ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.concurrency=220 + ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.tavily_api_key=$TAVILY_API_KEY + ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.agent_max_turns=${oc.env:GDPVAL_MAX_TURNS,250} + ++gdpval_resources_server.resources_servers.gdpval.reward_mode=${gdpval.reward_mode} + ++gdpval_resources_server.resources_servers.gdpval.persist_raw_judge_responses=true + ++gdpval_resources_server.resources_servers.gdpval.preconvert_max_concurrent=30 + ++gdpval_resources_server.resources_servers.gdpval.preconvert_office_to_pdf=true + ++gdpval_resources_server.resources_servers.gdpval.judge_responses_create_params_overrides.model=${gdpval_judge.model} + +export: + # LITERAL values only (auto_export resolves this block at submit time in a scope + # without deployment/evaluation nodes; ${oc.env:...} is fine). Keep the sampling + # tags EQUAL to evaluation params above — they're the only MLflow record of them. + mlflow: + tracking_uri: ${oc.env:MLFLOW_TRACKING_URI} # from modelopttools:eval-config + experiment_name: ${oc.env:USER}/CHANGEME-served-model-name + description: 'CHANGEME-served-model-name | GDPVal rubric | T=1.0, top_p=0.95, num_repeats=1' + log_logs: true + only_required: false + tags: + framework: vllm + model: CHANGEME-served-model-name + benchmark: nemo_gym.gdpval + temperature: '1.0' + top_p: '0.95' diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md new file mode 100644 index 00000000000..f4297068243 --- /dev/null +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -0,0 +1,105 @@ +# GDPVal (NeMo Gym "Stirrup" agent) + +## Task Details + +- Reference: `references/gym-gdpval.md` (SIF build, gym machinery, deploy sizing, + scoring modes, failure modes) — **read it before editing a GDPVal config.** +- Upstream README: + + +GDPVal is an **agentic** benchmark: the Stirrup agent produces office/PDF +deliverables inside a per-task Apptainer code-exec sandbox, then a pairwise/rubric +judge (**Gemini 3.1 Pro**) scores them. It is the most resource-intensive benchmark +in the suite — **220 tasks**, `num_repeats=2` in the reviewed golden (= 440 +rollouts), each rollout using 4 judge trials. + +It runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task (NOT nel-next), so +Steps 1–9 apply — but with the branch differences below. + +## What makes GDPVal different (do NOT treat it as a normal `aa/` task) + +- **Standalone — one gym eval per config.** Never add GDPVal to a multi-task + `evaluation.tasks` list, and never add other tasks to a GDPVal config. +- **Apptainer SIF sandbox (self-contained).** Set `GDPVAL_SIF_DIR` in `.env`, then + run `.agents/scripts/gdpval-sif.sh` (uses `$GDPVAL_SIF_DIR`) — it **builds if + absent, reuses if present**, and never copies from another cluster. The config + bind-mounts `$GDPVAL_SIF_DIR` at **exactly** `/gdpval/sif/python-3.12.gdpval.sif` + (matches `GDPVAL_CONTAINER_PATH`). Missing/mispathed → the agent **silently** runs + code-exec unsandboxed and results are not comparable. Details in `references/gym-gdpval.md`. +- **Thinking mode is mandatory.** Non-thinking loses ~86% of pairwise judgements. + Serve the policy with its `--reasoning-parser` and force thinking on via the + adapter `chat_template_kwargs` (see the example). +- **Judge + web search + gym plumbing.** Needs `INFERENCE_API_KEY` (judge auth), + `TAVILY_API_KEY` (agent web search), `INFERENCE_JUDGE_URL` (judge host, from + `.env`), a `pre_cmd` that installs apptainer/squashfuse, and the co-located + `_gym_prepare.yaml` include. + +## Scoring modes + +Set `gdpval.reward_mode` (override: `-o gdpval.reward_mode=comparison`): + +- **`rubric`** (default) — standalone LLM-judge scoring; **no reference + deliverables** needed. Use this unless you specifically need pairwise-vs-baseline. +- **`comparison`** — pairwise scoring vs a reference model's deliverables. Also + mount the ref dir at `/gdpval/refs/test_ref` and set `gdpval.reference_elo` + (golden uses Kimi-K2.5-Thinking refs, elo=1290). Two-step baseline→comparison + flow in `references/gym-gdpval.md`. + +## Config + +**Do not copy a fragment into another config.** GDPVal is standalone — start from +the self-contained example and edit it: + +```text +recipes/examples/gym_gdpval/ + example_gym_gdpval.yaml # SLURM + single-node vLLM self-deploy template + _gym_prepare.yaml # co-located Hydra include (${gym_prepare.*}); travels with the yaml +``` + +Copy the **whole `gym_gdpval/` directory** to your workspace (the `- _gym_prepare` +default resolves relative to the config dir — copying the yaml alone breaks it). + +- **num_repeats — the right value depends on the flow, so check which one you're in:** + - **Multistage comparison** (the current golden for AA-comparable ELO): **1**, set + with a top-level `++num_repeats=1`, which *does* work. Recent Gym pins already + ship `num_repeats: 1` in `benchmarks/gdpval/config.yaml`, so the `sed` below is a + no-op there. + - **Rubric / older single-reference comparison:** the pre-multistage golden used + **2** (220 tasks × 2 = 440 rollouts). On old pins the per-dataset key could not be + set via `++` (OmegaConf `ListConfig` merge error), hence the `sed` in the task + `command:`; delete that line to keep 2. + + Do not carry a `=2` from an old single-reference config into a multistage run. +- **SIF ↔ Gym version (rebuild on bump):** the SIF is built from `gdpval.def` at + `install_on_the_fly.commit`. **If you change that commit, rebuild the SIF** with a + matching `gdpval-sif.sh --commit ` (to a new version-tagged filename, then + repoint `GDPVAL_CONTAINER_PATH`) — the def's base image + package stack change + across commits, and running a new gym with an old SIF makes the agent's generated + code fail imports in the sandbox → silently degraded deliverables. See + `references/gym-gdpval.md` → "Rebuild the SIF when the Gym version changes". +- **Deployment:** single-node vLLM in the template; the full 220×2 run of a large + MoE typically needs multi-node — see `references/gym-gdpval.md`. +- Required `.env` keys: `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, + `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR` (see `recipes/env.example`). + `NEMO_EVALUATOR_TRUST_PRE_CMD=1` is needed because the config has a `pre_cmd`. + +## Canary + +Validate the SIF sandbox + judge + gym plumbing on a couple of tasks before the +full run: + +```bash +nel run --config example_gym_gdpval.yaml --env-file .env \ + -o ++evaluation.nemo_evaluator_config.config.params.limit_samples=2 +``` + +Inspect logs for the SIF fallback warning, judge auth/429s, and Ray/gym shutdown +hangs (see `references/gym-gdpval.md` → failure modes). + +## Score Extraction + +GDPVal reports a **win-rate / ELO** against the reference (comparison mode) or a +rubric score (rubric mode); the run's aggregate metric is logged under the +`nemo_gym.gdpval` benchmark in MLflow. Read the run's +`{output_dir}/evaluator_rollouts.jsonl` + `nemo_gym_logs/` for per-task rewards, +and the persisted judge responses under `PERSIST_DELIVERABLES_DIR`. diff --git a/.agents/skills/evaluation/references/gym-gdpval.md b/.agents/skills/evaluation/references/gym-gdpval.md new file mode 100644 index 00000000000..358839cfa04 --- /dev/null +++ b/.agents/skills/evaluation/references/gym-gdpval.md @@ -0,0 +1,261 @@ +# GDPVal (NeMo Gym "Stirrup" agent) — reference for the gym / agentic path + +GDPVal runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task, but it is +mechanically unlike the `aa/` nemo-skills tasks: the Stirrup agent produces +office/PDF **deliverables** in a per-task **Apptainer** code-exec sandbox, a +pairwise/rubric **judge** (Gemini 3.1 Pro) scores them, and NeMo Gym is pulled and +run **inline in the eval container** (`install_on_the_fly`) via `ng_prepare_benchmark` ++ `ng_e2e_collect_rollouts`. This file is the shared machinery; the config template +is `recipes/examples/gym_gdpval/` and the per-task pointer is +`recipes/tasks/aa_gym/gdpval.md`. + +## Where each piece runs + +| Component | Where | +|---|---| +| Policy model (under test) | your self-deployed vLLM endpoint (SLURM GPU node) — or an external endpoint | +| NeMo Gym + Stirrup agent orchestration | inside the **eval** container (`nemo_gym` task), pulled via `install_on_the_fly` | +| Per-task code-exec | **Apptainer SIF** launched by the agent inside the eval container | +| Judge (pairwise/rubric) | external OpenAI-compatible endpoint (`gdpval_judge`, e.g. Gemini 3.1 Pro) | +| Agent web search | Tavily (`TAVILY_API_KEY`) | + +## Apptainer SIF sandbox (self-contained: build-if-absent, reuse-if-present) + +The Stirrup agent runs each task's generated code in an Apptainer SIF, bind-mounted +into the eval container at **exactly** `/gdpval/sif/python-3.12.gdpval.sif` (the path +`GDPVAL_CONTAINER_PATH` points at). If it's missing or at a different path, the agent +**silently falls back to non-sandboxed local exec** — the run "succeeds" but the +numbers aren't comparable, so verify the SIF at canary. + +**The skill builds the SIF on the target cluster — it never copies one from +another cluster.** That is the self-contained default and the right behaviour when +you have no prebuilt sandbox. + +> **If your site already provides a prebuilt GDPVal SIF, prefer it over building — +> build only when it is absent.** A self-built SIF resolves its pip stack at *your* +> build time, so it can drift from the sandbox a published reference set was +> generated in, which matters for comparison-mode runs. To use a provided one, mount +> its dir at `/gdpval/sif` and point `GDPVAL_CONTAINER_PATH` at the provided +> filename. **NVIDIA-internal users:** `modelopttools:eval-config` (Step 3c) records +> the provided SIF's location and the check-then-fall-back procedure — if you have +> that skill, consult it before building. Without it, building (below) is correct. + +Ship the SIF via the idempotent helper, which builds it if absent and reuses it if +already present: + +```bash +# GDPVAL_SIF_DIR (.env) — persistent shared-FS dir on the TARGET cluster; the config +# bind-mounts this same dir at /gdpval/sif. Preferred: run the ~30-min build on the +# CPU partition, not a login node. `set -a && source .env` first so it's set. +srun -p cpu -t 01:00:00 --pty \ + .agents/scripts/gdpval-sif.sh # defaults to $GDPVAL_SIF_DIR (or pass a dir) +``` + +`gdpval-sif.sh` builds from the NeMo Gym `gdpval.def` at the pinned commit (keep +`GDPVAL_GYM_COMMIT` in sync with the config's `install_on_the_fly.commit`), writes +the SIF into that dir, and is flock-guarded + atomic so concurrent runs never +double-build. Re-running is a no-op once the SIF exists — that's the "reuse the +built one" path. It needs `apptainer`/`singularity` on the build host with +fakeroot/unprivileged build support and network egress; run it on a login or CPU +node (outside enroot, where fakeroot works), **not** inside the eval job. + +**Build vs run are separate.** The helper *builds* the SIF (once, off-GPU). The +eval then *runs* the prebuilt SIF inside the eval container — the golden-validated +path. The eval image doesn't ship apptainer, so the config's `pre_cmd` installs the +apptainer **runtime** + squashfuse (FUSE-mounts the SIF; falls back to slower +per-call extraction if `/dev/fuse` is absent). `pre_cmd` runs arbitrary commands → +the launching shell needs `export NEMO_EVALUATOR_TRUST_PRE_CMD=1`. The +apptainer-under-pyxis nesting is the least-validated part of the SLURM path — check +the canary logs for the "falling back to local exec" warning and apptainer mount +errors. (Local/Docker executor instead needs `--privileged` + the SIF bind mount via +`execution.extra_docker_args`; a rootless `--security-opt` + `/dev/fuse` variant is +in the upstream README appendix.) + +### Rebuild the SIF when the Gym version changes (SIF ↔ commit coupling) + +The SIF is **versioned with the Gym repo**: it's built from `gdpval.def` at +`install_on_the_fly.commit`, and that def changes across commits. Example — between +`2502893977` and the golden `049b1fd0`, the base went **python-3.12 → 3.13** and the +stack gained TeX Live, chromium/playwright, polars/duckdb, xgboost, geospatial +(gdal/proj/geos), and audio/video libs. The newer Stirrup agent's prompt advertises +that richer runtime, so the model's generated code reaches for those libs. + +**So whenever you bump `install_on_the_fly.commit`, rebuild the SIF from the matching +commit.** Run the new gym with an old SIF and the generated code fails its imports +*inside the sandbox* — deliverables silently degrade (missing figures/tables/docs) → +junk scores, with no hard error in the eval. Rebuild to a **version-tagged filename** +so the old SIF isn't clobbered (a running job keeps working), then repoint +`GDPVAL_CONTAINER_PATH` + the `/gdpval/sif` mount at the new file: + +```bash +# build the SIF for the NEW gym commit under a distinct name (old SIF stays intact) +GDPVAL_SIF_NAME=python-3.13.gdpval.sif \ + .agents/scripts/gdpval-sif.sh --commit "$GDPVAL_SIF_DIR" +# then set GDPVAL_CONTAINER_PATH=/gdpval/sif/python-3.13.gdpval.sif in the config +``` + +Rule of thumb: **`install_on_the_fly.commit` and the SIF move together.** Any bump +that alters `gdpval.def` (base image or the apt/pip stack) needs a rebuild; a bump +that leaves `gdpval.def` byte-identical does not — diff the def at the two commits +(`raw.githubusercontent.com/NVIDIA-NeMo/Gym//responses_api_agents/stirrup_agent/containers/gdpval.def`) +to be sure. Note the def already disables apt's sandbox internally as of `049b1fd0`, +so it builds cleanly under the helper's unprivileged path. + +## The `_gym_prepare.yaml` include (why it exists) + +`nemo_gym` tasks interpolate two shared snippets into the task `command:`: +`${gym_prepare.prepare}` (activate the baked Gym venv, checkout the +`install_on_the_fly` pin, repair the image's incomplete per-server venvs, front the +main venv on `PYTHONPATH`) and `${gym_prepare.run}` (data prep + +`ng_e2e_collect_rollouts`, run in its own `setsid` session so the whole server/Ray +process tree can be reaped by process group — otherwise orphaned Ray workers hold +the launcher's stdout open and the run **hangs in post-eval**). Both compensate for +the eval image's deployment-oriented packaging and Gym's incomplete shutdown; remove +once the image ships complete ray-consistent venvs. + +**The include is co-located, not central.** Hydra resolves `- _gym_prepare` relative +to the run config's directory, so `_gym_prepare.yaml` must sit next to your config — +copy the whole `recipes/examples/gym_gdpval/` dir, not the yaml alone. + +## Deployment sizing + +GDPVal is heavy: 220 tasks × `num_repeats` rollouts, each a long multi-turn agent +episode with code-exec + judge calls (`request_timeout: 36000`). The example +self-deploys single-node vLLM, which is fine for a canary or a small policy. For the +**full run of a large MoE** (e.g. MiniMax-M2.7), the reviewed golden uses **multi-node +`vllm_ray`** (16 × 4-GPU HSG = 64 GPUs, `walltime 04:00:00`). To scale up: + ++ Switch `defaults: - deployment: vllm_ray` and add nodes (`execution.num_nodes`); + see `references/multi-node.md` for the Ray TP/PP layout. ++ `parallelism` (`16384`) is **gym-internal concurrency**, not a server cap. The + real throttles are the agent's `stirrup_agent.concurrency` and the judge's + `max_concurrent_requests` — raise those only after the judge logs are clean of 429s. ++ Long runs exceed 4h; rely on NEL's walltime dependency-chain resume + (`resume_from_cache=true` is already set). See SKILL Step 4 + `run-validation.md`. + +## Scoring modes — rubric vs comparison + ++ **`rubric`** (template default) — the judge scores each deliverable standalone + against a rubric; **no reference deliverables** needed. ++ **`comparison`** — pairwise: the judge compares the policy's deliverable to a + **reference model's** deliverable and the result is an ELO-anchored win-rate. + Two-step flow: + 1. **Baseline:** run your reference model with `-o gdpval.reward_mode=rubric` to + generate baseline deliverables (they land under `PERSIST_DELIVERABLES_DIR`). + 2. **Comparison:** run the candidate with `-o gdpval.reward_mode=comparison`, mount + the baseline deliverables at `/gdpval/refs/test_ref` + (`execution.mounts.evaluation`), and set `gdpval.reference_elo` to the reference + model's ELO (golden: Kimi-K2.5-Thinking, elo=1290). + +### Comparison mode needs a newer Gym than the public image — override `container:` + +The public `nvcr.io/nvidia/eval-factory/nemo-gym:26.05` (== `latest` by digest) runs +**rubric** mode fine, but its Gym predates the multi-reference `reference_models` map. +In comparison mode `gdpval_resources_server` fails validation at startup with +`reward_mode=comparison requires reference_deliverables_dir to be set`, surfacing only +as the unhelpful `Process gdpval_resources_server finished unexpectedly!`. + +**You cannot fix this by bumping `install_on_the_fly.commit`.** That image bakes Gym as +a **non-git directory**, so the pin is silently ignored — the prepare step logs +`=== /opt/Gym is not a git repo; using baked-in Gym version ===` and you get the baked +build regardless. Treat the pin as inert unless the log shows `=== NeMo Gym commit ===` +followed by a SHA, and don't attribute run-to-run behaviour changes to it. (In +particular, a head-server `address already in use` on its fixed port 11000 is a +**transient collision** — resubmitting clears it; it is not a Gym-version symptom.) + +To run comparison mode you must **override the task's `container:`** with an image +whose Gym has `reference_models`. NVIDIA-internal users: the image path, the canonical +reference set, and the matching gym overrides are in the `modelopttools:eval-config` +skill (Step 3c) — internal cluster paths deliberately live only there. + +## Preflight — what NEL validates, and what it does NOT + +NEL validates mount paths at **submit** time (`_collect_mount_paths` + +`_validate_remote_paths_exist`): it ssh's to the cluster, runs `test -d` on every +mount source, and `raise ValueError` listing the missing ones **before** any +`sbatch` — so a missing reference dir or cache costs you nothing. Three gaps to know: + +| Artifact | Missing → | Loud? | +|---|---|---| +| mounted dirs (refs, caches, SIF **dir**, checkpoint) | `ValueError` at submit, no job queued | ✅ pre-allocation | +| **the SIF file inside that dir** | **agent silently runs code-exec unsandboxed** | ❌ **silent** | +| task `container:` (image / `.sqsh`) | not collected for validation → pyxis import failure | ⚠️ only after allocation | + +1. **`test -d` proves the directory, not the SIF.** A `$GDPVAL_SIF_DIR` that exists but + holds the *wrong* filename (e.g. `python-3.12…` after bumping to a 3.13 def) passes + validation, and the run then silently degrades. Guard with the verify-only mode: + + ```bash + .agents/scripts/gdpval-sif.sh --check # uses $GDPVAL_SIF_DIR; exit 1 + lists what IS there + ``` + + Keep `GDPVAL_SIF_NAME` / the helper's default in sync with the config's + `GDPVAL_CONTAINER_PATH`; they are the same string in two places. +2. **`--dry-run` skips remote validation entirely** (it never opens the ssh + connection). A clean dry-run says nothing about whether your mounts exist — run + the preflight separately. +3. **The container is never checked.** A wrong/rotated image path fails at pyxis + import, i.e. after the allocation is granted. Verify it with `ls -l` first + (comparison mode's internal image especially — see `modelopttools:eval-config`). + +## Env vars + +| Var | Prefix | Purpose | +|---|---|---| +| `HF_TOKEN` | host | model/dataset downloads | +| `INFERENCE_API_KEY` | host | **judge** auth (and policy if external) | +| `TAVILY_API_KEY` | host | Stirrup agent web search | +| `DUMMY_API_KEY` | lit:dummy | self-deployed vLLM policy key | +| `GDPVAL_CONTAINER_PATH` | lit | SIF path — must equal the SIF bind-mount target | +| `GDPVAL_REF_FILES_DIR` | lit:/gdpval_ref_files | shared-FS ref-file staging (node-local /tmp breaks multi-node Ray) | +| `PERSIST_DELIVERABLES_DIR` | lit | where deliverables persist (see MLflow note) | +| `GDPVAL_MAX_TURNS` | lit (optional) | Stirrup turn cap (default 100; golden uses 250) | +| `NEL_INVOCATION_ID` | runtime | run id | + +`INFERENCE_JUDGE_URL` is the judge host — config (from `.env`), substituted as the +literal `` placeholder in `gdpval_judge.base_url`, **not** +`${oc.env:...}`. Judge `model_id` is hardcoded in the config (swap for an equivalent +on your endpoint). The upstream OSS recipe uses a separate `GDPVAL_JUDGE_API_KEY`; +this template reuses the shared `INFERENCE_API_KEY` for the judge. + +`GDPVAL_SIF_DIR` (`.env`) is host-side config, **not** a container env var: it's the +persistent SIF cache dir the helper builds into and the config bind-mounts at +`/gdpval/sif`. `gdpval-sif.sh` reads `$GDPVAL_SIF_DIR` directly (its default target); +the config mount is a **literal `` placeholder** you substitute with +that same path — mount KEYS aren't interpolated, so don't emit `${oc.env:...}` there +(same rule as the judge URLs). One `.env` value feeds both, so the build path and the +run path can't drift. + +## MLflow export — the deliverables trap + +Deliverables can be large. The mlflow exporter excludes any artifact dir whose +basename matches `*cache*`, so the template sets +`PERSIST_DELIVERABLES_DIR=/results/gdpval/deliverables_cache`: the deliverables stay +under `/results` for inspection but are **not** auto-uploaded. Drop the `_cache` +suffix only if you actually want them uploaded. Everything else about auto-export is +standard (SKILL Step 1 shortcut #4): `auto_export.destinations: [mlflow]` + +`cpu_partition` + a literal-valued `export.mlflow` block (tag `benchmark: +nemo_gym.gdpval`). + +## num_repeats workaround (OmegaConf) + +`++gdpval.*.num_repeats=N` hits an OmegaConf `ListConfig` merge error, so the count +is patched by editing the checked-out file inside the task `command:`: +`sed -i 's/num_repeats: 2$/num_repeats: 1/' benchmarks/gdpval/config.yaml`. The Gym +default is 2 (the golden). The template seds it to **1** to halve cost; **remove the +sed line to keep 2 for golden-comparable / reported scores.** + +## Failure modes to check at canary + ++ **Silent unsandboxed exec** — grep the eval log for the SIF fallback warning / + apptainer mount errors; confirm `GDPVAL_CONTAINER_PATH` == the mount target. ++ **Judge 401 / 429** — wrong `INFERENCE_JUDGE_URL` / key, or `max_concurrent_requests` + too high for the judge endpoint. ++ **Empty reasoning / low win-rate** — thinking mode off. Confirm + `chat_template_kwargs.enable_thinking: true` (right toggle key for the family) + + the policy's `--reasoning-parser`. ++ **Run hangs in post-eval** — orphaned Ray/gym processes holding stdout; that's what + the `${gym_prepare.run}` setsid + process-group reap prevents. If it still hangs, + the `_gym_prepare.yaml` include didn't travel with the config. ++ **Multi-node ref-file errors** — `GDPVAL_REF_FILES_DIR` on node-local storage; + point it at a shared-FS staging dir. diff --git a/.agents/skills/evaluation/references/quantization-benchmarks.md b/.agents/skills/evaluation/references/quantization-benchmarks.md index 9b39fac806f..755d78a349a 100644 --- a/.agents/skills/evaluation/references/quantization-benchmarks.md +++ b/.agents/skills/evaluation/references/quantization-benchmarks.md @@ -3,15 +3,20 @@ When evaluating a quantized checkpoint, prioritize benchmarks that are sensitive to precision loss. The Artificial Analysis (AA) Index v2 suite under `recipes/tasks/aa/` is the default set for quantized-checkpoint validation. +**GDPVal** (`recipes/tasks/aa_gym/gdpval.md`) is also part of the AA suite, but a +different harness (NeMo Gym) — it runs as a **separate standalone config**, never +merged into the `aa/` multi-task list. **Scope rule:** - **Default quant validation** (when the user just says "evaluate this - quantized checkpoint"): use the AA suite plus the three always-include - benchmarks at `recipes/tasks/*.md` (MMLU-Pro, AIME 2025, LiveCodeBench). + quantized checkpoint"): use the AA suite — the `aa/` tasks **plus a standalone + GDPVal config** — plus the three always-include benchmarks at + `recipes/tasks/*.md` (MMLU-Pro, AIME 2025, LiveCodeBench). - **Explicit AA request** ("AA" / "Artificial Analysis" / "AA Index v2"): - use **only** `recipes/tasks/aa/`. Do not add the three always-include - tasks unless the user asks. See the callout at the bottom of this file. + use the `aa/` tasks **and** a companion standalone GDPVal config. Do not add + the three always-include tasks unless the user asks. See the callout at the + bottom of this file. ## Available task recipes @@ -28,6 +33,7 @@ to precision loss. The Artificial Analysis (AA) Index v2 suite under | `tasks/aa/mmmu_pro.md` | MMMU-Pro | Multimodal reasoning | VLM-only; usually Low/Medium when only the LLM is quantized (vision encoder/adapter typically stay BF16) | | `tasks/aa/tau2_bench_telecom.md` | Tau2-Bench Telecom | Agentic tool use (user-simulator + judge) | Medium-high — tool-call JSON is brittle, but user-sim + judge variance often dominates the signal | | `tasks/aa/omniscience.md` | AA-Omniscience | Knowledge reliability (`ns_omniscience`, nemo-skills, `num_repeats: 10`) — correct vs hallucinate vs abstain on obscure facts, judge-scored | Medium — measures the hallucination/abstention balance; aggressive precision loss can erode factual recall and shift the omni-index | +| `tasks/aa_gym/gdpval.md` | GDPVal (`nemo_gym` Stirrup agent, **standalone config**) | Agentic office/PDF deliverables in an Apptainer code-exec sandbox, pairwise/rubric judge | High — long-horizon agentic reasoning + code + judge; precision loss compounds across many turns. **Heaviest task**: multi-hour, often multi-node, needs the SIF sandbox + judge. Runs as its own config, never in the `aa/` list | ## Recommended sets by use case @@ -35,15 +41,17 @@ to precision loss. The Artificial Analysis (AA) Index v2 suite under |----------|-----------| | Quick sanity check | GPQA | | Standard quant validation (text LLM) | GPQA, SciCode, LCR | -| AA / Artificial Analysis suite (text LLM) | All `tasks/aa/` text tasks: GPQA, HLE, LCR, SciCode, IFBench, Tau2-Bench Telecom, AA-Omniscience | -| AA / Artificial Analysis suite (multimodal) | AA text suite + MMMU-Pro | +| AA / Artificial Analysis suite (text LLM) | All `tasks/aa/` text tasks: GPQA, HLE, LCR, SciCode, IFBench, Tau2-Bench Telecom, AA-Omniscience — **plus GDPVal** (`tasks/aa_gym/`, a separate standalone config) | +| AA / Artificial Analysis suite (multimodal) | AA text suite (incl. GDPVal) + MMMU-Pro | | Code-focused model | LiveCodeBench, SciCode | | Reasoning model | AIME 2025, GPQA, HLE | -> If the user asks for "AA" or "Artificial Analysis", generate **only** tasks -> under `recipes/tasks/aa/`. Do not silently add MMLU-Pro, AIME 2025, or -> LiveCodeBench — they live at `recipes/tasks/*.md` and are a separate -> always-include set. +> If the user asks for "AA" or "Artificial Analysis", generate the +> `recipes/tasks/aa/` tasks **plus a companion standalone GDPVal config** +> (`recipes/tasks/aa_gym/gdpval.md`) — GDPVal is part of the AA suite but a +> different harness, so it's its own config, never in the `aa/` `tasks` list. Do +> not silently add MMLU-Pro, AIME 2025, or LiveCodeBench — they live at +> `recipes/tasks/*.md` and are a separate always-include set. ## Notes for quantized-checkpoint runs @@ -62,6 +70,14 @@ to precision loss. The Artificial Analysis (AA) Index v2 suite under apples-to-apples comparison. - **IFBench** is the least quant-sensitive in the set but still useful as a regression check for aggressive formats (NVFP4, INT4-AWQ). +- **GDPVal** is part of the AA suite but the heaviest task and a separate + harness: it runs as its **own standalone `aa_gym` config** (never in the `aa/` + `tasks` list), needs the Apptainer SIF sandbox + judge, and is multi-hour / + often multi-node. Generate it alongside the `aa/` config; see + `recipes/tasks/aa_gym/gdpval.md` + `references/gym-gdpval.md`. Thinking mode is + mandatory (non-thinking loses ~86% of pairwise judgements). For low-variance + quant comparisons keep the golden `num_repeats: 2` — the example template + defaults to 1 to halve cost, so bump it back to 2 for quant validation. ## How to use