From 0c6a105b2a4d261f06ec86cdf6d7ebc087565b51 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Fri, 31 Jul 2026 21:42:40 +0000 Subject: [PATCH 01/16] [skill] evaluation: add GDPVal (NeMo Gym Stirrup agent) support GDPVal is an agentic AA benchmark: the Stirrup agent produces office/PDF deliverables in a per-task Apptainer code-exec sandbox, then a judge panel scores them. It runs on the 0.2.6 launcher as a `nemo_gym` task, but is standalone (one gym eval per config) and mechanically unlike the `aa/` nemo-skills tasks, so it gets its own branch in the skill. - recipes/tasks/aa_gym/gdpval.md: task recipe (standalone rule, scoring modes, canary, score extraction). - references/gym-gdpval.md: SIF sandbox, `_gym_prepare` machinery, deploy sizing, rubric-vs-comparison scoring, MLflow deliverables trap, failure modes, and the SIF<->Gym-version rebuild coupling. - recipes/examples/gym_gdpval/: self-contained SLURM + vLLM template plus the co-located `_gym_prepare.yaml` Hydra include. - scripts/gdpval-sif.sh: build-if-absent / reuse-if-present Apptainer SIF helper. Builds on the target cluster only (never copies across clusters), flock-guarded and atomic, driven by $GDPVAL_SIF_DIR. - SKILL.md / quantization-benchmarks.md: GDPVal is part of the AA suite but a different harness, so it is generated as a companion standalone config and never merged into the `aa/` task list. - env.example: TAVILY_API_KEY (agent web search) and GDPVAL_SIF_DIR. Validated end-to-end on an aarch64 GB300 cluster: deploy -> SIF build+exec -> gym head server -> 220 rollouts + deliverables -> judge scoring. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .agents/scripts/gdpval-sif.sh | 129 ++++++++ .agents/skills/evaluation/SKILL.md | 26 +- .agents/skills/evaluation/recipes/env.example | 11 + .../examples/gym_gdpval/_gym_prepare.yaml | 104 ++++++ .../gym_gdpval/example_gym_gdpval.yaml | 301 ++++++++++++++++++ .../evaluation/recipes/tasks/aa_gym/gdpval.md | 98 ++++++ .../evaluation/references/gym-gdpval.md | 198 ++++++++++++ .../references/quantization-benchmarks.md | 36 ++- 8 files changed, 892 insertions(+), 11 deletions(-) create mode 100755 .agents/scripts/gdpval-sif.sh create mode 100644 .agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml create mode 100644 .agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml create mode 100644 .agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md create mode 100644 .agents/skills/evaluation/references/gym-gdpval.md diff --git a/.agents/scripts/gdpval-sif.sh b/.agents/scripts/gdpval-sif.sh new file mode 100755 index 00000000000..b37518961fe --- /dev/null +++ b/.agents/scripts/gdpval-sif.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# gdpval-sif.sh — ensure the GDPVal Stirrup Apptainer SIF exists on THIS cluster. +# +# Build-if-absent, reuse-if-present. Self-contained: the SIF is built and reused +# on the TARGET cluster's own filesystem — this NEVER copies a SIF from another +# cluster. Idempotent, so it's safe to run before every `nel run`; a subsequent +# run reuses the built SIF instantly. +# +# Usage: +# .agents/scripts/gdpval-sif.sh [] [--commit ] [--force] +# Persistent path on the target cluster's shared FS. +# DEFAULTS to $GDPVAL_SIF_DIR (from .env) when omitted. A +# directory -> /python-3.12.gdpval.sif; a *.sif path +# is used verbatim. Bind-mount this SAME dir into the eval +# container at /gdpval/sif (see recipes/examples/gym_gdpval/). +# --commit NeMo Gym commit whose gdpval.def to build. Keep in sync +# with the config's install_on_the_fly.commit. +# --force Rebuild even if the SIF already exists. +# +# Requires `apptainer` (or `singularity`) on PATH with unprivileged/fakeroot +# build support, plus network egress to GitHub/base image. Run on a node that has +# it — a login node, or (preferred for the ~30-min build) the CPU partition: +# srun -p cpu -t 01:00:00 --pty \ +# .agents/scripts/gdpval-sif.sh /lustre/<...>/gdpval/sif +# +# Env overrides: GDPVAL_GYM_COMMIT, GDPVAL_SIF_NAME, APPTAINER_BIN. +set -euo pipefail + +# Keep GDPVAL_GYM_COMMIT in sync with install_on_the_fly.commit in the config. +GDPVAL_GYM_COMMIT="${GDPVAL_GYM_COMMIT:-2502893977e9e9af84adc1fa8d38c9314208d3ee}" # pragma: allowlist secret +GDPVAL_SIF_NAME="${GDPVAL_SIF_NAME:-python-3.12.gdpval.sif}" +APPTAINER_BIN="${APPTAINER_BIN:-}" + +_log() { printf '\033[2m %s\033[0m\n' "$*" >&2; } +_die() { printf '\033[31mgdpval-sif: %s\033[0m\n' "$*" >&2; exit 1; } +_usage() { sed -n '/^# gdpval-sif\.sh/,/^set -euo/p' "$0" | sed 's/^# \{0,1\}//; /^set -euo/d'; } + +# --- parse args --- +target=""; force=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --commit) GDPVAL_GYM_COMMIT="${2:?--commit needs a value}"; shift 2 ;; + --force) force=1; shift ;; + -h|--help) _usage; exit 0 ;; + -*) _die "unknown flag: $1 (see --help)" ;; + *) [[ -z "$target" ]] || _die "unexpected extra arg: $1"; target="$1"; shift ;; + esac +done +# Default to $GDPVAL_SIF_DIR (.env) when no path is given, so agents run it hands-free. +target="${target:-${GDPVAL_SIF_DIR:-}}" +[[ -n "$target" ]] || { _usage; _die "no path given and GDPVAL_SIF_DIR is unset — pass a dir or set GDPVAL_SIF_DIR (see recipes/env.example)"; } + +# --- resolve dir vs *.sif --- +if [[ "$target" == *.sif ]]; then + sif="$target"; sif_dir="$(dirname "$target")" +else + sif_dir="$target"; sif="$sif_dir/$GDPVAL_SIF_NAME" +fi +mkdir -p "$sif_dir" || _die "cannot create SIF dir: $sif_dir" + +# --- reuse if present --- +if [[ -f "$sif" && "$force" -eq 0 ]]; then + _log "reusing existing SIF (no rebuild): $sif" + echo "$sif"; exit 0 +fi + +# --- locate apptainer/singularity --- +if [[ -z "$APPTAINER_BIN" ]]; then + APPTAINER_BIN="$(command -v apptainer || command -v singularity || true)" +fi +[[ -n "$APPTAINER_BIN" ]] || _die "apptainer/singularity not found on PATH. Run on a node that has it \ +(e.g. 'module load apptainer', or inside the eval image). This script does NOT copy a SIF from another cluster." + +def_url="https://raw.githubusercontent.com/NVIDIA-NeMo/Gym/${GDPVAL_GYM_COMMIT}/responses_api_agents/stirrup_agent/containers/gdpval.def" +tmp="${sif_dir}/.build.$$.${GDPVAL_SIF_NAME}" +def_local="${sif_dir}/.gdpval.$$.def" +lock="${sif_dir}/.gdpval-sif.lock" + +# --- build under a flock (double-checked) so concurrent runs don't double-build --- +exec 9>"$lock" || _die "cannot open lock file: $lock" +_log "acquiring build lock ($lock) ..." +flock 9 +# Re-check inside the lock: another builder may have finished while we waited. +if [[ -f "$sif" && "$force" -eq 0 ]]; then + _log "another builder produced it: $sif" + echo "$sif"; exit 0 +fi + +_log "building GDPVal SIF (this can take ~20-40 min)" +_log " gym commit: ${GDPVAL_GYM_COMMIT}" +_log " def: ${def_url}" +_log " dest: ${sif}" +rm -f "$tmp" "$def_local" +# apptainer build cannot take a remote def URL as its source — fetch the def to a +# local file first, then build from it. +if command -v curl >/dev/null 2>&1; then curl -fsSL "$def_url" -o "$def_local" +else wget -qO "$def_local" "$def_url"; fi +[ -s "$def_local" ] || { rm -f "$def_local"; _die "failed to download def from $def_url"; } +# Prefer --fakeroot (needs an /etc/subuid entry for the build user); fall back to an +# unprivileged build where fakeroot is unavailable. +if "$APPTAINER_BIN" build --fakeroot "$tmp" "$def_local"; then + : +elif "$APPTAINER_BIN" build "$tmp" "$def_local"; then + _log "built without --fakeroot (unprivileged mode)" +else + rm -f "$tmp" "$def_local" + _die "apptainer build failed (see output above)." +fi +rm -f "$def_local" + +# Atomic publish: a partial build never looks complete. +mv -f "$tmp" "$sif" || { rm -f "$tmp"; _die "failed to move built SIF into place: $sif"; } +_log "done: $sif" +echo "$sif" diff --git a/.agents/skills/evaluation/SKILL.md b/.agents/skills/evaluation/SKILL.md index 6776315949d..a35771b0839 100644 --- a/.agents/skills/evaluation/SKILL.md +++ b/.agents/skills/evaluation/SKILL.md @@ -49,6 +49,29 @@ Steps 1–9 below are the 0.2.6 path — use them for everything else. --- +### GDPVal (NeMo Gym "Stirrup" agent) path — branch here too + +GDPVal **does** run on the 0.2.6 `nel` launcher (as a `nemo_gym` task, not +nel-next), so Steps 1–9 apply — but it is mechanically special and **standalone** +(one gym eval per config; never mix it with `aa/` tasks). If the user asks for +GDPVal: + +1. Read **`references/gym-gdpval.md`** (Apptainer SIF sandbox, `_gym_prepare.yaml` + machinery, deploy sizing, rubric-vs-comparison scoring, MLflow deliverables trap, + failure modes) + **`recipes/tasks/aa_gym/gdpval.md`**. +2. Start from the self-contained **`recipes/examples/gym_gdpval/`** dir — copy the + **whole dir** (the `_gym_prepare.yaml` include must travel next to the config). +3. Prerequisite: set `GDPVAL_SIF_DIR` in `.env`, then ensure the SIF exists with + `.agents/scripts/gdpval-sif.sh` (uses `$GDPVAL_SIF_DIR`; build-if-absent, + reuse-if-present, no cross-cluster copy). The config bind-mounts `$GDPVAL_SIF_DIR` + at exactly `/gdpval/sif/python-3.12.gdpval.sif`, or the agent silently runs + unsandboxed. `.env` needs `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, + `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR`, and `NEMO_EVALUATOR_TRUST_PRE_CMD=1` (the + config has a `pre_cmd`). Thinking mode is mandatory (non-thinking loses ~86%). +4. Dry-run → canary (`limit_samples=2`, verify the SIF sandbox + judge) → full. + +--- + ### Step 1 — Prerequisites Run `nel --version`; if missing, instruct `pip install nemo-evaluator-launcher`. If user has an existing config, skip to Step 8 (optionally review for `???` and quantization flags first). @@ -62,8 +85,9 @@ Run `nel --version`; if missing, instruct `pip install nemo-evaluator-launcher`. - AA Index v2 suite (default for quantized-checkpoint validation, see `references/quantization-benchmarks.md`): `recipes/tasks/aa/{gpqa_diamond,hle,lcr,scicode,ifbench,mmmu_pro,tau2_bench_telecom,omniscience}.md` - Optional: `recipes/tasks/mmlu_pro.md`, `recipes/tasks/aime_2025.md`, `recipes/tasks/livecodebench.md` - **nel-next only** (different evaluator — see the nel-next section below, NOT the 0.2.6 steps): shared reference `references/nel-next.md` + per-benchmark recipes `recipes/tasks/aa_next/{terminal_bench_2_1,swebench_verified}.md` (agentic). The `aa_next/` dir holds tasks that require nemo-evaluator-next (0.3.x); `aa/` is the 0.2.6 suite. +- **GDPVal (NeMo Gym / agentic)** — **part of the AA suite** but a 0.2.6 `nemo_gym` task on a different harness, so it's **standalone** (see the GDPVal branch above): recipe `recipes/tasks/aa_gym/gdpval.md` + shared reference `references/gym-gdpval.md` + self-contained example `recipes/examples/gym_gdpval/`. Generated as its **own config** from the example, **never merged into the `aa/` multi-task `tasks` list**. The `aa_gym/` dir holds the NeMo Gym Stirrup-agent tasks. -**AA rule:** If the user mentions "AA" / "Artificial Analysis", generate **only** tasks under `recipes/tasks/aa/`. Do not add MMLU-Pro, AIME 2025, or LiveCodeBench unless explicitly asked. +**AA rule:** If the user mentions "AA" / "Artificial Analysis", generate the `recipes/tasks/aa/` tasks (one multi-task config) **plus a companion standalone GDPVal config** (`recipes/tasks/aa_gym/gdpval.md`, via the GDPVal branch) — GDPVal is part of the AA suite but a different harness, so it's its own config, never added to the `aa/` `tasks` list. Do not add MMLU-Pro, AIME 2025, or LiveCodeBench unless explicitly asked. GDPVal is the heaviest AA task (standalone, multi-hour, needs the SIF sandbox + judge) — surface it and let the user opt out per run. **Shortcut path** (when task list is known up front, e.g. "run AA"): diff --git a/.agents/skills/evaluation/recipes/env.example b/.agents/skills/evaluation/recipes/env.example index 1e9f2e4750a..bf1d3515d1d 100644 --- a/.agents/skills/evaluation/recipes/env.example +++ b/.agents/skills/evaluation/recipes/env.example @@ -37,8 +37,19 @@ NEMO_EVALUATOR_TRUST_PRE_CMD=1 # /v1 base; tau2-bench needs the full /v1/chat/completions. # HLE + AA-LCR + AA-Omniscience judges (ns_hle_aa, ns_aa_lcr, ns_omniscience) — shared inference host +# GDPVal (nemo_gym) also reuses INFERENCE_JUDGE_URL for its pairwise judge. # INFERENCE_JUDGE_URL=https:///v1 +# GDPVal (nemo_gym Stirrup agent) — agent web search. Secret; exported and read +# by the harness. See recipes/tasks/aa_gym/gdpval.md + references/gym-gdpval.md. +# TAVILY_API_KEY= + +# GDPVal (nemo_gym) — persistent Apptainer SIF cache dir on the TARGET cluster's +# shared FS (a path, not a secret). .agents/scripts/gdpval-sif.sh builds the SIF +# here if absent and reuses it otherwise; the config bind-mounts this dir at +# /gdpval/sif. Convention: a per-user .cache dir. +# GDPVAL_SIF_DIR=//.cache/gdpval/sif + # Tau2 (tau2_bench_telecom) — judger + user-simulator model_ids are hardcoded in # the recipe; only the shared endpoint URL comes from here # TAU2_ENDPOINT_URL=https:///v1/chat/completions # user + judger diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml new file mode 100644 index 00000000000..9239d073761 --- /dev/null +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Shared prepare/teardown snippets — include via: defaults: [_gym_prepare, _self_] +# +# Compensates for the eval image's deployment-oriented packaging when run +# standalone. Interpolate into each benchmark's command: +# ${gym_prepare.prepare} at the start (activate venv, checkout pin, repair venvs) +# ${gym_prepare.run} as the last line (data prep + rollout collection run in +# its own session, then process-group reaped so the run +# finalizes — see the run: comment for why) +# Remove once the eval image ships complete, ray-consistent venvs and Gym's +# shutdown reaps server process groups. + +gym_prepare: + # Common command preamble shared by every benchmark: activate the baked Gym + # venv, optionally checkout the install_on_the_fly pin (only when /opt/Gym is a + # git repo; baked images aren't), then repair the image's incomplete per-server + # venvs. The repair: (1) rewrites each component requirements.txt — drops the + # editable "-e nemo-gym[dev]" line (it forces a ray>=2.55.1 re-resolve vs the + # image's pinned ray 2.49.2, breaking venv-less servers like arena_judge) and + # ensures ray==2.49.2 + tqdm; (2) installs the fixed requirements into each + # baked (skeleton) sub-venv; (3) fronts the main venv on PYTHONPATH so nemo_gym + # + framework deps resolve for server processes. + # NOTE: avoid bash ${VAR} here — OmegaConf parses ${...}; $(...) / $r / $v are fine. + # NOTE: every step is guarded (|| true / -q grep) so it is safe under set -e. + prepare: |- + set -ex + cd /opt/Gym + export UV_CACHE_DIR=/opt/cache/uv + source .venv/bin/activate + # install_on_the_fly: checkout the pin when /opt/Gym is a git repo; some + # images bake Gym at a fixed version (not a git repo) — use it as-is. + if [ -d .git ]; then + git remote add oss_pin "{{config.params.extra.nemo_gym.install_on_the_fly.url}}" 2>/dev/null || true + git fetch oss_pin + git checkout "{{config.params.extra.nemo_gym.install_on_the_fly.commit}}" + echo "=== NeMo Gym commit ===" && git rev-parse HEAD + else + echo "=== /opt/Gym is not a git repo; using baked-in Gym version ===" + fi + for r in /opt/Gym/responses_api_models/*/requirements.txt \ + /opt/Gym/responses_api_agents/*/requirements.txt \ + /opt/Gym/resources_servers/*/requirements.txt; do + [ -f "$r" ] || continue + grep -vE '^[[:space:]]*-e ' "$r" > "$r.fixed" || true + grep -qiE '^ray([<>=[]|$)' "$r.fixed" 2>/dev/null || echo 'ray==2.49.2' >> "$r.fixed" + grep -qiE '^tqdm' "$r.fixed" 2>/dev/null || echo 'tqdm' >> "$r.fixed" + mv "$r.fixed" "$r" 2>/dev/null || true + done + for v in /opt/Gym/responses_api_models/*/.venv \ + /opt/Gym/responses_api_agents/*/.venv \ + /opt/Gym/resources_servers/*/.venv; do + [ -d "$v" ] || continue + d="$(dirname "$v")" + [ -f "$d/requirements.txt" ] && uv pip install --python "$v/bin/python" -q -r "$d/requirements.txt" || true + done + export PYTHONPATH="/opt/Gym:$(/opt/Gym/.venv/bin/python -c 'import site; print(site.getsitepackages()[0])')" + + # Data prep + rollout collection. Interpolate as the LAST line of each + # benchmark's command via ${gym_prepare.run} (after ${gym_prepare.prepare} and + # any benchmark-specific prep). + # + # Rollout collection is run in its OWN session (setsid) so its entire process + # tree — servers + their multiprocessing pools + Ray workers — can be reaped by + # process group. Why: ng_e2e's own cleanup (cli.py shutdown) only SIGINTs/SIGKILLs + # the tracked server PIDs after a 1s grace; it does NOT reap each server's child + # tree, so pools/Ray workers orphan and re-parent to the launcher (which is + # blocked reading our stdout) — they hold stdout open and the run never finalizes. + # (Invisible in the deployment flow because the whole node is discarded; only bites + # the deployment-free inline path here.) The inner shell records the session PGID + # (its own $$, before exec) so we can target it; real exit code is captured from + # wait and propagated via exit. + # + # Ray ALSO daemonizes gcs_server/raylet into their own session, so they escape the + # process-group reap; if left running they keep the launcher's stdout open and the + # run hangs in post-eval. So after the group reap we additionally stop Ray and kill + # its daemons by name (safe: matches only Ray daemons, not the launcher's python). + # Remove once Gym's shutdown reaps server process groups + Ray. + run: |- + ng_prepare_benchmark {{config.params.extra.nemo_gym.data_prep_params}} {{config.params.extra.nemo_gym.common_params}} + setsid --wait bash -c 'echo $$ > /tmp/gym_eval_pgid; exec ng_e2e_collect_rollouts {{config.params.extra.nemo_gym.collect_rollout_params}} {{config.params.extra.nemo_gym.common_params}}' & + __ev=$! + __rc=0; wait "$__ev" || __rc=$? + echo "Evaluator Gym finished!" + __pg=$(cat /tmp/gym_eval_pgid 2>/dev/null || echo) + [ -n "$__pg" ] && kill -9 -"$__pg" 2>/dev/null || true + timeout 60 ray stop --force >/dev/null 2>&1 || true + pkill -9 -f raylet >/dev/null 2>&1 || true + pkill -9 -f gcs_server >/dev/null 2>&1 || true + pkill -9 -f plasma_store >/dev/null 2>&1 || true + exit $__rc diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml new file mode 100644 index 00000000000..1860a743d03 --- /dev/null +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -0,0 +1,301 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================= +# Example: GDPVal (NeMo Gym "Stirrup" agent) — single-task gym eval template. +# +# GDPVal is an AGENTIC benchmark: the Stirrup agent produces office/PDF +# deliverables inside a per-task Apptainer code-exec sandbox, then a pairwise/ +# rubric judge (Gemini 3.1 Pro) scores them. It runs on the 0.2.6 `nel` launcher +# as a `nemo_gym` task (NOT nel-next), but it is STANDALONE — one gym eval per +# config, no other tasks. Read recipes/tasks/aa_gym/gdpval.md and +# references/gym-gdpval.md before editing this file. +# +# This template SELF-DEPLOYS a (quantized) checkpoint via vLLM on ONE SLURM node +# and runs GDPVal against it. For the full 220-task run of a large MoE you will +# likely need multi-node — see references/gym-gdpval.md (deployment sizing). +# +# PREREQUISITES (see references/gym-gdpval.md): +# 1. Set GDPVAL_SIF_DIR in .env (persistent SIF cache dir on this cluster), then +# ensure the SIF exists (build-if-absent, reuse-if-present — never copied from +# another cluster): +# srun -p cpu -t 01:00:00 --pty .agents/scripts/gdpval-sif.sh # uses $GDPVAL_SIF_DIR +# The mount below binds $GDPVAL_SIF_DIR at /gdpval/sif so the SIF lands at EXACTLY +# /gdpval/sif/python-3.13.gdpval.sif (matches GDPVAL_CONTAINER_PATH below). +# Without it, the agent SILENTLY falls back to non-sandboxed local exec. +# 2. .env has HF_TOKEN, INFERENCE_API_KEY (judge auth), TAVILY_API_KEY (agent web +# search), INFERENCE_JUDGE_URL (judge host), and GDPVAL_SIF_DIR (item 1). See +# recipes/env.example. +# 3. This file's `defaults` include `_gym_prepare` — the _gym_prepare.yaml in +# THIS directory MUST travel next to this config (Hydra resolves it +# relative to the config dir). Copy the whole gym_gdpval/ dir to your +# workspace, don't copy the yaml alone. +# +# Canary (validates SIF sandbox + judge + gym plumbing on a couple of tasks): +# nel run --config example_gym_gdpval.yaml --env-file .env \ +# -o ++evaluation.nemo_evaluator_config.config.params.limit_samples=2 +# ============================================================================= +defaults: + # slurm/default works anywhere; if your install ships a predefined + # internal/slurm/ config, prefer it (pre-fills hostname/partition/ + # gres — see SKILL.md Step 4). + - execution: slurm/default + - deployment: vllm + - _gym_prepare # provides ${gym_prepare.prepare} / ${gym_prepare.run} + - _self_ + +# GDPVal scoring mode. Override at runtime: -o gdpval.reward_mode=comparison +# rubric — standalone LLM-judge scoring; NO reference deliverables needed. (default) +# comparison — pairwise scoring vs a reference model's deliverables; also +# mount the ref dir at /gdpval/refs/test_ref (see mounts) and set +# reference_elo. See references/gym-gdpval.md for the two-step flow. +gdpval: + reward_mode: rubric + reference_elo: 1290 # comparison mode only — ELO of the reference model + +# GDPVal pairwise judge. base_url is config (from .env), not a secret, so no +# export needed; only api_key (INFERENCE_API_KEY) is exported and read by the +# harness. Keep the judge fixed across comparable runs. +gdpval_judge: + base_url: # from .env (/v1 base); shared inference host + model: gcp/google/gemini-3.1-pro-preview # Gemini 3.1 Pro; use an equivalent on your endpoint if needed + api_key: INFERENCE_API_KEY # doc only — the gym injects the VALUE as $INFERENCE_API_KEY + # in common_params (NOT ${gdpval_judge.api_key}, which passes + # the literal NAME "INFERENCE_API_KEY" → judge 500). + +cluster: + sbatch_comment: '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"480","reason":"benchmarking","description":"Eval benchmark low GPU utilization"}}' + +execution: + hostname: ??? + username: ${oc.env:USER} + account: ??? + output_dir: ??? # absolute host path; keeps deliverables + response cache across resumes + walltime: "04:00:00" + # gres: a predefined internal/slurm/ config sets this. On slurm/default + # it's gpu:8 — set to the node's GPU count (match --tensor/--data-parallel-size) + # or sbatch fails "Requested node configuration is not available". + mounts: + # mount_home ALWAYS false — see example_eval.yaml / SKILL Step 4 for why. + mount_home: false + deployment: + # Real HF cache -> /hf-cache (paired with HF_HOME below). + : /hf-cache + evaluation: + : /hf-cache + : /cache/uv + # GDPVal Stirrup SIF dir — substitute the literal $GDPVAL_SIF_DIR value (.env), + # the persistent shared-FS dir gdpval-sif.sh builds python-3.13.gdpval.sif into + # (build-if-absent/reuse). Use the literal path here, NOT ${oc.env:...}: mount + # KEYS are not interpolated (same rule as the judge URLs). Mounting the DIR lands + # the SIF at /gdpval/sif/python-3.13.gdpval.sif == GDPVAL_CONTAINER_PATH. + : /gdpval/sif + # Writable shared-FS staging for ref files. Node-local /tmp breaks multi-node Ray. + : /gdpval_ref_files + # comparison mode ONLY — reference deliverables dir (drop for default rubric mode): + # : /gdpval/refs/test_ref + auto_export: # REQUIRED trigger for MLflow upload (see example_eval.yaml). + destinations: + - mlflow + # Auto-export is a separate CPU-only sbatch; GPU-only partitions reject it. + cpu_partition: ??? + +deployment: + env_vars: + HF_TOKEN: host:HF_TOKEN + HF_HOME: lit:/hf-cache + # vLLM backend toggles go HERE (not in command). Uncomment per model card — + # e.g. NVFP4 MoE on Blackwell needs FlashInfer FP4 kernels: + # VLLM_USE_FLASHINFER_MOE_FP4: lit:1 + # VLLM_FLASHINFER_MOE_BACKEND: lit:throughput + checkpoint_path: ??? # prefer a path already on the cluster over hf_model_handle + hf_model_handle: + served_model_name: ??? + image: vllm/vllm-openai:v0.19.1 # bump to the EXACT model's recipes.vllm.ai minimum (SKILL Step 3) + # GDPVal REQUIRES thinking mode from the policy (non-thinking loses ~86% of + # pairwise judgements). Serve with the model's --reasoning-parser so vLLM emits + # a separate reasoning channel; thinking is forced on via the adapter_config + # chat_template_kwargs below. Add --enable-expert-parallel for MoE, and + # --trust-remote-code for custom-code models. + # After filling `parallelism`, append --max-num-seqs N (N = ceil(parallelism / data_parallel_size)). + command: >- + vllm serve /checkpoint + --served-model-name ${deployment.served_model_name} + --host 0.0.0.0 + --port ${deployment.port} + --tensor-parallel-size 1 + --data-parallel-size 1 + --max-model-len 131072 + --reasoning-parser + --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 128}' + --max-num-batched-tokens 8192 + --enable-chunked-prefill + +evaluation: + env_vars: + HF_TOKEN: host:HF_TOKEN + HF_HOME: lit:/hf-cache + DUMMY_API_KEY: lit:dummy # the deployed vLLM endpoint's (policy) key + INFERENCE_API_KEY: host:INFERENCE_API_KEY # GDPVal judge auth (shared inference host) + TAVILY_API_KEY: host:TAVILY_API_KEY # Stirrup agent web search + UV_CACHE_DIR: lit:/cache/uv + # Apptainer SIF for the Stirrup per-task code-exec sandbox (must match the mount above). + GDPVAL_CONTAINER_PATH: lit:/gdpval/sif/python-3.13.gdpval.sif + # Shared-FS staging for ref files; node-local /tmp breaks multi-node Ray. + GDPVAL_REF_FILES_DIR: lit:/gdpval_ref_files + # Deliverables land under /results (already mounted) so they persist. The + # "*cache*" basename matches the mlflow exporter's exclusion, so the (large) + # deliverables are NOT auto-uploaded — they stay on disk for inspection. + # Drop "_cache" if you WANT them uploaded as artifacts. + PERSIST_DELIVERABLES_DIR: lit:/results/gdpval/deliverables_cache + # Stirrup agent turn cap (optional; default 100). + # GDPVAL_MAX_TURNS: lit:100 + NEL_INVOCATION_ID: runtime:NEL_INVOCATION_ID + # Installs apptainer + squashfuse into the eval container (needs + # NEMO_EVALUATOR_TRUST_PRE_CMD=1 in the launching shell). See references/gym-gdpval.md. + pre_cmd: | + set -ex + apt-get update -qq + wget -q -O /tmp/apptainer.deb \ + https://github.com/apptainer/apptainer/releases/download/v1.4.2/apptainer_1.4.2_amd64.deb + apt-get install -y -qq /tmp/apptainer.deb && rm /tmp/apptainer.deb + apt-get install -y -qq squashfuse fuse3 || apt-get install -y -qq squashfuse fuse || true + mkdir -p /usr/local/var/apptainer/mnt/session + nemo_evaluator_config: + config: + params: + temperature: 1.0 + top_p: 0.95 + parallelism: 16384 # gym-internal concurrency, NOT a model-server cap + request_timeout: 36000 # gym rollouts are long-running + max_retries: 10 + target: + api_endpoint: + api_key_name: DUMMY_API_KEY + adapter_config: + use_system_prompt: false + process_reasoning_traces: true + params_to_remove: + - max_tokens + - max_completion_tokens + # Thinking mode MUST be on — non-thinking loses ~86% of pairwise + # judgements. Keep the toggle key your model family uses (enable_thinking + # -> Qwen3.x/GLM; thinking -> Kimi/DeepSeek); see SKILL "Reasoning adapter config". + params_to_add: + chat_template_kwargs: + enable_thinking: true + skip_special_tokens: false + use_caching: false + use_progress_tracking: true + tracking_requests_stats: true + log_failed_requests: true + use_request_logging: true + max_logged_requests: 10 + use_response_logging: true + max_logged_responses: 10 + # STANDALONE: exactly one gym task. Do NOT add other tasks to this list. + tasks: + - name: nemo_gym + container: nvcr.io/nvidia/eval-factory/nemo-gym:26.05 # pin a verified tag + nemo_evaluator_config: + config: + params: + extra: + nemo_gym: + install_on_the_fly: + url: https://github.com/NVIDIA-NeMo/Gym + # Gym version. BUMPING THIS REQUIRES REBUILDING THE SIF from the matching + # commit — gdpval.def (the sandbox) is versioned with the gym repo, and an + # old SIF + new gym silently degrades deliverables. Rebuild with + # `gdpval-sif.sh --commit ` to a new name + repoint GDPVAL_CONTAINER_PATH. + # See references/gym-gdpval.md "Rebuild the SIF when the Gym version changes". + # Current golden pin (updated GDPVal task-sampling algo — matters for + # multistage stage-1 selection, so keep it aligned with the goldens). + # Its gdpval.def is byte-identical to 049b1fd0…, so the python-3.13 SIF + # built from either commit is valid — no rebuild when moving between them. + # Do NOT go back to 2502893977… — on that commit the gym head server + # fails to bind its fixed port 11000 and the run hangs forever polling + # "Head server is not up yet". + commit: dd41196f620f2af99947d776cbe5da9439d2a08d # pragma: allowlist secret + command: | + ${gym_prepare.prepare} + + # Writable staging dir for ref files (bind-mounted, see execution). + mkdir -p /gdpval_ref_files + + # num_repeats: this template defaults to 1 (halves cost). The + # reviewed GOLDEN uses num_repeats=2 (220 tasks x 2 = 440 rollouts); + # for golden-comparable / REPORTED scores, DELETE the sed line below + # to keep the checked-in default of 2. num_repeats can NOT be set via + # a ++ override (OmegaConf ListConfig merge error) — patch the file. + sed -i 's/num_repeats: 2$/num_repeats: 1/' benchmarks/gdpval/config.yaml + + ${gym_prepare.run} + data_prep_params: >- + "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" + +hf_token=$HF_TOKEN + ++use_cached_prepared_benchmarks=true + collect_rollout_params: >- + "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" + ++port_range_low=63000 + ++port_range_high=64000 + ++global_aiohttp_connector_limit_per_host={{config.params.parallelism}} + ++uv_cache_dir=$UV_CACHE_DIR + ++skip_venv_if_present=true + ++output_jsonl_fpath={{config.output_dir}}/evaluator_rollouts.jsonl + ++nemo_gym_log_dir={{config.output_dir}}/nemo_gym_logs + ++overwrite_metrics_conflicts=true + ++split=benchmark + ++resume_from_cache=true + ++reuse_existing_data_preparation=true + ++upload_rollouts_to_wandb=false + ++responses_create_params.temperature={{config.params.temperature}} + ++responses_create_params.top_p={{config.params.top_p}} + common_params: >- + ++use_absolute_ip=true + ++policy_base_url={{target.api_endpoint.url}} + ++policy_api_key=$DUMMY_API_KEY + ++policy_model_name={{target.api_endpoint.model_id}} + ++gdpval_judge_model.responses_api_models.openai_model.openai_base_url=${gdpval_judge.base_url} + ++gdpval_judge_model.responses_api_models.openai_model.openai_model=${gdpval_judge.model} + ++gdpval_judge_model.responses_api_models.openai_model.openai_api_key=$INFERENCE_API_KEY + ++gdpval_judge_model.responses_api_models.openai_model.max_concurrent_requests=10 + ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.concurrency=220 + ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.tavily_api_key=$TAVILY_API_KEY + ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.agent_max_turns=${oc.env:GDPVAL_MAX_TURNS,250} + ++gdpval_resources_server.resources_servers.gdpval.reward_mode=${gdpval.reward_mode} + ++gdpval_resources_server.resources_servers.gdpval.reference_deliverables_dir=/gdpval/refs/test_ref + ++gdpval_resources_server.resources_servers.gdpval.reference_elo=${gdpval.reference_elo} + ++gdpval_resources_server.resources_servers.gdpval.persist_raw_judge_responses=true + ++gdpval_resources_server.resources_servers.gdpval.preconvert_max_concurrent=30 + ++gdpval_resources_server.resources_servers.gdpval.preconvert_office_to_pdf=true + ++gdpval_resources_server.resources_servers.gdpval.judge_responses_create_params_overrides.model=${gdpval_judge.model} + +export: + # LITERAL values only (auto_export resolves this block at submit time in a scope + # without deployment/evaluation nodes; ${oc.env:...} is fine). Keep the sampling + # tags EQUAL to evaluation params above — they're the only MLflow record of them. + mlflow: + tracking_uri: ${oc.env:MLFLOW_TRACKING_URI} # from modelopttools:eval-config + experiment_name: ${oc.env:USER}/CHANGEME-served-model-name + description: 'CHANGEME-served-model-name | GDPVal rubric | T=1.0, top_p=0.95, num_repeats=1' + log_logs: true + only_required: false + tags: + framework: vllm + model: CHANGEME-served-model-name + benchmark: nemo_gym.gdpval + temperature: '1.0' + top_p: '0.95' diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md new file mode 100644 index 00000000000..9ca8118dacc --- /dev/null +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -0,0 +1,98 @@ +# GDPVal (NeMo Gym "Stirrup" agent) + +## Task Details + +- Reference: `references/gym-gdpval.md` (SIF build, gym machinery, deploy sizing, + scoring modes, failure modes) — **read it before editing a GDPVal config.** +- Upstream README: + + +GDPVal is an **agentic** benchmark: the Stirrup agent produces office/PDF +deliverables inside a per-task Apptainer code-exec sandbox, then a pairwise/rubric +judge (**Gemini 3.1 Pro**) scores them. It is the most resource-intensive benchmark +in the suite — **220 tasks**, `num_repeats=2` in the reviewed golden (= 440 +rollouts), each rollout using 4 judge trials. + +It runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task (NOT nel-next), so +Steps 1–9 apply — but with the branch differences below. + +## What makes GDPVal different (do NOT treat it as a normal `aa/` task) + +- **Standalone — one gym eval per config.** Never add GDPVal to a multi-task + `evaluation.tasks` list, and never add other tasks to a GDPVal config. +- **Apptainer SIF sandbox (self-contained).** Set `GDPVAL_SIF_DIR` in `.env`, then + run `.agents/scripts/gdpval-sif.sh` (uses `$GDPVAL_SIF_DIR`) — it **builds if + absent, reuses if present**, and never copies from another cluster. The config + bind-mounts `$GDPVAL_SIF_DIR` at **exactly** `/gdpval/sif/python-3.12.gdpval.sif` + (matches `GDPVAL_CONTAINER_PATH`). Missing/mispathed → the agent **silently** runs + code-exec unsandboxed and results are not comparable. Details in `references/gym-gdpval.md`. +- **Thinking mode is mandatory.** Non-thinking loses ~86% of pairwise judgements. + Serve the policy with its `--reasoning-parser` and force thinking on via the + adapter `chat_template_kwargs` (see the example). +- **Judge + web search + gym plumbing.** Needs `INFERENCE_API_KEY` (judge auth), + `TAVILY_API_KEY` (agent web search), `INFERENCE_JUDGE_URL` (judge host, from + `.env`), a `pre_cmd` that installs apptainer/squashfuse, and the co-located + `_gym_prepare.yaml` include. + +## Scoring modes + +Set `gdpval.reward_mode` (override: `-o gdpval.reward_mode=comparison`): + +- **`rubric`** (default) — standalone LLM-judge scoring; **no reference + deliverables** needed. Use this unless you specifically need pairwise-vs-baseline. +- **`comparison`** — pairwise scoring vs a reference model's deliverables. Also + mount the ref dir at `/gdpval/refs/test_ref` and set `gdpval.reference_elo` + (golden uses Kimi-K2.5-Thinking refs, elo=1290). Two-step baseline→comparison + flow in `references/gym-gdpval.md`. + +## Config + +**Do not copy a fragment into another config.** GDPVal is standalone — start from +the self-contained example and edit it: + +```text +recipes/examples/gym_gdpval/ + example_gym_gdpval.yaml # SLURM + single-node vLLM self-deploy template + _gym_prepare.yaml # co-located Hydra include (${gym_prepare.*}); travels with the yaml +``` + +Copy the **whole `gym_gdpval/` directory** to your workspace (the `- _gym_prepare` +default resolves relative to the config dir — copying the yaml alone breaks it). + +- **num_repeats:** the example defaults to **1** (halves cost); the reviewed golden + uses **2**. For golden-comparable / reported scores, delete the `sed` line in the + task `command:` to keep 2. It can **not** be set via a `++` override (OmegaConf + `ListConfig` merge error) — the file is patched with `sed`. +- **SIF ↔ Gym version (rebuild on bump):** the SIF is built from `gdpval.def` at + `install_on_the_fly.commit`. **If you change that commit, rebuild the SIF** with a + matching `gdpval-sif.sh --commit ` (to a new version-tagged filename, then + repoint `GDPVAL_CONTAINER_PATH`) — the def's base image + package stack change + across commits, and running a new gym with an old SIF makes the agent's generated + code fail imports in the sandbox → silently degraded deliverables. See + `references/gym-gdpval.md` → "Rebuild the SIF when the Gym version changes". +- **Deployment:** single-node vLLM in the template; the full 220×2 run of a large + MoE typically needs multi-node — see `references/gym-gdpval.md`. +- Required `.env` keys: `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, + `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR` (see `recipes/env.example`). + `NEMO_EVALUATOR_TRUST_PRE_CMD=1` is needed because the config has a `pre_cmd`. + +## Canary + +Validate the SIF sandbox + judge + gym plumbing on a couple of tasks before the +full run: + +```bash +nel run --config example_gym_gdpval.yaml --env-file .env \ + -o ++evaluation.nemo_evaluator_config.config.params.limit_samples=2 +``` + +Inspect logs for the SIF fallback warning, judge auth/429s, and Ray/gym shutdown +hangs (see `references/gym-gdpval.md` → failure modes). + +## Score Extraction + +GDPVal reports a **win-rate / ELO** against the reference (comparison mode) or a +rubric score (rubric mode); the run's aggregate metric is logged under the +`nemo_gym.gdpval` benchmark in MLflow. Read the run's +`{output_dir}/evaluator_rollouts.jsonl` + `nemo_gym_logs/` for per-task rewards, +and the persisted judge responses under `PERSIST_DELIVERABLES_DIR`. diff --git a/.agents/skills/evaluation/references/gym-gdpval.md b/.agents/skills/evaluation/references/gym-gdpval.md new file mode 100644 index 00000000000..61e25038267 --- /dev/null +++ b/.agents/skills/evaluation/references/gym-gdpval.md @@ -0,0 +1,198 @@ +# GDPVal (NeMo Gym "Stirrup" agent) — reference for the gym / agentic path + +GDPVal runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task, but it is +mechanically unlike the `aa/` nemo-skills tasks: the Stirrup agent produces +office/PDF **deliverables** in a per-task **Apptainer** code-exec sandbox, a +pairwise/rubric **judge** (Gemini 3.1 Pro) scores them, and NeMo Gym is pulled and +run **inline in the eval container** (`install_on_the_fly`) via `ng_prepare_benchmark` ++ `ng_e2e_collect_rollouts`. This file is the shared machinery; the config template +is `recipes/examples/gym_gdpval/` and the per-task pointer is +`recipes/tasks/aa_gym/gdpval.md`. + +## Where each piece runs + +| Component | Where | +|---|---| +| Policy model (under test) | your self-deployed vLLM endpoint (SLURM GPU node) — or an external endpoint | +| NeMo Gym + Stirrup agent orchestration | inside the **eval** container (`nemo_gym` task), pulled via `install_on_the_fly` | +| Per-task code-exec | **Apptainer SIF** launched by the agent inside the eval container | +| Judge (pairwise/rubric) | external OpenAI-compatible endpoint (`gdpval_judge`, e.g. Gemini 3.1 Pro) | +| Agent web search | Tavily (`TAVILY_API_KEY`) | + +## Apptainer SIF sandbox (self-contained: build-if-absent, reuse-if-present) + +The Stirrup agent runs each task's generated code in an Apptainer SIF, bind-mounted +into the eval container at **exactly** `/gdpval/sif/python-3.12.gdpval.sif` (the path +`GDPVAL_CONTAINER_PATH` points at). If it's missing or at a different path, the agent +**silently falls back to non-sandboxed local exec** — the run "succeeds" but the +numbers aren't comparable, so verify the SIF at canary. + +**The skill builds the SIF on the target cluster — it never copies one from +another cluster.** Ship the SIF via the idempotent helper, which builds it if +absent and reuses it if already present: + +```bash +# GDPVAL_SIF_DIR (.env) — persistent shared-FS dir on the TARGET cluster; the config +# bind-mounts this same dir at /gdpval/sif. Preferred: run the ~30-min build on the +# CPU partition, not a login node. `set -a && source .env` first so it's set. +srun -p cpu -t 01:00:00 --pty \ + .agents/scripts/gdpval-sif.sh # defaults to $GDPVAL_SIF_DIR (or pass a dir) +``` + +`gdpval-sif.sh` builds from the NeMo Gym `gdpval.def` at the pinned commit (keep +`GDPVAL_GYM_COMMIT` in sync with the config's `install_on_the_fly.commit`), writes +the SIF into that dir, and is flock-guarded + atomic so concurrent runs never +double-build. Re-running is a no-op once the SIF exists — that's the "reuse the +built one" path. It needs `apptainer`/`singularity` on the build host with +fakeroot/unprivileged build support and network egress; run it on a login or CPU +node (outside enroot, where fakeroot works), **not** inside the eval job. + +**Build vs run are separate.** The helper *builds* the SIF (once, off-GPU). The +eval then *runs* the prebuilt SIF inside the eval container — the golden-validated +path. The eval image doesn't ship apptainer, so the config's `pre_cmd` installs the +apptainer **runtime** + squashfuse (FUSE-mounts the SIF; falls back to slower +per-call extraction if `/dev/fuse` is absent). `pre_cmd` runs arbitrary commands → +the launching shell needs `export NEMO_EVALUATOR_TRUST_PRE_CMD=1`. The +apptainer-under-pyxis nesting is the least-validated part of the SLURM path — check +the canary logs for the "falling back to local exec" warning and apptainer mount +errors. (Local/Docker executor instead needs `--privileged` + the SIF bind mount via +`execution.extra_docker_args`; a rootless `--security-opt` + `/dev/fuse` variant is +in the upstream README appendix.) + +### Rebuild the SIF when the Gym version changes (SIF ↔ commit coupling) + +The SIF is **versioned with the Gym repo**: it's built from `gdpval.def` at +`install_on_the_fly.commit`, and that def changes across commits. Example — between +`2502893977` and the golden `049b1fd0`, the base went **python-3.12 → 3.13** and the +stack gained TeX Live, chromium/playwright, polars/duckdb, xgboost, geospatial +(gdal/proj/geos), and audio/video libs. The newer Stirrup agent's prompt advertises +that richer runtime, so the model's generated code reaches for those libs. + +**So whenever you bump `install_on_the_fly.commit`, rebuild the SIF from the matching +commit.** Run the new gym with an old SIF and the generated code fails its imports +*inside the sandbox* — deliverables silently degrade (missing figures/tables/docs) → +junk scores, with no hard error in the eval. Rebuild to a **version-tagged filename** +so the old SIF isn't clobbered (a running job keeps working), then repoint +`GDPVAL_CONTAINER_PATH` + the `/gdpval/sif` mount at the new file: + +```bash +# build the SIF for the NEW gym commit under a distinct name (old SIF stays intact) +GDPVAL_SIF_NAME=python-3.13.gdpval.sif \ + .agents/scripts/gdpval-sif.sh --commit "$GDPVAL_SIF_DIR" +# then set GDPVAL_CONTAINER_PATH=/gdpval/sif/python-3.13.gdpval.sif in the config +``` + +Rule of thumb: **`install_on_the_fly.commit` and the SIF move together.** Any bump +that alters `gdpval.def` (base image or the apt/pip stack) needs a rebuild; a bump +that leaves `gdpval.def` byte-identical does not — diff the def at the two commits +(`raw.githubusercontent.com/NVIDIA-NeMo/Gym//responses_api_agents/stirrup_agent/containers/gdpval.def`) +to be sure. Note the def already disables apt's sandbox internally as of `049b1fd0`, +so it builds cleanly under the helper's unprivileged path. + +## The `_gym_prepare.yaml` include (why it exists) + +`nemo_gym` tasks interpolate two shared snippets into the task `command:`: +`${gym_prepare.prepare}` (activate the baked Gym venv, checkout the +`install_on_the_fly` pin, repair the image's incomplete per-server venvs, front the +main venv on `PYTHONPATH`) and `${gym_prepare.run}` (data prep + +`ng_e2e_collect_rollouts`, run in its own `setsid` session so the whole server/Ray +process tree can be reaped by process group — otherwise orphaned Ray workers hold +the launcher's stdout open and the run **hangs in post-eval**). Both compensate for +the eval image's deployment-oriented packaging and Gym's incomplete shutdown; remove +once the image ships complete ray-consistent venvs. + +**The include is co-located, not central.** Hydra resolves `- _gym_prepare` relative +to the run config's directory, so `_gym_prepare.yaml` must sit next to your config — +copy the whole `recipes/examples/gym_gdpval/` dir, not the yaml alone. + +## Deployment sizing + +GDPVal is heavy: 220 tasks × `num_repeats` rollouts, each a long multi-turn agent +episode with code-exec + judge calls (`request_timeout: 36000`). The example +self-deploys single-node vLLM, which is fine for a canary or a small policy. For the +**full run of a large MoE** (e.g. MiniMax-M2.7), the reviewed golden uses **multi-node +`vllm_ray`** (16 × 4-GPU HSG = 64 GPUs, `walltime 04:00:00`). To scale up: + ++ Switch `defaults: - deployment: vllm_ray` and add nodes (`execution.num_nodes`); + see `references/multi-node.md` for the Ray TP/PP layout. ++ `parallelism` (`16384`) is **gym-internal concurrency**, not a server cap. The + real throttles are the agent's `stirrup_agent.concurrency` and the judge's + `max_concurrent_requests` — raise those only after the judge logs are clean of 429s. ++ Long runs exceed 4h; rely on NEL's walltime dependency-chain resume + (`resume_from_cache=true` is already set). See SKILL Step 4 + `run-validation.md`. + +## Scoring modes — rubric vs comparison + ++ **`rubric`** (template default) — the judge scores each deliverable standalone + against a rubric; **no reference deliverables** needed. ++ **`comparison`** — pairwise: the judge compares the policy's deliverable to a + **reference model's** deliverable and the result is an ELO-anchored win-rate. + Two-step flow: + 1. **Baseline:** run your reference model with `-o gdpval.reward_mode=rubric` to + generate baseline deliverables (they land under `PERSIST_DELIVERABLES_DIR`). + 2. **Comparison:** run the candidate with `-o gdpval.reward_mode=comparison`, mount + the baseline deliverables at `/gdpval/refs/test_ref` + (`execution.mounts.evaluation`), and set `gdpval.reference_elo` to the reference + model's ELO (golden: Kimi-K2.5-Thinking, elo=1290). + +## Env vars + +| Var | Prefix | Purpose | +|---|---|---| +| `HF_TOKEN` | host | model/dataset downloads | +| `INFERENCE_API_KEY` | host | **judge** auth (and policy if external) | +| `TAVILY_API_KEY` | host | Stirrup agent web search | +| `DUMMY_API_KEY` | lit:dummy | self-deployed vLLM policy key | +| `GDPVAL_CONTAINER_PATH` | lit | SIF path — must equal the SIF bind-mount target | +| `GDPVAL_REF_FILES_DIR` | lit:/gdpval_ref_files | shared-FS ref-file staging (node-local /tmp breaks multi-node Ray) | +| `PERSIST_DELIVERABLES_DIR` | lit | where deliverables persist (see MLflow note) | +| `GDPVAL_MAX_TURNS` | lit (optional) | Stirrup turn cap (default 100; golden uses 250) | +| `NEL_INVOCATION_ID` | runtime | run id | + +`INFERENCE_JUDGE_URL` is the judge host — config (from `.env`), substituted as the +literal `` placeholder in `gdpval_judge.base_url`, **not** +`${oc.env:...}`. Judge `model_id` is hardcoded in the config (swap for an equivalent +on your endpoint). The upstream OSS recipe uses a separate `GDPVAL_JUDGE_API_KEY`; +this template reuses the shared `INFERENCE_API_KEY` for the judge. + +`GDPVAL_SIF_DIR` (`.env`) is host-side config, **not** a container env var: it's the +persistent SIF cache dir the helper builds into and the config bind-mounts at +`/gdpval/sif`. `gdpval-sif.sh` reads `$GDPVAL_SIF_DIR` directly (its default target); +the config mount is a **literal `` placeholder** you substitute with +that same path — mount KEYS aren't interpolated, so don't emit `${oc.env:...}` there +(same rule as the judge URLs). One `.env` value feeds both, so the build path and the +run path can't drift. + +## MLflow export — the deliverables trap + +Deliverables can be large. The mlflow exporter excludes any artifact dir whose +basename matches `*cache*`, so the template sets +`PERSIST_DELIVERABLES_DIR=/results/gdpval/deliverables_cache`: the deliverables stay +under `/results` for inspection but are **not** auto-uploaded. Drop the `_cache` +suffix only if you actually want them uploaded. Everything else about auto-export is +standard (SKILL Step 1 shortcut #4): `auto_export.destinations: [mlflow]` + +`cpu_partition` + a literal-valued `export.mlflow` block (tag `benchmark: +nemo_gym.gdpval`). + +## num_repeats workaround (OmegaConf) + +`++gdpval.*.num_repeats=N` hits an OmegaConf `ListConfig` merge error, so the count +is patched by editing the checked-out file inside the task `command:`: +`sed -i 's/num_repeats: 2$/num_repeats: 1/' benchmarks/gdpval/config.yaml`. The Gym +default is 2 (the golden). The template seds it to **1** to halve cost; **remove the +sed line to keep 2 for golden-comparable / reported scores.** + +## Failure modes to check at canary + ++ **Silent unsandboxed exec** — grep the eval log for the SIF fallback warning / + apptainer mount errors; confirm `GDPVAL_CONTAINER_PATH` == the mount target. ++ **Judge 401 / 429** — wrong `INFERENCE_JUDGE_URL` / key, or `max_concurrent_requests` + too high for the judge endpoint. ++ **Empty reasoning / low win-rate** — thinking mode off. Confirm + `chat_template_kwargs.enable_thinking: true` (right toggle key for the family) + + the policy's `--reasoning-parser`. ++ **Run hangs in post-eval** — orphaned Ray/gym processes holding stdout; that's what + the `${gym_prepare.run}` setsid + process-group reap prevents. If it still hangs, + the `_gym_prepare.yaml` include didn't travel with the config. ++ **Multi-node ref-file errors** — `GDPVAL_REF_FILES_DIR` on node-local storage; + point it at a shared-FS staging dir. diff --git a/.agents/skills/evaluation/references/quantization-benchmarks.md b/.agents/skills/evaluation/references/quantization-benchmarks.md index 9b39fac806f..755d78a349a 100644 --- a/.agents/skills/evaluation/references/quantization-benchmarks.md +++ b/.agents/skills/evaluation/references/quantization-benchmarks.md @@ -3,15 +3,20 @@ When evaluating a quantized checkpoint, prioritize benchmarks that are sensitive to precision loss. The Artificial Analysis (AA) Index v2 suite under `recipes/tasks/aa/` is the default set for quantized-checkpoint validation. +**GDPVal** (`recipes/tasks/aa_gym/gdpval.md`) is also part of the AA suite, but a +different harness (NeMo Gym) — it runs as a **separate standalone config**, never +merged into the `aa/` multi-task list. **Scope rule:** - **Default quant validation** (when the user just says "evaluate this - quantized checkpoint"): use the AA suite plus the three always-include - benchmarks at `recipes/tasks/*.md` (MMLU-Pro, AIME 2025, LiveCodeBench). + quantized checkpoint"): use the AA suite — the `aa/` tasks **plus a standalone + GDPVal config** — plus the three always-include benchmarks at + `recipes/tasks/*.md` (MMLU-Pro, AIME 2025, LiveCodeBench). - **Explicit AA request** ("AA" / "Artificial Analysis" / "AA Index v2"): - use **only** `recipes/tasks/aa/`. Do not add the three always-include - tasks unless the user asks. See the callout at the bottom of this file. + use the `aa/` tasks **and** a companion standalone GDPVal config. Do not add + the three always-include tasks unless the user asks. See the callout at the + bottom of this file. ## Available task recipes @@ -28,6 +33,7 @@ to precision loss. The Artificial Analysis (AA) Index v2 suite under | `tasks/aa/mmmu_pro.md` | MMMU-Pro | Multimodal reasoning | VLM-only; usually Low/Medium when only the LLM is quantized (vision encoder/adapter typically stay BF16) | | `tasks/aa/tau2_bench_telecom.md` | Tau2-Bench Telecom | Agentic tool use (user-simulator + judge) | Medium-high — tool-call JSON is brittle, but user-sim + judge variance often dominates the signal | | `tasks/aa/omniscience.md` | AA-Omniscience | Knowledge reliability (`ns_omniscience`, nemo-skills, `num_repeats: 10`) — correct vs hallucinate vs abstain on obscure facts, judge-scored | Medium — measures the hallucination/abstention balance; aggressive precision loss can erode factual recall and shift the omni-index | +| `tasks/aa_gym/gdpval.md` | GDPVal (`nemo_gym` Stirrup agent, **standalone config**) | Agentic office/PDF deliverables in an Apptainer code-exec sandbox, pairwise/rubric judge | High — long-horizon agentic reasoning + code + judge; precision loss compounds across many turns. **Heaviest task**: multi-hour, often multi-node, needs the SIF sandbox + judge. Runs as its own config, never in the `aa/` list | ## Recommended sets by use case @@ -35,15 +41,17 @@ to precision loss. The Artificial Analysis (AA) Index v2 suite under |----------|-----------| | Quick sanity check | GPQA | | Standard quant validation (text LLM) | GPQA, SciCode, LCR | -| AA / Artificial Analysis suite (text LLM) | All `tasks/aa/` text tasks: GPQA, HLE, LCR, SciCode, IFBench, Tau2-Bench Telecom, AA-Omniscience | -| AA / Artificial Analysis suite (multimodal) | AA text suite + MMMU-Pro | +| AA / Artificial Analysis suite (text LLM) | All `tasks/aa/` text tasks: GPQA, HLE, LCR, SciCode, IFBench, Tau2-Bench Telecom, AA-Omniscience — **plus GDPVal** (`tasks/aa_gym/`, a separate standalone config) | +| AA / Artificial Analysis suite (multimodal) | AA text suite (incl. GDPVal) + MMMU-Pro | | Code-focused model | LiveCodeBench, SciCode | | Reasoning model | AIME 2025, GPQA, HLE | -> If the user asks for "AA" or "Artificial Analysis", generate **only** tasks -> under `recipes/tasks/aa/`. Do not silently add MMLU-Pro, AIME 2025, or -> LiveCodeBench — they live at `recipes/tasks/*.md` and are a separate -> always-include set. +> If the user asks for "AA" or "Artificial Analysis", generate the +> `recipes/tasks/aa/` tasks **plus a companion standalone GDPVal config** +> (`recipes/tasks/aa_gym/gdpval.md`) — GDPVal is part of the AA suite but a +> different harness, so it's its own config, never in the `aa/` `tasks` list. Do +> not silently add MMLU-Pro, AIME 2025, or LiveCodeBench — they live at +> `recipes/tasks/*.md` and are a separate always-include set. ## Notes for quantized-checkpoint runs @@ -62,6 +70,14 @@ to precision loss. The Artificial Analysis (AA) Index v2 suite under apples-to-apples comparison. - **IFBench** is the least quant-sensitive in the set but still useful as a regression check for aggressive formats (NVFP4, INT4-AWQ). +- **GDPVal** is part of the AA suite but the heaviest task and a separate + harness: it runs as its **own standalone `aa_gym` config** (never in the `aa/` + `tasks` list), needs the Apptainer SIF sandbox + judge, and is multi-hour / + often multi-node. Generate it alongside the `aa/` config; see + `recipes/tasks/aa_gym/gdpval.md` + `references/gym-gdpval.md`. Thinking mode is + mandatory (non-thinking loses ~86% of pairwise judgements). For low-variance + quant comparisons keep the golden `num_repeats: 2` — the example template + defaults to 1 to halve cost, so bump it back to 2 for quant validation. ## How to use From 74dfa8b97dcc40daa700b0f88de62ce32dbb1155 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Fri, 31 Jul 2026 23:50:01 +0000 Subject: [PATCH 02/16] [skill] evaluation: GDPVal fixes from a validated end-to-end run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Corrections found while bringing GDPVal up on an aarch64 GB300 cluster, in both rubric and comparison mode. Correctness: - _gym_prepare: run the rollout command from a script written via a QUOTED heredoc instead of `bash -c '...'`. Gym params legitimately contain single quotes (comparison mode's ++multistage.stages='[{num_tasks: 45}, ...]' and ++...judge_panel='[{...}]'), which closed the wrapper early so Hydra got the value split on spaces and died with "no viable alternative at input '[{num_tasks:'". The quoted delimiter keeps $$ and $*_API_KEY unexpanded until run time. - example: never put '#' comments inside a folded (>-) params scalar. YAML keeps them as literal text; the block folds to one line and the first '#' comments out every override after it in the shell command. - example: arch-aware pre_cmd (Ubuntu PPA) — the hardcoded amd64 apptainer .deb cannot install on aarch64, which is most Blackwell/Grace clusters. - gdpval-sif.sh: align the default SIF name and gym commit with the template, so build and run no longer disagree by default. Scoping (rubric default; comparison is a conversion): - The template is now rubric-only and self-consistent: no dangling reference_elo / reference_deliverables_dir, which CONFLICT with comparison mode's reference_models map. Comparison needs a reference set, a newer gym image, and its own overrides -- documented, not half-wired here. Documented traps: - install_on_the_fly.commit is INERT on images that bake Gym as a non-git directory (the public nemo-gym image): the prepare step logs "/opt/Gym is not a git repo" and the pin is ignored. Do not attribute behaviour changes to it without "=== NeMo Gym commit ===" + a SHA in the log; a port-11000 head-server hang is a transient collision, not a version symptom. - What NEL validates and what it does not: mount dirs fail loudly before sbatch, but `test -d` cannot see a missing/misnamed SIF *file* (silent unsandboxed fallback) and the container is never checked; --dry-run skips remote validation entirely. Adds `gdpval-sif.sh --check` as the preflight. - num_repeats depends on the flow: multistage comparison uses 1 (top-level ++num_repeats works), pre-multistage rubric/single-ref used 2. - Prefer a site-provided SIF over building, when one exists: a self-built SIF resolves its pip stack at build time and can drift from the sandbox a published reference set was generated in. Validated on aws-cmh (GB300, aarch64): deploy -> SIF build+sandboxed exec -> gym head server -> rollouts + deliverables -> judge scoring, in rubric mode (219/220 tasks, 0 judge failures) and in comparison mode against a 9-reference set (verify_mode=comparison, per-reference win/loss/tie). Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .agents/scripts/gdpval-sif.sh | 33 +++++++-- .../examples/gym_gdpval/_gym_prepare.yaml | 15 ++++- .../gym_gdpval/example_gym_gdpval.yaml | 63 +++++++++++------ .../evaluation/recipes/tasks/aa_gym/gdpval.md | 15 +++-- .../evaluation/references/gym-gdpval.md | 67 ++++++++++++++++++- 5 files changed, 160 insertions(+), 33 deletions(-) diff --git a/.agents/scripts/gdpval-sif.sh b/.agents/scripts/gdpval-sif.sh index b37518961fe..00034ff5888 100755 --- a/.agents/scripts/gdpval-sif.sh +++ b/.agents/scripts/gdpval-sif.sh @@ -22,15 +22,20 @@ # run reuses the built SIF instantly. # # Usage: -# .agents/scripts/gdpval-sif.sh [] [--commit ] [--force] +# .agents/scripts/gdpval-sif.sh [] [--commit ] [--force|--check] # Persistent path on the target cluster's shared FS. # DEFAULTS to $GDPVAL_SIF_DIR (from .env) when omitted. A -# directory -> /python-3.12.gdpval.sif; a *.sif path +# directory -> /$GDPVAL_SIF_NAME (default python-3.13.gdpval.sif, +# matching the example config); a *.sif path # is used verbatim. Bind-mount this SAME dir into the eval # container at /gdpval/sif (see recipes/examples/gym_gdpval/). # --commit NeMo Gym commit whose gdpval.def to build. Keep in sync # with the config's install_on_the_fly.commit. # --force Rebuild even if the SIF already exists. +# --check Verify-only preflight: exit 0 if the expected SIF exists, +# nonzero (listing what IS there) if not. Never builds. +# Use before `nel run` — NEL's mount validation is `test -d` +# and cannot see a missing/misnamed SIF file. # # Requires `apptainer` (or `singularity`) on PATH with unprivileged/fakeroot # build support, plus network egress to GitHub/base image. Run on a node that has @@ -42,8 +47,8 @@ set -euo pipefail # Keep GDPVAL_GYM_COMMIT in sync with install_on_the_fly.commit in the config. -GDPVAL_GYM_COMMIT="${GDPVAL_GYM_COMMIT:-2502893977e9e9af84adc1fa8d38c9314208d3ee}" # pragma: allowlist secret -GDPVAL_SIF_NAME="${GDPVAL_SIF_NAME:-python-3.12.gdpval.sif}" +GDPVAL_GYM_COMMIT="${GDPVAL_GYM_COMMIT:-dd41196f620f2af99947d776cbe5da9439d2a08d}" # pragma: allowlist secret +GDPVAL_SIF_NAME="${GDPVAL_SIF_NAME:-python-3.13.gdpval.sif}" APPTAINER_BIN="${APPTAINER_BIN:-}" _log() { printf '\033[2m %s\033[0m\n' "$*" >&2; } @@ -51,11 +56,12 @@ _die() { printf '\033[31mgdpval-sif: %s\033[0m\n' "$*" >&2; exit 1; } _usage() { sed -n '/^# gdpval-sif\.sh/,/^set -euo/p' "$0" | sed 's/^# \{0,1\}//; /^set -euo/d'; } # --- parse args --- -target=""; force=0 +target=""; force=0; check=0 while [[ $# -gt 0 ]]; do case "$1" in --commit) GDPVAL_GYM_COMMIT="${2:?--commit needs a value}"; shift 2 ;; --force) force=1; shift ;; + --check) check=1; shift ;; -h|--help) _usage; exit 0 ;; -*) _die "unknown flag: $1 (see --help)" ;; *) [[ -z "$target" ]] || _die "unexpected extra arg: $1"; target="$1"; shift ;; @@ -71,6 +77,23 @@ if [[ "$target" == *.sif ]]; then else sif_dir="$target"; sif="$sif_dir/$GDPVAL_SIF_NAME" fi +# --- verify-only mode (preflight) --- +# NEL's submit-time mount validation runs `test -d`, so it only proves the SIF *dir* +# exists — a dir holding the WRONG sif name (e.g. python-3.12 after a gym bump to a +# 3.13 def) passes validation, and the Stirrup agent then SILENTLY falls back to +# non-sandboxed exec. Run this before submitting to fail loudly instead. +if [[ "$check" -eq 1 ]]; then + if [[ -f "$sif" ]]; then + _log "SIF present: $sif ($(du -h "$sif" 2>/dev/null | cut -f1))" + echo "$sif"; exit 0 + fi + printf '\033[31mgdpval-sif: MISSING expected SIF: %s\033[0m\n' "$sif" >&2 + [[ -d "$sif_dir" ]] && { echo " dir exists but does not contain it; found:" >&2 + ls -1 "$sif_dir"/*.sif 2>/dev/null | sed 's/^/ /' >&2 || echo " (no .sif files)" >&2; } + echo " Build it with: $0 ${sif_dir} (or --commit for a different def)" >&2 + exit 1 +fi + mkdir -p "$sif_dir" || _die "cannot create SIF dir: $sif_dir" # --- reuse if present --- diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml index 9239d073761..bd9fde37fbf 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml @@ -91,7 +91,20 @@ gym_prepare: # Remove once Gym's shutdown reaps server process groups + Ray. run: |- ng_prepare_benchmark {{config.params.extra.nemo_gym.data_prep_params}} {{config.params.extra.nemo_gym.common_params}} - setsid --wait bash -c 'echo $$ > /tmp/gym_eval_pgid; exec ng_e2e_collect_rollouts {{config.params.extra.nemo_gym.collect_rollout_params}} {{config.params.extra.nemo_gym.common_params}}' & + # The rollout command is written to a script and run from it, rather than passed + # to `bash -c '...'`. Gym params legitimately CONTAIN single quotes — comparison + # mode's ++multistage.stages='[{num_tasks: 45}, ...]' and ++...judge_panel='[{...}]' + # — and those would close a single-quoted `bash -c` wrapper early, so Hydra then + # receives the value split on spaces and dies with + # "no viable alternative at input '[{num_tasks:'". + # The heredoc delimiter is QUOTED ('GYM_RUN_EOF') so nothing expands while writing: + # $$ and $TAVILY_API_KEY / $INFERENCE_API_KEY land in the script literally and are + # expanded at run time by the shell that executes it, which is what we want. + cat > /tmp/gym_run.sh <<'GYM_RUN_EOF' + echo $$ > /tmp/gym_eval_pgid + exec ng_e2e_collect_rollouts {{config.params.extra.nemo_gym.collect_rollout_params}} {{config.params.extra.nemo_gym.common_params}} + GYM_RUN_EOF + setsid --wait bash /tmp/gym_run.sh & __ev=$! __rc=0; wait "$__ev" || __rc=$? echo "Evaluator Gym finished!" diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index 1860a743d03..f9df5023764 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -56,14 +56,17 @@ defaults: - _gym_prepare # provides ${gym_prepare.prepare} / ${gym_prepare.run} - _self_ -# GDPVal scoring mode. Override at runtime: -o gdpval.reward_mode=comparison -# rubric — standalone LLM-judge scoring; NO reference deliverables needed. (default) -# comparison — pairwise scoring vs a reference model's deliverables; also -# mount the ref dir at /gdpval/refs/test_ref (see mounts) and set -# reference_elo. See references/gym-gdpval.md for the two-step flow. +# GDPVal scoring mode. This template is RUBRIC-ONLY, deliberately: rubric needs no +# reference deliverables and runs on the public gym image, so it works standalone. +# rubric — standalone LLM-judge scoring, 0-1 reward per deliverable. (this template) +# comparison — pairwise vs anchored reference deliverables; the only mode that yields +# an AA-comparable ELO / win-rate. It needs a reference set, a newer gym +# image, and its own overrides — do NOT just flip reward_mode here. +# To run comparison, NVIDIA-internal users should follow `modelopttools:eval-config` +# Step 3c, which converts this config (container override + reference_models map + +# mounts + multistage). See references/gym-gdpval.md "Scoring modes". gdpval: reward_mode: rubric - reference_elo: 1290 # comparison mode only — ELO of the reference model # GDPVal pairwise judge. base_url is config (from .env), not a secret, so no # export needed; only api_key (INFERENCE_API_KEY) is exported and read by the @@ -104,8 +107,9 @@ execution: : /gdpval/sif # Writable shared-FS staging for ref files. Node-local /tmp breaks multi-node Ray. : /gdpval_ref_files - # comparison mode ONLY — reference deliverables dir (drop for default rubric mode): - # : /gdpval/refs/test_ref + # Comparison mode additionally mounts one dir per reference model at + # /gdpval/refs/ (both `deployment` and `evaluation`) — added by + # modelopttools:eval-config Step 3c, not here. auto_export: # REQUIRED trigger for MLflow upload (see example_eval.yaml). destinations: - mlflow @@ -165,13 +169,18 @@ evaluation: NEL_INVOCATION_ID: runtime:NEL_INVOCATION_ID # Installs apptainer + squashfuse into the eval container (needs # NEMO_EVALUATOR_TRUST_PRE_CMD=1 in the launching shell). See references/gym-gdpval.md. + # Installs the apptainer RUNTIME into the eval container (it is not baked in) so the + # Stirrup agent can exec the prebuilt SIF. ARCH-AWARE: Blackwell/Grace clusters are + # aarch64, so never hardcode an amd64 .deb — the Ubuntu PPA builds both arches. pre_cmd: | set -ex + export DEBIAN_FRONTEND=noninteractive apt-get update -qq - wget -q -O /tmp/apptainer.deb \ - https://github.com/apptainer/apptainer/releases/download/v1.4.2/apptainer_1.4.2_amd64.deb - apt-get install -y -qq /tmp/apptainer.deb && rm /tmp/apptainer.deb - apt-get install -y -qq squashfuse fuse3 || apt-get install -y -qq squashfuse fuse || true + apt-get install -y -qq software-properties-common squashfuse fuse3 ca-certificates || true + add-apt-repository -y ppa:apptainer/ppa + apt-get update -qq + apt-get install -y -qq apptainer + apptainer --version mkdir -p /usr/local/var/apptainer/mnt/session nemo_evaluator_config: config: @@ -208,6 +217,12 @@ evaluation: # STANDALONE: exactly one gym task. Do NOT add other tasks to this list. tasks: - name: nemo_gym + # Public image — fine for RUBRIC mode (this template's default). + # COMPARISON mode needs a NEWER Gym than this image ships and must OVERRIDE this + # line; see references/gym-gdpval.md "Scoring modes". NOTE: this image bakes Gym + # as a non-git directory, so `install_on_the_fly.commit` below is silently + # ignored here (the prepare step logs "/opt/Gym is not a git repo; using baked-in + # Gym version") — the pin only applies on images where /opt/Gym IS a git repo. container: nvcr.io/nvidia/eval-factory/nemo-gym:26.05 # pin a verified tag nemo_evaluator_config: config: @@ -221,13 +236,14 @@ evaluation: # old SIF + new gym silently degrades deliverables. Rebuild with # `gdpval-sif.sh --commit ` to a new name + repoint GDPVAL_CONTAINER_PATH. # See references/gym-gdpval.md "Rebuild the SIF when the Gym version changes". - # Current golden pin (updated GDPVal task-sampling algo — matters for - # multistage stage-1 selection, so keep it aligned with the goldens). - # Its gdpval.def is byte-identical to 049b1fd0…, so the python-3.13 SIF - # built from either commit is valid — no rebuild when moving between them. - # Do NOT go back to 2502893977… — on that commit the gym head server - # fails to bind its fixed port 11000 and the run hangs forever polling - # "Head server is not up yet". + # Current golden pin (updated GDPVal task-sampling algo, which drives + # multistage stage-1 selection). Its gdpval.def is byte-identical to + # 049b1fd0…, so a python-3.13 SIF built from either commit is valid — + # no rebuild when moving between them. + # NOTE: this pin is INERT on the public nemo-gym image (see the + # `container:` comment above) — it only takes effect on an image where + # /opt/Gym is a git repo. Confirm via "=== NeMo Gym commit ===" + SHA + # in the client log before crediting it with any behaviour change. commit: dd41196f620f2af99947d776cbe5da9439d2a08d # pragma: allowlist secret command: | ${gym_prepare.prepare} @@ -263,6 +279,13 @@ evaluation: ++upload_rollouts_to_wandb=false ++responses_create_params.temperature={{config.params.temperature}} ++responses_create_params.top_p={{config.params.top_p}} + # NOTE: no reference_* overrides below — rubric mode needs none, and the + # single-reference keys (reference_deliverables_dir / reference_elo) + # CONFLICT with comparison mode's reference_models map (added by + # modelopttools:eval-config Step 3c). + # Do NOT put '#' comments INSIDE this folded (>-) scalar: YAML keeps + # them as literal text, the block folds to one line, and the first '#' + # comments out every override after it in the shell command. common_params: >- ++use_absolute_ip=true ++policy_base_url={{target.api_endpoint.url}} @@ -276,8 +299,6 @@ evaluation: ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.tavily_api_key=$TAVILY_API_KEY ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.agent_max_turns=${oc.env:GDPVAL_MAX_TURNS,250} ++gdpval_resources_server.resources_servers.gdpval.reward_mode=${gdpval.reward_mode} - ++gdpval_resources_server.resources_servers.gdpval.reference_deliverables_dir=/gdpval/refs/test_ref - ++gdpval_resources_server.resources_servers.gdpval.reference_elo=${gdpval.reference_elo} ++gdpval_resources_server.resources_servers.gdpval.persist_raw_judge_responses=true ++gdpval_resources_server.resources_servers.gdpval.preconvert_max_concurrent=30 ++gdpval_resources_server.resources_servers.gdpval.preconvert_office_to_pdf=true diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index 9ca8118dacc..f4297068243 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -59,10 +59,17 @@ recipes/examples/gym_gdpval/ Copy the **whole `gym_gdpval/` directory** to your workspace (the `- _gym_prepare` default resolves relative to the config dir — copying the yaml alone breaks it). -- **num_repeats:** the example defaults to **1** (halves cost); the reviewed golden - uses **2**. For golden-comparable / reported scores, delete the `sed` line in the - task `command:` to keep 2. It can **not** be set via a `++` override (OmegaConf - `ListConfig` merge error) — the file is patched with `sed`. +- **num_repeats — the right value depends on the flow, so check which one you're in:** + - **Multistage comparison** (the current golden for AA-comparable ELO): **1**, set + with a top-level `++num_repeats=1`, which *does* work. Recent Gym pins already + ship `num_repeats: 1` in `benchmarks/gdpval/config.yaml`, so the `sed` below is a + no-op there. + - **Rubric / older single-reference comparison:** the pre-multistage golden used + **2** (220 tasks × 2 = 440 rollouts). On old pins the per-dataset key could not be + set via `++` (OmegaConf `ListConfig` merge error), hence the `sed` in the task + `command:`; delete that line to keep 2. + + Do not carry a `=2` from an old single-reference config into a multistage run. - **SIF ↔ Gym version (rebuild on bump):** the SIF is built from `gdpval.def` at `install_on_the_fly.commit`. **If you change that commit, rebuild the SIF** with a matching `gdpval-sif.sh --commit ` (to a new version-tagged filename, then diff --git a/.agents/skills/evaluation/references/gym-gdpval.md b/.agents/skills/evaluation/references/gym-gdpval.md index 61e25038267..358839cfa04 100644 --- a/.agents/skills/evaluation/references/gym-gdpval.md +++ b/.agents/skills/evaluation/references/gym-gdpval.md @@ -28,8 +28,20 @@ into the eval container at **exactly** `/gdpval/sif/python-3.12.gdpval.sif` (the numbers aren't comparable, so verify the SIF at canary. **The skill builds the SIF on the target cluster — it never copies one from -another cluster.** Ship the SIF via the idempotent helper, which builds it if -absent and reuses it if already present: +another cluster.** That is the self-contained default and the right behaviour when +you have no prebuilt sandbox. + +> **If your site already provides a prebuilt GDPVal SIF, prefer it over building — +> build only when it is absent.** A self-built SIF resolves its pip stack at *your* +> build time, so it can drift from the sandbox a published reference set was +> generated in, which matters for comparison-mode runs. To use a provided one, mount +> its dir at `/gdpval/sif` and point `GDPVAL_CONTAINER_PATH` at the provided +> filename. **NVIDIA-internal users:** `modelopttools:eval-config` (Step 3c) records +> the provided SIF's location and the check-then-fall-back procedure — if you have +> that skill, consult it before building. Without it, building (below) is correct. + +Ship the SIF via the idempotent helper, which builds it if absent and reuses it if +already present: ```bash # GDPVAL_SIF_DIR (.env) — persistent shared-FS dir on the TARGET cluster; the config @@ -135,6 +147,57 @@ self-deploys single-node vLLM, which is fine for a canary or a small policy. For (`execution.mounts.evaluation`), and set `gdpval.reference_elo` to the reference model's ELO (golden: Kimi-K2.5-Thinking, elo=1290). +### Comparison mode needs a newer Gym than the public image — override `container:` + +The public `nvcr.io/nvidia/eval-factory/nemo-gym:26.05` (== `latest` by digest) runs +**rubric** mode fine, but its Gym predates the multi-reference `reference_models` map. +In comparison mode `gdpval_resources_server` fails validation at startup with +`reward_mode=comparison requires reference_deliverables_dir to be set`, surfacing only +as the unhelpful `Process gdpval_resources_server finished unexpectedly!`. + +**You cannot fix this by bumping `install_on_the_fly.commit`.** That image bakes Gym as +a **non-git directory**, so the pin is silently ignored — the prepare step logs +`=== /opt/Gym is not a git repo; using baked-in Gym version ===` and you get the baked +build regardless. Treat the pin as inert unless the log shows `=== NeMo Gym commit ===` +followed by a SHA, and don't attribute run-to-run behaviour changes to it. (In +particular, a head-server `address already in use` on its fixed port 11000 is a +**transient collision** — resubmitting clears it; it is not a Gym-version symptom.) + +To run comparison mode you must **override the task's `container:`** with an image +whose Gym has `reference_models`. NVIDIA-internal users: the image path, the canonical +reference set, and the matching gym overrides are in the `modelopttools:eval-config` +skill (Step 3c) — internal cluster paths deliberately live only there. + +## Preflight — what NEL validates, and what it does NOT + +NEL validates mount paths at **submit** time (`_collect_mount_paths` + +`_validate_remote_paths_exist`): it ssh's to the cluster, runs `test -d` on every +mount source, and `raise ValueError` listing the missing ones **before** any +`sbatch` — so a missing reference dir or cache costs you nothing. Three gaps to know: + +| Artifact | Missing → | Loud? | +|---|---|---| +| mounted dirs (refs, caches, SIF **dir**, checkpoint) | `ValueError` at submit, no job queued | ✅ pre-allocation | +| **the SIF file inside that dir** | **agent silently runs code-exec unsandboxed** | ❌ **silent** | +| task `container:` (image / `.sqsh`) | not collected for validation → pyxis import failure | ⚠️ only after allocation | + +1. **`test -d` proves the directory, not the SIF.** A `$GDPVAL_SIF_DIR` that exists but + holds the *wrong* filename (e.g. `python-3.12…` after bumping to a 3.13 def) passes + validation, and the run then silently degrades. Guard with the verify-only mode: + + ```bash + .agents/scripts/gdpval-sif.sh --check # uses $GDPVAL_SIF_DIR; exit 1 + lists what IS there + ``` + + Keep `GDPVAL_SIF_NAME` / the helper's default in sync with the config's + `GDPVAL_CONTAINER_PATH`; they are the same string in two places. +2. **`--dry-run` skips remote validation entirely** (it never opens the ssh + connection). A clean dry-run says nothing about whether your mounts exist — run + the preflight separately. +3. **The container is never checked.** A wrong/rotated image path fails at pyxis + import, i.e. after the allocation is granted. Verify it with `ls -l` first + (comparison mode's internal image especially — see `modelopttools:eval-config`). + ## Env vars | Var | Prefix | Purpose | From 8a1aae4b423ddf7d966d22765344c5cb84c8efed Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Sun, 2 Aug 2026 23:36:14 +0000 Subject: [PATCH 03/16] [skill] evaluation: make GDPVal score extraction unambiguous Reading the final score off a finished GDPVal run was guesswork: the recipe just said the metric "is logged under nemo_gym.gdpval in MLflow". Two traps this fixes, both hit on a real run: - `artifacts/eval_factory_metrics.json` does NOT contain the GDPVal score. It holds only response_stats / reasoning / evaluation (request telemetry), so looking there and finding no ELO reads like the run failed to score. - A comparison run logs ~200 MLflow metrics of which ~126 are per-reference (/ref//...), so the three headline numbers are easy to miss in the UI. Documents the authoritative local file (artifacts/results.yml) with the exact metric paths for both modes, the MLflow key names (including the key_metrics/ duplicates), a copy-pasteable one-liner that reads the ELO without MLflow, and sanity checks (judged count, num_stages/num_references, unique task_id count vs 220) so a score computed on a short task set is not quoted as final. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .../evaluation/recipes/tasks/aa_gym/gdpval.md | 49 +++++++++++++++++-- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index f4297068243..bc1ed09ca18 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -98,8 +98,47 @@ hangs (see `references/gym-gdpval.md` → failure modes). ## Score Extraction -GDPVal reports a **win-rate / ELO** against the reference (comparison mode) or a -rubric score (rubric mode); the run's aggregate metric is logged under the -`nemo_gym.gdpval` benchmark in MLflow. Read the run's -`{output_dir}/evaluator_rollouts.jsonl` + `nemo_gym_logs/` for per-task rewards, -and the persisted judge responses under `PERSIST_DELIVERABLES_DIR`. +> **The GDPVal score is NOT in `artifacts/eval_factory_metrics.json`.** That file +> holds only `response_stats` / `reasoning` / `evaluation` (request-level telemetry). +> Looking there and finding no ELO does not mean the run failed to score. + +The final numbers live in **`artifacts/results.yml`** (authoritative, local) and are +mirrored to MLflow. Read them by metric name: + +| Mode | Metric (results.yml → `groups.nemo_gym.metrics..scores..value`) | +| --- | --- | +| comparison | `gdpval_stirrup_agent/comparison/eval_elo` ← **the headline** | +| comparison | `gdpval_stirrup_agent/comparison/normalized_elo` (AA 0–1 scale) | +| comparison | `gdpval_stirrup_agent/comparison/win_rate`, `/judged`, `/wins`, `/losses`, `/ties` | +| comparison | per-reference: `gdpval_stirrup_agent/comparison/ref//{win_rate,wins,losses,ties,judged}` | +| comparison | per-stage estimate: `gdpval_stirrup_agent/comparison/stage_0/eval_elo` (stage 1, all refs) — the **final** value is the top-level one, from the last stage | +| rubric | mean of `reward` across `artifacts/evaluator_rollouts.jsonl` (per-rollout 0–1) | + +```bash +# final ELO from the local results file (no MLflow needed) +python3 -c " +import yaml,sys +m=yaml.safe_load(open('results.yml'))['groups']['nemo_gym']['metrics'] +for k in ('eval_elo','normalized_elo','win_rate'): + n=f'gdpval_stirrup_agent/comparison/{k}' + print(k, '=', m[n]['scores'][n]['value'])" +``` + +In **MLflow** the same values are prefixed `nemo_gym_` and duplicated under a +`key_metrics/` path — query these exact keys rather than browsing the UI, because a +comparison run logs **~200 metrics and most of them are per-reference**, so the +headline is easy to miss: + +```text +nemo_gym_gdpval_stirrup_agent/key_metrics/comparison/eval_elo +nemo_gym_gdpval_stirrup_agent/key_metrics/comparison/normalized_elo +nemo_gym_gdpval_stirrup_agent/key_metrics/comparison/win_rate +``` + +Sanity checks before quoting a score: `…/comparison/judged` should be large (a few +hundred+), `num_stages`/`num_references` should match your multistage config, and the +unique `task_id` count in `evaluator_rollouts.jsonl` should be close to 220 — a short +count means tasks were lost (e.g. across a walltime resume) and the ELO is computed on +fewer tasks than the references were. Per-task detail is in +`evaluator_rollouts.jsonl` + `nemo_gym_logs/`; raw judge responses are under +`PERSIST_DELIVERABLES_DIR`. From ffa0ea9c144562347c28cb551c357763b2cfbc86 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Sun, 2 Aug 2026 23:39:17 +0000 Subject: [PATCH 04/16] [skill] evaluation: report normalized_elo as the GDPVal score normalized_elo is the AA 0-1 scale and the number that is comparable across models and to the published AA index; eval_elo is the same Bradley-Terry fit on the raw Elo axis (normalized_elo = (eval_elo - 500) / 2000). Make the recipe say which one to report instead of leaving the reader to choose: normalized_elo leads the metric table, the extraction one-liner, and the MLflow key list, with eval_elo demoted to supporting detail. Also fixes three places where the recipe contradicted itself after the rubric/comparison split: - the header asserted num_repeats=2 "in the reviewed golden" while the Config section (correctly) makes it flow-dependent; - the SIF filename still said python-3.12 while the template ships 3.13; - "Scoring modes" still described the old single-reference two-step flow (mount at /gdpval/refs/test_ref, set reference_elo=1290), which no longer matches the multi-reference design and would send a reader down a path that fails at startup. It now states that comparison is a conversion, not a flag flip, and points at the conversion checklist. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .../evaluation/recipes/tasks/aa_gym/gdpval.md | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index bc1ed09ca18..4749b9e9e03 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -10,8 +10,9 @@ GDPVal is an **agentic** benchmark: the Stirrup agent produces office/PDF deliverables inside a per-task Apptainer code-exec sandbox, then a pairwise/rubric judge (**Gemini 3.1 Pro**) scores them. It is the most resource-intensive benchmark -in the suite — **220 tasks**, `num_repeats=2` in the reviewed golden (= 440 -rollouts), each rollout using 4 judge trials. +in the suite — **220 tasks**, 4 judge trials per rollout. Repeat count depends on +the flow (multistage comparison uses 1; the pre-multistage golden used 2) — see +the num_repeats note under Config. It runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task (NOT nel-next), so Steps 1–9 apply — but with the branch differences below. @@ -23,7 +24,7 @@ Steps 1–9 apply — but with the branch differences below. - **Apptainer SIF sandbox (self-contained).** Set `GDPVAL_SIF_DIR` in `.env`, then run `.agents/scripts/gdpval-sif.sh` (uses `$GDPVAL_SIF_DIR`) — it **builds if absent, reuses if present**, and never copies from another cluster. The config - bind-mounts `$GDPVAL_SIF_DIR` at **exactly** `/gdpval/sif/python-3.12.gdpval.sif` + bind-mounts `$GDPVAL_SIF_DIR` at **exactly** `/gdpval/sif/python-3.13.gdpval.sif` (matches `GDPVAL_CONTAINER_PATH`). Missing/mispathed → the agent **silently** runs code-exec unsandboxed and results are not comparable. Details in `references/gym-gdpval.md`. - **Thinking mode is mandatory.** Non-thinking loses ~86% of pairwise judgements. @@ -36,14 +37,17 @@ Steps 1–9 apply — but with the branch differences below. ## Scoring modes -Set `gdpval.reward_mode` (override: `-o gdpval.reward_mode=comparison`): - -- **`rubric`** (default) — standalone LLM-judge scoring; **no reference - deliverables** needed. Use this unless you specifically need pairwise-vs-baseline. -- **`comparison`** — pairwise scoring vs a reference model's deliverables. Also - mount the ref dir at `/gdpval/refs/test_ref` and set `gdpval.reference_elo` - (golden uses Kimi-K2.5-Thinking refs, elo=1290). Two-step baseline→comparison - flow in `references/gym-gdpval.md`. +- **`rubric`** — the template default. Standalone LLM-judge scoring against each + task's rubric; **no reference deliverables**, runs on the public gym image. Gives a + 0–1 reward, **not** an ELO (ELO is undefined without an opponent). +- **`comparison`** — pairwise vs anchored reference deliverables; the **only** mode + that yields the AA-comparable `normalized_elo`. It is a conversion, not a flag + flip: it needs a reference set, a gym image whose Gym has the `reference_models` + map, ref mounts on both stages, and multistage overrides. **Do not just set + `reward_mode=comparison`** — the server exits at startup with + `reward_mode=comparison requires reference_deliverables_dir to be set`. + NVIDIA-internal users: `modelopttools:eval-config` Step 3c is the conversion + checklist. See `references/gym-gdpval.md` → "Scoring modes". ## Config @@ -77,7 +81,7 @@ default resolves relative to the config dir — copying the yaml alone breaks it across commits, and running a new gym with an old SIF makes the agent's generated code fail imports in the sandbox → silently degraded deliverables. See `references/gym-gdpval.md` → "Rebuild the SIF when the Gym version changes". -- **Deployment:** single-node vLLM in the template; the full 220×2 run of a large +- **Deployment:** single-node vLLM in the template; a full 220-task run of a large MoE typically needs multi-node — see `references/gym-gdpval.md`. - Required `.env` keys: `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR` (see `recipes/env.example`). @@ -102,24 +106,29 @@ hangs (see `references/gym-gdpval.md` → failure modes). > holds only `response_stats` / `reasoning` / `evaluation` (request-level telemetry). > Looking there and finding no ELO does not mean the run failed to score. +**The reported GDPVal score is `normalized_elo`** — the AA 0–1 scale, comparable +across models and to the published AA index. `eval_elo` is the same fit on the raw +Elo axis (`normalized_elo = (eval_elo - 500) / 2000`); quote it as supporting +detail, not as the score. + The final numbers live in **`artifacts/results.yml`** (authoritative, local) and are mirrored to MLflow. Read them by metric name: | Mode | Metric (results.yml → `groups.nemo_gym.metrics..scores..value`) | | --- | --- | -| comparison | `gdpval_stirrup_agent/comparison/eval_elo` ← **the headline** | -| comparison | `gdpval_stirrup_agent/comparison/normalized_elo` (AA 0–1 scale) | +| comparison | `gdpval_stirrup_agent/comparison/normalized_elo` ← **REPORT THIS** (AA 0–1 scale) | +| comparison | `gdpval_stirrup_agent/comparison/eval_elo` (raw Elo; supporting detail) | | comparison | `gdpval_stirrup_agent/comparison/win_rate`, `/judged`, `/wins`, `/losses`, `/ties` | | comparison | per-reference: `gdpval_stirrup_agent/comparison/ref//{win_rate,wins,losses,ties,judged}` | | comparison | per-stage estimate: `gdpval_stirrup_agent/comparison/stage_0/eval_elo` (stage 1, all refs) — the **final** value is the top-level one, from the last stage | | rubric | mean of `reward` across `artifacts/evaluator_rollouts.jsonl` (per-rollout 0–1) | ```bash -# final ELO from the local results file (no MLflow needed) +# final score from the local results file (no MLflow needed) python3 -c " import yaml,sys m=yaml.safe_load(open('results.yml'))['groups']['nemo_gym']['metrics'] -for k in ('eval_elo','normalized_elo','win_rate'): +for k in ('normalized_elo','eval_elo','win_rate'): n=f'gdpval_stirrup_agent/comparison/{k}' print(k, '=', m[n]['scores'][n]['value'])" ``` @@ -130,8 +139,8 @@ comparison run logs **~200 metrics and most of them are per-reference**, so the headline is easy to miss: ```text +nemo_gym_gdpval_stirrup_agent/key_metrics/comparison/normalized_elo <- report this nemo_gym_gdpval_stirrup_agent/key_metrics/comparison/eval_elo -nemo_gym_gdpval_stirrup_agent/key_metrics/comparison/normalized_elo nemo_gym_gdpval_stirrup_agent/key_metrics/comparison/win_rate ``` From fb3c05854bbeea94aa4b8b91aae3d7784017784d Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 05:30:16 +0000 Subject: [PATCH 05/16] [skill] evaluation: fix GDPVal reference contradictions found by a regen test Regenerating a config from the skills alone (agent blocked from seeing the known-good answer) reproduced every score-determining setting exactly, but surfaced contradictions the shared reference had been left holding. - The SIF path said `/gdpval/sif/python-3.12.gdpval.sif` while SKILL.md, the task recipe and the template all say 3.13. This is the one string the file itself stresses must match in two places, and a mismatch degrades silently to non-sandboxed exec -- the worst possible thing to be stale. - The num_repeats section still ended with "remove the sed line to keep 2 for golden-comparable / reported scores", which is wrong for the multistage comparison flow (1) that the recipe and eval-config both specify. Since SKILL.md sends readers to this reference first, it was the answer they would find. Now split by flow. Also documents three things a generating agent had to guess: - `--max-num-seqs` must come from `stirrup_agent.concurrency`, not from `parallelism` (which is gym-internal); the generic rule yields an absurd cap. - `max_new_tokens` legitimately does not apply on the gym path (the adapter puts it in params_to_remove), so the six-field template is not violated. - temperature/top_p should match whatever the reference deliverables were generated with, since a sampling difference shows up in a pairwise ELO as if it were a quality difference. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .../evaluation/references/gym-gdpval.md | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/.agents/skills/evaluation/references/gym-gdpval.md b/.agents/skills/evaluation/references/gym-gdpval.md index 358839cfa04..d76befcfa83 100644 --- a/.agents/skills/evaluation/references/gym-gdpval.md +++ b/.agents/skills/evaluation/references/gym-gdpval.md @@ -22,7 +22,7 @@ is `recipes/examples/gym_gdpval/` and the per-task pointer is ## Apptainer SIF sandbox (self-contained: build-if-absent, reuse-if-present) The Stirrup agent runs each task's generated code in an Apptainer SIF, bind-mounted -into the eval container at **exactly** `/gdpval/sif/python-3.12.gdpval.sif` (the path +into the eval container at **exactly** `/gdpval/sif/python-3.13.gdpval.sif` (the path `GDPVAL_CONTAINER_PATH` points at). If it's missing or at a different path, the agent **silently falls back to non-sandboxed local exec** — the run "succeeds" but the numbers aren't comparable, so verify the SIF at canary. @@ -130,6 +130,17 @@ self-deploys single-node vLLM, which is fine for a canary or a small policy. For + `parallelism` (`16384`) is **gym-internal concurrency**, not a server cap. The real throttles are the agent's `stirrup_agent.concurrency` and the judge's `max_concurrent_requests` — raise those only after the judge logs are clean of 429s. ++ **`--max-num-seqs`: derive it from `stirrup_agent.concurrency`, NOT `parallelism`.** + SKILL Step 3/4's `ceil(parallelism / DP)` rule assumes `parallelism` is the in-flight + request count; on the gym path it is not, and applying it literally gives an absurd + cap. Use `ceil(stirrup_agent.concurrency / DP)` — e.g. 220 / DP 4 → 55, round to 64. ++ **`max_new_tokens` does not apply here.** Step 3 mandates it (with a model-card + lookup) for the six-field params template, but the GDPVal adapter puts `max_tokens` / + `max_completion_tokens` in `params_to_remove`, so this config has five params and no + `max_new_tokens`. That is correct, not an omission — skip that Step 3 subsection. ++ **Match `temperature` / `top_p` to whatever the reference deliverables were generated + with.** A pairwise ELO compares your deliverables against theirs, so a sampling + difference lands in the score as if it were a quality difference. + Long runs exceed 4h; rely on NEL's walltime dependency-chain resume (`resume_from_cache=true` is already set). See SKILL Step 4 + `run-validation.md`. @@ -237,13 +248,19 @@ standard (SKILL Step 1 shortcut #4): `auto_export.destinations: [mlflow]` + `cpu_partition` + a literal-valued `export.mlflow` block (tag `benchmark: nemo_gym.gdpval`). -## num_repeats workaround (OmegaConf) +## num_repeats — depends on the flow -`++gdpval.*.num_repeats=N` hits an OmegaConf `ListConfig` merge error, so the count -is patched by editing the checked-out file inside the task `command:`: -`sed -i 's/num_repeats: 2$/num_repeats: 1/' benchmarks/gdpval/config.yaml`. The Gym -default is 2 (the golden). The template seds it to **1** to halve cost; **remove the -sed line to keep 2 for golden-comparable / reported scores.** ++ **Multistage comparison** (the current golden, and the only path to an + AA-comparable score): **1**, set with a top-level `++num_repeats=1`, which *does* + work. Recent Gym pins already ship `num_repeats: 1` in + `benchmarks/gdpval/config.yaml`, so the `sed` below is a no-op there. ++ **Rubric / pre-multistage single-reference:** the old golden used **2** (220 × 2 = + 440 rollouts). On those pins the per-dataset key could not be set via `++` (an + OmegaConf `ListConfig` merge error), so the count was patched in the task + `command:` with `sed -i 's/num_repeats: 2$/num_repeats: 1/' benchmarks/gdpval/config.yaml` + — delete that line to keep 2. + +Do **not** carry a `=2` from an old single-reference config into a multistage run. ## Failure modes to check at canary From 923217131c887e0807dd3acb11ff79e03798c39e Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 05:37:55 +0000 Subject: [PATCH 06/16] [skill] evaluation: GDPVal num_repeats is 1, set via ++num_repeats Checked the designated golden (Qwen3.6-35B-A3B): it uses num_repeats=1, set as a plain top-level `++num_repeats=1` in the multistage block, with no `sed` patching anywhere. The `=2` in the skill came from the older MiniMax-M2.7 single-reference golden, which predates multistage. So standardise on 1 instead of making the reader work out which flow they are in: - template: drop the `sed 's/num_repeats: 2$/num_repeats: 1/'` workaround and its "the reviewed GOLDEN uses 2 / delete this line for reported scores" comment (both wrong now), and set `++num_repeats=1` the way the golden does. The pinned Gym already ships 1, so the override is belt-and-braces. - recipe + reference: collapse the two-branch explanation to "use 1", with the pre-multistage 2 kept only as a one-line historical note so nobody carries it forward from an old config. This also removes the last claim that num_repeats cannot be set via `++` -- it can, on current pins. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .../gym_gdpval/example_gym_gdpval.yaml | 10 +++------ .../evaluation/recipes/tasks/aa_gym/gdpval.md | 18 ++++----------- .../evaluation/references/gym-gdpval.md | 22 ++++++++----------- 3 files changed, 16 insertions(+), 34 deletions(-) diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index f9df5023764..22d70d3b8bb 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -251,13 +251,6 @@ evaluation: # Writable staging dir for ref files (bind-mounted, see execution). mkdir -p /gdpval_ref_files - # num_repeats: this template defaults to 1 (halves cost). The - # reviewed GOLDEN uses num_repeats=2 (220 tasks x 2 = 440 rollouts); - # for golden-comparable / REPORTED scores, DELETE the sed line below - # to keep the checked-in default of 2. num_repeats can NOT be set via - # a ++ override (OmegaConf ListConfig merge error) — patch the file. - sed -i 's/num_repeats: 2$/num_repeats: 1/' benchmarks/gdpval/config.yaml - ${gym_prepare.run} data_prep_params: >- "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" @@ -279,6 +272,8 @@ evaluation: ++upload_rollouts_to_wandb=false ++responses_create_params.temperature={{config.params.temperature}} ++responses_create_params.top_p={{config.params.top_p}} + # num_repeats=1 — the value both current goldens use. The pinned Gym + # already ships 1, so this override is belt-and-braces, not a change. # NOTE: no reference_* overrides below — rubric mode needs none, and the # single-reference keys (reference_deliverables_dir / reference_elo) # CONFLICT with comparison mode's reference_models map (added by @@ -298,6 +293,7 @@ evaluation: ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.concurrency=220 ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.tavily_api_key=$TAVILY_API_KEY ++gdpval_stirrup_agent.responses_api_agents.stirrup_agent.agent_max_turns=${oc.env:GDPVAL_MAX_TURNS,250} + ++num_repeats=1 ++gdpval_resources_server.resources_servers.gdpval.reward_mode=${gdpval.reward_mode} ++gdpval_resources_server.resources_servers.gdpval.persist_raw_judge_responses=true ++gdpval_resources_server.resources_servers.gdpval.preconvert_max_concurrent=30 diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index 4749b9e9e03..c3b3eed33e3 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -10,9 +10,7 @@ GDPVal is an **agentic** benchmark: the Stirrup agent produces office/PDF deliverables inside a per-task Apptainer code-exec sandbox, then a pairwise/rubric judge (**Gemini 3.1 Pro**) scores them. It is the most resource-intensive benchmark -in the suite — **220 tasks**, 4 judge trials per rollout. Repeat count depends on -the flow (multistage comparison uses 1; the pre-multistage golden used 2) — see -the num_repeats note under Config. +in the suite — **220 tasks**, `num_repeats=1`, 4 judge trials per rollout. It runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task (NOT nel-next), so Steps 1–9 apply — but with the branch differences below. @@ -63,17 +61,9 @@ recipes/examples/gym_gdpval/ Copy the **whole `gym_gdpval/` directory** to your workspace (the `- _gym_prepare` default resolves relative to the config dir — copying the yaml alone breaks it). -- **num_repeats — the right value depends on the flow, so check which one you're in:** - - **Multistage comparison** (the current golden for AA-comparable ELO): **1**, set - with a top-level `++num_repeats=1`, which *does* work. Recent Gym pins already - ship `num_repeats: 1` in `benchmarks/gdpval/config.yaml`, so the `sed` below is a - no-op there. - - **Rubric / older single-reference comparison:** the pre-multistage golden used - **2** (220 tasks × 2 = 440 rollouts). On old pins the per-dataset key could not be - set via `++` (OmegaConf `ListConfig` merge error), hence the `sed` in the task - `command:`; delete that line to keep 2. - - Do not carry a `=2` from an old single-reference config into a multistage run. +- **num_repeats: 1.** Both current goldens use it, set with a top-level + `++num_repeats=1` (which works — no `sed` needed; recent Gym pins already ship 1). + Pre-multistage single-reference configs used 2; do not carry that over. - **SIF ↔ Gym version (rebuild on bump):** the SIF is built from `gdpval.def` at `install_on_the_fly.commit`. **If you change that commit, rebuild the SIF** with a matching `gdpval-sif.sh --commit ` (to a new version-tagged filename, then diff --git a/.agents/skills/evaluation/references/gym-gdpval.md b/.agents/skills/evaluation/references/gym-gdpval.md index d76befcfa83..1d8d1da0fb0 100644 --- a/.agents/skills/evaluation/references/gym-gdpval.md +++ b/.agents/skills/evaluation/references/gym-gdpval.md @@ -248,19 +248,15 @@ standard (SKILL Step 1 shortcut #4): `auto_export.destinations: [mlflow]` + `cpu_partition` + a literal-valued `export.mlflow` block (tag `benchmark: nemo_gym.gdpval`). -## num_repeats — depends on the flow - -+ **Multistage comparison** (the current golden, and the only path to an - AA-comparable score): **1**, set with a top-level `++num_repeats=1`, which *does* - work. Recent Gym pins already ship `num_repeats: 1` in - `benchmarks/gdpval/config.yaml`, so the `sed` below is a no-op there. -+ **Rubric / pre-multistage single-reference:** the old golden used **2** (220 × 2 = - 440 rollouts). On those pins the per-dataset key could not be set via `++` (an - OmegaConf `ListConfig` merge error), so the count was patched in the task - `command:` with `sed -i 's/num_repeats: 2$/num_repeats: 1/' benchmarks/gdpval/config.yaml` - — delete that line to keep 2. - -Do **not** carry a `=2` from an old single-reference config into a multistage run. +## num_repeats + +**Use 1.** Both current goldens do, set with a top-level `++num_repeats=1` — it +works, and recent Gym pins already ship `num_repeats: 1` in +`benchmarks/gdpval/config.yaml`, so no `sed` patching is needed. + +Historical only: pre-multistage single-reference configs used 2 (220 × 2 = 440 +rollouts) and patched it with `sed` because the per-dataset key could not be set +via `++` on those pins. Do not carry a `=2` into a current run. ## Failure modes to check at canary From 64f696a4c77a45c057cf447b68da8695c414b2ee Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 05:51:42 +0000 Subject: [PATCH 07/16] [skill] evaluation: document the GDPVal judge panel OSS-side The judge details were living in the internal eval-config skill, but nothing about them is internal: the panel composition, the failure modes and the throttle values are properties of GDPVal itself, useful to anyone running it. Only the concrete endpoint URL is site-specific, and the public skill already has a convention for that (`` from `.env`). Adds a Judge section to the reference covering: - rubric uses one judge; comparison uses a 3-member panel sampled per trial, with judge_sampling_seed making the sampling reproducible; - inject the key's VALUE, not its name -- an interpolation that resolves to the literal string "INFERENCE_API_KEY" makes the proxy reply "LiteLLM Virtual Key expected", which the gym wraps as an opaque 500 that reads like a judge outage rather than a config error; - do not set judge_responses_create_params_overrides.model, which collapses the panel to a single judge and silently changes the methodology; - the judge rate-limits before the served model does, so max_concurrent_requests=10 / concurrency=220 are the golden values to raise only after clean 429 logs. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .../evaluation/references/gym-gdpval.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.agents/skills/evaluation/references/gym-gdpval.md b/.agents/skills/evaluation/references/gym-gdpval.md index 1d8d1da0fb0..a12b83c3d1e 100644 --- a/.agents/skills/evaluation/references/gym-gdpval.md +++ b/.agents/skills/evaluation/references/gym-gdpval.md @@ -179,6 +179,25 @@ whose Gym has `reference_models`. NVIDIA-internal users: the image path, the can reference set, and the matching gym overrides are in the `modelopttools:eval-config` skill (Step 3c) — internal cluster paths deliberately live only there. +## Judge + +Rubric mode uses a single judge. **Comparison mode uses a 3-member panel** — +`openai/gpt-5.5`, `gcp/google/gemini-3.1-pro-preview`, +`aws/anthropic/bedrock-claude-opus-4-8` — one **sampled per trial**, all routed +through the single `gdpval_judge_model` proxy (`` from `.env`). +`++...judge_sampling_seed=42` makes that sampling reproducible. + ++ **Inject the key's VALUE, not its name:** `openai_api_key=$INFERENCE_API_KEY`. + Passing an env-var *name* (e.g. via a `${...api_key}` interpolation that resolves to + the literal string `INFERENCE_API_KEY`) makes the proxy reply `LiteLLM Virtual Key + expected`, which the gym wraps as an opaque **500** — it looks like a judge outage, + not a config error. One key covers all three panel members. ++ **Do not set `judge_responses_create_params_overrides.model`.** Pinning a model + collapses the panel to a single judge, silently changing the scoring methodology. ++ **Throttles:** judge `max_concurrent_requests=10` and Stirrup `concurrency=220` are + the golden values — the judge rate-limits long before the served model does, so raise + these only after the judge logs are clean of 429s. + ## Preflight — what NEL validates, and what it does NOT NEL validates mount paths at **submit** time (`_collect_mount_paths` + From 4bf96e8b8f93f50287c0aa322219dbd82036676f Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 16:53:28 +0000 Subject: [PATCH 08/16] [skill] evaluation: compress the GDPVal docs (436 -> 341 lines) Same treatment the internal eval-config skill got: these are references an agent acts from, not essays. Cut narrative and cross-file duplication, keep every fact that prevents a mistake. reference (293 -> 225): - Merge the two SIF sections into one. The build-vs-run narrative, the privileged/rootless aside and the worked rebuild example were explanation; what remains is the rule (prefer a provided SIF, else build on the target cluster, never copy between clusters), the silent-fallback warning, the arch-aware pre_cmd note, and the diff-the-def check before a rebuild. - Scoring modes: drop the two-step baseline walkthrough, keep the rubric-vs- comparison distinction, the startup error string, and the pointer to the conversion checklist. recipe (143 -> 116): it had grown a second copy of the reference's SIF, scoring-mode and num_repeats material. Make it the thin entry point it should be -- what GDPVal is, the standalone rule, the config pointer, canary, and score extraction (the part that is genuinely unique to it) -- and delegate the mechanics to the reference in one line. Verified after the cut that the load-bearing facts survive: the silent-fallback warning, normalized_elo + results.yml extraction, reference_models, the sif helper, TRUST_PRE_CMD, num_repeats=1, the LiteLLM key-value trap, the `test -d` validation gap and the don't-pin-the-judge rule. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .../evaluation/recipes/tasks/aa_gym/gdpval.md | 75 +++------ .../evaluation/references/gym-gdpval.md | 144 +++++------------- 2 files changed, 62 insertions(+), 157 deletions(-) diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index c3b3eed33e3..8558d06e508 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -15,67 +15,40 @@ in the suite — **220 tasks**, `num_repeats=1`, 4 judge trials per rollout. It runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task (NOT nel-next), so Steps 1–9 apply — but with the branch differences below. -## What makes GDPVal different (do NOT treat it as a normal `aa/` task) +## What makes GDPVal different (not a normal `aa/` task) -- **Standalone — one gym eval per config.** Never add GDPVal to a multi-task +- **Standalone** — one gym eval per config. Never add GDPVal to a multi-task `evaluation.tasks` list, and never add other tasks to a GDPVal config. -- **Apptainer SIF sandbox (self-contained).** Set `GDPVAL_SIF_DIR` in `.env`, then - run `.agents/scripts/gdpval-sif.sh` (uses `$GDPVAL_SIF_DIR`) — it **builds if - absent, reuses if present**, and never copies from another cluster. The config - bind-mounts `$GDPVAL_SIF_DIR` at **exactly** `/gdpval/sif/python-3.13.gdpval.sif` - (matches `GDPVAL_CONTAINER_PATH`). Missing/mispathed → the agent **silently** runs - code-exec unsandboxed and results are not comparable. Details in `references/gym-gdpval.md`. -- **Thinking mode is mandatory.** Non-thinking loses ~86% of pairwise judgements. - Serve the policy with its `--reasoning-parser` and force thinking on via the - adapter `chat_template_kwargs` (see the example). -- **Judge + web search + gym plumbing.** Needs `INFERENCE_API_KEY` (judge auth), - `TAVILY_API_KEY` (agent web search), `INFERENCE_JUDGE_URL` (judge host, from - `.env`), a `pre_cmd` that installs apptainer/squashfuse, and the co-located - `_gym_prepare.yaml` include. - -## Scoring modes - -- **`rubric`** — the template default. Standalone LLM-judge scoring against each - task's rubric; **no reference deliverables**, runs on the public gym image. Gives a - 0–1 reward, **not** an ELO (ELO is undefined without an opponent). -- **`comparison`** — pairwise vs anchored reference deliverables; the **only** mode - that yields the AA-comparable `normalized_elo`. It is a conversion, not a flag - flip: it needs a reference set, a gym image whose Gym has the `reference_models` - map, ref mounts on both stages, and multistage overrides. **Do not just set - `reward_mode=comparison`** — the server exits at startup with - `reward_mode=comparison requires reference_deliverables_dir to be set`. - NVIDIA-internal users: `modelopttools:eval-config` Step 3c is the conversion - checklist. See `references/gym-gdpval.md` → "Scoring modes". +- **Apptainer SIF sandbox** — prefer a site-provided SIF; otherwise + `.agents/scripts/gdpval-sif.sh` builds one into `$GDPVAL_SIF_DIR` (build-if-absent, + never copied between clusters). Missing/misnamed → **silent** unsandboxed exec. +- **Thinking mode is mandatory** — non-thinking loses ~86% of pairwise judgements. + Serve with the model's `--reasoning-parser` and force it on via the adapter's + `chat_template_kwargs`. +- **Scoring:** `rubric` (template default, no references, no ELO) vs `comparison` + (the AA-comparable `normalized_elo`; a conversion, not a flag flip). +- Needs `INFERENCE_API_KEY`, `TAVILY_API_KEY`, `INFERENCE_JUDGE_URL`, + `GDPVAL_SIF_DIR` in `.env`, plus `NEMO_EVALUATOR_TRUST_PRE_CMD=1` (the config has a + `pre_cmd`). + +All of the above — SIF handling, the SIF↔Gym-commit coupling, scoring modes, judge +panel, preflight and failure modes — is detailed in **`references/gym-gdpval.md`**. +Read it before editing a GDPVal config. ## Config -**Do not copy a fragment into another config.** GDPVal is standalone — start from -the self-contained example and edit it: +Start from the self-contained example and edit it — **do not** copy a fragment into +another config: ```text recipes/examples/gym_gdpval/ - example_gym_gdpval.yaml # SLURM + single-node vLLM self-deploy template - _gym_prepare.yaml # co-located Hydra include (${gym_prepare.*}); travels with the yaml + example_gym_gdpval.yaml # SLURM + single-node vLLM self-deploy template (rubric) + _gym_prepare.yaml # co-located Hydra include; MUST travel with the yaml ``` -Copy the **whole `gym_gdpval/` directory** to your workspace (the `- _gym_prepare` -default resolves relative to the config dir — copying the yaml alone breaks it). - -- **num_repeats: 1.** Both current goldens use it, set with a top-level - `++num_repeats=1` (which works — no `sed` needed; recent Gym pins already ship 1). - Pre-multistage single-reference configs used 2; do not carry that over. -- **SIF ↔ Gym version (rebuild on bump):** the SIF is built from `gdpval.def` at - `install_on_the_fly.commit`. **If you change that commit, rebuild the SIF** with a - matching `gdpval-sif.sh --commit ` (to a new version-tagged filename, then - repoint `GDPVAL_CONTAINER_PATH`) — the def's base image + package stack change - across commits, and running a new gym with an old SIF makes the agent's generated - code fail imports in the sandbox → silently degraded deliverables. See - `references/gym-gdpval.md` → "Rebuild the SIF when the Gym version changes". -- **Deployment:** single-node vLLM in the template; a full 220-task run of a large - MoE typically needs multi-node — see `references/gym-gdpval.md`. -- Required `.env` keys: `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, - `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR` (see `recipes/env.example`). - `NEMO_EVALUATOR_TRUST_PRE_CMD=1` is needed because the config has a `pre_cmd`. +Copy the **whole dir** (the `- _gym_prepare` default resolves relative to the config +dir). `num_repeats=1` — already set by the template via `++num_repeats=1`; both +current goldens use it. A full 220-task run of a large MoE typically needs multi-node. ## Canary diff --git a/.agents/skills/evaluation/references/gym-gdpval.md b/.agents/skills/evaluation/references/gym-gdpval.md index a12b83c3d1e..2eff10bab84 100644 --- a/.agents/skills/evaluation/references/gym-gdpval.md +++ b/.agents/skills/evaluation/references/gym-gdpval.md @@ -19,87 +19,41 @@ is `recipes/examples/gym_gdpval/` and the per-task pointer is | Judge (pairwise/rubric) | external OpenAI-compatible endpoint (`gdpval_judge`, e.g. Gemini 3.1 Pro) | | Agent web search | Tavily (`TAVILY_API_KEY`) | -## Apptainer SIF sandbox (self-contained: build-if-absent, reuse-if-present) +## Apptainer SIF sandbox The Stirrup agent runs each task's generated code in an Apptainer SIF, bind-mounted -into the eval container at **exactly** `/gdpval/sif/python-3.13.gdpval.sif` (the path -`GDPVAL_CONTAINER_PATH` points at). If it's missing or at a different path, the agent -**silently falls back to non-sandboxed local exec** — the run "succeeds" but the -numbers aren't comparable, so verify the SIF at canary. - -**The skill builds the SIF on the target cluster — it never copies one from -another cluster.** That is the self-contained default and the right behaviour when -you have no prebuilt sandbox. - -> **If your site already provides a prebuilt GDPVal SIF, prefer it over building — -> build only when it is absent.** A self-built SIF resolves its pip stack at *your* -> build time, so it can drift from the sandbox a published reference set was -> generated in, which matters for comparison-mode runs. To use a provided one, mount -> its dir at `/gdpval/sif` and point `GDPVAL_CONTAINER_PATH` at the provided -> filename. **NVIDIA-internal users:** `modelopttools:eval-config` (Step 3c) records -> the provided SIF's location and the check-then-fall-back procedure — if you have -> that skill, consult it before building. Without it, building (below) is correct. - -Ship the SIF via the idempotent helper, which builds it if absent and reuses it if -already present: +at **exactly** the path `GDPVAL_CONTAINER_PATH` names (template: +`/gdpval/sif/python-3.13.gdpval.sif`). Missing or misnamed → the agent **silently** +runs code-exec unsandboxed; the run "succeeds" but the numbers aren't comparable. -```bash -# GDPVAL_SIF_DIR (.env) — persistent shared-FS dir on the TARGET cluster; the config -# bind-mounts this same dir at /gdpval/sif. Preferred: run the ~30-min build on the -# CPU partition, not a login node. `set -a && source .env` first so it's set. -srun -p cpu -t 01:00:00 --pty \ - .agents/scripts/gdpval-sif.sh # defaults to $GDPVAL_SIF_DIR (or pass a dir) -``` - -`gdpval-sif.sh` builds from the NeMo Gym `gdpval.def` at the pinned commit (keep -`GDPVAL_GYM_COMMIT` in sync with the config's `install_on_the_fly.commit`), writes -the SIF into that dir, and is flock-guarded + atomic so concurrent runs never -double-build. Re-running is a no-op once the SIF exists — that's the "reuse the -built one" path. It needs `apptainer`/`singularity` on the build host with -fakeroot/unprivileged build support and network egress; run it on a login or CPU -node (outside enroot, where fakeroot works), **not** inside the eval job. - -**Build vs run are separate.** The helper *builds* the SIF (once, off-GPU). The -eval then *runs* the prebuilt SIF inside the eval container — the golden-validated -path. The eval image doesn't ship apptainer, so the config's `pre_cmd` installs the -apptainer **runtime** + squashfuse (FUSE-mounts the SIF; falls back to slower -per-call extraction if `/dev/fuse` is absent). `pre_cmd` runs arbitrary commands → -the launching shell needs `export NEMO_EVALUATOR_TRUST_PRE_CMD=1`. The -apptainer-under-pyxis nesting is the least-validated part of the SLURM path — check -the canary logs for the "falling back to local exec" warning and apptainer mount -errors. (Local/Docker executor instead needs `--privileged` + the SIF bind mount via -`execution.extra_docker_args`; a rootless `--security-opt` + `/dev/fuse` variant is -in the upstream README appendix.) - -### Rebuild the SIF when the Gym version changes (SIF ↔ commit coupling) - -The SIF is **versioned with the Gym repo**: it's built from `gdpval.def` at -`install_on_the_fly.commit`, and that def changes across commits. Example — between -`2502893977` and the golden `049b1fd0`, the base went **python-3.12 → 3.13** and the -stack gained TeX Live, chromium/playwright, polars/duckdb, xgboost, geospatial -(gdal/proj/geos), and audio/video libs. The newer Stirrup agent's prompt advertises -that richer runtime, so the model's generated code reaches for those libs. - -**So whenever you bump `install_on_the_fly.commit`, rebuild the SIF from the matching -commit.** Run the new gym with an old SIF and the generated code fails its imports -*inside the sandbox* — deliverables silently degrade (missing figures/tables/docs) → -junk scores, with no hard error in the eval. Rebuild to a **version-tagged filename** -so the old SIF isn't clobbered (a running job keeps working), then repoint -`GDPVAL_CONTAINER_PATH` + the `/gdpval/sif` mount at the new file: +**If your site provides a prebuilt SIF, use it** — a self-built one resolves its pip +stack at *your* build time and can drift from the sandbox a published reference set +was generated in. Mount the provided dir at `/gdpval/sif` and point +`GDPVAL_CONTAINER_PATH` at its filename. (NVIDIA-internal: `modelopttools:eval-config` +Step 3c has the path.) Otherwise build it on the target cluster — never copy a SIF +between clusters: ```bash -# build the SIF for the NEW gym commit under a distinct name (old SIF stays intact) -GDPVAL_SIF_NAME=python-3.13.gdpval.sif \ - .agents/scripts/gdpval-sif.sh --commit "$GDPVAL_SIF_DIR" -# then set GDPVAL_CONTAINER_PATH=/gdpval/sif/python-3.13.gdpval.sif in the config +srun -p cpu -t 01:00:00 --pty .agents/scripts/gdpval-sif.sh # uses $GDPVAL_SIF_DIR ``` -Rule of thumb: **`install_on_the_fly.commit` and the SIF move together.** Any bump -that alters `gdpval.def` (base image or the apt/pip stack) needs a rebuild; a bump -that leaves `gdpval.def` byte-identical does not — diff the def at the two commits -(`raw.githubusercontent.com/NVIDIA-NeMo/Gym//responses_api_agents/stirrup_agent/containers/gdpval.def`) -to be sure. Note the def already disables apt's sandbox internally as of `049b1fd0`, -so it builds cleanly under the helper's unprivileged path. +`gdpval-sif.sh` is idempotent (flock-guarded, atomic): it builds from `gdpval.def` at +the pinned commit if absent and is a no-op once present. It needs +apptainer/singularity with unprivileged-build support and network egress — run it on a +login or CPU node, **not** inside the eval job. The eval image doesn't ship apptainer, +so the config's `pre_cmd` installs the **runtime** (arch-aware: use the Ubuntu PPA, not +an amd64 `.deb` — most Blackwell/Grace clusters are aarch64), which needs +`NEMO_EVALUATOR_TRUST_PRE_CMD=1`. + +**The SIF is versioned with the Gym repo.** `gdpval.def` changes across commits (e.g. +`2502893977` → `049b1fd0` moved python 3.12 → 3.13 and added TeX Live, playwright, +polars, geospatial), and the newer agent's prompt advertises that richer runtime. So +when you bump `install_on_the_fly.commit`, **diff the def at the two commits** +(`raw.githubusercontent.com/NVIDIA-NeMo/Gym//responses_api_agents/stirrup_agent/containers/gdpval.def`); +if it changed, rebuild to a **new version-tagged filename** (`GDPVAL_SIF_NAME=… gdpval-sif.sh --commit `) +and repoint `GDPVAL_CONTAINER_PATH`. Running a new gym on an old SIF makes the +generated code fail its imports *inside the sandbox* — deliverables silently degrade +with no error in the eval. ## The `_gym_prepare.yaml` include (why it exists) @@ -146,38 +100,16 @@ self-deploys single-node vLLM, which is fine for a canary or a small policy. For ## Scoring modes — rubric vs comparison -+ **`rubric`** (template default) — the judge scores each deliverable standalone - against a rubric; **no reference deliverables** needed. -+ **`comparison`** — pairwise: the judge compares the policy's deliverable to a - **reference model's** deliverable and the result is an ELO-anchored win-rate. - Two-step flow: - 1. **Baseline:** run your reference model with `-o gdpval.reward_mode=rubric` to - generate baseline deliverables (they land under `PERSIST_DELIVERABLES_DIR`). - 2. **Comparison:** run the candidate with `-o gdpval.reward_mode=comparison`, mount - the baseline deliverables at `/gdpval/refs/test_ref` - (`execution.mounts.evaluation`), and set `gdpval.reference_elo` to the reference - model's ELO (golden: Kimi-K2.5-Thinking, elo=1290). - -### Comparison mode needs a newer Gym than the public image — override `container:` - -The public `nvcr.io/nvidia/eval-factory/nemo-gym:26.05` (== `latest` by digest) runs -**rubric** mode fine, but its Gym predates the multi-reference `reference_models` map. -In comparison mode `gdpval_resources_server` fails validation at startup with -`reward_mode=comparison requires reference_deliverables_dir to be set`, surfacing only -as the unhelpful `Process gdpval_resources_server finished unexpectedly!`. - -**You cannot fix this by bumping `install_on_the_fly.commit`.** That image bakes Gym as -a **non-git directory**, so the pin is silently ignored — the prepare step logs -`=== /opt/Gym is not a git repo; using baked-in Gym version ===` and you get the baked -build regardless. Treat the pin as inert unless the log shows `=== NeMo Gym commit ===` -followed by a SHA, and don't attribute run-to-run behaviour changes to it. (In -particular, a head-server `address already in use` on its fixed port 11000 is a -**transient collision** — resubmitting clears it; it is not a Gym-version symptom.) - -To run comparison mode you must **override the task's `container:`** with an image -whose Gym has `reference_models`. NVIDIA-internal users: the image path, the canonical -reference set, and the matching gym overrides are in the `modelopttools:eval-config` -skill (Step 3c) — internal cluster paths deliberately live only there. ++ **`rubric`** (template default) — judge scores each deliverable against its rubric. + 0–1 reward, **no ELO** (undefined without an opponent). Runs on the public gym image. ++ **`comparison`** — pairwise vs anchored reference deliverables; the **only** mode + yielding the AA-comparable `normalized_elo`. It is a conversion, not a flag flip: + it needs a reference set, a gym image whose Gym has the `reference_models` map, ref + mounts on **both** deployment and evaluation, and multistage overrides. Setting + `reward_mode=comparison` alone exits at startup with + `reward_mode=comparison requires reference_deliverables_dir to be set`, surfaced + only as `Process gdpval_resources_server finished unexpectedly!`. + NVIDIA-internal: `modelopttools:eval-config` Step 3c is the conversion checklist. ## Judge From c199fc1bb3a9302e4d8d8728d61b35ad59678dcf Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 17:08:28 +0000 Subject: [PATCH 09/16] [skill] evaluation: fix three GDPVal traps found by a post-compression regen test Regenerating a config from the compressed docs alone reproduced every score-determining setting exactly, but surfaced three defects. All three fail silently, which is why they survived earlier passes. 1. SKILL.md's GDPVal branch still named `python-3.12.gdpval.sif` while the template, recipe and reference had all moved to 3.13 -- and a stale 3.12 SIF in an otherwise-correct dir passes NEL's `test -d` preflight, so the agent drops to unsandboxed code-exec with no error. The same step also presented *building* a SIF as mandatory, contradicting the prefer-a-provided-SIF rule the other two files state. Now: prefer provided, build as fallback, verify the filename with `gdpval-sif.sh --check`. 2. The template shipped `++...judge_responses_create_params_overrides.model=${gdpval_judge.model}`. Harmless in rubric mode (one judge), but on conversion to comparison it collapses the 3-member judge panel to a single judge and silently changes the scoring methodology. A "splice these overrides in" conversion checklist structurally cannot catch a line that must be *removed*, so remove it from the template instead: the judge model is already set by `openai_model=${gdpval_judge.model}`. 3. The template carried the generic `--max-num-seqs = ceil(parallelism / DP)` rule inline. On the gym path `parallelism` is gym-internal, not an in-flight request count, so that yields 4096. The correction existed only in the reference; put the right rule (`ceil(stirrup_agent.concurrency / DP)`) in the artifact people actually edit. Also makes the score-extraction snippet's `results.yml` path explicit instead of a bare filename. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .agents/skills/evaluation/SKILL.md | 12 +++++++----- .../examples/gym_gdpval/example_gym_gdpval.yaml | 8 ++++++-- .../skills/evaluation/recipes/tasks/aa_gym/gdpval.md | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.agents/skills/evaluation/SKILL.md b/.agents/skills/evaluation/SKILL.md index a35771b0839..767e3546ea6 100644 --- a/.agents/skills/evaluation/SKILL.md +++ b/.agents/skills/evaluation/SKILL.md @@ -61,11 +61,13 @@ GDPVal: failure modes) + **`recipes/tasks/aa_gym/gdpval.md`**. 2. Start from the self-contained **`recipes/examples/gym_gdpval/`** dir — copy the **whole dir** (the `_gym_prepare.yaml` include must travel next to the config). -3. Prerequisite: set `GDPVAL_SIF_DIR` in `.env`, then ensure the SIF exists with - `.agents/scripts/gdpval-sif.sh` (uses `$GDPVAL_SIF_DIR`; build-if-absent, - reuse-if-present, no cross-cluster copy). The config bind-mounts `$GDPVAL_SIF_DIR` - at exactly `/gdpval/sif/python-3.12.gdpval.sif`, or the agent silently runs - unsandboxed. `.env` needs `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, +3. Prerequisite — the Apptainer SIF. **If your site provides one, use it** + (NVIDIA-internal: `modelopttools:eval-config` Step 3c); otherwise set + `GDPVAL_SIF_DIR` in `.env` and build with `.agents/scripts/gdpval-sif.sh` + (build-if-absent, no cross-cluster copy). Either way the mounted dir must contain + the file `GDPVAL_CONTAINER_PATH` names (template: `python-3.13.gdpval.sif`) — a + name mismatch passes NEL's `test -d` check and the agent then silently runs + unsandboxed. Verify with `gdpval-sif.sh --check`. `.env` needs `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR`, and `NEMO_EVALUATOR_TRUST_PRE_CMD=1` (the config has a `pre_cmd`). Thinking mode is mandatory (non-thinking loses ~86%). 4. Dry-run → canary (`limit_samples=2`, verify the SIF sandbox + judge) → full. diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index 22d70d3b8bb..73c5a371e18 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -133,7 +133,9 @@ deployment: # a separate reasoning channel; thinking is forced on via the adapter_config # chat_template_kwargs below. Add --enable-expert-parallel for MoE, and # --trust-remote-code for custom-code models. - # After filling `parallelism`, append --max-num-seqs N (N = ceil(parallelism / data_parallel_size)). + # --max-num-seqs: on the GYM path derive it from the agent's concurrency, NOT from + # `parallelism` (which is gym-internal, not an in-flight request count — the generic + # SKILL rule would give an absurd cap). N = ceil(stirrup_agent.concurrency / DP). command: >- vllm serve /checkpoint --served-model-name ${deployment.served_model_name} @@ -272,6 +274,9 @@ evaluation: ++upload_rollouts_to_wandb=false ++responses_create_params.temperature={{config.params.temperature}} ++responses_create_params.top_p={{config.params.top_p}} + # No judge_responses_create_params_overrides.model here: the judge model is + # already set by openai_model=${gdpval_judge.model}, and pinning it a second + # way silently collapses comparison mode's 3-member judge panel to one judge. # num_repeats=1 — the value both current goldens use. The pinned Gym # already ships 1, so this override is belt-and-braces, not a change. # NOTE: no reference_* overrides below — rubric mode needs none, and the @@ -298,7 +303,6 @@ evaluation: ++gdpval_resources_server.resources_servers.gdpval.persist_raw_judge_responses=true ++gdpval_resources_server.resources_servers.gdpval.preconvert_max_concurrent=30 ++gdpval_resources_server.resources_servers.gdpval.preconvert_office_to_pdf=true - ++gdpval_resources_server.resources_servers.gdpval.judge_responses_create_params_overrides.model=${gdpval_judge.model} export: # LITERAL values only (auto_export resolves this block at submit time in a scope diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index 8558d06e508..bad0e872d2f 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -90,7 +90,7 @@ mirrored to MLflow. Read them by metric name: # final score from the local results file (no MLflow needed) python3 -c " import yaml,sys -m=yaml.safe_load(open('results.yml'))['groups']['nemo_gym']['metrics'] +m=yaml.safe_load(open('//nemo_gym.0/artifacts/results.yml'))['groups']['nemo_gym']['metrics'] for k in ('normalized_elo','eval_elo','win_rate'): n=f'gdpval_stirrup_agent/comparison/{k}' print(k, '=', m[n]['scores'][n]['value'])" From b906c84fc9f646ee572ac6bc491304c31b996615 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 18:42:18 +0000 Subject: [PATCH 10/16] [skill] evaluation: GDPVal fixes from PR review + the reviewed golden config Cross-checked the open review findings against the reviewed golden GDPVal run (Qwen3.6-35B-A3B). Two of my own claims were wrong; the rest are real bugs. Contradicted by the golden: - quantization-benchmarks.md still told the quant-validation path to "keep the golden num_repeats: 2 ... bump it back to 2", while the recipe and reference say 1. The golden uses 1, so the primary use case in this repo was being told to double a multi-hour judge-metered run against guidance elsewhere. - I had documented that max_new_tokens "does not apply" on the gym path. The golden sets it alongside the adapter's params_to_remove, so the Step 3 model-card lookup applies as normal. Real defects in code I wrote: - gdpval-sif.sh: the unprivileged fallback could never succeed, because a failed --fakeroot attempt leaves a partial $tmp and apptainer refuses an existing destination. Clear it before falling back. - gdpval-sif.sh --check: `[[ -d ... ]] && { ... }` is a non-final command under `set -e`, so when the dir is missing the script exited before printing the "Build it with:" hint -- exactly when the hint matters. The "(no .sif files)" branch was also dead, since sed exits 0 on empty input. - The template exposed GDPVAL_MAX_TURNS as a container env var, which cannot affect agent_max_turns: that value is an ${oc.env:...} resolved at submit time against the launching shell. The golden does not set it either. - The score-extraction snippet was comparison-only, but the shipped template is rubric-only, so it KeyErrors on the default config. Added the rubric form. Checked and NOT changed: `--env-file` is a real nel 0.2.6 flag; and both `use_reasoning` and `process_reasoning_traces` exist in the installed adapter config, so the template's field is valid even though the golden uses the other. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .agents/scripts/gdpval-sif.sh | 11 ++++++++--- .../examples/gym_gdpval/example_gym_gdpval.yaml | 2 -- .../skills/evaluation/recipes/tasks/aa_gym/gdpval.md | 11 +++++++++-- .agents/skills/evaluation/references/gym-gdpval.md | 8 ++++---- .../evaluation/references/quantization-benchmarks.md | 5 ++--- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.agents/scripts/gdpval-sif.sh b/.agents/scripts/gdpval-sif.sh index 00034ff5888..7f6493d7f2d 100755 --- a/.agents/scripts/gdpval-sif.sh +++ b/.agents/scripts/gdpval-sif.sh @@ -88,8 +88,11 @@ if [[ "$check" -eq 1 ]]; then echo "$sif"; exit 0 fi printf '\033[31mgdpval-sif: MISSING expected SIF: %s\033[0m\n' "$sif" >&2 - [[ -d "$sif_dir" ]] && { echo " dir exists but does not contain it; found:" >&2 - ls -1 "$sif_dir"/*.sif 2>/dev/null | sed 's/^/ /' >&2 || echo " (no .sif files)" >&2; } + if [[ -d "$sif_dir" ]]; then + echo " dir exists but does not contain it; found:" >&2 + if ls -1 "$sif_dir"/*.sif >/dev/null 2>&1; then ls -1 "$sif_dir"/*.sif | sed 's/^/ /' >&2 + else echo " (no .sif files)" >&2; fi + fi echo " Build it with: $0 ${sif_dir} (or --commit for a different def)" >&2 exit 1 fi @@ -138,7 +141,9 @@ else wget -qO "$def_local" "$def_url"; fi # unprivileged build where fakeroot is unavailable. if "$APPTAINER_BIN" build --fakeroot "$tmp" "$def_local"; then : -elif "$APPTAINER_BIN" build "$tmp" "$def_local"; then +# A failed --fakeroot attempt can leave a partial $tmp behind, and apptainer refuses an +# existing destination — clear it or the unprivileged fallback can never succeed. +elif rm -f "$tmp" && "$APPTAINER_BIN" build "$tmp" "$def_local"; then _log "built without --fakeroot (unprivileged mode)" else rm -f "$tmp" "$def_local" diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index 73c5a371e18..86438c14956 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -166,8 +166,6 @@ evaluation: # deliverables are NOT auto-uploaded — they stay on disk for inspection. # Drop "_cache" if you WANT them uploaded as artifacts. PERSIST_DELIVERABLES_DIR: lit:/results/gdpval/deliverables_cache - # Stirrup agent turn cap (optional; default 100). - # GDPVAL_MAX_TURNS: lit:100 NEL_INVOCATION_ID: runtime:NEL_INVOCATION_ID # Installs apptainer + squashfuse into the eval container (needs # NEMO_EVALUATOR_TRUST_PRE_CMD=1 in the launching shell). See references/gym-gdpval.md. diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index bad0e872d2f..2d6b068eb13 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -87,13 +87,20 @@ mirrored to MLflow. Read them by metric name: | rubric | mean of `reward` across `artifacts/evaluator_rollouts.jsonl` (per-rollout 0–1) | ```bash -# final score from the local results file (no MLflow needed) +# COMPARISON mode — final score from the local results file (no MLflow needed) python3 -c " -import yaml,sys +import yaml m=yaml.safe_load(open('//nemo_gym.0/artifacts/results.yml'))['groups']['nemo_gym']['metrics'] for k in ('normalized_elo','eval_elo','win_rate'): n=f'gdpval_stirrup_agent/comparison/{k}' print(k, '=', m[n]['scores'][n]['value'])" + +# RUBRIC mode (the template default) — there is no ELO; average the per-rollout reward +python3 -c " +import json +r=[json.loads(l).get('reward') for l in open('//nemo_gym.0/artifacts/evaluator_rollouts.jsonl')] +r=[x for x in r if isinstance(x,(int,float))] +print('mean reward =', sum(r)/len(r), 'over', len(r), 'rollouts')" ``` In **MLflow** the same values are prefixed `nemo_gym_` and duplicated under a diff --git a/.agents/skills/evaluation/references/gym-gdpval.md b/.agents/skills/evaluation/references/gym-gdpval.md index 2eff10bab84..4ae24460112 100644 --- a/.agents/skills/evaluation/references/gym-gdpval.md +++ b/.agents/skills/evaluation/references/gym-gdpval.md @@ -88,10 +88,10 @@ self-deploys single-node vLLM, which is fine for a canary or a small policy. For SKILL Step 3/4's `ceil(parallelism / DP)` rule assumes `parallelism` is the in-flight request count; on the gym path it is not, and applying it literally gives an absurd cap. Use `ceil(stirrup_agent.concurrency / DP)` — e.g. 220 / DP 4 → 55, round to 64. -+ **`max_new_tokens` does not apply here.** Step 3 mandates it (with a model-card - lookup) for the six-field params template, but the GDPVal adapter puts `max_tokens` / - `max_completion_tokens` in `params_to_remove`, so this config has five params and no - `max_new_tokens`. That is correct, not an omission — skip that Step 3 subsection. ++ **`max_new_tokens`:** the reviewed golden **does** set it alongside the adapter's + `params_to_remove: [max_tokens, max_completion_tokens]`, so do the Step 3 model-card + lookup as normal. The template omits it (five params) because the adapter strips the + per-request cap anyway; adding it back matches the golden and is harmless. + **Match `temperature` / `top_p` to whatever the reference deliverables were generated with.** A pairwise ELO compares your deliverables against theirs, so a sampling difference lands in the score as if it were a quality difference. diff --git a/.agents/skills/evaluation/references/quantization-benchmarks.md b/.agents/skills/evaluation/references/quantization-benchmarks.md index 755d78a349a..978d51843f0 100644 --- a/.agents/skills/evaluation/references/quantization-benchmarks.md +++ b/.agents/skills/evaluation/references/quantization-benchmarks.md @@ -75,9 +75,8 @@ merged into the `aa/` multi-task list. `tasks` list), needs the Apptainer SIF sandbox + judge, and is multi-hour / often multi-node. Generate it alongside the `aa/` config; see `recipes/tasks/aa_gym/gdpval.md` + `references/gym-gdpval.md`. Thinking mode is - mandatory (non-thinking loses ~86% of pairwise judgements). For low-variance - quant comparisons keep the golden `num_repeats: 2` — the example template - defaults to 1 to halve cost, so bump it back to 2 for quant validation. + mandatory (non-thinking loses ~86% of pairwise judgements). `num_repeats` is **1** — + the value both current goldens use, already set by the template; do not raise it. ## How to use From 7e074f819c78833f248629a9aada6fb57a82a8ff Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 18:50:59 +0000 Subject: [PATCH 11/16] [skill] evaluation: inline _gym_prepare, making the GDPVal example self-contained The include was described as "shared by every benchmark", but there is one gym benchmark and one consumer, and the split created a footgun that then had to be warned about in five separate places -- SKILL.md, the recipe, the reference, the config header, and the failure-modes list ("if it still hangs, the include didn't travel with the config"). Copying the yaml without its sibling silently produced an unresolvable ${gym_prepare.*} interpolation. Inline both shell blocks into the task `command:` and delete the file. The example is now a single self-contained yaml like example_eval.yaml and example_eval_next.yaml, and the "copy the whole dir" instruction disappears from every doc that carried it. The reference keeps the *why* under a "Gym prepare / reap" heading, since the blocks are still unobvious: the venv repair works around the eval image's deployment-oriented packaging, and the setsid + process-group reap works around Gym's incomplete shutdown (orphaned Ray workers otherwise hold the launcher's stdout open and the run hangs in post-eval). Both are marked for removal once upstream fixes land. Verified: the template parses, the inlined command passes `bash -n` with quote-containing params substituted, and no `#` leaked into a folded scalar. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .agents/skills/evaluation/SKILL.md | 6 +- .../examples/gym_gdpval/_gym_prepare.yaml | 117 ------------------ .../gym_gdpval/example_gym_gdpval.yaml | 63 ++++++++-- .../evaluation/recipes/tasks/aa_gym/gdpval.md | 8 +- .../evaluation/references/gym-gdpval.md | 38 +++--- 5 files changed, 83 insertions(+), 149 deletions(-) delete mode 100644 .agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml diff --git a/.agents/skills/evaluation/SKILL.md b/.agents/skills/evaluation/SKILL.md index 767e3546ea6..a097744bb70 100644 --- a/.agents/skills/evaluation/SKILL.md +++ b/.agents/skills/evaluation/SKILL.md @@ -56,11 +56,11 @@ nel-next), so Steps 1–9 apply — but it is mechanically special and **standal (one gym eval per config; never mix it with `aa/` tasks). If the user asks for GDPVal: -1. Read **`references/gym-gdpval.md`** (Apptainer SIF sandbox, `_gym_prepare.yaml` +1. Read **`references/gym-gdpval.md`** (Apptainer SIF sandbox, gym prepare/reap machinery, deploy sizing, rubric-vs-comparison scoring, MLflow deliverables trap, failure modes) + **`recipes/tasks/aa_gym/gdpval.md`**. -2. Start from the self-contained **`recipes/examples/gym_gdpval/`** dir — copy the - **whole dir** (the `_gym_prepare.yaml` include must travel next to the config). +2. Start from **`recipes/examples/gym_gdpval/example_gym_gdpval.yaml`** — a single + self-contained file. 3. Prerequisite — the Apptainer SIF. **If your site provides one, use it** (NVIDIA-internal: `modelopttools:eval-config` Step 3c); otherwise set `GDPVAL_SIF_DIR` in `.env` and build with `.agents/scripts/gdpval-sif.sh` diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml deleted file mode 100644 index bd9fde37fbf..00000000000 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/_gym_prepare.yaml +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Shared prepare/teardown snippets — include via: defaults: [_gym_prepare, _self_] -# -# Compensates for the eval image's deployment-oriented packaging when run -# standalone. Interpolate into each benchmark's command: -# ${gym_prepare.prepare} at the start (activate venv, checkout pin, repair venvs) -# ${gym_prepare.run} as the last line (data prep + rollout collection run in -# its own session, then process-group reaped so the run -# finalizes — see the run: comment for why) -# Remove once the eval image ships complete, ray-consistent venvs and Gym's -# shutdown reaps server process groups. - -gym_prepare: - # Common command preamble shared by every benchmark: activate the baked Gym - # venv, optionally checkout the install_on_the_fly pin (only when /opt/Gym is a - # git repo; baked images aren't), then repair the image's incomplete per-server - # venvs. The repair: (1) rewrites each component requirements.txt — drops the - # editable "-e nemo-gym[dev]" line (it forces a ray>=2.55.1 re-resolve vs the - # image's pinned ray 2.49.2, breaking venv-less servers like arena_judge) and - # ensures ray==2.49.2 + tqdm; (2) installs the fixed requirements into each - # baked (skeleton) sub-venv; (3) fronts the main venv on PYTHONPATH so nemo_gym - # + framework deps resolve for server processes. - # NOTE: avoid bash ${VAR} here — OmegaConf parses ${...}; $(...) / $r / $v are fine. - # NOTE: every step is guarded (|| true / -q grep) so it is safe under set -e. - prepare: |- - set -ex - cd /opt/Gym - export UV_CACHE_DIR=/opt/cache/uv - source .venv/bin/activate - # install_on_the_fly: checkout the pin when /opt/Gym is a git repo; some - # images bake Gym at a fixed version (not a git repo) — use it as-is. - if [ -d .git ]; then - git remote add oss_pin "{{config.params.extra.nemo_gym.install_on_the_fly.url}}" 2>/dev/null || true - git fetch oss_pin - git checkout "{{config.params.extra.nemo_gym.install_on_the_fly.commit}}" - echo "=== NeMo Gym commit ===" && git rev-parse HEAD - else - echo "=== /opt/Gym is not a git repo; using baked-in Gym version ===" - fi - for r in /opt/Gym/responses_api_models/*/requirements.txt \ - /opt/Gym/responses_api_agents/*/requirements.txt \ - /opt/Gym/resources_servers/*/requirements.txt; do - [ -f "$r" ] || continue - grep -vE '^[[:space:]]*-e ' "$r" > "$r.fixed" || true - grep -qiE '^ray([<>=[]|$)' "$r.fixed" 2>/dev/null || echo 'ray==2.49.2' >> "$r.fixed" - grep -qiE '^tqdm' "$r.fixed" 2>/dev/null || echo 'tqdm' >> "$r.fixed" - mv "$r.fixed" "$r" 2>/dev/null || true - done - for v in /opt/Gym/responses_api_models/*/.venv \ - /opt/Gym/responses_api_agents/*/.venv \ - /opt/Gym/resources_servers/*/.venv; do - [ -d "$v" ] || continue - d="$(dirname "$v")" - [ -f "$d/requirements.txt" ] && uv pip install --python "$v/bin/python" -q -r "$d/requirements.txt" || true - done - export PYTHONPATH="/opt/Gym:$(/opt/Gym/.venv/bin/python -c 'import site; print(site.getsitepackages()[0])')" - - # Data prep + rollout collection. Interpolate as the LAST line of each - # benchmark's command via ${gym_prepare.run} (after ${gym_prepare.prepare} and - # any benchmark-specific prep). - # - # Rollout collection is run in its OWN session (setsid) so its entire process - # tree — servers + their multiprocessing pools + Ray workers — can be reaped by - # process group. Why: ng_e2e's own cleanup (cli.py shutdown) only SIGINTs/SIGKILLs - # the tracked server PIDs after a 1s grace; it does NOT reap each server's child - # tree, so pools/Ray workers orphan and re-parent to the launcher (which is - # blocked reading our stdout) — they hold stdout open and the run never finalizes. - # (Invisible in the deployment flow because the whole node is discarded; only bites - # the deployment-free inline path here.) The inner shell records the session PGID - # (its own $$, before exec) so we can target it; real exit code is captured from - # wait and propagated via exit. - # - # Ray ALSO daemonizes gcs_server/raylet into their own session, so they escape the - # process-group reap; if left running they keep the launcher's stdout open and the - # run hangs in post-eval. So after the group reap we additionally stop Ray and kill - # its daemons by name (safe: matches only Ray daemons, not the launcher's python). - # Remove once Gym's shutdown reaps server process groups + Ray. - run: |- - ng_prepare_benchmark {{config.params.extra.nemo_gym.data_prep_params}} {{config.params.extra.nemo_gym.common_params}} - # The rollout command is written to a script and run from it, rather than passed - # to `bash -c '...'`. Gym params legitimately CONTAIN single quotes — comparison - # mode's ++multistage.stages='[{num_tasks: 45}, ...]' and ++...judge_panel='[{...}]' - # — and those would close a single-quoted `bash -c` wrapper early, so Hydra then - # receives the value split on spaces and dies with - # "no viable alternative at input '[{num_tasks:'". - # The heredoc delimiter is QUOTED ('GYM_RUN_EOF') so nothing expands while writing: - # $$ and $TAVILY_API_KEY / $INFERENCE_API_KEY land in the script literally and are - # expanded at run time by the shell that executes it, which is what we want. - cat > /tmp/gym_run.sh <<'GYM_RUN_EOF' - echo $$ > /tmp/gym_eval_pgid - exec ng_e2e_collect_rollouts {{config.params.extra.nemo_gym.collect_rollout_params}} {{config.params.extra.nemo_gym.common_params}} - GYM_RUN_EOF - setsid --wait bash /tmp/gym_run.sh & - __ev=$! - __rc=0; wait "$__ev" || __rc=$? - echo "Evaluator Gym finished!" - __pg=$(cat /tmp/gym_eval_pgid 2>/dev/null || echo) - [ -n "$__pg" ] && kill -9 -"$__pg" 2>/dev/null || true - timeout 60 ray stop --force >/dev/null 2>&1 || true - pkill -9 -f raylet >/dev/null 2>&1 || true - pkill -9 -f gcs_server >/dev/null 2>&1 || true - pkill -9 -f plasma_store >/dev/null 2>&1 || true - exit $__rc diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index 86438c14956..2c161d37611 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -38,10 +38,6 @@ # 2. .env has HF_TOKEN, INFERENCE_API_KEY (judge auth), TAVILY_API_KEY (agent web # search), INFERENCE_JUDGE_URL (judge host), and GDPVAL_SIF_DIR (item 1). See # recipes/env.example. -# 3. This file's `defaults` include `_gym_prepare` — the _gym_prepare.yaml in -# THIS directory MUST travel next to this config (Hydra resolves it -# relative to the config dir). Copy the whole gym_gdpval/ dir to your -# workspace, don't copy the yaml alone. # # Canary (validates SIF sandbox + judge + gym plumbing on a couple of tasks): # nel run --config example_gym_gdpval.yaml --env-file .env \ @@ -53,7 +49,6 @@ defaults: # gres — see SKILL.md Step 4). - execution: slurm/default - deployment: vllm - - _gym_prepare # provides ${gym_prepare.prepare} / ${gym_prepare.run} - _self_ # GDPVal scoring mode. This template is RUBRIC-ONLY, deliberately: rubric needs no @@ -246,12 +241,66 @@ evaluation: # in the client log before crediting it with any behaviour change. commit: dd41196f620f2af99947d776cbe5da9439d2a08d # pragma: allowlist secret command: | - ${gym_prepare.prepare} + set -ex + cd /opt/Gym + export UV_CACHE_DIR=/opt/cache/uv + source .venv/bin/activate + # install_on_the_fly: checkout the pin when /opt/Gym is a git repo; some + # images bake Gym at a fixed version (not a git repo) — use it as-is. + if [ -d .git ]; then + git remote add oss_pin "{{config.params.extra.nemo_gym.install_on_the_fly.url}}" 2>/dev/null || true + git fetch oss_pin + git checkout "{{config.params.extra.nemo_gym.install_on_the_fly.commit}}" + echo "=== NeMo Gym commit ===" && git rev-parse HEAD + else + echo "=== /opt/Gym is not a git repo; using baked-in Gym version ===" + fi + for r in /opt/Gym/responses_api_models/*/requirements.txt \ + /opt/Gym/responses_api_agents/*/requirements.txt \ + /opt/Gym/resources_servers/*/requirements.txt; do + [ -f "$r" ] || continue + grep -vE '^[[:space:]]*-e ' "$r" > "$r.fixed" || true + grep -qiE '^ray([<>=[]|$)' "$r.fixed" 2>/dev/null || echo 'ray==2.49.2' >> "$r.fixed" + grep -qiE '^tqdm' "$r.fixed" 2>/dev/null || echo 'tqdm' >> "$r.fixed" + mv "$r.fixed" "$r" 2>/dev/null || true + done + for v in /opt/Gym/responses_api_models/*/.venv \ + /opt/Gym/responses_api_agents/*/.venv \ + /opt/Gym/resources_servers/*/.venv; do + [ -d "$v" ] || continue + d="$(dirname "$v")" + [ -f "$d/requirements.txt" ] && uv pip install --python "$v/bin/python" -q -r "$d/requirements.txt" || true + done + export PYTHONPATH="/opt/Gym:$(/opt/Gym/.venv/bin/python -c 'import site; print(site.getsitepackages()[0])')" # Writable staging dir for ref files (bind-mounted, see execution). mkdir -p /gdpval_ref_files - ${gym_prepare.run} + ng_prepare_benchmark {{config.params.extra.nemo_gym.data_prep_params}} {{config.params.extra.nemo_gym.common_params}} + # The rollout command is written to a script and run from it, rather than passed + # to `bash -c '...'`. Gym params legitimately CONTAIN single quotes — comparison + # mode's ++multistage.stages='[{num_tasks: 45}, ...]' and ++...judge_panel='[{...}]' + # — and those would close a single-quoted `bash -c` wrapper early, so Hydra then + # receives the value split on spaces and dies with + # "no viable alternative at input '[{num_tasks:'". + # The heredoc delimiter is QUOTED ('GYM_RUN_EOF') so nothing expands while writing: + # $$ and $TAVILY_API_KEY / $INFERENCE_API_KEY land in the script literally and are + # expanded at run time by the shell that executes it, which is what we want. + cat > /tmp/gym_run.sh <<'GYM_RUN_EOF' + echo $$ > /tmp/gym_eval_pgid + exec ng_e2e_collect_rollouts {{config.params.extra.nemo_gym.collect_rollout_params}} {{config.params.extra.nemo_gym.common_params}} + GYM_RUN_EOF + setsid --wait bash /tmp/gym_run.sh & + __ev=$! + __rc=0; wait "$__ev" || __rc=$? + echo "Evaluator Gym finished!" + __pg=$(cat /tmp/gym_eval_pgid 2>/dev/null || echo) + [ -n "$__pg" ] && kill -9 -"$__pg" 2>/dev/null || true + timeout 60 ray stop --force >/dev/null 2>&1 || true + pkill -9 -f raylet >/dev/null 2>&1 || true + pkill -9 -f gcs_server >/dev/null 2>&1 || true + pkill -9 -f plasma_store >/dev/null 2>&1 || true + exit $__rc data_prep_params: >- "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" +hf_token=$HF_TOKEN diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index 2d6b068eb13..68ade7390ab 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -41,13 +41,11 @@ Start from the self-contained example and edit it — **do not** copy a fragment another config: ```text -recipes/examples/gym_gdpval/ - example_gym_gdpval.yaml # SLURM + single-node vLLM self-deploy template (rubric) - _gym_prepare.yaml # co-located Hydra include; MUST travel with the yaml +recipes/examples/gym_gdpval/example_gym_gdpval.yaml # SLURM + single-node vLLM, + # rubric mode, self-contained ``` -Copy the **whole dir** (the `- _gym_prepare` default resolves relative to the config -dir). `num_repeats=1` — already set by the template via `++num_repeats=1`; both +`num_repeats=1` — already set by the template via `++num_repeats=1`; both current goldens use it. A full 220-task run of a large MoE typically needs multi-node. ## Canary diff --git a/.agents/skills/evaluation/references/gym-gdpval.md b/.agents/skills/evaluation/references/gym-gdpval.md index 4ae24460112..c684124c7bd 100644 --- a/.agents/skills/evaluation/references/gym-gdpval.md +++ b/.agents/skills/evaluation/references/gym-gdpval.md @@ -55,21 +55,26 @@ and repoint `GDPVAL_CONTAINER_PATH`. Running a new gym on an old SIF makes the generated code fail its imports *inside the sandbox* — deliverables silently degrade with no error in the eval. -## The `_gym_prepare.yaml` include (why it exists) - -`nemo_gym` tasks interpolate two shared snippets into the task `command:`: -`${gym_prepare.prepare}` (activate the baked Gym venv, checkout the -`install_on_the_fly` pin, repair the image's incomplete per-server venvs, front the -main venv on `PYTHONPATH`) and `${gym_prepare.run}` (data prep + -`ng_e2e_collect_rollouts`, run in its own `setsid` session so the whole server/Ray -process tree can be reaped by process group — otherwise orphaned Ray workers hold -the launcher's stdout open and the run **hangs in post-eval**). Both compensate for -the eval image's deployment-oriented packaging and Gym's incomplete shutdown; remove -once the image ships complete ray-consistent venvs. - -**The include is co-located, not central.** Hydra resolves `- _gym_prepare` relative -to the run config's directory, so `_gym_prepare.yaml` must sit next to your config — -copy the whole `recipes/examples/gym_gdpval/` dir, not the yaml alone. +## Gym prepare / reap (why the task `command:` is long) + +The task `command:` carries two workaround blocks, inlined in the template: + +1. **prepare** — activate the baked Gym venv, checkout the `install_on_the_fly` pin + (only if `/opt/Gym` is a git repo), repair the image's incomplete per-server venvs + (drop the editable `-e nemo-gym[dev]` line, which forces a ray re-resolve that + breaks venv-less servers; pin `ray==2.49.2` + `tqdm`), and front the main venv on + `PYTHONPATH`. +2. **run** — data prep, then `ng_e2e_collect_rollouts` executed from a script written + via a **quoted heredoc** and launched under `setsid`, so the whole server/Ray + process tree can be reaped by process group. Without that reap, orphaned Ray + workers hold the launcher's stdout open and the run **hangs in post-eval**; the + quoted heredoc keeps `$$` and `$*_API_KEY` unexpanded until run time and survives + params that contain single quotes (comparison mode's `stages='[{...}]'`). + +Both compensate for the eval image's deployment-oriented packaging and Gym's +incomplete shutdown — remove them once the image ships complete ray-consistent venvs +and Gym reaps its own process groups. Avoid bash `${VAR}` inside these blocks: +OmegaConf parses `${...}`. `$(...)`, `$$` and `$VAR` are fine. ## Deployment sizing @@ -219,7 +224,6 @@ via `++` on those pins. Do not carry a `=2` into a current run. `chat_template_kwargs.enable_thinking: true` (right toggle key for the family) + the policy's `--reasoning-parser`. + **Run hangs in post-eval** — orphaned Ray/gym processes holding stdout; that's what - the `${gym_prepare.run}` setsid + process-group reap prevents. If it still hangs, - the `_gym_prepare.yaml` include didn't travel with the config. + the setsid + process-group reap in the task `command:` prevents. + **Multi-node ref-file errors** — `GDPVAL_REF_FILES_DIR` on node-local storage; point it at a shared-FS staging dir. From 9102f876e169c754fa0d1f6c22f46bc476980c10 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 19:04:05 +0000 Subject: [PATCH 12/16] [skill] evaluation: GDPVal follow-ups from the post-inlining validation A regen from the now-self-contained template reproduced every score-determining setting and copied the inlined `command:` verbatim, which was the point of the inlining. Two things it surfaced are worth fixing. - The prepare block hardcoded `export UV_CACHE_DIR=/opt/cache/uv`, overriding the `UV_CACHE_DIR` env var and the `/cache/uv` mount that the template sets up two screens earlier -- so the mounted uv cache was silently bypassed for the venv repair, and `++uv_cache_dir=$UV_CACHE_DIR` then pointed at a container-local dir. Only set it when it is not already provided. (No `${VAR}` braces -- OmegaConf parses those.) - Document the one case where the SIF <-> gym-commit coupling rule does not apply: a site-provided SIF used with a site-provided gym image. Those images bake Gym as a non-git dir, so the pin is inert and the two artifacts are already matched -- even though the SIF filename can encode a different SHA than `install_on_the_fly.commit`, which reads like a violation of the rule stated just above it. Not changed: a review claim that MLflow auto-export is rejected on aws-cmh because `sbatch_extra_flags.qos` reaches the export sbatch. Evidence says otherwise -- the export job ran on partition `cpu` and COMPLETED (2m20s), precisely because those flags are not propagated to it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .../recipes/examples/gym_gdpval/example_gym_gdpval.yaml | 5 ++++- .agents/skills/evaluation/references/gym-gdpval.md | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index 2c161d37611..1ff00ab8b9f 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -243,7 +243,10 @@ evaluation: command: | set -ex cd /opt/Gym - export UV_CACHE_DIR=/opt/cache/uv + # Honour a mounted uv cache if one was injected (UV_CACHE_DIR env var + + # /cache/uv mount); fall back to the image-local dir. No bash ${VAR} + # braces here — OmegaConf parses ${...}. + [ -n "$UV_CACHE_DIR" ] || export UV_CACHE_DIR=/opt/cache/uv source .venv/bin/activate # install_on_the_fly: checkout the pin when /opt/Gym is a git repo; some # images bake Gym at a fixed version (not a git repo) — use it as-is. diff --git a/.agents/skills/evaluation/references/gym-gdpval.md b/.agents/skills/evaluation/references/gym-gdpval.md index c684124c7bd..999d6bd6ee0 100644 --- a/.agents/skills/evaluation/references/gym-gdpval.md +++ b/.agents/skills/evaluation/references/gym-gdpval.md @@ -55,6 +55,13 @@ and repoint `GDPVAL_CONTAINER_PATH`. Running a new gym on an old SIF makes the generated code fail its imports *inside the sandbox* — deliverables silently degrade with no error in the eval. +**Exception — a site-provided SIF paired with a site-provided gym image.** Those +images typically bake Gym as a non-git dir, so `install_on_the_fly.commit` is inert +and the two provided artifacts are already matched to each other; that pairing is the +coherent one even when the SIF filename encodes a different SHA than your pin. You +cannot check the image's baked Gym from the config — confirm from the client log +(`=== NeMo Gym commit ===` + a SHA, or the "not a git repo" line). + ## Gym prepare / reap (why the task `command:` is long) The task `command:` carries two workaround blocks, inlined in the template: From 62fbb10ddf0cd03426d5a26f707681ae8c1444a2 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 19:51:35 +0000 Subject: [PATCH 13/16] [skill] evaluation: preserve inherited PYTHONPATH in the GDPVal prepare block The prepare block overwrote PYTHONPATH outright, discarding anything the base image or launcher had set. Prepend instead. The usual "${PYTHONPATH:+:$PYTHONPATH}" idiom cannot be used here because OmegaConf parses ${...} in the surrounding YAML, so this uses a plain $VAR test. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .../recipes/examples/gym_gdpval/example_gym_gdpval.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index 1ff00ab8b9f..502d9986e52 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -274,7 +274,11 @@ evaluation: d="$(dirname "$v")" [ -f "$d/requirements.txt" ] && uv pip install --python "$v/bin/python" -q -r "$d/requirements.txt" || true done - export PYTHONPATH="/opt/Gym:$(/opt/Gym/.venv/bin/python -c 'import site; print(site.getsitepackages()[0])')" + # Preserve any inherited PYTHONPATH. Plain $VAR only — OmegaConf parses ${...}, + # so the usual "${PYTHONPATH:+:$PYTHONPATH}" idiom cannot be used here. + _gym_sp="$(/opt/Gym/.venv/bin/python -c 'import site; print(site.getsitepackages()[0])')" + if [ -n "$PYTHONPATH" ]; then export PYTHONPATH="/opt/Gym:$_gym_sp:$PYTHONPATH" + else export PYTHONPATH="/opt/Gym:$_gym_sp"; fi # Writable staging dir for ref files (bind-mounted, see execution). mkdir -p /gdpval_ref_files From 4bc6937d9f7ccc5af3390f7116ac66cf3276f362 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 20:51:10 +0000 Subject: [PATCH 14/16] [skill] evaluation: stop GDPVal leaking secrets to the log; fix the fake canary Two findings from review, both confirmed against a real run. 1. Secret leak. The task `command:` opens with `set -ex`, and `set -x` traces commands AFTER expansion -- so the `ng_prepare_benchmark` trace line wrote $HF_TOKEN, $INFERENCE_API_KEY and $TAVILY_API_KEY in plaintext into the eval log, which lives on shared FS and is uploaded to MLflow when export.mlflow.log_logs is true. Verified in a completed run's client log: the traced line carries the expanded token, not `$HF_TOKEN`. Wrap that one call in `set +x` / `set -x`. The rollout call below it was already safe -- it goes through the quoted heredoc, so its secrets stay literal until a shell that never sets -x expands them; this call just wasn't covered by the same pattern. 2. `limit_samples` is inert on the gym path -- the gym does its own data prep and rollout collection, so the launcher-level limiter is ignored and you get the full 220-task run. This was observed during the validated run but the finding was lost in the doc-compression pass while the instruction depending on it survived in three places, so following the documented canary launches the heaviest benchmark in the suite believing it is a two-task smoke test. Replaced with the real procedure: launch, then treat the first ~20-30 minutes as the canary (SIF-sandbox line, judge auth, rollouts flowing), with the grep commands to check; note stage 1 of multistage as a natural checkpoint. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .agents/skills/evaluation/SKILL.md | 5 ++++- .../gym_gdpval/example_gym_gdpval.yaml | 16 +++++++++++--- .../evaluation/recipes/tasks/aa_gym/gdpval.md | 22 +++++++++++++------ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/.agents/skills/evaluation/SKILL.md b/.agents/skills/evaluation/SKILL.md index a097744bb70..bed737b3155 100644 --- a/.agents/skills/evaluation/SKILL.md +++ b/.agents/skills/evaluation/SKILL.md @@ -70,7 +70,10 @@ GDPVal: unsandboxed. Verify with `gdpval-sif.sh --check`. `.env` needs `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR`, and `NEMO_EVALUATOR_TRUST_PRE_CMD=1` (the config has a `pre_cmd`). Thinking mode is mandatory (non-thinking loses ~86%). -4. Dry-run → canary (`limit_samples=2`, verify the SIF sandbox + judge) → full. +4. Dry-run → launch. **`limit_samples` is inert on the gym path** (the gym runs all + 220 tasks regardless), so there is no cheap canary: watch the real run's first + ~20–30 min for the SIF-sandbox line and judge auth, and cancel if wrong. See the + recipe's Canary section. --- diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index 502d9986e52..181fda3e337 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -39,9 +39,11 @@ # search), INFERENCE_JUDGE_URL (judge host), and GDPVAL_SIF_DIR (item 1). See # recipes/env.example. # -# Canary (validates SIF sandbox + judge + gym plumbing on a couple of tasks): -# nel run --config example_gym_gdpval.yaml --env-file .env \ -# -o ++evaluation.nemo_evaluator_config.config.params.limit_samples=2 +# NOTE: `limit_samples` is INERT on the gym path — the gym runs all 220 tasks no +# matter what, so there is no cheap smoke test. Launch the real run and watch its +# first ~20-30 min (SIF-sandbox line, judge auth, rollouts appearing); see the +# recipe's Canary section. +# nel run --config example_gym_gdpval.yaml --env-file .env # ============================================================================= defaults: # slurm/default works anywhere; if your install ships a predefined @@ -283,7 +285,15 @@ evaluation: # Writable staging dir for ref files (bind-mounted, see execution). mkdir -p /gdpval_ref_files + # SECRETS: `set -x` traces commands AFTER expansion, and these param blobs + # carry $HF_TOKEN / $INFERENCE_API_KEY / $TAVILY_API_KEY. Without this the + # trace line writes all three in plaintext into the eval log — which lives on + # shared FS and is uploaded to MLflow when export.mlflow.log_logs is true. + # (The rollout call below is already safe: it goes through the quoted heredoc, + # so its secrets stay literal until a shell that never sets -x expands them.) + set +x ng_prepare_benchmark {{config.params.extra.nemo_gym.data_prep_params}} {{config.params.extra.nemo_gym.common_params}} + set -x # The rollout command is written to a script and run from it, rather than passed # to `bash -c '...'`. Gym params legitimately CONTAIN single quotes — comparison # mode's ++multistage.stages='[{num_tasks: 45}, ...]' and ++...judge_panel='[{...}]' diff --git a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index 68ade7390ab..00b2ff73b96 100644 --- a/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/.agents/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -48,18 +48,26 @@ recipes/examples/gym_gdpval/example_gym_gdpval.yaml # SLURM + single-node vLLM `num_repeats=1` — already set by the template via `++num_repeats=1`; both current goldens use it. A full 220-task run of a large MoE typically needs multi-node. -## Canary +## Canary — `limit_samples` does NOT work here -Validate the SIF sandbox + judge + gym plumbing on a couple of tasks before the -full run: +**`++…params.limit_samples=N` is inert on the gym path.** The gym does its own data +prep and rollout collection, so the launcher-level limiter is ignored: you get the +full 220-task run. Do not use it believing you launched a two-task smoke test — this +is the heaviest benchmark in the suite. + +There is no cheap sample-limited canary. Instead, **launch the real run and treat its +first ~20–30 minutes as the canary**, cancelling if any of these is wrong: ```bash -nel run --config example_gym_gdpval.yaml --env-file .env \ - -o ++evaluation.nemo_evaluator_config.config.params.limit_samples=2 +RD=//nemo_gym.0 +grep -c "Using Apptainer container" $RD/logs/client-*.log # sandbox actually used +grep -c "falling back\|not a git repo" $RD/logs/client-*.log # unsandboxed / inert pin +grep -ciE " 401 | 403 |Internal Server Error" $RD/artifacts/nemo_gym_logs/gdpval_judge_model.log +wc -l $RD/artifacts/evaluator_rollouts.jsonl # rollouts flowing ``` -Inspect logs for the SIF fallback warning, judge auth/429s, and Ray/gym shutdown -hangs (see `references/gym-gdpval.md` → failure modes). +In comparison mode stage 1 (45 tasks) is a natural early checkpoint — an ELO estimate +appears before the full 220-task stage 2 starts. ## Score Extraction From 507007b56174a62625bbfae76a23c69bb4f3d8ea Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 21:03:13 +0000 Subject: [PATCH 15/16] [skill] evaluation: GDPVal robustness follow-ups from review Lower-severity items raised alongside the secret-leak and canary findings. gdpval-sif.sh: - Add an EXIT trap so an interrupted or failed build leaves no `.build.*` / `.def` leftovers in the shared SIF dir. ($tmp is renamed on success, so the trap only ever removes leftovers.) - `--check` validated with `-f` only, which passes on a truncated or 0-byte file -- exactly what an interrupted copy leaves behind, and the failure it is supposed to prevent is silent. Now also requires a plausible size (>100 MB; a real GDPVal SIF is 1-4 GB) and, when apptainer is available, that `apptainer inspect` succeeds. Verified on-cluster: a real SIF passes, a 0-byte one is rejected with a rebuild hint. - Document that `--check` inspects the filesystem it RUNS ON, so it must be run on the cluster (srun/ssh) rather than the submitting box. template: - Scope the Ray reap to our own uid. `pkill -9 -f raylet` matches every raylet on the node, including other users'/jobs' daemons on a shared node. - Note that the apptainer PPA install is deliberately unpinned (it is the only source with current arm64 builds; pinning risks unavailability) and that the resolved version is echoed into the run log for reproducibility. env.example: - Document `GDPVAL_MAX_TURNS`. It is read at SUBMIT time from the launching shell via ${oc.env:...}, so it belongs with the other exported values; setting it as a container env var has no effect (that trap was removed earlier). Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .agents/scripts/gdpval-sif.sh | 21 ++++++++++++++++++- .agents/skills/evaluation/recipes/env.example | 6 ++++++ .../gym_gdpval/example_gym_gdpval.yaml | 11 +++++++--- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.agents/scripts/gdpval-sif.sh b/.agents/scripts/gdpval-sif.sh index 7f6493d7f2d..585dc0fdbc0 100755 --- a/.agents/scripts/gdpval-sif.sh +++ b/.agents/scripts/gdpval-sif.sh @@ -33,7 +33,11 @@ # with the config's install_on_the_fly.commit. # --force Rebuild even if the SIF already exists. # --check Verify-only preflight: exit 0 if the expected SIF exists, -# nonzero (listing what IS there) if not. Never builds. +# is non-trivial in size, and is a readable SIF. NOTE: it +# inspects the filesystem it RUNS ON — run it on the cluster +# (srun/ssh), not the submitting box, or you validate the +# wrong filesystem. +# Never builds. # Use before `nel run` — NEL's mount validation is `test -d` # and cannot see a missing/misnamed SIF file. # @@ -84,6 +88,18 @@ fi # non-sandboxed exec. Run this before submitting to fail loudly instead. if [[ "$check" -eq 1 ]]; then if [[ -f "$sif" ]]; then + # -f alone would pass on a truncated or 0-byte file (e.g. an interrupted copy). + # A real GDPVal SIF is ~1-4 GB; anything under 100 MB is not one. + _sz=$(stat -c %s "$sif" 2>/dev/null || echo 0) + if [[ "$_sz" -lt 104857600 ]]; then + printf '\033[31mgdpval-sif: %s exists but is only %s bytes — truncated/incomplete\033[0m\n' "$sif" "$_sz" >&2 + echo " Rebuild with: $0 --force ${sif_dir}" >&2 + exit 1 + fi + if command -v apptainer >/dev/null 2>&1 && ! apptainer inspect "$sif" >/dev/null 2>&1; then + printf '\033[31mgdpval-sif: %s is not a readable SIF (apptainer inspect failed)\033[0m\n' "$sif" >&2 + exit 1 + fi _log "SIF present: $sif ($(du -h "$sif" 2>/dev/null | cut -f1))" echo "$sif"; exit 0 fi @@ -131,6 +147,9 @@ _log "building GDPVal SIF (this can take ~20-40 min)" _log " gym commit: ${GDPVAL_GYM_COMMIT}" _log " def: ${def_url}" _log " dest: ${sif}" +# Leave no temp artefacts if we are killed or exit early. $tmp is renamed on success, +# so this only ever removes leftovers. +trap 'rm -f "$tmp" "$def_local"' EXIT rm -f "$tmp" "$def_local" # apptainer build cannot take a remote def URL as its source — fetch the def to a # local file first, then build from it. diff --git a/.agents/skills/evaluation/recipes/env.example b/.agents/skills/evaluation/recipes/env.example index bf1d3515d1d..09bd7888155 100644 --- a/.agents/skills/evaluation/recipes/env.example +++ b/.agents/skills/evaluation/recipes/env.example @@ -50,6 +50,12 @@ NEMO_EVALUATOR_TRUST_PRE_CMD=1 # /gdpval/sif. Convention: a per-user .cache dir. # GDPVAL_SIF_DIR=//.cache/gdpval/sif +# GDPVal (nemo_gym) — Stirrup agent turn cap. Read at SUBMIT time from the +# launching shell (the config uses ${oc.env:GDPVAL_MAX_TURNS,250}), so it must be +# exported before `nel run`; setting it as a container env var has no effect. +# Default 250 (the golden value); lower it only to shorten a debugging run. +# GDPVAL_MAX_TURNS=250 + # Tau2 (tau2_bench_telecom) — judger + user-simulator model_ids are hardcoded in # the recipe; only the shared endpoint URL comes from here # TAU2_ENDPOINT_URL=https:///v1/chat/completions # user + judger diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index 181fda3e337..ec098b8c2d2 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -176,6 +176,9 @@ evaluation: apt-get install -y -qq software-properties-common squashfuse fuse3 ca-certificates || true add-apt-repository -y ppa:apptainer/ppa apt-get update -qq + # Deliberately unpinned: the PPA is the only source with current arm64 builds, and + # pinning risks the version being unavailable there. The resolved version is echoed + # below so the run log records exactly what was installed. apt-get install -y -qq apptainer apptainer --version mkdir -p /usr/local/var/apptainer/mnt/session @@ -314,9 +317,11 @@ evaluation: __pg=$(cat /tmp/gym_eval_pgid 2>/dev/null || echo) [ -n "$__pg" ] && kill -9 -"$__pg" 2>/dev/null || true timeout 60 ray stop --force >/dev/null 2>&1 || true - pkill -9 -f raylet >/dev/null 2>&1 || true - pkill -9 -f gcs_server >/dev/null 2>&1 || true - pkill -9 -f plasma_store >/dev/null 2>&1 || true + # Scope to OUR uid — an unscoped pkill would take down Ray daemons + # belonging to other users/jobs sharing the node. + pkill -9 -u "$(id -u)" -f raylet >/dev/null 2>&1 || true + pkill -9 -u "$(id -u)" -f gcs_server >/dev/null 2>&1 || true + pkill -9 -u "$(id -u)" -f plasma_store >/dev/null 2>&1 || true exit $__rc data_prep_params: >- "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" From 3f8c7dd5f3dc6bf5c6bfee778cffc65d259749c3 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 3 Aug 2026 21:16:41 +0000 Subject: [PATCH 16/16] [skill] evaluation: compress the GDPVal template (391 -> 360 lines) After the last compression pass the two markdown files held at ~365 lines, but the template had grown to 391 with 168 comment lines against 214 config lines -- each review fix added a three-to-six line explanation next to the line it fixed. Cut the prose the recipe and reference already own (the 32-line header is now 16) and reduce eight multi-line comment blocks to one or two lines each. What stays is the part that prevents an edit mistake *at that line*: never `#` inside the folded scalar, mount keys are not interpolated, `set +x` around the traced call, the api_key is a VALUE not a NAME, plain `$VAR` only because OmegaConf parses `${...}`, and the uid-scoped pkill. The reasoning behind each moves to references/gym-gdpval.md, which is where a reader already is. Verified after the cut: the YAML parses, the inlined command still passes `bash -n` with quote-containing params substituted, no `#` leaked into any folded scalar, and all nine load-bearing markers are still present. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- .../gym_gdpval/example_gym_gdpval.yaml | 87 ++++++------------- 1 file changed, 28 insertions(+), 59 deletions(-) diff --git a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index ec098b8c2d2..76cd40dad10 100644 --- a/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/.agents/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -14,36 +14,21 @@ # limitations under the License. # # ============================================================================= -# Example: GDPVal (NeMo Gym "Stirrup" agent) — single-task gym eval template. +# GDPVal (NeMo Gym "Stirrup" agent) — STANDALONE gym eval, RUBRIC mode. +# Self-deploys a checkpoint via vLLM on one SLURM node. One gym eval per config. # -# GDPVal is an AGENTIC benchmark: the Stirrup agent produces office/PDF -# deliverables inside a per-task Apptainer code-exec sandbox, then a pairwise/ -# rubric judge (Gemini 3.1 Pro) scores them. It runs on the 0.2.6 `nel` launcher -# as a `nemo_gym` task (NOT nel-next), but it is STANDALONE — one gym eval per -# config, no other tasks. Read recipes/tasks/aa_gym/gdpval.md and -# references/gym-gdpval.md before editing this file. +# Read recipes/tasks/aa_gym/gdpval.md + references/gym-gdpval.md first — they +# cover the SIF sandbox, scoring modes, judge panel, preflight and failure modes. # -# This template SELF-DEPLOYS a (quantized) checkpoint via vLLM on ONE SLURM node -# and runs GDPVal against it. For the full 220-task run of a large MoE you will -# likely need multi-node — see references/gym-gdpval.md (deployment sizing). +# Before running: `.env` needs HF_TOKEN, INFERENCE_API_KEY, TAVILY_API_KEY, +# INFERENCE_JUDGE_URL, GDPVAL_SIF_DIR, NEMO_EVALUATOR_TRUST_PRE_CMD=1; and the +# SIF must exist (prefer a site-provided one, else +# `srun -p cpu -t 01:00:00 --pty .agents/scripts/gdpval-sif.sh`). # -# PREREQUISITES (see references/gym-gdpval.md): -# 1. Set GDPVAL_SIF_DIR in .env (persistent SIF cache dir on this cluster), then -# ensure the SIF exists (build-if-absent, reuse-if-present — never copied from -# another cluster): -# srun -p cpu -t 01:00:00 --pty .agents/scripts/gdpval-sif.sh # uses $GDPVAL_SIF_DIR -# The mount below binds $GDPVAL_SIF_DIR at /gdpval/sif so the SIF lands at EXACTLY -# /gdpval/sif/python-3.13.gdpval.sif (matches GDPVAL_CONTAINER_PATH below). -# Without it, the agent SILENTLY falls back to non-sandboxed local exec. -# 2. .env has HF_TOKEN, INFERENCE_API_KEY (judge auth), TAVILY_API_KEY (agent web -# search), INFERENCE_JUDGE_URL (judge host), and GDPVAL_SIF_DIR (item 1). See -# recipes/env.example. -# -# NOTE: `limit_samples` is INERT on the gym path — the gym runs all 220 tasks no -# matter what, so there is no cheap smoke test. Launch the real run and watch its -# first ~20-30 min (SIF-sandbox line, judge auth, rollouts appearing); see the -# recipe's Canary section. # nel run --config example_gym_gdpval.yaml --env-file .env +# +# `limit_samples` is INERT here — the gym always runs all 220 tasks. There is no +# cheap smoke test; see the recipe's Canary section. # ============================================================================= defaults: # slurm/default works anywhere; if your install ships a predefined @@ -71,9 +56,8 @@ gdpval: gdpval_judge: base_url: # from .env (/v1 base); shared inference host model: gcp/google/gemini-3.1-pro-preview # Gemini 3.1 Pro; use an equivalent on your endpoint if needed - api_key: INFERENCE_API_KEY # doc only — the gym injects the VALUE as $INFERENCE_API_KEY - # in common_params (NOT ${gdpval_judge.api_key}, which passes - # the literal NAME "INFERENCE_API_KEY" → judge 500). + api_key: INFERENCE_API_KEY # doc only; common_params injects the VALUE ($INFERENCE_API_KEY). + # An env-var NAME reaches the proxy as a literal -> opaque judge 500. cluster: sbatch_comment: '{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"480","reason":"benchmarking","description":"Eval benchmark low GPU utilization"}}' @@ -96,11 +80,8 @@ execution: evaluation: : /hf-cache : /cache/uv - # GDPVal Stirrup SIF dir — substitute the literal $GDPVAL_SIF_DIR value (.env), - # the persistent shared-FS dir gdpval-sif.sh builds python-3.13.gdpval.sif into - # (build-if-absent/reuse). Use the literal path here, NOT ${oc.env:...}: mount - # KEYS are not interpolated (same rule as the judge URLs). Mounting the DIR lands - # the SIF at /gdpval/sif/python-3.13.gdpval.sif == GDPVAL_CONTAINER_PATH. + # SIF dir -> /gdpval/sif, so the file lands at GDPVAL_CONTAINER_PATH below. + # Literal path only: mount KEYS are never interpolated. : /gdpval/sif # Writable shared-FS staging for ref files. Node-local /tmp breaks multi-node Ray. : /gdpval_ref_files @@ -176,9 +157,8 @@ evaluation: apt-get install -y -qq software-properties-common squashfuse fuse3 ca-certificates || true add-apt-repository -y ppa:apptainer/ppa apt-get update -qq - # Deliberately unpinned: the PPA is the only source with current arm64 builds, and - # pinning risks the version being unavailable there. The resolved version is echoed - # below so the run log records exactly what was installed. + # Unpinned on purpose: the PPA is the only source with current arm64 builds. The + # resolved version is echoed below for the record. apt-get install -y -qq apptainer apptainer --version mkdir -p /usr/local/var/apptainer/mnt/session @@ -248,9 +228,7 @@ evaluation: command: | set -ex cd /opt/Gym - # Honour a mounted uv cache if one was injected (UV_CACHE_DIR env var + - # /cache/uv mount); fall back to the image-local dir. No bash ${VAR} - # braces here — OmegaConf parses ${...}. + # Honour a mounted uv cache if injected, else image-local. No ${VAR} braces. [ -n "$UV_CACHE_DIR" ] || export UV_CACHE_DIR=/opt/cache/uv source .venv/bin/activate # install_on_the_fly: checkout the pin when /opt/Gym is a git repo; some @@ -279,8 +257,7 @@ evaluation: d="$(dirname "$v")" [ -f "$d/requirements.txt" ] && uv pip install --python "$v/bin/python" -q -r "$d/requirements.txt" || true done - # Preserve any inherited PYTHONPATH. Plain $VAR only — OmegaConf parses ${...}, - # so the usual "${PYTHONPATH:+:$PYTHONPATH}" idiom cannot be used here. + # Preserve inherited PYTHONPATH. Plain $VAR only (OmegaConf parses ${...}). _gym_sp="$(/opt/Gym/.venv/bin/python -c 'import site; print(site.getsitepackages()[0])')" if [ -n "$PYTHONPATH" ]; then export PYTHONPATH="/opt/Gym:$_gym_sp:$PYTHONPATH" else export PYTHONPATH="/opt/Gym:$_gym_sp"; fi @@ -288,12 +265,9 @@ evaluation: # Writable staging dir for ref files (bind-mounted, see execution). mkdir -p /gdpval_ref_files - # SECRETS: `set -x` traces commands AFTER expansion, and these param blobs - # carry $HF_TOKEN / $INFERENCE_API_KEY / $TAVILY_API_KEY. Without this the - # trace line writes all three in plaintext into the eval log — which lives on - # shared FS and is uploaded to MLflow when export.mlflow.log_logs is true. - # (The rollout call below is already safe: it goes through the quoted heredoc, - # so its secrets stay literal until a shell that never sets -x expands them.) + # SECRETS: `set -x` traces AFTER expansion, so without this the params' + # $HF_TOKEN / $INFERENCE_API_KEY / $TAVILY_API_KEY land in the log (and in + # MLflow via log_logs). The rollout call below is already safe via the heredoc. set +x ng_prepare_benchmark {{config.params.extra.nemo_gym.data_prep_params}} {{config.params.extra.nemo_gym.common_params}} set -x @@ -317,8 +291,7 @@ evaluation: __pg=$(cat /tmp/gym_eval_pgid 2>/dev/null || echo) [ -n "$__pg" ] && kill -9 -"$__pg" 2>/dev/null || true timeout 60 ray stop --force >/dev/null 2>&1 || true - # Scope to OUR uid — an unscoped pkill would take down Ray daemons - # belonging to other users/jobs sharing the node. + # uid-scoped: an unscoped pkill would kill other jobs' Ray daemons. pkill -9 -u "$(id -u)" -f raylet >/dev/null 2>&1 || true pkill -9 -u "$(id -u)" -f gcs_server >/dev/null 2>&1 || true pkill -9 -u "$(id -u)" -f plasma_store >/dev/null 2>&1 || true @@ -346,15 +319,11 @@ evaluation: # No judge_responses_create_params_overrides.model here: the judge model is # already set by openai_model=${gdpval_judge.model}, and pinning it a second # way silently collapses comparison mode's 3-member judge panel to one judge. - # num_repeats=1 — the value both current goldens use. The pinned Gym - # already ships 1, so this override is belt-and-braces, not a change. - # NOTE: no reference_* overrides below — rubric mode needs none, and the - # single-reference keys (reference_deliverables_dir / reference_elo) - # CONFLICT with comparison mode's reference_models map (added by - # modelopttools:eval-config Step 3c). - # Do NOT put '#' comments INSIDE this folded (>-) scalar: YAML keeps - # them as literal text, the block folds to one line, and the first '#' - # comments out every override after it in the shell command. + # num_repeats=1 — both current goldens. No reference_* overrides here: + # the single-reference keys conflict with comparison mode's reference_models + # map (added by modelopttools:eval-config Step 3c). + # NEVER put '#' inside this folded (>-) scalar — it folds to one line and + # comments out every override after it. common_params: >- ++use_absolute_ip=true ++policy_base_url={{target.api_endpoint.url}}