[skill] evaluation: add GDPVal (NeMo Gym Stirrup agent) support

.agents/scripts/gdpval-sif.sh

@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# gdpval-sif.sh — ensure the GDPVal Stirrup Apptainer SIF exists on THIS cluster.
+#
+# Build-if-absent, reuse-if-present. Self-contained: the SIF is built and reused
+# on the TARGET cluster's own filesystem — this NEVER copies a SIF from another
+# cluster. Idempotent, so it's safe to run before every `nel run`; a subsequent
+# run reuses the built SIF instantly.
+#
+# Usage:
+#   .agents/scripts/gdpval-sif.sh [<sif-dir-or-file>] [--commit <sha>] [--force|--check]
+#     <sif-dir-or-file>  Persistent path on the target cluster's shared FS.
+#                        DEFAULTS to $GDPVAL_SIF_DIR (from .env) when omitted. A
+#                        directory -> <dir>/$GDPVAL_SIF_NAME (default python-3.13.gdpval.sif,
+#                        matching the example config); a *.sif path
+#                        is used verbatim. Bind-mount this SAME dir into the eval
+#                        container at /gdpval/sif (see recipes/examples/gym_gdpval/).
+#     --commit <sha>     NeMo Gym commit whose gdpval.def to build. Keep in sync
+#                        with the config's install_on_the_fly.commit.
+#     --force            Rebuild even if the SIF already exists.
+#     --check            Verify-only preflight: exit 0 if the expected SIF exists,
+#                        is non-trivial in size, and is a readable SIF. NOTE: it
+#                        inspects the filesystem it RUNS ON — run it on the cluster
+#                        (srun/ssh), not the submitting box, or you validate the
+#                        wrong filesystem.
+#                        Never builds.
+#                        Use before `nel run` — NEL's mount validation is `test -d`
+#                        and cannot see a missing/misnamed SIF file.
+#
+# Requires `apptainer` (or `singularity`) on PATH with unprivileged/fakeroot
+# build support, plus network egress to GitHub/base image. Run on a node that has
+# it — a login node, or (preferred for the ~30-min build) the CPU partition:
+#   srun -p cpu -t 01:00:00 --pty \
+#     .agents/scripts/gdpval-sif.sh /lustre/<...>/gdpval/sif
+#
+# Env overrides: GDPVAL_GYM_COMMIT, GDPVAL_SIF_NAME, APPTAINER_BIN.
+set -euo pipefail
+
+# Keep GDPVAL_GYM_COMMIT in sync with install_on_the_fly.commit in the config.
+GDPVAL_GYM_COMMIT="${GDPVAL_GYM_COMMIT:-dd41196f620f2af99947d776cbe5da9439d2a08d}"  # pragma: allowlist secret
+GDPVAL_SIF_NAME="${GDPVAL_SIF_NAME:-python-3.13.gdpval.sif}"
+APPTAINER_BIN="${APPTAINER_BIN:-}"
+
+_log() { printf '\033[2m  %s\033[0m\n' "$*" >&2; }
+_die() { printf '\033[31mgdpval-sif: %s\033[0m\n' "$*" >&2; exit 1; }
+_usage() { sed -n '/^# gdpval-sif\.sh/,/^set -euo/p' "$0" | sed 's/^# \{0,1\}//; /^set -euo/d'; }
+
+# --- parse args ---
+target=""; force=0; check=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --commit) GDPVAL_GYM_COMMIT="${2:?--commit needs a value}"; shift 2 ;;
+    --force)  force=1; shift ;;
+    --check)  check=1; shift ;;
+    -h|--help) _usage; exit 0 ;;
+    -*) _die "unknown flag: $1 (see --help)" ;;
+    *) [[ -z "$target" ]] || _die "unexpected extra arg: $1"; target="$1"; shift ;;
+  esac
+done
+# Default to $GDPVAL_SIF_DIR (.env) when no path is given, so agents run it hands-free.
+target="${target:-${GDPVAL_SIF_DIR:-}}"
+[[ -n "$target" ]] || { _usage; _die "no path given and GDPVAL_SIF_DIR is unset — pass a dir or set GDPVAL_SIF_DIR (see recipes/env.example)"; }
+
+# --- resolve dir vs *.sif ---
+if [[ "$target" == *.sif ]]; then
+  sif="$target"; sif_dir="$(dirname "$target")"
+else
+  sif_dir="$target"; sif="$sif_dir/$GDPVAL_SIF_NAME"
+fi
+# --- verify-only mode (preflight) ---
+# NEL's submit-time mount validation runs `test -d`, so it only proves the SIF *dir*
+# exists — a dir holding the WRONG sif name (e.g. python-3.12 after a gym bump to a
+# 3.13 def) passes validation, and the Stirrup agent then SILENTLY falls back to
+# non-sandboxed exec. Run this before submitting to fail loudly instead.
+if [[ "$check" -eq 1 ]]; then
+  if [[ -f "$sif" ]]; then
+    # -f alone would pass on a truncated or 0-byte file (e.g. an interrupted copy).
+    # A real GDPVal SIF is ~1-4 GB; anything under 100 MB is not one.
+    _sz=$(stat -c %s "$sif" 2>/dev/null || echo 0)
+    if [[ "$_sz" -lt 104857600 ]]; then
+      printf '\033[31mgdpval-sif: %s exists but is only %s bytes — truncated/incomplete\033[0m\n' "$sif" "$_sz" >&2
+      echo "  Rebuild with: $0 --force ${sif_dir}" >&2
+      exit 1
+    fi
+    if command -v apptainer >/dev/null 2>&1 && ! apptainer inspect "$sif" >/dev/null 2>&1; then
+      printf '\033[31mgdpval-sif: %s is not a readable SIF (apptainer inspect failed)\033[0m\n' "$sif" >&2
+      exit 1
+    fi
+    _log "SIF present: $sif ($(du -h "$sif" 2>/dev/null | cut -f1))"
+    echo "$sif"; exit 0
+  fi
+  printf '\033[31mgdpval-sif: MISSING expected SIF: %s\033[0m\n' "$sif" >&2
+  if [[ -d "$sif_dir" ]]; then
+    echo "  dir exists but does not contain it; found:" >&2
+    if ls -1 "$sif_dir"/*.sif >/dev/null 2>&1; then ls -1 "$sif_dir"/*.sif | sed 's/^/    /' >&2
+    else echo "    (no .sif files)" >&2; fi
+  fi
+  echo "  Build it with: $0 ${sif_dir}   (or --commit <gym-sha> for a different def)" >&2
+  exit 1
+fi
+
+mkdir -p "$sif_dir" || _die "cannot create SIF dir: $sif_dir"
+
+# --- reuse if present ---
+if [[ -f "$sif" && "$force" -eq 0 ]]; then
+  _log "reusing existing SIF (no rebuild): $sif"
+  echo "$sif"; exit 0
+fi
+
+# --- locate apptainer/singularity ---
+if [[ -z "$APPTAINER_BIN" ]]; then
+  APPTAINER_BIN="$(command -v apptainer || command -v singularity || true)"
+fi
+[[ -n "$APPTAINER_BIN" ]] || _die "apptainer/singularity not found on PATH. Run on a node that has it \
+(e.g. 'module load apptainer', or inside the eval image). This script does NOT copy a SIF from another cluster."
+
+def_url="https://raw.githubusercontent.com/NVIDIA-NeMo/Gym/${GDPVAL_GYM_COMMIT}/responses_api_agents/stirrup_agent/containers/gdpval.def"
+tmp="${sif_dir}/.build.$$.${GDPVAL_SIF_NAME}"
+def_local="${sif_dir}/.gdpval.$$.def"
+lock="${sif_dir}/.gdpval-sif.lock"
+
+# --- build under a flock (double-checked) so concurrent runs don't double-build ---
+exec 9>"$lock" || _die "cannot open lock file: $lock"
+_log "acquiring build lock ($lock) ..."
+flock 9
+# Re-check inside the lock: another builder may have finished while we waited.
+if [[ -f "$sif" && "$force" -eq 0 ]]; then
+  _log "another builder produced it: $sif"
+  echo "$sif"; exit 0
+fi
+
+_log "building GDPVal SIF (this can take ~20-40 min)"
+_log "  gym commit: ${GDPVAL_GYM_COMMIT}"
+_log "  def:        ${def_url}"
+_log "  dest:       ${sif}"
+# Leave no temp artefacts if we are killed or exit early. $tmp is renamed on success,
+# so this only ever removes leftovers.
+trap 'rm -f "$tmp" "$def_local"' EXIT
+rm -f "$tmp" "$def_local"
+# apptainer build cannot take a remote def URL as its source — fetch the def to a
+# local file first, then build from it.
+if command -v curl >/dev/null 2>&1; then curl -fsSL "$def_url" -o "$def_local"
+else wget -qO "$def_local" "$def_url"; fi
+[ -s "$def_local" ] || { rm -f "$def_local"; _die "failed to download def from $def_url"; }
+# Prefer --fakeroot (needs an /etc/subuid entry for the build user); fall back to an
+# unprivileged build where fakeroot is unavailable.
+if "$APPTAINER_BIN" build --fakeroot "$tmp" "$def_local"; then
+  :
+# A failed --fakeroot attempt can leave a partial $tmp behind, and apptainer refuses an
+# existing destination — clear it or the unprivileged fallback can never succeed.
+elif rm -f "$tmp" && "$APPTAINER_BIN" build "$tmp" "$def_local"; then
+  _log "built without --fakeroot (unprivileged mode)"
+else
+  rm -f "$tmp" "$def_local"
+  _die "apptainer build failed (see output above)."
+fi
+rm -f "$def_local"
+
+# Atomic publish: a partial build never looks complete.
+mv -f "$tmp" "$sif" || { rm -f "$tmp"; _die "failed to move built SIF into place: $sif"; }
+_log "done: $sif"
+echo "$sif"

.agents/skills/evaluation/SKILL.md

@@ -49,6 +49,34 @@ Steps 1–9 below are the 0.2.6 path — use them for everything else.
 
 ---
 
+### GDPVal (NeMo Gym "Stirrup" agent) path — branch here too
+
+GDPVal **does** run on the 0.2.6 `nel` launcher (as a `nemo_gym` task, not
+nel-next), so Steps 1–9 apply — but it is mechanically special and **standalone**
+(one gym eval per config; never mix it with `aa/` tasks). If the user asks for
+GDPVal:
+
+1. Read **`references/gym-gdpval.md`** (Apptainer SIF sandbox, gym prepare/reap
+   machinery, deploy sizing, rubric-vs-comparison scoring, MLflow deliverables trap,
+   failure modes) + **`recipes/tasks/aa_gym/gdpval.md`**.
+2. Start from **`recipes/examples/gym_gdpval/example_gym_gdpval.yaml`** — a single
+   self-contained file.
+3. Prerequisite — the Apptainer SIF. **If your site provides one, use it**
+   (NVIDIA-internal: `modelopttools:eval-config` Step 3c); otherwise set
+   `GDPVAL_SIF_DIR` in `.env` and build with `.agents/scripts/gdpval-sif.sh`
+   (build-if-absent, no cross-cluster copy). Either way the mounted dir must contain
+   the file `GDPVAL_CONTAINER_PATH` names (template: `python-3.13.gdpval.sif`) — a
+   name mismatch passes NEL's `test -d` check and the agent then silently runs
+   unsandboxed. Verify with `gdpval-sif.sh --check`. `.env` needs `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`,
+   `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR`, and `NEMO_EVALUATOR_TRUST_PRE_CMD=1` (the
+   config has a `pre_cmd`). Thinking mode is mandatory (non-thinking loses ~86%).
+4. Dry-run → launch. **`limit_samples` is inert on the gym path** (the gym runs all
+   220 tasks regardless), so there is no cheap canary: watch the real run's first
+   ~20–30 min for the SIF-sandbox line and judge auth, and cancel if wrong. See the
+   recipe's Canary section.
+
+---
+
 ### Step 1 — Prerequisites
 
 Run `nel --version`; if missing, instruct `pip install nemo-evaluator-launcher`. If user has an existing config, skip to Step 8 (optionally review for `???` and quantization flags first).
@@ -62,8 +90,9 @@ Run `nel --version`; if missing, instruct `pip install nemo-evaluator-launcher`.
 - AA Index v2 suite (default for quantized-checkpoint validation, see `references/quantization-benchmarks.md`): `recipes/tasks/aa/{gpqa_diamond,hle,lcr,scicode,ifbench,mmmu_pro,tau2_bench_telecom,omniscience}.md`
 - Optional: `recipes/tasks/mmlu_pro.md`, `recipes/tasks/aime_2025.md`, `recipes/tasks/livecodebench.md`
 - **nel-next only** (different evaluator — see the nel-next section below, NOT the 0.2.6 steps): shared reference `references/nel-next.md` + per-benchmark recipes `recipes/tasks/aa_next/{terminal_bench_2_1,swebench_verified}.md` (agentic). The `aa_next/` dir holds tasks that require nemo-evaluator-next (0.3.x); `aa/` is the 0.2.6 suite.
+- **GDPVal (NeMo Gym / agentic)** — **part of the AA suite** but a 0.2.6 `nemo_gym` task on a different harness, so it's **standalone** (see the GDPVal branch above): recipe `recipes/tasks/aa_gym/gdpval.md` + shared reference `references/gym-gdpval.md` + self-contained example `recipes/examples/gym_gdpval/`. Generated as its **own config** from the example, **never merged into the `aa/` multi-task `tasks` list**. The `aa_gym/` dir holds the NeMo Gym Stirrup-agent tasks.
 
-**AA rule:** If the user mentions "AA" / "Artificial Analysis", generate **only** tasks under `recipes/tasks/aa/`. Do not add MMLU-Pro, AIME 2025, or LiveCodeBench unless explicitly asked.
+**AA rule:** If the user mentions "AA" / "Artificial Analysis", generate the `recipes/tasks/aa/` tasks (one multi-task config) **plus a companion standalone GDPVal config** (`recipes/tasks/aa_gym/gdpval.md`, via the GDPVal branch) — GDPVal is part of the AA suite but a different harness, so it's its own config, never added to the `aa/` `tasks` list. Do not add MMLU-Pro, AIME 2025, or LiveCodeBench unless explicitly asked. GDPVal is the heaviest AA task (standalone, multi-hour, needs the SIF sandbox + judge) — surface it and let the user opt out per run.
 
 **Shortcut path** (when task list is known up front, e.g. "run AA"):
 

.agents/skills/evaluation/recipes/env.example

@@ -37,8 +37,25 @@ NEMO_EVALUATOR_TRUST_PRE_CMD=1
 # /v1 base; tau2-bench needs the full /v1/chat/completions.
 
 # HLE + AA-LCR + AA-Omniscience judges (ns_hle_aa, ns_aa_lcr, ns_omniscience) — shared inference host
+# GDPVal (nemo_gym) also reuses INFERENCE_JUDGE_URL for its pairwise judge.
 # INFERENCE_JUDGE_URL=https://<your-inference-host>/v1
 
+# GDPVal (nemo_gym Stirrup agent) — agent web search. Secret; exported and read
+# by the harness. See recipes/tasks/aa_gym/gdpval.md + references/gym-gdpval.md.
+# TAVILY_API_KEY=
+
+# GDPVal (nemo_gym) — persistent Apptainer SIF cache dir on the TARGET cluster's
+# shared FS (a path, not a secret). .agents/scripts/gdpval-sif.sh builds the SIF
+# here if absent and reuses it otherwise; the config bind-mounts this dir at
+# /gdpval/sif. Convention: a per-user .cache dir.
+# GDPVAL_SIF_DIR=<shared-fs>/<user>/.cache/gdpval/sif
+
+# GDPVal (nemo_gym) — Stirrup agent turn cap. Read at SUBMIT time from the
+# launching shell (the config uses ${oc.env:GDPVAL_MAX_TURNS,250}), so it must be
+# exported before `nel run`; setting it as a container env var has no effect.
+# Default 250 (the golden value); lower it only to shorten a debugging run.
+# GDPVAL_MAX_TURNS=250
+
 # Tau2 (tau2_bench_telecom) — judger + user-simulator model_ids are hardcoded in
 # the recipe; only the shared endpoint URL comes from here
 # TAU2_ENDPOINT_URL=https://<your-inference-host>/v1/chat/completions   # user + judger